diff --git a/spring-ai-vector-store/src/main/java/org/springframework/ai/vectorstore/filter/FilterExpressionTextParser.java b/spring-ai-vector-store/src/main/java/org/springframework/ai/vectorstore/filter/FilterExpressionTextParser.java index 45d1bb6d70..f2b54b3c46 100644 --- a/spring-ai-vector-store/src/main/java/org/springframework/ai/vectorstore/filter/FilterExpressionTextParser.java +++ b/spring-ai-vector-store/src/main/java/org/springframework/ai/vectorstore/filter/FilterExpressionTextParser.java @@ -231,7 +231,13 @@ private String unescapeStringValue(String in) { @Override public Filter.Operand visitIntegerConstant(FiltersParser.IntegerConstantContext ctx) { - return new Filter.Value(Integer.valueOf(ctx.getText())); + String literal = ctx.getText(); + try { + return new Filter.Value(Integer.valueOf(literal)); + } + catch (NumberFormatException ex) { + return new Filter.Value(Long.valueOf(literal)); + } } @Override diff --git a/spring-ai-vector-store/src/test/java/org/springframework/ai/vectorstore/filter/FilterExpressionTextParserTests.java b/spring-ai-vector-store/src/test/java/org/springframework/ai/vectorstore/filter/FilterExpressionTextParserTests.java index bf227923a7..4590f0fab3 100644 --- a/spring-ai-vector-store/src/test/java/org/springframework/ai/vectorstore/filter/FilterExpressionTextParserTests.java +++ b/spring-ai-vector-store/src/test/java/org/springframework/ai/vectorstore/filter/FilterExpressionTextParserTests.java @@ -66,6 +66,28 @@ public void testEQ() { assertThat(this.parser.getCache().get("WHERE " + "country == 'BG'")).isEqualTo(exp); } + @Test + public void testIntegerLiterals() { + // Literals within the Integer range remain Integer + Expression exp = this.parser.parse("timestamp == " + Integer.MAX_VALUE); + assertThat(exp.right()).isEqualTo(new Value(Integer.MAX_VALUE)); + + // Literals beyond the Integer range fall back to Long + exp = this.parser.parse("timestamp == " + Long.MAX_VALUE); + assertThat(exp.right()).isEqualTo(new Value(Long.MAX_VALUE)); + + exp = this.parser.parse("timestamp == " + Long.MIN_VALUE); + assertThat(exp.right()).isEqualTo(new Value(Long.MIN_VALUE)); + + // The explicit 'L' suffix keeps producing Long values + exp = this.parser.parse("timestamp == 9223372036854775807L"); + assertThat(exp.right()).isEqualTo(new Value(Long.MAX_VALUE)); + + // Large literals are supported inside IN lists + exp = this.parser.parse("id in [" + Long.MAX_VALUE + ", 1]"); + assertThat(exp.right()).isEqualTo(new Value(List.of(Long.MAX_VALUE, 1))); + } + @Test public void tesEqAndGte() { // genre == "drama" AND year >= 2020 diff --git a/vector-stores/spring-ai-mariadb-store/pom.xml b/vector-stores/spring-ai-mariadb-store/pom.xml index 0684cf5d68..9b6e54f480 100644 --- a/vector-stores/spring-ai-mariadb-store/pom.xml +++ b/vector-stores/spring-ai-mariadb-store/pom.xml @@ -52,6 +52,12 @@ test + + org.springframework.boot + spring-boot-testcontainers + test + + org.testcontainers testcontainers diff --git a/vector-stores/spring-ai-mariadb-store/src/main/java/org/springframework/ai/vectorstore/mariadb/MariaDBSchemaValidator.java b/vector-stores/spring-ai-mariadb-store/src/main/java/org/springframework/ai/vectorstore/mariadb/MariaDBSchemaValidator.java index e42d1f1ffd..fdbc0e18d6 100644 --- a/vector-stores/spring-ai-mariadb-store/src/main/java/org/springframework/ai/vectorstore/mariadb/MariaDBSchemaValidator.java +++ b/vector-stores/spring-ai-mariadb-store/src/main/java/org/springframework/ai/vectorstore/mariadb/MariaDBSchemaValidator.java @@ -45,13 +45,21 @@ public MariaDBSchemaValidator(JdbcTemplate jdbcTemplate) { this.jdbcTemplate = jdbcTemplate; } - private boolean isTableExists(@Nullable String schemaName, String tableName) { + private String resolveSchemaName(@Nullable String schemaName) { + if (schemaName != null) { + return schemaName; + } + String currentSchema = this.jdbcTemplate.queryForObject("SELECT SCHEMA()", String.class); + Assert.state(currentSchema != null, "No schema name configured and no schema selected on the connection"); + return currentSchema; + } + + private boolean isTableExists(String schemaName, String tableName) { // schema and table are expected to be escaped String sql = "SELECT 1 FROM INFORMATION_SCHEMA.TABLES WHERE TABLE_SCHEMA = ? AND TABLE_NAME = ?"; try { // Query for a single integer value, if it exists, table exists - this.jdbcTemplate.queryForObject(sql, Integer.class, (schemaName == null) ? "SCHEMA()" : schemaName, - tableName); + this.jdbcTemplate.queryForObject(sql, Integer.class, schemaName, tableName); return true; } catch (DataAccessException e) { @@ -62,16 +70,17 @@ private boolean isTableExists(@Nullable String schemaName, String tableName) { void validateTableSchema(@Nullable String schemaName, String tableName, String idFieldName, String contentFieldName, String metadataFieldName, String embeddingFieldName, int embeddingDimensions) { - if (!isTableExists(schemaName, tableName)) { + String resolvedSchemaName = resolveSchemaName(schemaName); + + if (!isTableExists(resolvedSchemaName, tableName)) { throw new IllegalStateException( - String.format("Table '%s' does not exist in schema '%s'", tableName, schemaName)); + String.format("Table '%s' does not exist in schema '%s'", tableName, resolvedSchemaName)); } // ensure server support VECTORs try { // Query for a single integer value, if it exists, database support vector - this.jdbcTemplate.queryForObject("SELECT vec_distance_euclidean(x'0000803f', x'0000803f')", Integer.class, - schemaName, tableName); + this.jdbcTemplate.queryForObject("SELECT vec_distance_euclidean(x'0000803f', x'0000803f')", Integer.class); } catch (DataAccessException e) { if (logger.isErrorEnabled()) { @@ -87,7 +96,8 @@ void validateTableSchema(@Nullable String schemaName, String tableName, String i try { if (logger.isInfoEnabled()) { - logger.info("Validating MariaDBStore schema for table: " + tableName + " in schema: " + schemaName); + logger.info( + "Validating MariaDBStore schema for table: " + tableName + " in schema: " + resolvedSchemaName); } List expectedColumns = new ArrayList<>(); @@ -100,11 +110,12 @@ void validateTableSchema(@Nullable String schemaName, String tableName, String i // Include the schema name in the query to target the correct table String query = "SELECT COLUMN_NAME, DATA_TYPE FROM INFORMATION_SCHEMA.COLUMNS " + "WHERE TABLE_SCHEMA = ? AND TABLE_NAME = ?"; - List> columns = this.jdbcTemplate.queryForList(query, schemaName, tableName); + List> columns = this.jdbcTemplate.queryForList(query, resolvedSchemaName, + tableName); if (columns.isEmpty()) { throw new IllegalStateException("Error while validating table schema, Table " + tableName - + " does not exist in schema " + schemaName); + + " does not exist in schema " + resolvedSchemaName); } // Check each column against expected fields @@ -142,9 +153,8 @@ void validateTableSchema(@Nullable String schemaName, String tableName, String i %s JSON, %s VECTOR(%d) NOT NULL, VECTOR INDEX (%s) - ) ENGINE=InnoDB""", schemaName == null ? tableName : schemaName + "." + tableName, - idFieldName, contentFieldName, metadataFieldName, embeddingFieldName, - embeddingDimensions, embeddingFieldName) + ) ENGINE=InnoDB""", resolvedSchemaName + "." + tableName, idFieldName, contentFieldName, + metadataFieldName, embeddingFieldName, embeddingDimensions, embeddingFieldName) + "\n" + "Please adjust these commands based on your specific configuration and the" + " capabilities of your vector database system."); } diff --git a/vector-stores/spring-ai-mariadb-store/src/test/java/org/springframework/ai/vectorstore/mariadb/MariaDBSchemaValidatorIT.java b/vector-stores/spring-ai-mariadb-store/src/test/java/org/springframework/ai/vectorstore/mariadb/MariaDBSchemaValidatorIT.java new file mode 100644 index 0000000000..2f40475d62 --- /dev/null +++ b/vector-stores/spring-ai-mariadb-store/src/test/java/org/springframework/ai/vectorstore/mariadb/MariaDBSchemaValidatorIT.java @@ -0,0 +1,98 @@ +/* + * Copyright 2023-present the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.ai.vectorstore.mariadb; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.testcontainers.containers.MariaDBContainer; +import org.testcontainers.junit.jupiter.Container; +import org.testcontainers.junit.jupiter.Testcontainers; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.jdbc.test.autoconfigure.JdbcTest; +import org.springframework.boot.testcontainers.service.connection.ServiceConnection; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.jdbc.core.JdbcTemplate; + +import static org.assertj.core.api.Assertions.assertThatIllegalStateException; +import static org.assertj.core.api.Assertions.assertThatNoException; + +/** + * @author Junhyeok Park + */ +@Testcontainers +@JdbcTest +public class MariaDBSchemaValidatorIT { + + @ServiceConnection + @Container + @SuppressWarnings("resource") + static MariaDBContainer mariadbContainer = new MariaDBContainer<>(MariaDBImage.DEFAULT_IMAGE) + .withUsername("mariadb") + .withPassword("mariadbpwd") + .withDatabaseName("testdb"); + + @Autowired + JdbcTemplate jdbcTemplate; + + @Autowired + MariaDBSchemaValidator schemaValidator; + + @BeforeEach + void createVectorStoreTable() { + this.jdbcTemplate.execute(""" + CREATE TABLE IF NOT EXISTS vector_store ( + id UUID NOT NULL DEFAULT uuid() PRIMARY KEY, + content TEXT, + metadata JSON, + embedding VECTOR(1536) NOT NULL, + VECTOR INDEX (embedding) + ) ENGINE=InnoDB"""); + } + + @Test + void validateTableSchemaWithDefaultSchema() { + assertThatNoException().isThrownBy(() -> this.schemaValidator.validateTableSchema(null, "vector_store", "id", + "content", "metadata", "embedding", 1536)); + } + + @Test + void validateTableSchemaWithExplicitSchemaName() { + assertThatNoException().isThrownBy(() -> this.schemaValidator.validateTableSchema("testdb", "vector_store", + "id", "content", "metadata", "embedding", 1536)); + } + + @Test + void validateTableSchemaFailsOnMissingTableWithDefaultSchema() { + assertThatIllegalStateException() + .isThrownBy(() -> this.schemaValidator.validateTableSchema(null, "missing_vector_store", "id", "content", + "metadata", "embedding", 1536)) + .withMessageContaining("Table 'missing_vector_store' does not exist in schema 'testdb'"); + } + + @Configuration(proxyBeanMethods = false) + static class Config { + + @Bean + MariaDBSchemaValidator schemaValidator(JdbcTemplate jdbcTemplate) { + return new MariaDBSchemaValidator(jdbcTemplate); + } + + } + +}