Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -231,7 +231,13 @@ private String unescapeStringValue(String in) {

@Override
public Filter.Operand visitIntegerConstant(FiltersParser.IntegerConstantContext ctx) {
return new Filter.Value(Integer.valueOf(ctx.getText()));
String literal = ctx.getText();
try {
return new Filter.Value(Integer.valueOf(literal));
}
catch (NumberFormatException ex) {
return new Filter.Value(Long.valueOf(literal));
}
}

@Override
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -66,6 +66,28 @@ public void testEQ() {
assertThat(this.parser.getCache().get("WHERE " + "country == 'BG'")).isEqualTo(exp);
}

@Test
public void testIntegerLiterals() {
// Literals within the Integer range remain Integer
Expression exp = this.parser.parse("timestamp == " + Integer.MAX_VALUE);
assertThat(exp.right()).isEqualTo(new Value(Integer.MAX_VALUE));

// Literals beyond the Integer range fall back to Long
exp = this.parser.parse("timestamp == " + Long.MAX_VALUE);
assertThat(exp.right()).isEqualTo(new Value(Long.MAX_VALUE));

exp = this.parser.parse("timestamp == " + Long.MIN_VALUE);
assertThat(exp.right()).isEqualTo(new Value(Long.MIN_VALUE));

// The explicit 'L' suffix keeps producing Long values
exp = this.parser.parse("timestamp == 9223372036854775807L");
assertThat(exp.right()).isEqualTo(new Value(Long.MAX_VALUE));

// Large literals are supported inside IN lists
exp = this.parser.parse("id in [" + Long.MAX_VALUE + ", 1]");
assertThat(exp.right()).isEqualTo(new Value(List.of(Long.MAX_VALUE, 1)));
}

@Test
public void tesEqAndGte() {
// genre == "drama" AND year >= 2020
Expand Down
6 changes: 6 additions & 0 deletions vector-stores/spring-ai-mariadb-store/pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -52,6 +52,12 @@
<scope>test</scope>
</dependency>

<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-testcontainers</artifactId>
<scope>test</scope>
</dependency>

<dependency>
<groupId>org.testcontainers</groupId>
<artifactId>testcontainers</artifactId>
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -45,13 +45,21 @@ public MariaDBSchemaValidator(JdbcTemplate jdbcTemplate) {
this.jdbcTemplate = jdbcTemplate;
}

private boolean isTableExists(@Nullable String schemaName, String tableName) {
private String resolveSchemaName(@Nullable String schemaName) {
if (schemaName != null) {
return schemaName;
}
String currentSchema = this.jdbcTemplate.queryForObject("SELECT SCHEMA()", String.class);
Assert.state(currentSchema != null, "No schema name configured and no schema selected on the connection");
return currentSchema;
}

private boolean isTableExists(String schemaName, String tableName) {
// schema and table are expected to be escaped
String sql = "SELECT 1 FROM INFORMATION_SCHEMA.TABLES WHERE TABLE_SCHEMA = ? AND TABLE_NAME = ?";
try {
// Query for a single integer value, if it exists, table exists
this.jdbcTemplate.queryForObject(sql, Integer.class, (schemaName == null) ? "SCHEMA()" : schemaName,
tableName);
this.jdbcTemplate.queryForObject(sql, Integer.class, schemaName, tableName);
return true;
}
catch (DataAccessException e) {
Expand All @@ -62,16 +70,17 @@ private boolean isTableExists(@Nullable String schemaName, String tableName) {
void validateTableSchema(@Nullable String schemaName, String tableName, String idFieldName, String contentFieldName,
String metadataFieldName, String embeddingFieldName, int embeddingDimensions) {

if (!isTableExists(schemaName, tableName)) {
String resolvedSchemaName = resolveSchemaName(schemaName);

if (!isTableExists(resolvedSchemaName, tableName)) {
throw new IllegalStateException(
String.format("Table '%s' does not exist in schema '%s'", tableName, schemaName));
String.format("Table '%s' does not exist in schema '%s'", tableName, resolvedSchemaName));
}

// ensure server support VECTORs
try {
// Query for a single integer value, if it exists, database support vector
this.jdbcTemplate.queryForObject("SELECT vec_distance_euclidean(x'0000803f', x'0000803f')", Integer.class,
schemaName, tableName);
this.jdbcTemplate.queryForObject("SELECT vec_distance_euclidean(x'0000803f', x'0000803f')", Integer.class);
}
catch (DataAccessException e) {
if (logger.isErrorEnabled()) {
Expand All @@ -87,7 +96,8 @@ void validateTableSchema(@Nullable String schemaName, String tableName, String i

try {
if (logger.isInfoEnabled()) {
logger.info("Validating MariaDBStore schema for table: " + tableName + " in schema: " + schemaName);
logger.info(
"Validating MariaDBStore schema for table: " + tableName + " in schema: " + resolvedSchemaName);
}

List<String> expectedColumns = new ArrayList<>();
Expand All @@ -100,11 +110,12 @@ void validateTableSchema(@Nullable String schemaName, String tableName, String i
// Include the schema name in the query to target the correct table
String query = "SELECT COLUMN_NAME, DATA_TYPE FROM INFORMATION_SCHEMA.COLUMNS "
+ "WHERE TABLE_SCHEMA = ? AND TABLE_NAME = ?";
List<Map<String, @Nullable Object>> columns = this.jdbcTemplate.queryForList(query, schemaName, tableName);
List<Map<String, @Nullable Object>> columns = this.jdbcTemplate.queryForList(query, resolvedSchemaName,
tableName);

if (columns.isEmpty()) {
throw new IllegalStateException("Error while validating table schema, Table " + tableName
+ " does not exist in schema " + schemaName);
+ " does not exist in schema " + resolvedSchemaName);
}

// Check each column against expected fields
Expand Down Expand Up @@ -142,9 +153,8 @@ void validateTableSchema(@Nullable String schemaName, String tableName, String i
%s JSON,
%s VECTOR(%d) NOT NULL,
VECTOR INDEX (%s)
) ENGINE=InnoDB""", schemaName == null ? tableName : schemaName + "." + tableName,
idFieldName, contentFieldName, metadataFieldName, embeddingFieldName,
embeddingDimensions, embeddingFieldName)
) ENGINE=InnoDB""", resolvedSchemaName + "." + tableName, idFieldName, contentFieldName,
metadataFieldName, embeddingFieldName, embeddingDimensions, embeddingFieldName)
+ "\n" + "Please adjust these commands based on your specific configuration and the"
+ " capabilities of your vector database system.");
}
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,98 @@
/*
* Copyright 2023-present the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

package org.springframework.ai.vectorstore.mariadb;

import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.testcontainers.containers.MariaDBContainer;
import org.testcontainers.junit.jupiter.Container;
import org.testcontainers.junit.jupiter.Testcontainers;

import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.jdbc.test.autoconfigure.JdbcTest;
import org.springframework.boot.testcontainers.service.connection.ServiceConnection;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.jdbc.core.JdbcTemplate;

import static org.assertj.core.api.Assertions.assertThatIllegalStateException;
import static org.assertj.core.api.Assertions.assertThatNoException;

/**
* @author Junhyeok Park
*/
@Testcontainers
@JdbcTest
public class MariaDBSchemaValidatorIT {

@ServiceConnection
@Container
@SuppressWarnings("resource")
static MariaDBContainer<?> mariadbContainer = new MariaDBContainer<>(MariaDBImage.DEFAULT_IMAGE)
.withUsername("mariadb")
.withPassword("mariadbpwd")
.withDatabaseName("testdb");

@Autowired
JdbcTemplate jdbcTemplate;

@Autowired
MariaDBSchemaValidator schemaValidator;

@BeforeEach
void createVectorStoreTable() {
this.jdbcTemplate.execute("""
CREATE TABLE IF NOT EXISTS vector_store (
id UUID NOT NULL DEFAULT uuid() PRIMARY KEY,
content TEXT,
metadata JSON,
embedding VECTOR(1536) NOT NULL,
VECTOR INDEX (embedding)
) ENGINE=InnoDB""");
}

@Test
void validateTableSchemaWithDefaultSchema() {
assertThatNoException().isThrownBy(() -> this.schemaValidator.validateTableSchema(null, "vector_store", "id",
"content", "metadata", "embedding", 1536));
}

@Test
void validateTableSchemaWithExplicitSchemaName() {
assertThatNoException().isThrownBy(() -> this.schemaValidator.validateTableSchema("testdb", "vector_store",
"id", "content", "metadata", "embedding", 1536));
}

@Test
void validateTableSchemaFailsOnMissingTableWithDefaultSchema() {
assertThatIllegalStateException()
.isThrownBy(() -> this.schemaValidator.validateTableSchema(null, "missing_vector_store", "id", "content",
"metadata", "embedding", 1536))
.withMessageContaining("Table 'missing_vector_store' does not exist in schema 'testdb'");
}

@Configuration(proxyBeanMethods = false)
static class Config {

@Bean
MariaDBSchemaValidator schemaValidator(JdbcTemplate jdbcTemplate) {
return new MariaDBSchemaValidator(jdbcTemplate);
}

}

}
Loading