Fulltext fixes

bin/tasks/load.php

@@ -10,6 +10,7 @@
 use Utopia\Database\Adapter\MySQL;
 use Utopia\Database\Helpers\Permission;
 use Utopia\Database\Helpers\Role;
+use Utopia\Database\Validator\Authorization;
 use Utopia\Mongo\Client;
 use Swoole\Database\PDOConfig;
 use Swoole\Database\PDOPool;
@@ -230,7 +231,13 @@ function createSchema(Database $database): void
         $database->delete($database->getDefaultDatabase());
     }
     $database->create();
-    $database->createCollection('articles');
+
+    Authorization::setRole(Role::any()->toString());
+    $database->createCollection('articles', permissions: [
+        Permission::create(Role::any()),
+        Permission::read(Role::any()),
+    ]);
+
     $database->createAttribute('articles', 'author', Database::VAR_STRING, 256, true);
     $database->createAttribute('articles', 'created', Database::VAR_DATETIME, 0, true, null, false, false, null, [], ['datetime']);
     $database->createAttribute('articles', 'text', Database::VAR_STRING, 5000, true);
@@ -247,20 +254,19 @@ function addArticle($database, Generator $faker): void
 
         '$permissions' => [
             Permission::read(Role::any()),
-            Permission::read(Role::user($faker->randomNumber(4))),
-            Permission::read(Role::user($faker->randomNumber(4))),
-            Permission::read(Role::user($faker->randomNumber(4))),
-            Permission::read(Role::user($faker->randomNumber(4))),
-            Permission::read(Role::user($faker->randomNumber(4))),
-            Permission::create(Role::user($faker->randomNumber(4))),
-            Permission::create(Role::user($faker->randomNumber(4))),
-            Permission::create(Role::user($faker->randomNumber(4))),
-            Permission::update(Role::user($faker->randomNumber(4))),
-            Permission::update(Role::user($faker->randomNumber(4))),
-            Permission::update(Role::user($faker->randomNumber(4))),
-            Permission::delete(Role::user($faker->randomNumber(4))),
-            Permission::delete(Role::user($faker->randomNumber(4))),
-            Permission::delete(Role::user($faker->randomNumber(4))),
+            Permission::read(Role::user($faker->randomNumber(9))),
+            Permission::read(Role::user($faker->randomNumber(9))),
+            Permission::read(Role::user($faker->randomNumber(9))),
+            Permission::read(Role::user($faker->randomNumber(9))),
+            Permission::create(Role::user($faker->randomNumber(9))),
+            Permission::create(Role::user($faker->randomNumber(9))),
+            Permission::create(Role::user($faker->randomNumber(9))),
+            Permission::update(Role::user($faker->randomNumber(9))),
+            Permission::update(Role::user($faker->randomNumber(9))),
+            Permission::update(Role::user($faker->randomNumber(9))),
+            Permission::delete(Role::user($faker->randomNumber(9))),
+            Permission::delete(Role::user($faker->randomNumber(9))),
+            Permission::delete(Role::user($faker->randomNumber(9))),
         ],
         'author' => $faker->name(),
         'created' => \Utopia\Database\DateTime::format($faker->dateTime()),

src/Database/Adapter/MariaDB.php

@@ -1229,15 +1229,7 @@ protected function getSQLCondition(Query $query): string
 
         switch ($query->getMethod()) {
             case Query::TYPE_SEARCH:
-                /**
-                 * Replace reserved chars with space.
-                 */
-                $value = trim(str_replace(['@', '+', '-', '*'], ' ', $query->getValues()[0]));
-                /**
-                 * Prepend wildcard by default on the back.
-                 */
-                $value = $this->getSQLValue($query->getMethod(), $value);
-                return 'MATCH('.$attribute.') AGAINST ('.$this->getPDO()->quote($value).' IN BOOLEAN MODE)';
+                return "MATCH(table_main.{$attribute}) AGAINST (:{$placeholder}_0 IN BOOLEAN MODE)";
 
             case Query::TYPE_BETWEEN:
                 return "table_main.{$attribute} BETWEEN :{$placeholder}_0 AND :{$placeholder}_1";

src/Database/Adapter/Postgres.php

@@ -1240,9 +1240,7 @@ protected function getSQLCondition(Query $query): string
 
         switch ($query->getMethod()) {
             case Query::TYPE_SEARCH:
-                $value = trim(str_replace(['@', '+', '-', '*', '.'], '|', $query->getValues()[0]));
-                $value = $this->getSQLValue($query->getMethod(), $value);
-                return "to_tsvector(regexp_replace({$attribute}, '[^\w]+',' ','g')) @@ to_tsquery(trim(REGEXP_REPLACE({$value}, '\|+','|','g'),'|'))";
+                return "to_tsvector(regexp_replace({$attribute}, '[^\w]+',' ','g')) @@ websearch_to_tsquery(:{$placeholder}_0)";
 
             case Query::TYPE_BETWEEN:
                 return "table_main.{$attribute} BETWEEN :{$placeholder}_0 AND :{$placeholder}_1";
@@ -1261,6 +1259,24 @@ protected function getSQLCondition(Query $query): string
         }
     }
 
+    /**
+     * @param string $value
+     * @return string
+     */
+    protected function getFulltextValue(string $value): string
+    {
+        $exact = str_ends_with($value, '"') && str_starts_with($value, '"');
+        $value = str_replace(['@', '+', '-', '*', '.', "'", '"'], ' ', $value);
+        $value = preg_replace('/\s+/', ' ', $value); // Remove multiple whitespaces
+        $value = trim($value);
+
+        if (!$exact) {
+            $value = str_replace(' ', ' or ', $value);
+        }
+
+        return "'" . $value . "'";
+    }
+
     /**
      * Get SQL Type
      *

src/Database/Adapter/SQL.php

@@ -681,20 +681,51 @@ public function getSupportForRelationships(): bool
      * @param mixed $stmt
      * @param Query $query
      * @return void
+     * @throws Exception
      */
     protected function bindConditionValue(mixed $stmt, Query $query): void
     {
-        if (in_array($query->getMethod(), [Query::TYPE_SEARCH, Query::TYPE_SELECT])) {
+        if ($query->getMethod() == Query::TYPE_SELECT) {
             return;
         }
 
         foreach ($query->getValues() as $key => $value) {
+            $value = match ($query->getMethod()) {
+                Query::TYPE_STARTS_WITH => $this->escapeWildcards($value) . '%',
+                Query::TYPE_ENDS_WITH => '%' . $this->escapeWildcards($value),
+                Query::TYPE_SEARCH => $this->getFulltextValue($value),
+                default => $value
+            };
+
             $placeholder = $this->getSQLPlaceholder($query).'_'.$key;
-            $value = $this->getSQLValue($query->getMethod(), $value);
             $stmt->bindValue($placeholder, $value, $this->getPDOType($value));
         }
     }
 
+    /**
+     * @param string $value
+     * @return string
+     */
+    protected function getFulltextValue(string $value): string
+    {
+        $exact = str_ends_with($value, '"') && str_starts_with($value, '"');
+
+        /** Replace reserved chars with space. */
+        $specialChars = '@,+,-,*,),(,<,>,~,"';
+        $value = str_replace(explode(',', $specialChars), ' ', $value);
+        $value = preg_replace('/\s+/', ' ', $value); // Remove multiple whitespaces
+        $value = trim($value);
+
+        if ($exact) {
+            $value = '"' . $value . '"';
+        } else {
+            /** Prepend wildcard by default on the back. */
+            $value .= '*';
+        }
+
+        return $value;
+    }
+
     /**
      * Get SQL Operator
      *
@@ -745,21 +776,15 @@ protected function getSQLPlaceholder(Query $query): string
         return \md5($json);
     }
 
-    protected function getSQLValue(string $method, mixed $value): mixed
+    public function escapeWildcards(string $value): string
     {
-        switch ($method) {
-            case Query::TYPE_STARTS_WITH:
-                $value = $this->escapeWildcards($value);
-                return "$value%";
-            case Query::TYPE_ENDS_WITH:
-                $value = $this->escapeWildcards($value);
-                return "%$value";
-            case Query::TYPE_SEARCH:
-                $value = $this->escapeWildcards($value);
-                return "'$value*'";
-            default:
-                return $value;
+        $wildcards = ['%', '_', '[', ']', '^', '-', '.', '*', '+', '?', '(', ')', '{', '}', '|'];
+
+        foreach ($wildcards as $wildcard) {
+            $value = \str_replace($wildcard, "\\$wildcard", $value);
         }
+
+        return $value;
     }
 
     /**

tests/Database/Base.php

@@ -1607,6 +1607,70 @@ public function testFindFulltext(): void
         $this->assertEquals(true, true); // Test must do an assertion
     }
 
+    public function testFindFulltextSpecialChars(): void
+    {
+        if (!static::getDatabase()->getAdapter()->getSupportForFulltextIndex()) {
+            $this->expectNotToPerformAssertions();
+            return;
+        }
+
+        $collection = 'full_text';
+        static::getDatabase()->createCollection($collection, permissions: [
+            Permission::create(Role::any()),
+            Permission::update(Role::users())
+        ]);
+
+        $this->assertTrue(static::getDatabase()->createAttribute($collection, 'ft', Database::VAR_STRING, 128, true));
+        $this->assertTrue(static::getDatabase()->createIndex($collection, 'ft-index', Database::INDEX_FULLTEXT, ['ft']));
+
+        static::getDatabase()->createDocument($collection, new Document([
+            '$permissions' => [Permission::read(Role::any())],
+            'ft' => 'Alf: chapter_4@nasa.com'
+        ]));
+
+        $documents = static::getDatabase()->find($collection, [
+            Query::search('ft', 'chapter_4'),
+        ]);
+        $this->assertEquals(1, count($documents));
+
+        static::getDatabase()->createDocument($collection, new Document([
+            '$permissions' => [Permission::read(Role::any())],
+            'ft' => 'al@ba.io +-*)(<>~'
+        ]));
+
+        $documents = static::getDatabase()->find($collection, [
+            Query::search('ft', 'al@ba.io'), // === al ba io*
+        ]);
+
+        if (static::getDatabase()->getAdapter()->getSupportForFulltextWildcardIndex()) {
+            $this->assertEquals(0, count($documents));
+        } else {
+            $this->assertEquals(1, count($documents));
+        }
+
+        static::getDatabase()->createDocument($collection, new Document([
+            '$permissions' => [Permission::read(Role::any())],
+            'ft' => 'donald duck'
+        ]));
+
+        static::getDatabase()->createDocument($collection, new Document([
+            '$permissions' => [Permission::read(Role::any())],
+            'ft' => 'donald trump'
+        ]));
+
+        $documents = static::getDatabase()->find($collection, [
+            Query::search('ft', 'donald trump'),
+            Query::orderAsc('ft'),
+        ]);
+        $this->assertEquals(2, count($documents));
+
+        $documents = static::getDatabase()->find($collection, [
+            Query::search('ft', '"donald trump"'), // Exact match
+        ]);
+
+        $this->assertEquals(1, count($documents));
+    }
+
     public function testFindMultipleConditions(): void
     {
         /**