Skip to content

Fix empty $wpdb->collate resulting in utf8mb4_0900_ai_ci tables #511

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wojtekn wants to merge 4 commits into
base: trunk
Choose a base branch
from
Open

Fix empty $wpdb->collate resulting in utf8mb4_0900_ai_ci tables #511

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
8319a29
Determine wpdb charset and collation after connecting
wojtekn Sep 24, 2026
370adde
Add utf8mb4_unicode_520_ci to the emulated collations
wojtekn Sep 24, 2026
7cd15a0
Inherit column collations from the table default
wojtekn Sep 24, 2026
6d003d6
Use the connection charset when stripping invalid text without a charset
wojtekn Sep 24, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
139 changes: 109 additions & 30 deletions packages/mysql-on-sqlite/src/sqlite/class-wp-sqlite-information-schema-builder.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -450,6 +450,7 @@ public function get_computed_information_schema_table_definition( string $table_
('utf8_unicode_ci', 'utf8', 192, '', 'Yes', 8, 'PAD SPACE'),
('utf8mb4_bin', 'utf8mb4', 46, '', 'Yes', 1, 'PAD SPACE'),
('utf8mb4_unicode_ci', 'utf8mb4', 224, '', 'Yes', 8, 'PAD SPACE'),
('utf8mb4_unicode_520_ci', 'utf8mb4', 246, '', 'Yes', 8, 'PAD SPACE'),
('utf8mb4_0900_ai_ci', 'utf8mb4', 255, 'Yes', 'Yes', 0, 'NO PAD')
)";
default:
Expand Down Expand Up @@ -559,7 +560,10 @@ public function record_create_table( WP_Parser_Node $node ): void {
$table_name,
$column_name,
$column_node,
$column_position
$column_position,
function () use ( $table_collation ) {
return $table_collation;
}
);

try {
Expand Down Expand Up @@ -865,7 +869,15 @@ private function record_add_column(
array( self::SAVED_DATABASE_NAME, $table_name )
)->fetchColumn();

$column_data = $this->extract_column_data( $table_name, $column_name, $node, (int) $position + 1 );
$column_data = $this->extract_column_data(
$table_name,
$column_name,
$node,
(int) $position + 1,
function () use ( $table_is_temporary, $table_name ) {
return $this->get_recorded_table_collation( $table_is_temporary, $table_name );
}
);
try {
$this->insert_values(
$this->get_table_name( $table_is_temporary, 'columns' ),
Expand Down Expand Up @@ -915,7 +927,15 @@ private function record_change_column(
string $new_column_name,
WP_Parser_Node $node
): void {
$column_data = $this->extract_column_data( $table_name, $new_column_name, $node, 0 );
$column_data = $this->extract_column_data(
$table_name,
$new_column_name,
$node,
0,
function () use ( $table_is_temporary, $table_name ) {
return $this->get_recorded_table_collation( $table_is_temporary, $table_name );
}
);
unset( $column_data['ordinal_position'] );
$this->update_values(
$this->get_table_name( $table_is_temporary, 'columns' ),
Expand Down Expand Up @@ -1498,13 +1518,21 @@ private function record_drop_check_constraint(
/**
* Analyze "columnDefinition" or "fieldDefinition" AST node and extract column data.
*
* @param string $table_name The table name.
* @param string $column_name The column name.
* @param WP_Parser_Node $node The "columnDefinition" or "fieldDefinition" AST node.
* @param int $position The ordinal position of the column in the table.
* @return array Column data for the information schema.
* @param string $table_name The table name.
* @param string $column_name The column name.
* @param WP_Parser_Node $node The "columnDefinition" or "fieldDefinition" AST node.
* @param int $position The ordinal position of the column in the table.
* @param callable $get_table_collation Returns the default collation of the table.
* Called only when the column inherits it.
* @return array Column data for the information schema.
*/
private function extract_column_data( string $table_name, string $column_name, WP_Parser_Node $node, int $position ): array {
private function extract_column_data(
string $table_name,
string $column_name,
WP_Parser_Node $node,
int $position,
callable $get_table_collation
): array {
list ( $data_type, $column_type ) = $this->get_column_data_types( $node );

$default = $this->get_column_default( $node, $data_type, $column_name );
Expand All @@ -1513,7 +1541,7 @@ private function extract_column_data( string $table_name, string $column_name, W
$extra = $this->get_column_extra( $node );
$comment = $this->get_column_comment( $node );

list ( $charset, $collation ) = $this->get_column_charset_and_collation( $node, $data_type );
list ( $charset, $collation ) = $this->get_column_charset_and_collation( $node, $data_type, $get_table_collation );
list ( $char_length, $octet_length ) = $this->get_column_lengths( $node, $data_type, $charset );
list ( $precision, $scale ) = $this->get_column_numeric_attributes( $node, $data_type );
$datetime_precision = $this->get_column_datetime_precision( $node, $data_type );
Expand Down Expand Up @@ -2017,12 +2045,44 @@ private function get_table_engine( WP_Parser_Node $node ): string {
* @return string The table collation as stored in information schema.
*/
private function get_table_collation( WP_Parser_Node $node ): string {
$collate_node = $node->get_first_descendant_node( 'collationName' );
if ( null === $collate_node ) {
// @TODO: Use default DB collation or DB_CHARSET & DB_COLLATE.
return 'utf8mb4_0900_ai_ci';
// Only table options apply; column definitions may specify their own collation.
$options_node = $node->get_first_descendant_node( 'createTableOptions' );

$collation_node = $options_node ? $options_node->get_first_descendant_node( 'defaultCollation' ) : null;
if ( null !== $collation_node ) {
return strtolower( $this->get_value( $collation_node->get_first_child_node( 'collationName' ) ) );
}

$charset_node = $options_node ? $options_node->get_first_descendant_node( 'defaultCharset' ) : null;
if ( null !== $charset_node ) {
$charset = strtolower( $this->get_value( $charset_node->get_first_child_node( 'charsetName' ) ) );
return $this->get_charset_default_collation( 'utf8mb3' === $charset ? 'utf8' : $charset );
}
return strtolower( $this->get_value( $collate_node ) );

// @TODO: Use the default collation of the database.
return 'utf8mb4_0900_ai_ci';
}

/**
* Get the table collation that is recorded in the information schema.
*
* @param bool $table_is_temporary Whether the table is temporary.
* @param string $table_name The table name.
* @return string The table collation as stored in information schema.
*/
private function get_recorded_table_collation( bool $table_is_temporary, string $table_name ): string {
$collation = $this->connection->query(
'
SELECT table_collation
FROM ' . $this->connection->quote_identifier( $this->get_table_name( $table_is_temporary, 'tables' ) ) . '
WHERE table_schema = ?
AND table_name = ?
',
array( self::SAVED_DATABASE_NAME, $table_name )
)->fetchColumn();

// @TODO: Use the default collation of the database.
return false === $collation || null === $collation ? 'utf8mb4_0900_ai_ci' : $collation;
}

/**
Expand Down Expand Up @@ -2442,11 +2502,12 @@ private function get_column_data_types( WP_Parser_Node $node ): array {
/**
* Extract column charset and collation from the "columnDefinition" or "fieldDefinition" AST node.
*
* @param WP_Parser_Node $node The "columnDefinition" or "fieldDefinition" AST node.
* @param string $data_type The column data type as stored in information schema.
* @return array{ string|null, string|null } The column charset and collation as stored in information schema.
* @param WP_Parser_Node $node The "columnDefinition" or "fieldDefinition" AST node.
* @param string $data_type The column data type as stored in information schema.
* @param callable $get_table_collation Returns the default collation of the table.
* @return array{ string|null, string|null } The column charset and collation as stored in information schema.
*/
private function get_column_charset_and_collation( WP_Parser_Node $node, string $data_type ): array {
private function get_column_charset_and_collation( WP_Parser_Node $node, string $data_type, callable $get_table_collation ): array {
if ( ! (
'char' === $data_type
|| 'varchar' === $data_type
Expand Down Expand Up @@ -2507,30 +2568,48 @@ private function get_column_charset_and_collation( WP_Parser_Node $node, string
$collation = strtolower( $this->get_value( $collation_node ) );
}

// Defaults.
// @TODO: These are hardcoded now. We should get them from table/DB.
// When neither charset nor collation is set, the table defaults are used.
if ( null === $charset && null === $collation ) {
$charset = 'utf8mb4';
$table_collation = $get_table_collation();
$charset = $this->get_collation_charset( $table_collation );
if ( ! $is_binary ) {
$collation = $table_collation;
}
// @TODO: "BINARY" (seems to change varchar to varbinary).
// @TODO: "DEFAULT"
}

// If only one of charset/collation is set, the other one is derived.
if ( null === $collation ) {
if ( $is_binary ) {
$collation = $charset . '_bin';
} elseif ( isset( self::CHARSET_DEFAULT_COLLATION_MAP[ $charset ] ) ) {
$collation = self::CHARSET_DEFAULT_COLLATION_MAP[ $charset ];
} else {
$collation = $charset . '_general_ci';
}
$collation = $is_binary ? $charset . '_bin' : $this->get_charset_default_collation( $charset );
} elseif ( null === $charset ) {
$charset = substr( $collation, 0, strpos( $collation, '_' ) );
$charset = $this->get_collation_charset( $collation );
}

return array( $charset, $collation );
}

/**
* Get the default collation for a given charset.
*
* @param string $charset The charset name.
* @return string The default collation of the charset.
*/
private function get_charset_default_collation( string $charset ): string {
return self::CHARSET_DEFAULT_COLLATION_MAP[ $charset ] ?? $charset . '_general_ci';
}

/**
* Get the charset for a given collation.
*
* @param string $collation The collation name.
* @return string The charset of the collation.
*/
private function get_collation_charset( string $collation ): string {
$separator_position = strpos( $collation, '_' );
return false === $separator_position ? $collation : substr( $collation, 0, $separator_position );
}

/**
* Extract column length info from the "columnDefinition" or "fieldDefinition" AST node.
*
Expand Down
117 changes: 114 additions & 3 deletions packages/mysql-on-sqlite/tests/WP_MySQL_On_SQLite_Metadata_Tests.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -128,6 +128,107 @@ public function testInfromationSchemaCharacterSets(): void {
);
}

public function testTableCollationFromTableOptions(): void {
$this->assertQuery( 'CREATE TABLE t1 (a TEXT)' );
$this->assertQuery( 'CREATE TABLE t2 (a TEXT COLLATE utf8mb4_bin)' );
$this->assertQuery( 'CREATE TABLE t3 (a TEXT) DEFAULT CHARSET=latin1' );
$this->assertQuery( 'CREATE TABLE t4 (a TEXT) DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_520_ci' );

$result = $this->assertQuery(
"SELECT table_name, table_collation FROM information_schema.tables WHERE table_schema = 'wp' ORDER BY table_name"
);
$this->assertEquals(
array(
(object) array(
'TABLE_NAME' => 't1',
'TABLE_COLLATION' => 'utf8mb4_0900_ai_ci',
),
(object) array(
'TABLE_NAME' => 't2',
'TABLE_COLLATION' => 'utf8mb4_0900_ai_ci',
),
(object) array(
'TABLE_NAME' => 't3',
'TABLE_COLLATION' => 'latin1_swedish_ci',
),
(object) array(
'TABLE_NAME' => 't4',
'TABLE_COLLATION' => 'utf8mb4_unicode_520_ci',
),
),
$result
);
}

public function testColumnCollationInheritedFromTable(): void {
$this->assertQuery(
'CREATE TABLE t (
a VARCHAR(10),
b VARCHAR(10) BINARY,
c VARCHAR(10) CHARACTER SET utf8mb4,
d VARCHAR(10) COLLATE utf8mb4_bin,
e INT
) DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_520_ci'
);
$this->assertQuery( 'ALTER TABLE t ADD COLUMN f TEXT' );
$this->assertQuery( 'ALTER TABLE t MODIFY COLUMN e VARCHAR(10)' );
$this->assertQuery( 'CREATE TABLE t2 (a TEXT) DEFAULT CHARSET=latin1' );

$result = $this->assertQuery(
"SELECT table_name, column_name, character_set_name, collation_name
FROM information_schema.columns
WHERE table_schema = 'wp'
ORDER BY table_name, column_name"
);
$this->assertEquals(
array(
(object) array(
'TABLE_NAME' => 't',
'COLUMN_NAME' => 'a',
'CHARACTER_SET_NAME' => 'utf8mb4',
'COLLATION_NAME' => 'utf8mb4_unicode_520_ci',
),
(object) array(
'TABLE_NAME' => 't',
'COLUMN_NAME' => 'b',
'CHARACTER_SET_NAME' => 'utf8mb4',
'COLLATION_NAME' => 'utf8mb4_bin',
),
(object) array(
'TABLE_NAME' => 't',
'COLUMN_NAME' => 'c',
'CHARACTER_SET_NAME' => 'utf8mb4',
'COLLATION_NAME' => 'utf8mb4_0900_ai_ci',
),
(object) array(
'TABLE_NAME' => 't',
'COLUMN_NAME' => 'd',
'CHARACTER_SET_NAME' => 'utf8mb4',
'COLLATION_NAME' => 'utf8mb4_bin',
),
(object) array(
'TABLE_NAME' => 't',
'COLUMN_NAME' => 'e',
'CHARACTER_SET_NAME' => 'utf8mb4',
'COLLATION_NAME' => 'utf8mb4_unicode_520_ci',
),
(object) array(
'TABLE_NAME' => 't',
'COLUMN_NAME' => 'f',
'CHARACTER_SET_NAME' => 'utf8mb4',
'COLLATION_NAME' => 'utf8mb4_unicode_520_ci',
),
(object) array(
'TABLE_NAME' => 't2',
'COLUMN_NAME' => 'a',
'CHARACTER_SET_NAME' => 'latin1',
'COLLATION_NAME' => 'latin1_swedish_ci',
),
),
$result
);
}

public function testInfromationSchemaCollations(): void {
$result = $this->assertQuery( 'SELECT * FROM INFORMATION_SCHEMA.COLLATIONS ORDER BY COLLATION_NAME' );
$this->assertEquals(
Expand Down Expand Up @@ -186,6 +287,15 @@ public function testInfromationSchemaCollations(): void {
'SORTLEN' => '1',
'PAD_ATTRIBUTE' => 'PAD SPACE',
),
(object) array(
'COLLATION_NAME' => 'utf8mb4_unicode_520_ci',
'CHARACTER_SET_NAME' => 'utf8mb4',
'ID' => '246',
'IS_DEFAULT' => '',
'IS_COMPILED' => 'Yes',
'SORTLEN' => '8',
'PAD_ATTRIBUTE' => 'PAD SPACE',
),
(object) array(
'COLLATION_NAME' => 'utf8mb4_unicode_ci',
'CHARACTER_SET_NAME' => 'utf8mb4',
Expand Down Expand Up @@ -516,19 +626,20 @@ public function testShowCollation(): void {
// Simple.
$this->assertQuery( 'SHOW COLLATION' );
$actual = $this->last_result;
$this->assertCount( 7, $actual );
$this->assertCount( 8, $actual );
$this->assertEquals( 'binary', $actual[0]->Collation );
$this->assertEquals( 'utf8_bin', $actual[1]->Collation );
$this->assertEquals( 'utf8_general_ci', $actual[2]->Collation );
$this->assertEquals( 'utf8_unicode_ci', $actual[3]->Collation );
$this->assertEquals( 'utf8mb4_bin', $actual[4]->Collation );
$this->assertEquals( 'utf8mb4_unicode_ci', $actual[5]->Collation );
$this->assertEquals( 'utf8mb4_0900_ai_ci', $actual[6]->Collation );
$this->assertEquals( 'utf8mb4_unicode_520_ci', $actual[6]->Collation );
$this->assertEquals( 'utf8mb4_0900_ai_ci', $actual[7]->Collation );

// With LIKE clause.
$this->assertQuery( "SHOW COLLATION LIKE 'utf8%'" );
$actual = $this->last_result;
$this->assertCount( 6, $actual );
$this->assertCount( 7, $actual );
$this->assertEquals( 'utf8_bin', $actual[0]->Collation );
$this->assertEquals( 'utf8_general_ci', $actual[1]->Collation );
$this->assertEquals( 'utf8_unicode_ci', $actual[2]->Collation );
Expand Down
2 changes: 1 addition & 1 deletion packages/mysql-on-sqlite/tests/WP_MySQL_On_SQLite_Tests.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4083,7 +4083,7 @@ public function testFoundRowsWithoutSqlCalcFoundRows(): void {
// SHOW COLLATION
$this->assertQuery( 'SHOW COLLATION' );
$result = $this->assertQuery( 'SELECT FOUND_ROWS()' );
$this->assertSame( '7', $result[0]->{'FOUND_ROWS()'} );
$this->assertSame( '8', $result[0]->{'FOUND_ROWS()'} );

// SHOW DATABASES
$this->assertQuery( 'SHOW DATABASES' );
Expand Down
1 change: 1 addition & 0 deletions packages/mysql-on-sqlite/tests/WP_MySQL_On_SQLite_Translation_Tests.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1031,6 +1031,7 @@ public function testAlterTableAddMultipleColumns(): void {
'INSERT INTO `_wp_sqlite_mysql_information_schema_columns` (`table_schema`, `table_name`, `column_name`, `ordinal_position`, `column_default`, `is_nullable`, `data_type`, `character_maximum_length`, `character_octet_length`, `numeric_precision`, `numeric_scale`, `datetime_precision`, `character_set_name`, `collation_name`, `column_type`, `column_key`, `extra`, `privileges`, `column_comment`, `generation_expression`, `srs_id`)'
. " VALUES ('sqlite_database', 't', 'a', 2, null, 'YES', 'int', null, null, 10, 0, null, null, null, 'int', '', '', 'select,insert,update,references', '', '', null)",
"SELECT MAX(ordinal_position) FROM `_wp_sqlite_mysql_information_schema_columns` WHERE table_schema = 'sqlite_database' AND table_name = 't'",
"SELECT table_collation FROM `_wp_sqlite_mysql_information_schema_tables` WHERE table_schema = 'sqlite_database' AND table_name = 't'",
'INSERT INTO `_wp_sqlite_mysql_information_schema_columns` (`table_schema`, `table_name`, `column_name`, `ordinal_position`, `column_default`, `is_nullable`, `data_type`, `character_maximum_length`, `character_octet_length`, `numeric_precision`, `numeric_scale`, `datetime_precision`, `character_set_name`, `collation_name`, `column_type`, `column_key`, `extra`, `privileges`, `column_comment`, `generation_expression`, `srs_id`)'
. " VALUES ('sqlite_database', 't', 'b', 3, null, 'YES', 'text', 65535, 65535, null, null, null, 'utf8mb4', 'utf8mb4_0900_ai_ci', 'text', '', '', 'select,insert,update,references', '', '', null)",
"SELECT MAX(ordinal_position) FROM `_wp_sqlite_mysql_information_schema_columns` WHERE table_schema = 'sqlite_database' AND table_name = 't'",
Expand Down
Loading
Loading