MDEV-27653 long uniques don't work with unicode collations

This commit is contained in:
Alexander Barkov 2022-10-28 13:43:51 +04:00
parent 9924466b3b
commit 284ac6f2b7
27 changed files with 668 additions and 66 deletions

View File

@ -11379,3 +11379,178 @@ a
# #
# End of 10.3 tests # End of 10.3 tests
# #
#
# Start of 10.4 tests
#
#
# MDEV-27653 long uniques don't work with unicode collations
#
SET NAMES utf8mb3;
CREATE TABLE t1 (
a CHAR(30) COLLATE utf8mb3_general_ci,
UNIQUE KEY(a) USING HASH
);
SHOW CREATE TABLE t1;
Table Create Table
t1 CREATE TABLE `t1` (
`a` char(30) CHARACTER SET utf8 COLLATE utf8_general_ci DEFAULT NULL,
UNIQUE KEY `a` (`a`) USING HASH
) ENGINE=MyISAM DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci
INSERT INTO t1 VALUES ('a');
INSERT INTO t1 VALUES ('ä');
ERROR 23000: Duplicate entry 'ä' for key 'a'
SELECT * FROM t1;
a
a
DROP TABLE t1;
CREATE TABLE t1 (
a CHAR(30) COLLATE utf8mb3_general_ci,
UNIQUE KEY(a(10)) USING HASH
);
SHOW CREATE TABLE t1;
Table Create Table
t1 CREATE TABLE `t1` (
`a` char(30) CHARACTER SET utf8 COLLATE utf8_general_ci DEFAULT NULL,
UNIQUE KEY `a` (`a`(10)) USING HASH
) ENGINE=MyISAM DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci
INSERT INTO t1 VALUES ('a');
INSERT INTO t1 VALUES ('ä');
ERROR 23000: Duplicate entry 'ä' for key 'a'
SELECT * FROM t1;
a
a
DROP TABLE t1;
CREATE TABLE t1 (
a VARCHAR(30) COLLATE utf8mb3_general_ci,
UNIQUE KEY(a) USING HASH
);
SHOW CREATE TABLE t1;
Table Create Table
t1 CREATE TABLE `t1` (
`a` varchar(30) CHARACTER SET utf8 COLLATE utf8_general_ci DEFAULT NULL,
UNIQUE KEY `a` (`a`) USING HASH
) ENGINE=MyISAM DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci
INSERT INTO t1 VALUES ('a');
INSERT INTO t1 VALUES ('ä');
ERROR 23000: Duplicate entry 'ä' for key 'a'
SELECT * FROM t1;
a
a
DROP TABLE t1;
CREATE TABLE t1 (
a VARCHAR(30) COLLATE utf8mb3_general_ci,
UNIQUE KEY(a(10)) USING HASH
);
SHOW CREATE TABLE t1;
Table Create Table
t1 CREATE TABLE `t1` (
`a` varchar(30) CHARACTER SET utf8 COLLATE utf8_general_ci DEFAULT NULL,
UNIQUE KEY `a` (`a`(10)) USING HASH
) ENGINE=MyISAM DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci
INSERT INTO t1 VALUES ('a');
INSERT INTO t1 VALUES ('ä');
ERROR 23000: Duplicate entry 'ä' for key 'a'
SELECT * FROM t1;
a
a
DROP TABLE t1;
CREATE TABLE t1 (a TEXT COLLATE utf8mb3_general_ci UNIQUE);
SHOW CREATE TABLE t1;
Table Create Table
t1 CREATE TABLE `t1` (
`a` text CHARACTER SET utf8 COLLATE utf8_general_ci DEFAULT NULL,
UNIQUE KEY `a` (`a`) USING HASH
) ENGINE=MyISAM DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci
INSERT INTO t1 VALUES ('a');
INSERT INTO t1 VALUES ('ä');
ERROR 23000: Duplicate entry 'ä' for key 'a'
SELECT * FROM t1;
a
a
DROP TABLE t1;
CREATE TABLE t1 (
a LONGTEXT COLLATE utf8mb3_general_ci,
UNIQUE KEY(a(10)) USING HASH
);
SHOW CREATE TABLE t1;
Table Create Table
t1 CREATE TABLE `t1` (
`a` longtext CHARACTER SET utf8 COLLATE utf8_general_ci DEFAULT NULL,
UNIQUE KEY `a` (`a`(10)) USING HASH
) ENGINE=MyISAM DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci
INSERT INTO t1 VALUES ('a');
INSERT INTO t1 VALUES ('ä');
ERROR 23000: Duplicate entry 'ä' for key 'a'
SELECT * FROM t1;
a
a
DROP TABLE t1;
SHOW CREATE TABLE t1;
Table Create Table
t1 CREATE TABLE `t1` (
`a` text CHARACTER SET utf8 COLLATE utf8_general_ci DEFAULT NULL,
UNIQUE KEY `a` (`a`) USING HASH
) ENGINE=MyISAM DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci
SELECT a, OCTET_LENGTH(a) FROM t1 ORDER BY BINARY a;
a OCTET_LENGTH(a)
a 1
ä 2
CHECK TABLE t1;
Table Op Msg_type Msg_text
test.t1 check error Upgrade required. Please do "REPAIR TABLE `t1`" or dump/reload to fix it!
INSERT INTO t1 VALUES ('A');
ERROR 23000: Duplicate entry 'A' for key 'a'
INSERT INTO t1 VALUES ('Ä');
ERROR 23000: Duplicate entry 'Ä' for key 'a'
INSERT INTO t1 VALUES ('Ấ');
SELECT a, OCTET_LENGTH(a) FROM t1 ORDER BY BINARY a;
a OCTET_LENGTH(a)
a 1
ä 2
Ấ 3
CHECK TABLE t1;
Table Op Msg_type Msg_text
test.t1 check error Upgrade required. Please do "REPAIR TABLE `t1`" or dump/reload to fix it!
ALTER TABLE t1 FORCE;
ERROR 23000: Duplicate entry 'ä' for key 'a'
DELETE FROM t1 WHERE OCTET_LENGTH(a)>1;
ALTER TABLE t1 FORCE;
INSERT INTO t1 VALUES ('ä');
ERROR 23000: Duplicate entry 'ä' for key 'a'
DROP TABLE t1;
SHOW CREATE TABLE t1;
Table Create Table
t1 CREATE TABLE `t1` (
`a` text CHARACTER SET utf8 COLLATE utf8_general_ci DEFAULT NULL,
UNIQUE KEY `a` (`a`) USING HASH
) ENGINE=MyISAM DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci
SELECT a, OCTET_LENGTH(a) FROM t1 ORDER BY BINARY a;
a OCTET_LENGTH(a)
a 1
ä 2
ALTER IGNORE TABLE t1 FORCE;
SELECT a, OCTET_LENGTH(a) FROM t1 ORDER BY BINARY a;
a OCTET_LENGTH(a)
a 1
DROP TABLE t1;
SHOW CREATE TABLE t1;
Table Create Table
t1 CREATE TABLE `t1` (
`a` text CHARACTER SET utf8 COLLATE utf8_general_ci DEFAULT NULL,
UNIQUE KEY `a` (`a`) USING HASH
) ENGINE=MyISAM DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci
SELECT a, OCTET_LENGTH(a) FROM t1 ORDER BY BINARY a;
a OCTET_LENGTH(a)
a 1
ä 2
REPAIR TABLE t1;
Table Op Msg_type Msg_text
test.t1 repair Warning Number of rows changed from 2 to 1
test.t1 repair status OK
SELECT a, OCTET_LENGTH(a) FROM t1 ORDER BY BINARY a;
a OCTET_LENGTH(a)
a 1
DROP TABLE t1;
#
# End of 10.4 tests
#

View File

@ -2310,3 +2310,161 @@ VALUES (_latin1 0xDF) UNION VALUES(_utf8'a' COLLATE utf8_bin);
--echo # --echo #
--echo # End of 10.3 tests --echo # End of 10.3 tests
--echo # --echo #
--echo #
--echo # Start of 10.4 tests
--echo #
--echo #
--echo # MDEV-27653 long uniques don't work with unicode collations
--echo #
SET NAMES utf8mb3;
# CHAR
CREATE TABLE t1 (
a CHAR(30) COLLATE utf8mb3_general_ci,
UNIQUE KEY(a) USING HASH
);
SHOW CREATE TABLE t1;
INSERT INTO t1 VALUES ('a');
--error ER_DUP_ENTRY
INSERT INTO t1 VALUES ('ä');
SELECT * FROM t1;
DROP TABLE t1;
CREATE TABLE t1 (
a CHAR(30) COLLATE utf8mb3_general_ci,
UNIQUE KEY(a(10)) USING HASH
);
SHOW CREATE TABLE t1;
INSERT INTO t1 VALUES ('a');
--error ER_DUP_ENTRY
INSERT INTO t1 VALUES ('ä');
SELECT * FROM t1;
DROP TABLE t1;
# VARCHAR
CREATE TABLE t1 (
a VARCHAR(30) COLLATE utf8mb3_general_ci,
UNIQUE KEY(a) USING HASH
);
SHOW CREATE TABLE t1;
INSERT INTO t1 VALUES ('a');
--error ER_DUP_ENTRY
INSERT INTO t1 VALUES ('ä');
SELECT * FROM t1;
DROP TABLE t1;
CREATE TABLE t1 (
a VARCHAR(30) COLLATE utf8mb3_general_ci,
UNIQUE KEY(a(10)) USING HASH
);
SHOW CREATE TABLE t1;
INSERT INTO t1 VALUES ('a');
--error ER_DUP_ENTRY
INSERT INTO t1 VALUES ('ä');
SELECT * FROM t1;
DROP TABLE t1;
# TEXT
CREATE TABLE t1 (a TEXT COLLATE utf8mb3_general_ci UNIQUE);
SHOW CREATE TABLE t1;
INSERT INTO t1 VALUES ('a');
--error ER_DUP_ENTRY
INSERT INTO t1 VALUES ('ä');
SELECT * FROM t1;
DROP TABLE t1;
CREATE TABLE t1 (
a LONGTEXT COLLATE utf8mb3_general_ci,
UNIQUE KEY(a(10)) USING HASH
);
SHOW CREATE TABLE t1;
INSERT INTO t1 VALUES ('a');
--error ER_DUP_ENTRY
INSERT INTO t1 VALUES ('ä');
SELECT * FROM t1;
DROP TABLE t1;
# Testing upgrade:
# Prior to MDEV-27653, the UNIQUE HASH function errorneously
# took into account string octet length.
# Old tables should still open and work, but with wrong results.
copy_file std_data/mysql_upgrade/mdev27653_100422_myisam_text.frm $MYSQLD_DATADIR/test/t1.frm;
copy_file std_data/mysql_upgrade/mdev27653_100422_myisam_text.MYD $MYSQLD_DATADIR/test/t1.MYD;
copy_file std_data/mysql_upgrade/mdev27653_100422_myisam_text.MYI $MYSQLD_DATADIR/test/t1.MYI;
SHOW CREATE TABLE t1;
SELECT a, OCTET_LENGTH(a) FROM t1 ORDER BY BINARY a;
CHECK TABLE t1;
# There is already a one byte value 'a' in the table
--error ER_DUP_ENTRY
INSERT INTO t1 VALUES ('A');
# There is already a two-byte value 'ä' in the table
--error ER_DUP_ENTRY
INSERT INTO t1 VALUES ('Ä');
# There were no three-byte values in the table so far.
# The below value violates UNIQUE, but it gets inserted.
# This is wrong but expected for a pre-MDEV-27653 table.
INSERT INTO t1 VALUES ('Ấ');
SELECT a, OCTET_LENGTH(a) FROM t1 ORDER BY BINARY a;
CHECK TABLE t1;
# ALTER FORCE fails: it tries to rebuild the table
# with a correct UNIQUE HASH function, but there are duplicates!
--error ER_DUP_ENTRY
ALTER TABLE t1 FORCE;
# Let's remove all duplicate values, so only the one-byte 'a' stays.
# ALTER..FORCE should work after that.
DELETE FROM t1 WHERE OCTET_LENGTH(a)>1;
ALTER TABLE t1 FORCE;
# Make sure that 'a' and 'ä' cannot co-exists any more,
# because the table was recreated with a correct UNIQUE HASH function.
--error ER_DUP_ENTRY
INSERT INTO t1 VALUES ('ä');
DROP TABLE t1;
#
# Testing an old table with ALTER IGNORE.
# The table is expected to rebuild with a new hash function,
# duplicates go away.
#
copy_file std_data/mysql_upgrade/mdev27653_100422_myisam_text.frm $MYSQLD_DATADIR/test/t1.frm;
copy_file std_data/mysql_upgrade/mdev27653_100422_myisam_text.MYD $MYSQLD_DATADIR/test/t1.MYD;
copy_file std_data/mysql_upgrade/mdev27653_100422_myisam_text.MYI $MYSQLD_DATADIR/test/t1.MYI;
SHOW CREATE TABLE t1;
SELECT a, OCTET_LENGTH(a) FROM t1 ORDER BY BINARY a;
ALTER IGNORE TABLE t1 FORCE;
SELECT a, OCTET_LENGTH(a) FROM t1 ORDER BY BINARY a;
DROP TABLE t1;
#
# Testing an old table with REPAIR.
# The table is expected to rebuild with a new hash function,
# duplicates go away.
#
copy_file std_data/mysql_upgrade/mdev27653_100422_myisam_text.frm $MYSQLD_DATADIR/test/t1.frm;
copy_file std_data/mysql_upgrade/mdev27653_100422_myisam_text.MYD $MYSQLD_DATADIR/test/t1.MYD;
copy_file std_data/mysql_upgrade/mdev27653_100422_myisam_text.MYI $MYSQLD_DATADIR/test/t1.MYI;
SHOW CREATE TABLE t1;
SELECT a, OCTET_LENGTH(a) FROM t1 ORDER BY BINARY a;
REPAIR TABLE t1;
SELECT a, OCTET_LENGTH(a) FROM t1 ORDER BY BINARY a;
DROP TABLE t1;
--echo #
--echo # End of 10.4 tests
--echo #

View File

@ -1320,5 +1320,27 @@ CASE WHEN a THEN DEFAULT(a) END
DROP TABLE t1; DROP TABLE t1;
SET timestamp=DEFAULT; SET timestamp=DEFAULT;
# #
# MDEV-27653 long uniques don't work with unicode collations
#
CREATE TABLE t1 (a timestamp, UNIQUE KEY(a) USING HASH);
SET time_zone='+00:00';
INSERT INTO t1 VALUES ('2001-01-01 10:20:30');
SET time_zone='+01:00';
INSERT INTO t1 SELECT MAX(a) FROM t1;
ERROR 23000: Duplicate entry '2001-01-01 11:20:30' for key 'a'
SELECT * FROM t1;
a
2001-01-01 11:20:30
DROP TABLE t1;
CREATE TABLE t1 (a timestamp, UNIQUE KEY(a) USING HASH);
SET time_zone='+00:00';
INSERT INTO t1 VALUES ('2001-01-01 10:20:30');
SET time_zone='+01:00';
CHECK TABLE t1;
Table Op Msg_type Msg_text
test.t1 check status OK
DROP TABLE t1;
SET time_zone=DEFAULT;
#
# End of 10.4 tests # End of 10.4 tests
# #

View File

@ -878,6 +878,27 @@ DROP TABLE t1;
SET timestamp=DEFAULT; SET timestamp=DEFAULT;
--echo #
--echo # MDEV-27653 long uniques don't work with unicode collations
--echo #
CREATE TABLE t1 (a timestamp, UNIQUE KEY(a) USING HASH);
SET time_zone='+00:00';
INSERT INTO t1 VALUES ('2001-01-01 10:20:30');
SET time_zone='+01:00';
--error ER_DUP_ENTRY
INSERT INTO t1 SELECT MAX(a) FROM t1;
SELECT * FROM t1;
DROP TABLE t1;
CREATE TABLE t1 (a timestamp, UNIQUE KEY(a) USING HASH);
SET time_zone='+00:00';
INSERT INTO t1 VALUES ('2001-01-01 10:20:30');
SET time_zone='+01:00';
CHECK TABLE t1;
DROP TABLE t1;
SET time_zone=DEFAULT;
--echo # --echo #
--echo # End of 10.4 tests --echo # End of 10.4 tests
--echo # --echo #

View File

@ -1792,18 +1792,11 @@ Field::Field(uchar *ptr_arg,uint32 length_arg,uchar *null_ptr_arg,
} }
void Field::hash(ulong *nr, ulong *nr2) void Field::hash_not_null(Hasher *hasher)
{ {
if (is_null()) DBUG_ASSERT(marked_for_read());
{ DBUG_ASSERT(!is_null());
*nr^= (*nr << 1) | 1; hasher->add(sort_charset(), ptr, pack_length());
}
else
{
uint len= pack_length();
CHARSET_INFO *cs= sort_charset();
cs->coll->hash_sort(cs, ptr, len, nr, nr2);
}
} }
size_t size_t
@ -8180,18 +8173,12 @@ bool Field_varstring::is_equal(const Column_definition &new_field) const
} }
void Field_varstring::hash(ulong *nr, ulong *nr2) void Field_varstring::hash_not_null(Hasher *hasher)
{ {
if (is_null()) DBUG_ASSERT(marked_for_read());
{ DBUG_ASSERT(!is_null());
*nr^= (*nr << 1) | 1;
}
else
{
uint len= length_bytes == 1 ? (uint) *ptr : uint2korr(ptr); uint len= length_bytes == 1 ? (uint) *ptr : uint2korr(ptr);
CHARSET_INFO *cs= charset(); hasher->add(charset(), ptr + length_bytes, len);
cs->coll->hash_sort(cs, ptr + length_bytes, len, nr, nr2);
}
} }
@ -8553,6 +8540,17 @@ oom_error:
} }
void Field_blob::hash_not_null(Hasher *hasher)
{
DBUG_ASSERT(marked_for_read());
DBUG_ASSERT(!is_null());
char *blob;
memcpy(&blob, ptr + packlength, sizeof(char*));
if (blob)
hasher->add(Field_blob::charset(), blob, get_length(ptr));
}
double Field_blob::val_real(void) double Field_blob::val_real(void)
{ {
DBUG_ASSERT(marked_for_read()); DBUG_ASSERT(marked_for_read());
@ -9901,20 +9899,27 @@ Field_bit::Field_bit(uchar *ptr_arg, uint32 len_arg, uchar *null_ptr_arg,
} }
void Field_bit::hash(ulong *nr, ulong *nr2) /*
This method always calculates hash over 8 bytes.
This is different from how the HEAP engine calculate hash:
HEAP takes into account the actual octet size, so say for BIT(18)
it calculates hash over three bytes only:
- the incomplete byte with bits 16..17
- the two full bytes with bits 0..15
See hp_rec_hashnr(), hp_hashnr() for details.
The HEAP way is more efficient, especially for short lengths.
Let's consider fixing Field_bit eventually to do it in the HEAP way,
with proper measures to upgrade partitioned tables easy.
*/
void Field_bit::hash_not_null(Hasher *hasher)
{ {
if (is_null()) DBUG_ASSERT(marked_for_read());
{ DBUG_ASSERT(!is_null());
*nr^= (*nr << 1) | 1;
}
else
{
CHARSET_INFO *cs= &my_charset_bin;
longlong value= Field_bit::val_int(); longlong value= Field_bit::val_int();
uchar tmp[8]; uchar tmp[8];
mi_int8store(tmp,value); mi_int8store(tmp,value);
cs->coll->hash_sort(cs, tmp, 8, nr, nr2); hasher->add(&my_charset_bin, tmp, 8);
}
} }

View File

@ -1628,7 +1628,14 @@ public:
key_map get_possible_keys(); key_map get_possible_keys();
/* Hash value */ /* Hash value */
virtual void hash(ulong *nr, ulong *nr2); void hash(Hasher *hasher)
{
if (is_null())
hasher->add_null();
else
hash_not_null(hasher);
}
virtual void hash_not_null(Hasher *hasher);
/** /**
Get the upper limit of the MySQL integral and floating-point type. Get the upper limit of the MySQL integral and floating-point type.
@ -3744,7 +3751,7 @@ public:
uchar *new_ptr, uint32 length, uchar *new_ptr, uint32 length,
uchar *new_null_ptr, uint new_null_bit); uchar *new_null_ptr, uint new_null_bit);
bool is_equal(const Column_definition &new_field) const; bool is_equal(const Column_definition &new_field) const;
void hash(ulong *nr, ulong *nr2); void hash_not_null(Hasher *hasher);
uint length_size() const { return length_bytes; } uint length_size() const { return length_bytes; }
void print_key_value(String *out, uint32 length); void print_key_value(String *out, uint32 length);
private: private:
@ -3951,6 +3958,7 @@ public:
bool make_empty_rec_store_default_value(THD *thd, Item *item); bool make_empty_rec_store_default_value(THD *thd, Item *item);
int store(const char *to, size_t length, CHARSET_INFO *charset); int store(const char *to, size_t length, CHARSET_INFO *charset);
using Field_str::store; using Field_str::store;
void hash_not_null(Hasher *hasher);
double val_real(void); double val_real(void);
longlong val_int(void); longlong val_int(void);
String *val_str(String*,String *); String *val_str(String*,String *);
@ -4576,7 +4584,7 @@ public:
if (bit_ptr) if (bit_ptr)
bit_ptr= ADD_TO_PTR(bit_ptr, ptr_diff, uchar*); bit_ptr= ADD_TO_PTR(bit_ptr, ptr_diff, uchar*);
} }
void hash(ulong *nr, ulong *nr2); void hash_not_null(Hasher *hasher);
SEL_ARG *get_mm_leaf(RANGE_OPT_PARAM *param, KEY_PART *key_part, SEL_ARG *get_mm_leaf(RANGE_OPT_PARAM *param, KEY_PART *key_part,
const Item_bool_func *cond, const Item_bool_func *cond,

View File

@ -9850,8 +9850,7 @@ uint8 ha_partition::table_cache_type()
uint32 ha_partition::calculate_key_hash_value(Field **field_array) uint32 ha_partition::calculate_key_hash_value(Field **field_array)
{ {
ulong nr1= 1; Hasher hasher;
ulong nr2= 4;
bool use_51_hash; bool use_51_hash;
use_51_hash= MY_TEST((*field_array)->table->part_info->key_algorithm == use_51_hash= MY_TEST((*field_array)->table->part_info->key_algorithm ==
partition_info::KEY_ALGORITHM_51); partition_info::KEY_ALGORITHM_51);
@ -9878,13 +9877,12 @@ uint32 ha_partition::calculate_key_hash_value(Field **field_array)
{ {
if (field->is_null()) if (field->is_null())
{ {
nr1^= (nr1 << 1) | 1; hasher.add_null();
continue; continue;
} }
/* Force this to my_hash_sort_bin, which was used in 5.1! */ /* Force this to my_hash_sort_bin, which was used in 5.1! */
uint len= field->pack_length(); uint len= field->pack_length();
my_charset_bin.coll->hash_sort(&my_charset_bin, field->ptr, len, hasher.add(&my_charset_bin, field->ptr, len);
&nr1, &nr2);
/* Done with this field, continue with next one. */ /* Done with this field, continue with next one. */
continue; continue;
} }
@ -9902,13 +9900,12 @@ uint32 ha_partition::calculate_key_hash_value(Field **field_array)
{ {
if (field->is_null()) if (field->is_null())
{ {
nr1^= (nr1 << 1) | 1; hasher.add_null();
continue; continue;
} }
/* Force this to my_hash_sort_bin, which was used in 5.1! */ /* Force this to my_hash_sort_bin, which was used in 5.1! */
uint len= field->pack_length(); uint len= field->pack_length();
my_charset_latin1.coll->hash_sort(&my_charset_latin1, field->ptr, hasher.add(&my_charset_latin1, field->ptr, len);
len, &nr1, &nr2);
continue; continue;
} }
/* New types in mysql-5.6. */ /* New types in mysql-5.6. */
@ -9935,9 +9932,9 @@ uint32 ha_partition::calculate_key_hash_value(Field **field_array)
} }
/* fall through, use collation based hashing. */ /* fall through, use collation based hashing. */
} }
field->hash(&nr1, &nr2); field->hash(&hasher);
} while (*(++field_array)); } while (*(++field_array));
return (uint32) nr1; return (uint32) hasher.finalize();
} }

View File

@ -4145,6 +4145,35 @@ int handler::check_collation_compatibility()
} }
int handler::check_long_hash_compatibility() const
{
if (!table->s->old_long_hash_function())
return 0;
KEY *key= table->key_info;
KEY *key_end= key + table->s->keys;
for ( ; key < key_end; key++)
{
if (key->algorithm == HA_KEY_ALG_LONG_HASH)
{
/*
The old (pre-MDEV-27653) hash function was wrong.
So the long hash unique constraint can have some
duplicate records. REPAIR TABLE can't fix this,
it will fail on a duplicate key error.
Only "ALTER IGNORE TABLE .. FORCE" can fix this.
So we need to return HA_ADMIN_NEEDS_ALTER here,
(not HA_ADMIN_NEEDS_UPGRADE which is used elsewhere),
to properly send the error message text corresponding
to ER_TABLE_NEEDS_REBUILD (rather than to ER_TABLE_NEEDS_UPGRADE)
to the user.
*/
return HA_ADMIN_NEEDS_ALTER;
}
}
return 0;
}
int handler::ha_check_for_upgrade(HA_CHECK_OPT *check_opt) int handler::ha_check_for_upgrade(HA_CHECK_OPT *check_opt)
{ {
int error; int error;
@ -4183,6 +4212,9 @@ int handler::ha_check_for_upgrade(HA_CHECK_OPT *check_opt)
if (unlikely((error= check_collation_compatibility()))) if (unlikely((error= check_collation_compatibility())))
return error; return error;
if (unlikely((error= check_long_hash_compatibility())))
return error;
return check_for_upgrade(check_opt); return check_for_upgrade(check_opt);
} }

View File

@ -3271,6 +3271,7 @@ public:
} }
int check_collation_compatibility(); int check_collation_compatibility();
int check_long_hash_compatibility() const;
int ha_check_for_upgrade(HA_CHECK_OPT *check_opt); int ha_check_for_upgrade(HA_CHECK_OPT *check_opt);
/** to be actually called to get 'check()' functionality*/ /** to be actually called to get 'check()' functionality*/
int ha_check(THD *thd, HA_CHECK_OPT *check_opt); int ha_check(THD *thd, HA_CHECK_OPT *check_opt);

View File

@ -1271,6 +1271,12 @@ public:
*/ */
inline ulonglong val_uint() { return (ulonglong) val_int(); } inline ulonglong val_uint() { return (ulonglong) val_int(); }
virtual bool hash_not_null(Hasher *hasher)
{
DBUG_ASSERT(0);
return true;
}
/* /*
Return string representation of this item object. Return string representation of this item object.
@ -3460,6 +3466,13 @@ public:
{ {
return Sql_mode_dependency(0, field->value_depends_on_sql_mode()); return Sql_mode_dependency(0, field->value_depends_on_sql_mode());
} }
bool hash_not_null(Hasher *hasher)
{
if (field->is_null())
return true;
field->hash_not_null(hasher);
return false;
}
longlong val_int_endpoint(bool left_endp, bool *incl_endp); longlong val_int_endpoint(bool left_endp, bool *incl_endp);
bool get_date(THD *thd, MYSQL_TIME *ltime, date_mode_t fuzzydate); bool get_date(THD *thd, MYSQL_TIME *ltime, date_mode_t fuzzydate);
bool get_date_result(THD *thd, MYSQL_TIME *ltime,date_mode_t fuzzydate); bool get_date_result(THD *thd, MYSQL_TIME *ltime,date_mode_t fuzzydate);

View File

@ -1765,7 +1765,7 @@ static void calc_hash_for_unique(ulong &nr1, ulong &nr2, String *str)
cs->coll->hash_sort(cs, (uchar *)str->ptr(), str->length(), &nr1, &nr2); cs->coll->hash_sort(cs, (uchar *)str->ptr(), str->length(), &nr1, &nr2);
} }
longlong Item_func_hash::val_int() longlong Item_func_hash_mariadb_100403::val_int()
{ {
DBUG_EXECUTE_IF("same_long_unique_hash", return 9;); DBUG_EXECUTE_IF("same_long_unique_hash", return 9;);
unsigned_flag= true; unsigned_flag= true;
@ -1786,6 +1786,24 @@ longlong Item_func_hash::val_int()
} }
longlong Item_func_hash::val_int()
{
DBUG_EXECUTE_IF("same_long_unique_hash", return 9;);
unsigned_flag= true;
Hasher hasher;
for(uint i= 0;i<arg_count;i++)
{
if (args[i]->hash_not_null(&hasher))
{
null_value= 1;
return 0;
}
}
null_value= 0;
return (longlong) hasher.finalize();
}
bool Item_func_hash::fix_length_and_dec() bool Item_func_hash::fix_length_and_dec()
{ {
decimals= 0; decimals= 0;

View File

@ -1078,6 +1078,18 @@ public:
const char *func_name() const { return "<hash>"; } const char *func_name() const { return "<hash>"; }
}; };
class Item_func_hash_mariadb_100403: public Item_func_hash
{
public:
Item_func_hash_mariadb_100403(THD *thd, List<Item> &item)
:Item_func_hash(thd, item)
{}
longlong val_int();
Item *get_copy(THD *thd)
{ return get_item_copy<Item_func_hash_mariadb_100403>(thd, this); }
const char *func_name() const { return "<hash_mariadb_100403>"; }
};
class Item_longlong_func: public Item_int_func class Item_longlong_func: public Item_int_func
{ {
public: public:

View File

@ -1630,6 +1630,18 @@ bool Item_func_ucase::fix_length_and_dec()
} }
bool Item_func_left::hash_not_null(Hasher *hasher)
{
StringBuffer<STRING_BUFFER_USUAL_SIZE> buf;
String *str= val_str(&buf);
DBUG_ASSERT((str == NULL) == null_value);
if (!str)
return true;
hasher->add(collation.collation, str->ptr(), str->length());
return false;
}
String *Item_func_left::val_str(String *str) String *Item_func_left::val_str(String *str)
{ {
DBUG_ASSERT(fixed == 1); DBUG_ASSERT(fixed == 1);

View File

@ -461,6 +461,7 @@ class Item_func_left :public Item_str_func
String tmp_value; String tmp_value;
public: public:
Item_func_left(THD *thd, Item *a, Item *b): Item_str_func(thd, a, b) {} Item_func_left(THD *thd, Item *a, Item *b): Item_str_func(thd, a, b) {}
bool hash_not_null(Hasher *hasher);
String *val_str(String *); String *val_str(String *);
bool fix_length_and_dec(); bool fix_length_and_dec();
const char *func_name() const { return "left"; } const char *func_name() const { return "left"; }

View File

@ -35,7 +35,8 @@
#include "wsrep_mysqld.h" #include "wsrep_mysqld.h"
/* Prepare, run and cleanup for mysql_recreate_table() */ /* Prepare, run and cleanup for mysql_recreate_table() */
static bool admin_recreate_table(THD *thd, TABLE_LIST *table_list) static bool admin_recreate_table(THD *thd, TABLE_LIST *table_list,
Recreate_info *recreate_info)
{ {
bool result_code; bool result_code;
DBUG_ENTER("admin_recreate_table"); DBUG_ENTER("admin_recreate_table");
@ -56,7 +57,7 @@ static bool admin_recreate_table(THD *thd, TABLE_LIST *table_list)
DEBUG_SYNC(thd, "ha_admin_try_alter"); DEBUG_SYNC(thd, "ha_admin_try_alter");
tmp_disable_binlog(thd); // binlogging is done by caller if wanted tmp_disable_binlog(thd); // binlogging is done by caller if wanted
result_code= (thd->open_temporary_tables(table_list) || result_code= (thd->open_temporary_tables(table_list) ||
mysql_recreate_table(thd, table_list, false)); mysql_recreate_table(thd, table_list, recreate_info, false));
reenable_binlog(thd); reenable_binlog(thd);
/* /*
mysql_recreate_table() can push OK or ERROR. mysql_recreate_table() can push OK or ERROR.
@ -516,6 +517,7 @@ static bool mysql_admin_table(THD* thd, TABLE_LIST* tables,
bool open_error; bool open_error;
bool collect_eis= FALSE; bool collect_eis= FALSE;
bool open_for_modify= org_open_for_modify; bool open_for_modify= org_open_for_modify;
Recreate_info recreate_info;
DBUG_PRINT("admin", ("table: '%s'.'%s'", db, table->table_name.str)); DBUG_PRINT("admin", ("table: '%s'.'%s'", db, table->table_name.str));
DEBUG_SYNC(thd, "admin_command_kill_before_modify"); DEBUG_SYNC(thd, "admin_command_kill_before_modify");
@ -776,7 +778,8 @@ static bool mysql_admin_table(THD* thd, TABLE_LIST* tables,
{ {
/* We use extra_open_options to be able to open crashed tables */ /* We use extra_open_options to be able to open crashed tables */
thd->open_options|= extra_open_options; thd->open_options|= extra_open_options;
result_code= admin_recreate_table(thd, table); result_code= admin_recreate_table(thd, table, &recreate_info) ?
HA_ADMIN_FAILED : HA_ADMIN_OK;
thd->open_options&= ~extra_open_options; thd->open_options&= ~extra_open_options;
goto send_result; goto send_result;
} }
@ -947,12 +950,31 @@ static bool mysql_admin_table(THD* thd, TABLE_LIST* tables,
repair was not implemented and we need to upgrade the table repair was not implemented and we need to upgrade the table
to a new version so we recreate the table with ALTER TABLE to a new version so we recreate the table with ALTER TABLE
*/ */
result_code= admin_recreate_table(thd, table); result_code= admin_recreate_table(thd, table, &recreate_info);
} }
send_result: send_result:
lex->cleanup_after_one_table_open(); lex->cleanup_after_one_table_open();
thd->clear_error(); // these errors shouldn't get client thd->clear_error(); // these errors shouldn't get client
if (recreate_info.records_duplicate())
{
protocol->prepare_for_resend();
protocol->store(table_name, system_charset_info);
protocol->store((char*) operator_name, system_charset_info);
protocol->store(warning_level_names[Sql_condition::WARN_LEVEL_WARN].str,
warning_level_names[Sql_condition::WARN_LEVEL_WARN].length,
system_charset_info);
char buf[80];
size_t length= my_snprintf(buf, sizeof(buf),
"Number of rows changed from %u to %u",
(uint) recreate_info.records_processed(),
(uint) recreate_info.records_copied());
protocol->store(buf, length, system_charset_info);
if (protocol->write())
goto err;
}
{ {
Diagnostics_area::Sql_condition_iterator it= Diagnostics_area::Sql_condition_iterator it=
thd->get_stmt_da()->sql_conditions(); thd->get_stmt_da()->sql_conditions();
@ -1063,7 +1085,7 @@ send_result_message:
table->next_local= table->next_global= 0; table->next_local= table->next_global= 0;
tmp_disable_binlog(thd); // binlogging is done by caller if wanted tmp_disable_binlog(thd); // binlogging is done by caller if wanted
result_code= admin_recreate_table(thd, table); result_code= admin_recreate_table(thd, table, &recreate_info);
reenable_binlog(thd); reenable_binlog(thd);
trans_commit_stmt(thd); trans_commit_stmt(thd);
trans_commit(thd); trans_commit(thd);
@ -1409,6 +1431,7 @@ bool Sql_cmd_optimize_table::execute(THD *thd)
LEX *m_lex= thd->lex; LEX *m_lex= thd->lex;
TABLE_LIST *first_table= m_lex->first_select_lex()->table_list.first; TABLE_LIST *first_table= m_lex->first_select_lex()->table_list.first;
bool res= TRUE; bool res= TRUE;
Recreate_info recreate_info;
DBUG_ENTER("Sql_cmd_optimize_table::execute"); DBUG_ENTER("Sql_cmd_optimize_table::execute");
if (check_table_access(thd, SELECT_ACL | INSERT_ACL, first_table, if (check_table_access(thd, SELECT_ACL | INSERT_ACL, first_table,
@ -1417,7 +1440,7 @@ bool Sql_cmd_optimize_table::execute(THD *thd)
WSREP_TO_ISOLATION_BEGIN_WRTCHK(NULL, NULL, first_table); WSREP_TO_ISOLATION_BEGIN_WRTCHK(NULL, NULL, first_table);
res= (specialflag & SPECIAL_NO_NEW_FUNC) ? res= (specialflag & SPECIAL_NO_NEW_FUNC) ?
mysql_recreate_table(thd, first_table, true) : mysql_recreate_table(thd, first_table, &recreate_info, true) :
mysql_admin_table(thd, first_table, &m_lex->check_opt, mysql_admin_table(thd, first_table, &m_lex->check_opt,
"optimize", TL_WRITE, 1, 0, 0, 0, "optimize", TL_WRITE, 1, 0, 0, 0,
&handler::ha_optimize, 0, true); &handler::ha_optimize, 0, true);

View File

@ -527,9 +527,11 @@ bool Sql_cmd_alter_table::execute(THD *thd)
thd->work_part_info= 0; thd->work_part_info= 0;
#endif #endif
Recreate_info recreate_info;
result= mysql_alter_table(thd, &select_lex->db, &lex->name, result= mysql_alter_table(thd, &select_lex->db, &lex->name,
&create_info, &create_info,
first_table, first_table,
&recreate_info,
&alter_info, &alter_info,
select_lex->order_list.elements, select_lex->order_list.elements,
select_lex->order_list.first, select_lex->order_list.first,

View File

@ -7939,3 +7939,16 @@ bool THD::timestamp_to_TIME(MYSQL_TIME *ltime, my_time_t ts,
} }
return 0; return 0;
} }
void THD::my_ok_with_recreate_info(const Recreate_info &info,
ulong warn_count)
{
char buf[80];
my_snprintf(buf, sizeof(buf),
ER_THD(this, ER_INSERT_INFO),
(ulong) info.records_processed(),
(ulong) info.records_duplicate(),
warn_count);
my_ok(this, info.records_processed(), 0L, buf);
}

View File

@ -232,6 +232,29 @@ public:
}; };
class Recreate_info
{
ha_rows m_records_copied;
ha_rows m_records_duplicate;
public:
Recreate_info()
:m_records_copied(0),
m_records_duplicate(0)
{ }
Recreate_info(ha_rows records_copied,
ha_rows records_duplicate)
:m_records_copied(records_copied),
m_records_duplicate(records_duplicate)
{ }
ha_rows records_copied() const { return m_records_copied; }
ha_rows records_duplicate() const { return m_records_duplicate; }
ha_rows records_processed() const
{
return m_records_copied + m_records_duplicate;
}
};
#define TC_HEURISTIC_RECOVER_COMMIT 1 #define TC_HEURISTIC_RECOVER_COMMIT 1
#define TC_HEURISTIC_RECOVER_ROLLBACK 2 #define TC_HEURISTIC_RECOVER_ROLLBACK 2
extern ulong tc_heuristic_recover; extern ulong tc_heuristic_recover;
@ -3943,6 +3966,8 @@ public:
inline bool vio_ok() const { return TRUE; } inline bool vio_ok() const { return TRUE; }
inline bool is_connected() { return TRUE; } inline bool is_connected() { return TRUE; }
#endif #endif
void my_ok_with_recreate_info(const Recreate_info &info, ulong warn_count);
/** /**
Mark the current error as fatal. Warning: this does not Mark the current error as fatal. Warning: this does not
set any error, it sets a property of the error, so must be set any error, it sets a property of the error, so must be

View File

@ -4240,8 +4240,10 @@ mysql_execute_command(THD *thd)
create_info.row_type= ROW_TYPE_NOT_USED; create_info.row_type= ROW_TYPE_NOT_USED;
create_info.default_table_charset= thd->variables.collation_database; create_info.default_table_charset= thd->variables.collation_database;
Recreate_info recreate_info;
res= mysql_alter_table(thd, &first_table->db, &first_table->table_name, res= mysql_alter_table(thd, &first_table->db, &first_table->table_name,
&create_info, first_table, &alter_info, &create_info, first_table,
&recreate_info, &alter_info,
0, (ORDER*) 0, 0); 0, (ORDER*) 0, 0);
break; break;
} }

View File

@ -9619,6 +9619,7 @@ bool mysql_alter_table(THD *thd, const LEX_CSTRING *new_db,
const LEX_CSTRING *new_name, const LEX_CSTRING *new_name,
HA_CREATE_INFO *create_info, HA_CREATE_INFO *create_info,
TABLE_LIST *table_list, TABLE_LIST *table_list,
Recreate_info *recreate_info,
Alter_info *alter_info, Alter_info *alter_info,
uint order_num, ORDER *order, bool ignore) uint order_num, ORDER *order, bool ignore)
{ {
@ -10687,11 +10688,10 @@ end_inplace:
} }
end_temporary: end_temporary:
my_snprintf(alter_ctx.tmp_buff, sizeof(alter_ctx.tmp_buff), *recreate_info= Recreate_info(copied, deleted);
ER_THD(thd, ER_INSERT_INFO), thd->my_ok_with_recreate_info(*recreate_info,
(ulong) (copied + deleted), (ulong) deleted, (ulong) thd->get_stmt_da()->
(ulong) thd->get_stmt_da()->current_statement_warn_count()); current_statement_warn_count());
my_ok(thd, copied + deleted, 0L, alter_ctx.tmp_buff);
DEBUG_SYNC(thd, "alter_table_inplace_trans_commit"); DEBUG_SYNC(thd, "alter_table_inplace_trans_commit");
DBUG_RETURN(false); DBUG_RETURN(false);
@ -11208,7 +11208,8 @@ copy_data_between_tables(THD *thd, TABLE *from, TABLE *to,
Like mysql_alter_table(). Like mysql_alter_table().
*/ */
bool mysql_recreate_table(THD *thd, TABLE_LIST *table_list, bool table_copy) bool mysql_recreate_table(THD *thd, TABLE_LIST *table_list,
Recreate_info *recreate_info, bool table_copy)
{ {
HA_CREATE_INFO create_info; HA_CREATE_INFO create_info;
Alter_info alter_info; Alter_info alter_info;
@ -11233,8 +11234,10 @@ bool mysql_recreate_table(THD *thd, TABLE_LIST *table_list, bool table_copy)
Alter_info::ALTER_TABLE_ALGORITHM_COPY); Alter_info::ALTER_TABLE_ALGORITHM_COPY);
bool res= mysql_alter_table(thd, &null_clex_str, &null_clex_str, &create_info, bool res= mysql_alter_table(thd, &null_clex_str, &null_clex_str, &create_info,
table_list, &alter_info, 0, table_list, recreate_info, &alter_info, 0,
(ORDER *) 0, 0); (ORDER *) 0,
// Ignore duplicate records on REPAIR
thd->lex->sql_command == SQLCOM_REPAIR);
table_list->next_global= next_table; table_list->next_global= next_table;
DBUG_RETURN(res); DBUG_RETURN(res);
} }

View File

@ -220,13 +220,15 @@ bool mysql_trans_commit_alter_copy_data(THD *thd);
bool mysql_alter_table(THD *thd, const LEX_CSTRING *new_db, const LEX_CSTRING *new_name, bool mysql_alter_table(THD *thd, const LEX_CSTRING *new_db, const LEX_CSTRING *new_name,
HA_CREATE_INFO *create_info, HA_CREATE_INFO *create_info,
TABLE_LIST *table_list, TABLE_LIST *table_list,
class Recreate_info *recreate_info,
Alter_info *alter_info, Alter_info *alter_info,
uint order_num, ORDER *order, bool ignore); uint order_num, ORDER *order, bool ignore);
bool mysql_compare_tables(TABLE *table, bool mysql_compare_tables(TABLE *table,
Alter_info *alter_info, Alter_info *alter_info,
HA_CREATE_INFO *create_info, HA_CREATE_INFO *create_info,
bool *metadata_equal); bool *metadata_equal);
bool mysql_recreate_table(THD *thd, TABLE_LIST *table_list, bool table_copy); bool mysql_recreate_table(THD *thd, TABLE_LIST *table_list,
class Recreate_info *recreate_info, bool table_copy);
bool mysql_create_like_table(THD *thd, TABLE_LIST *table, bool mysql_create_like_table(THD *thd, TABLE_LIST *table,
TABLE_LIST *src_table, TABLE_LIST *src_table,
Table_specification_st *create_info); Table_specification_st *create_info);

View File

@ -110,6 +110,32 @@ enum scalar_comparison_op
}; };
class Hasher
{
ulong m_nr1;
ulong m_nr2;
public:
Hasher(): m_nr1(1), m_nr2(4)
{ }
void add_null()
{
m_nr1^= (m_nr1 << 1) | 1;
}
void add(CHARSET_INFO *cs, const uchar *str, size_t length)
{
cs->coll->hash_sort(cs, str, length, &m_nr1, &m_nr2);
}
void add(CHARSET_INFO *cs, const char *str, size_t length)
{
add(cs, (const uchar *) str, length);
}
uint32 finalize() const
{
return (uint32) m_nr1;
}
};
/* /*
A helper class to store column attributes that are inherited A helper class to store column attributes that are inherited
by columns (from the table level) when not specified explicitly. by columns (from the table level) when not specified explicitly.

View File

@ -1056,6 +1056,18 @@ static void mysql57_calculate_null_position(TABLE_SHARE *share,
} }
} }
Item_func_hash *TABLE_SHARE::make_long_hash_func(THD *thd,
MEM_ROOT *mem_root,
List<Item> *field_list)
const
{
if (old_long_hash_function())
return new (mem_root) Item_func_hash_mariadb_100403(thd, *field_list);
return new (mem_root) Item_func_hash(thd, *field_list);
}
/** Parse TABLE_SHARE::vcol_defs /** Parse TABLE_SHARE::vcol_defs
unpack_vcol_info_from_frm unpack_vcol_info_from_frm
@ -1267,7 +1279,10 @@ bool parse_vcol_defs(THD *thd, MEM_ROOT *mem_root, TABLE *table,
list_item= new (mem_root) Item_field(thd, keypart->field); list_item= new (mem_root) Item_field(thd, keypart->field);
field_list->push_back(list_item, mem_root); field_list->push_back(list_item, mem_root);
} }
Item_func_hash *hash_item= new(mem_root)Item_func_hash(thd, *field_list);
Item_func_hash *hash_item= table->s->make_long_hash_func(thd, mem_root,
field_list);
Virtual_column_info *v= new (mem_root) Virtual_column_info(); Virtual_column_info *v= new (mem_root) Virtual_column_info();
field->vcol_info= v; field->vcol_info= v;
field->vcol_info->expr= hash_item; field->vcol_info->expr= hash_item;

View File

@ -52,6 +52,7 @@ class Item; /* Needed by ORDER */
typedef Item (*Item_ptr); typedef Item (*Item_ptr);
class Item_subselect; class Item_subselect;
class Item_field; class Item_field;
class Item_func_hash;
class GRANT_TABLE; class GRANT_TABLE;
class st_select_lex_unit; class st_select_lex_unit;
class st_select_lex; class st_select_lex;
@ -1137,6 +1138,21 @@ struct TABLE_SHARE
void free_frm_image(const uchar *frm); void free_frm_image(const uchar *frm);
void set_overlapped_keys(); void set_overlapped_keys();
bool old_long_hash_function() const
{
return mysql_version < 100428 ||
(mysql_version >= 100500 && mysql_version < 100519) ||
(mysql_version >= 100600 && mysql_version < 100612) ||
(mysql_version >= 100700 && mysql_version < 100708) ||
(mysql_version >= 100800 && mysql_version < 100807) ||
(mysql_version >= 100900 && mysql_version < 100905) ||
(mysql_version >= 101000 && mysql_version < 101003) ||
(mysql_version >= 101100 && mysql_version < 101102);
}
Item_func_hash *make_long_hash_func(THD *thd,
MEM_ROOT *mem_root,
List<Item> *field_list) const;
}; };
/* not NULL, but cannot be dereferenced */ /* not NULL, but cannot be dereferenced */