From e1b9be5417668b184893148adea5f1f0e3a8d00f Mon Sep 17 00:00:00 2001 From: Alexander Barkov Date: Tue, 29 Dec 2015 14:17:31 +0400 Subject: [PATCH 1/3] MDEV-9319 ALTER from a bigger to a smaller blob type truncates too much data --- mysql-test/r/ctype_utf8.result | 16 +++++++++++++++ mysql-test/r/ctype_utf8mb4.result | 16 +++++++++++++++ mysql-test/r/myisam-blob.result | 4 +++- mysql-test/r/type_blob.result | 33 +++++++++++++++++++++++++++++++ mysql-test/t/ctype_utf8.test | 11 +++++++++++ mysql-test/t/ctype_utf8mb4.test | 12 +++++++++++ mysql-test/t/type_blob.test | 25 +++++++++++++++++++++++ sql/field.cc | 29 +++++++++++++++++++++++++++ sql/field.h | 11 ++++++----- sql/field_conv.cc | 9 +-------- 10 files changed, 152 insertions(+), 14 deletions(-) diff --git a/mysql-test/r/ctype_utf8.result b/mysql-test/r/ctype_utf8.result index 91dbe853a38..dfc872cc734 100644 --- a/mysql-test/r/ctype_utf8.result +++ b/mysql-test/r/ctype_utf8.result @@ -5804,5 +5804,21 @@ c0 c1 c2 c3 c4 2012-06-11 15:18:24 2012-06-11 15:18:24 2012-06-11 15:18:24 2012-06-11 15:18:24 2012-06-11 15:18:24 DROP TABLE t2, t1; # +# MDEV-9319 ALTER from a bigger to a smaller blob type truncates too much data +# +SET NAMES utf8; +CREATE TABLE t1 (a TEXT CHARACTER SET utf8); +INSERT INTO t1 VALUES (REPEAT('Ａ',100)); +SELECT OCTET_LENGTH(a) FROM t1; +OCTET_LENGTH(a) +300 +ALTER TABLE t1 MODIFY a TINYTEXT CHARACTER SET utf8; +Warnings: +Warning 1265 Data truncated for column 'a' at row 1 +SELECT OCTET_LENGTH(a),a FROM t1; +OCTET_LENGTH(a) a +255 ＡＡＡＡＡＡＡＡＡＡＡＡＡＡＡＡＡＡＡＡＡＡＡＡＡＡＡＡＡＡＡＡＡＡＡＡＡＡＡＡＡＡＡＡＡＡＡＡＡＡＡＡＡＡＡＡＡＡＡＡＡＡＡＡＡＡＡＡＡＡＡＡＡＡＡＡＡＡＡＡＡＡＡＡＡ +DROP TABLE t1; +# # End of 5.5 tests # diff --git a/mysql-test/r/ctype_utf8mb4.result b/mysql-test/r/ctype_utf8mb4.result index 64556308de2..29749d3f07b 100644 --- a/mysql-test/r/ctype_utf8mb4.result +++ b/mysql-test/r/ctype_utf8mb4.result @@ -2606,6 +2606,22 @@ Warning 1292 Truncated incorrect INTEGER value: 'a' DROP TABLE t1; # End of test for Bug#13581962,Bug#14096619 # +# MDEV-9319 ALTER from a bigger to a smaller blob type truncates too much data +# +SET NAMES utf8mb4; +CREATE TABLE t1 (a TEXT CHARACTER SET utf8mb4); +INSERT INTO t1 VALUES (REPEAT('😎',100)); +SELECT OCTET_LENGTH(a) FROM t1; +OCTET_LENGTH(a) +400 +ALTER TABLE t1 MODIFY a TINYTEXT CHARACTER SET utf8mb4; +Warnings: +Warning 1265 Data truncated for column 'a' at row 1 +SELECT OCTET_LENGTH(a),a FROM t1; +OCTET_LENGTH(a) a +252 😎😎😎😎😎😎😎😎😎😎😎😎😎😎😎😎😎😎😎😎😎😎😎😎😎😎😎😎😎😎😎😎😎😎😎😎😎😎😎😎😎😎😎😎😎😎😎😎😎😎😎😎😎😎😎😎😎😎😎😎😎😎😎 +DROP TABLE t1; +# # End of 5.5 tests # # diff --git a/mysql-test/r/myisam-blob.result b/mysql-test/r/myisam-blob.result index 43db7c8badd..6b41a244621 100644 --- a/mysql-test/r/myisam-blob.result +++ b/mysql-test/r/myisam-blob.result @@ -29,9 +29,11 @@ select length(data) from t1; length(data) 18874368 alter table t1 modify data blob; +Warnings: +Warning 1265 Data truncated for column 'data' at row 1 select length(data) from t1; length(data) -0 +65535 drop table t1; CREATE TABLE t1 (data BLOB) ENGINE=myisam; INSERT INTO t1 (data) VALUES (NULL); diff --git a/mysql-test/r/type_blob.result b/mysql-test/r/type_blob.result index 723efbdcb23..f49b2a7d5ef 100644 --- a/mysql-test/r/type_blob.result +++ b/mysql-test/r/type_blob.result @@ -1000,9 +1000,42 @@ COUNT(*) DROP FUNCTION f1; DROP TABLE t1; End of 5.1 tests +# +# Start of 5.5 tests +# CREATE TABLE t1 ( f1 blob, f2 blob ); INSERT INTO t1 VALUES ('',''); SELECT f1,f2,"found row" FROM t1 WHERE f1 = f2 ; f1 f2 found row found row DROP TABLE t1; +# +# MDEV-9319 ALTER from a bigger to a smaller blob type truncates too much data +# +CREATE TABLE t1 (a MEDIUMBLOB); +INSERT INTO t1 VALUES (REPEAT(0x61,128000)); +SELECT LENGTH(a) FROM t1; +LENGTH(a) +128000 +ALTER TABLE t1 MODIFY a BLOB; +Warnings: +Warning 1265 Data truncated for column 'a' at row 1 +SELECT LENGTH(a) FROM t1; +LENGTH(a) +65535 +DROP TABLE t1; +CREATE TABLE t1 (a BLOB); +INSERT INTO t1 VALUES (REPEAT(0x61,65000)); +SELECT LENGTH(a) FROM t1; +LENGTH(a) +65000 +ALTER TABLE t1 MODIFY a TINYBLOB; +Warnings: +Warning 1265 Data truncated for column 'a' at row 1 +SELECT LENGTH(a) FROM t1; +LENGTH(a) +255 +DROP TABLE t1; +# +# End of 5.5 tests +# diff --git a/mysql-test/t/ctype_utf8.test b/mysql-test/t/ctype_utf8.test index 8cd70e9261a..fd70292c6c8 100644 --- a/mysql-test/t/ctype_utf8.test +++ b/mysql-test/t/ctype_utf8.test @@ -1648,6 +1648,17 @@ SHOW CREATE TABLE t2; SELECT * FROM t2; DROP TABLE t2, t1; +--echo # +--echo # MDEV-9319 ALTER from a bigger to a smaller blob type truncates too much data +--echo # +SET NAMES utf8; +CREATE TABLE t1 (a TEXT CHARACTER SET utf8); +INSERT INTO t1 VALUES (REPEAT('Ａ',100)); +SELECT OCTET_LENGTH(a) FROM t1; +ALTER TABLE t1 MODIFY a TINYTEXT CHARACTER SET utf8; +SELECT OCTET_LENGTH(a),a FROM t1; +DROP TABLE t1; + --echo # --echo # End of 5.5 tests --echo # diff --git a/mysql-test/t/ctype_utf8mb4.test b/mysql-test/t/ctype_utf8mb4.test index fda20ca0ec5..66f5a3ba5ac 100644 --- a/mysql-test/t/ctype_utf8mb4.test +++ b/mysql-test/t/ctype_utf8mb4.test @@ -1826,6 +1826,18 @@ DROP TABLE t1; --echo # End of test for Bug#13581962,Bug#14096619 +--echo # +--echo # MDEV-9319 ALTER from a bigger to a smaller blob type truncates too much data +--echo # +SET NAMES utf8mb4; +CREATE TABLE t1 (a TEXT CHARACTER SET utf8mb4); +INSERT INTO t1 VALUES (REPEAT('😎',100)); +SELECT OCTET_LENGTH(a) FROM t1; +ALTER TABLE t1 MODIFY a TINYTEXT CHARACTER SET utf8mb4; +SELECT OCTET_LENGTH(a),a FROM t1; +DROP TABLE t1; + + --echo # --echo # End of 5.5 tests --echo # diff --git a/mysql-test/t/type_blob.test b/mysql-test/t/type_blob.test index 66dd71d2305..d5f6fc6ed32 100644 --- a/mysql-test/t/type_blob.test +++ b/mysql-test/t/type_blob.test @@ -643,6 +643,10 @@ DROP TABLE t1; --echo End of 5.1 tests +--echo # +--echo # Start of 5.5 tests +--echo # + # # Problem when comparing blobs #778901 # @@ -651,3 +655,24 @@ CREATE TABLE t1 ( f1 blob, f2 blob ); INSERT INTO t1 VALUES ('',''); SELECT f1,f2,"found row" FROM t1 WHERE f1 = f2 ; DROP TABLE t1; + +--echo # +--echo # MDEV-9319 ALTER from a bigger to a smaller blob type truncates too much data +--echo # +CREATE TABLE t1 (a MEDIUMBLOB); +INSERT INTO t1 VALUES (REPEAT(0x61,128000)); +SELECT LENGTH(a) FROM t1; +ALTER TABLE t1 MODIFY a BLOB; +SELECT LENGTH(a) FROM t1; +DROP TABLE t1; + +CREATE TABLE t1 (a BLOB); +INSERT INTO t1 VALUES (REPEAT(0x61,65000)); +SELECT LENGTH(a) FROM t1; +ALTER TABLE t1 MODIFY a TINYBLOB; +SELECT LENGTH(a) FROM t1; +DROP TABLE t1; + +--echo # +--echo # End of 5.5 tests +--echo # diff --git a/sql/field.cc b/sql/field.cc index aeeacf7f88d..ceea0893a3f 100644 --- a/sql/field.cc +++ b/sql/field.cc @@ -7068,6 +7068,35 @@ uint32 Field_blob::get_length(const uchar *pos, uint packlength_arg) } +/** + Copy a value from another BLOB field of the same character set. + This method is used by Copy_field, e.g. during ALTER TABLE. +*/ +int Field_blob::copy_value(Field_blob *from) +{ + DBUG_ASSERT(field_charset == from->charset()); + int rc= 0; + uint32 length= from->get_length(); + uchar *data; + from->get_ptr(&data); + if (packlength < from->packlength) + { + int well_formed_errors; + set_if_smaller(length, Field_blob::max_data_length()); + length= field_charset->cset->well_formed_len(field_charset, + (const char *) data, + (const char *) data + length, + length, &well_formed_errors); + rc= report_if_important_data((const char *) data + length, + (const char *) data + from->get_length(), + true); + } + store_length(length); + bmove(ptr + packlength, (uchar*) &data, sizeof(char*)); + return rc; +} + + int Field_blob::store(const char *from,uint length,CHARSET_INFO *cs) { ASSERT_COLUMN_MARKED_FOR_WRITE_OR_COMPUTED; diff --git a/sql/field.h b/sql/field.h index 4c79847228e..49f94179a92 100644 --- a/sql/field.h +++ b/sql/field.h @@ -1830,6 +1830,11 @@ protected: */ String value; + void store_length(uchar *i_ptr, uint i_packlength, uint32 i_number); + inline void store_length(uint32 number) + { + store_length(ptr, packlength, number); + } public: Field_blob(uchar *ptr_arg, uchar *null_ptr_arg, uchar null_bit_arg, enum utype unireg_check_arg, const char *field_name_arg, @@ -1902,11 +1907,6 @@ public: int reset(void) { bzero(ptr, packlength+sizeof(uchar*)); return 0; } void reset_fields() { bzero((uchar*) &value,sizeof(value)); } uint32 get_field_buffer_size(void) { return value.alloced_length(); } - void store_length(uchar *i_ptr, uint i_packlength, uint32 i_number); - inline void store_length(uint32 number) - { - store_length(ptr, packlength, number); - } inline uint32 get_length(uint row_offset= 0) { return get_length(ptr+row_offset, this->packlength); } uint32 get_length(const uchar *ptr, uint packlength); @@ -1935,6 +1935,7 @@ public: { set_ptr_offset(0, length, data); } + int copy_value(Field_blob *from); uint get_key_image(uchar *buff,uint length, imagetype type); void set_key_image(const uchar *buff,uint length); void sql_type(String &str) const; diff --git a/sql/field_conv.cc b/sql/field_conv.cc index d24f31e4fa1..3fdc1639266 100644 --- a/sql/field_conv.cc +++ b/sql/field_conv.cc @@ -332,9 +332,7 @@ static void do_copy_next_number(Copy_field *copy) static void do_copy_blob(Copy_field *copy) { - ulong length=((Field_blob*) copy->from_field)->get_length(); - ((Field_blob*) copy->to_field)->store_length(length); - memcpy(copy->to_ptr, copy->from_ptr, sizeof(char*)); + ((Field_blob*) copy->to_field)->copy_value(((Field_blob*) copy->from_field)); } static void do_conv_blob(Copy_field *copy) @@ -709,12 +707,7 @@ Copy_field::get_copy_func(Field *to,Field *from) if (!(from->flags & BLOB_FLAG) || from->charset() != to->charset()) return do_conv_blob; if (from_length != to_length) - { - // Correct pointer to point at char pointer - to_ptr+= to_length - to->table->s->blob_ptr_size; - from_ptr+= from_length- from->table->s->blob_ptr_size; return do_copy_blob; - } } else { From 93b078cc85f4ff11a0dca661d0089b1200d94981 Mon Sep 17 00:00:00 2001 From: Sergey Vojtovich Date: Sun, 27 Dec 2015 15:40:34 +0400 Subject: [PATCH 2/3] MDEV-9128 - Compiling on IBM System Z fails Provided IBM System Z have outdated compiler version, which supports gcc sync builtins but not gcc atomic builtins. It also has weak memory model. InnoDB attempted to verify if __sync_lock_test_and_set() is available by checking IB_STRONG_MEMORY_MODEL. This macro has nothing to do with availability of __sync_lock_test_and_set(), the right one is HAVE_ATOMIC_BUILTINS. --- storage/xtradb/include/os0sync.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/storage/xtradb/include/os0sync.h b/storage/xtradb/include/os0sync.h index 56c094547b1..db996b096bb 100644 --- a/storage/xtradb/include/os0sync.h +++ b/storage/xtradb/include/os0sync.h @@ -488,7 +488,7 @@ os_atomic_clear(volatile lock_word_t* ptr) __atomic_clear(ptr, __ATOMIC_RELEASE); } -# elif defined(IB_STRONG_MEMORY_MODEL) +# elif defined(HAVE_ATOMIC_BUILTINS) /** Do an atomic test and set. @param[in,out] ptr Memory location to set to non-zero From f31a89191f6b1679d9f8b584aa95fe006182ee7d Mon Sep 17 00:00:00 2001 From: Sergey Vojtovich Date: Wed, 18 Nov 2015 15:51:20 +0400 Subject: [PATCH 3/3] MDEV-9128 - Compiling on IBM System Z fails Provided IBM System Z have outdated compiler version, which supports gcc sync builtins but not gcc atomic builtins. It also has weak memory model. InnoDB attempted to verify if __sync_lock_test_and_set() is available by checking IB_STRONG_MEMORY_MODEL. This macro has nothing to do with availability of __sync_lock_test_and_set(), the right one is HAVE_ATOMIC_BUILTINS. --- storage/innobase/include/os0sync.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/storage/innobase/include/os0sync.h b/storage/innobase/include/os0sync.h index 8e4b4f41aae..e192a3a6c94 100644 --- a/storage/innobase/include/os0sync.h +++ b/storage/innobase/include/os0sync.h @@ -487,7 +487,7 @@ os_atomic_clear(volatile lock_word_t* ptr) __atomic_clear(ptr, __ATOMIC_RELEASE); } -# elif defined(IB_STRONG_MEMORY_MODEL) +# elif defined(HAVE_ATOMIC_BUILTINS) /** Do an atomic test and set. @param[in,out] ptr Memory location to set to non-zero