MDEV-16325 CREATE..SELECT..UNION creates a wrong field type for old varchar

- Adding Type_handler::traditional_merge_field_type()
- Removing real_type_to_type(), field_merge_type()
- Making Type_handler_var_string to merge as VARCHAR
- Additionally, fixing Field_string::print() to add the "/*old*/"
  comment into the data type for the old VARCHAR.
  This is similar to what MDEV-8267 earlier did for old DECIMAL.
- Adding tests

mysql-test/main/type_varchar.result

@@ -12,7 +12,7 @@ t1	CREATE TABLE `t1` (
 show create table vchar;
 Table	Create Table
 vchar	CREATE TABLE `vchar` (
-  `v` varchar(30) DEFAULT NULL,
+  `v` varchar(30)/*old*/ DEFAULT NULL,
   `c` char(3) DEFAULT NULL,
   `e` enum('abc','def','ghi') DEFAULT NULL,
   `t` text DEFAULT NULL

mysql-test/main/type_varchar_mysql41.result

@@ -0,0 +1,113 @@
+#
+# MDEV-16325 CREATE..SELECT..UNION creates a wrong field type for old varchar
+#
+CREATE PROCEDURE p1(col VARCHAR(32))
+BEGIN
+EXECUTE IMMEDIATE REPLACE('CREATE TABLE t2_simple AS SELECT col FROM t1old','col',col);
+SHOW CREATE TABLE t2_simple;
+DROP TABLE t2_simple;
+EXECUTE IMMEDIATE REPLACE('CREATE TABLE t2_union_vv AS SELECT col FROM t1old UNION SELECT col FROM t1old','col',col);
+SHOW CREATE TABLE t2_union_vv;
+DROP TABLE t2_union_vv;
+EXECUTE IMMEDIATE REPLACE('CREATE TABLE t2_union_vn AS SELECT col FROM t1old UNION SELECT NULL','col',col);
+SHOW CREATE TABLE t2_union_vn;
+DROP TABLE t2_union_vn;
+EXECUTE IMMEDIATE REPLACE('CREATE TABLE t2_union_nv AS SELECT NULL AS col UNION SELECT col FROM t1old','col',col);
+SHOW CREATE TABLE t2_union_nv;
+DROP TABLE t2_union_nv;
+EXECUTE IMMEDIATE REPLACE('CREATE TABLE t2 AS SELECT
+    COALESCE(col),
+    COALESCE(col,col),
+    COALESCE(col,NULL),
+    COALESCE(NULL,col)
+  FROM t1old', 'col', col);
+SHOW CREATE TABLE t2;
+DROP TABLE t2;
+EXECUTE IMMEDIATE REPLACE('CREATE TABLE t2 AS SELECT
+    LEAST(col,col),
+    LEAST(col,NULL),
+    LEAST(NULL,col)
+  FROM t1old','col',col);
+SHOW CREATE TABLE t2;
+DROP TABLE t2;
+END;
+$$
+TRUNCATE TABLE t1old;
+SHOW CREATE TABLE t1old;
+Table	Create Table
+t1old	CREATE TABLE `t1old` (
+  `v` varchar(30)/*old*/ DEFAULT NULL,
+  `c` char(3) DEFAULT NULL,
+  `e` enum('abc','def','ghi') DEFAULT NULL,
+  `t` text DEFAULT NULL
+) ENGINE=MyISAM DEFAULT CHARSET=latin1
+CALL p1('v');
+Table	Create Table
+t2_simple	CREATE TABLE `t2_simple` (
+  `v` varchar(30) DEFAULT NULL
+) ENGINE=MyISAM DEFAULT CHARSET=latin1
+Table	Create Table
+t2_union_vv	CREATE TABLE `t2_union_vv` (
+  `v` varchar(30) DEFAULT NULL
+) ENGINE=MyISAM DEFAULT CHARSET=latin1
+Table	Create Table
+t2_union_vn	CREATE TABLE `t2_union_vn` (
+  `v` varchar(30) DEFAULT NULL
+) ENGINE=MyISAM DEFAULT CHARSET=latin1
+Table	Create Table
+t2_union_nv	CREATE TABLE `t2_union_nv` (
+  `v` varchar(30) DEFAULT NULL
+) ENGINE=MyISAM DEFAULT CHARSET=latin1
+Table	Create Table
+t2	CREATE TABLE `t2` (
+  `COALESCE(v)` varchar(30) DEFAULT NULL,
+  `COALESCE(v,v)` varchar(30) DEFAULT NULL,
+  `COALESCE(v,NULL)` varchar(30) DEFAULT NULL,
+  `COALESCE(NULL,v)` varchar(30) DEFAULT NULL
+) ENGINE=MyISAM DEFAULT CHARSET=latin1
+Table	Create Table
+t2	CREATE TABLE `t2` (
+  `LEAST(v,v)` varchar(30) DEFAULT NULL,
+  `LEAST(v,NULL)` varchar(30) DEFAULT NULL,
+  `LEAST(NULL,v)` varchar(30) DEFAULT NULL
+) ENGINE=MyISAM DEFAULT CHARSET=latin1
+DROP TABLE t1old;
+TRUNCATE TABLE t1old;
+SHOW CREATE TABLE t1old;
+Table	Create Table
+t1old	CREATE TABLE `t1old` (
+  `a` varbinary(255)/*old*/ DEFAULT NULL,
+  `b` varchar(255)/*old*/ DEFAULT NULL
+) ENGINE=MyISAM DEFAULT CHARSET=latin1
+CALL p1('a');
+Table	Create Table
+t2_simple	CREATE TABLE `t2_simple` (
+  `a` varbinary(255) DEFAULT NULL
+) ENGINE=MyISAM DEFAULT CHARSET=latin1
+Table	Create Table
+t2_union_vv	CREATE TABLE `t2_union_vv` (
+  `a` varbinary(255) DEFAULT NULL
+) ENGINE=MyISAM DEFAULT CHARSET=latin1
+Table	Create Table
+t2_union_vn	CREATE TABLE `t2_union_vn` (
+  `a` varbinary(255) DEFAULT NULL
+) ENGINE=MyISAM DEFAULT CHARSET=latin1
+Table	Create Table
+t2_union_nv	CREATE TABLE `t2_union_nv` (
+  `a` varbinary(255) DEFAULT NULL
+) ENGINE=MyISAM DEFAULT CHARSET=latin1
+Table	Create Table
+t2	CREATE TABLE `t2` (
+  `COALESCE(a)` varbinary(255) DEFAULT NULL,
+  `COALESCE(a,a)` varbinary(255) DEFAULT NULL,
+  `COALESCE(a,NULL)` varbinary(255) DEFAULT NULL,
+  `COALESCE(NULL,a)` varbinary(255) DEFAULT NULL
+) ENGINE=MyISAM DEFAULT CHARSET=latin1
+Table	Create Table
+t2	CREATE TABLE `t2` (
+  `LEAST(a,a)` varbinary(255) DEFAULT NULL,
+  `LEAST(a,NULL)` varbinary(255) DEFAULT NULL,
+  `LEAST(NULL,a)` varbinary(255) DEFAULT NULL
+) ENGINE=MyISAM DEFAULT CHARSET=latin1
+DROP TABLE t1old;
+DROP PROCEDURE p1;

mysql-test/main/type_varchar_mysql41.test

@@ -0,0 +1,59 @@
+--echo #
+--echo # MDEV-16325 CREATE..SELECT..UNION creates a wrong field type for old varchar
+--echo #
+
+
+DELIMITER $$;
+CREATE PROCEDURE p1(col VARCHAR(32))
+BEGIN
+  EXECUTE IMMEDIATE REPLACE('CREATE TABLE t2_simple AS SELECT col FROM t1old','col',col);
+  SHOW CREATE TABLE t2_simple;
+  DROP TABLE t2_simple;
+
+  EXECUTE IMMEDIATE REPLACE('CREATE TABLE t2_union_vv AS SELECT col FROM t1old UNION SELECT col FROM t1old','col',col);
+  SHOW CREATE TABLE t2_union_vv;
+  DROP TABLE t2_union_vv;
+
+  EXECUTE IMMEDIATE REPLACE('CREATE TABLE t2_union_vn AS SELECT col FROM t1old UNION SELECT NULL','col',col);
+  SHOW CREATE TABLE t2_union_vn;
+  DROP TABLE t2_union_vn;
+
+  EXECUTE IMMEDIATE REPLACE('CREATE TABLE t2_union_nv AS SELECT NULL AS col UNION SELECT col FROM t1old','col',col);
+  SHOW CREATE TABLE t2_union_nv;
+  DROP TABLE t2_union_nv;
+
+  EXECUTE IMMEDIATE REPLACE('CREATE TABLE t2 AS SELECT
+    COALESCE(col),
+    COALESCE(col,col),
+    COALESCE(col,NULL),
+    COALESCE(NULL,col)
+  FROM t1old', 'col', col);
+  SHOW CREATE TABLE t2;
+  DROP TABLE t2;
+
+  EXECUTE IMMEDIATE REPLACE('CREATE TABLE t2 AS SELECT
+    LEAST(col,col),
+    LEAST(col,NULL),
+    LEAST(NULL,col)
+  FROM t1old','col',col);
+  SHOW CREATE TABLE t2;
+  DROP TABLE t2;
+END;
+$$
+DELIMITER ;$$
+
+let $MYSQLD_DATADIR= `SELECT @@datadir`;
+copy_file $MYSQL_TEST_DIR/std_data/vchar.frm $MYSQLD_DATADIR/test/t1old.frm;
+TRUNCATE TABLE t1old;
+SHOW CREATE TABLE t1old;
+CALL p1('v');
+DROP TABLE t1old;
+
+let $MYSQLD_DATADIR= `SELECT @@datadir`;
+copy_file $MYSQL_TEST_DIR/std_data/bug19371.frm $MYSQLD_DATADIR/test/t1old.frm;
+TRUNCATE TABLE t1old;
+SHOW CREATE TABLE t1old;
+CALL p1('a');
+DROP TABLE t1old;
+
+DROP PROCEDURE p1;

mysql-test/main/varbinary.result

@@ -82,8 +82,8 @@ drop table t1;
 show create table t1;
 Table	Create Table
 t1	CREATE TABLE `t1` (
-  `a` varbinary(255) DEFAULT NULL,
+  `a` varbinary(255)/*old*/ DEFAULT NULL,
-  `b` varchar(255) DEFAULT NULL
+  `b` varchar(255)/*old*/ DEFAULT NULL
 ) ENGINE=MyISAM DEFAULT CHARSET=latin1
 select length(a), length(b) from t1;
 length(a)	length(b)

sql/field.cc

@@ -80,15 +80,14 @@ const int FIELDTYPE_LAST=      254;
 const int FIELDTYPE_NUM=       FIELDTYPE_TEAR_FROM + (FIELDTYPE_LAST -
                                                       FIELDTYPE_TEAR_TO);
 
-static inline int field_type2index (enum_field_types field_type)
+static inline int merge_type2index(enum_field_types merge_type)
 {
-  DBUG_ASSERT(real_type_to_type(field_type) < FIELDTYPE_TEAR_FROM ||
+  DBUG_ASSERT(merge_type < FIELDTYPE_TEAR_FROM ||
-              real_type_to_type(field_type) > FIELDTYPE_TEAR_TO);
+              merge_type > FIELDTYPE_TEAR_TO);
-  DBUG_ASSERT(field_type <= FIELDTYPE_LAST);
+  DBUG_ASSERT(merge_type <= FIELDTYPE_LAST);
-  field_type= real_type_to_type(field_type);
+  if (merge_type < FIELDTYPE_TEAR_FROM)
-  if (field_type < FIELDTYPE_TEAR_FROM)
+    return merge_type;
-    return field_type;
+  return FIELDTYPE_TEAR_FROM + (merge_type - FIELDTYPE_TEAR_TO) - 1;
-  return FIELDTYPE_TEAR_FROM + (field_type - FIELDTYPE_TEAR_TO) - 1;
 }
 
 
@@ -913,31 +912,16 @@ static enum_field_types field_types_merge_rules [FIELDTYPE_NUM][FIELDTYPE_NUM]=
   }
 };
 
-/**
-  Return type of which can carry value of both given types in UNION result.
-
-  @param a  type for merging
-  @param b  type for merging
-
-  @return
-    type of field
-*/
-
-enum_field_types Field::field_type_merge(enum_field_types a,
-                                         enum_field_types b)
-{
-  return field_types_merge_rules[field_type2index(a)]
-                                [field_type2index(b)];
-}
 
 const Type_handler *
 Type_handler::aggregate_for_result_traditional(const Type_handler *a,
                                                const Type_handler *b)
 {
-  enum_field_types ta= a->real_field_type();
+  enum_field_types ta= a->traditional_merge_field_type();
-  enum_field_types tb= b->real_field_type();
+  enum_field_types tb= b->traditional_merge_field_type();
-  return
+  enum_field_types res= field_types_merge_rules[merge_type2index(ta)]
-    Type_handler::get_handler_by_real_type(Field::field_type_merge(ta, tb));
+                                               [merge_type2index(tb)];
+  return Type_handler::get_handler_by_real_type(res);
 }
 
 
@@ -7296,11 +7280,12 @@ void Field_string::sql_type(String &res) const
   size_t length;
 
   length= cs->cset->snprintf(cs,(char*) res.ptr(),
-                             res.alloced_length(), "%s(%d)",
+                             res.alloced_length(), "%s(%d)%s",
                              (type() == MYSQL_TYPE_VAR_STRING ?
                               (has_charset() ? "varchar" : "varbinary") :
 			      (has_charset() ? "char" : "binary")),
-                             (int) field_length / charset()->mbmaxlen);
+                             (int) field_length / charset()->mbmaxlen,
+                             type() == MYSQL_TYPE_VAR_STRING ? "/*old*/" : "");
   res.length(length);
   if ((thd->variables.sql_mode & (MODE_MYSQL323 | MODE_MYSQL40)) &&
       has_charset() && (charset()->state & MY_CS_BINSORT))

sql/field.h

@@ -467,31 +467,6 @@ inline bool is_temporal_type_with_date(enum_field_types type)
 }
 
 
-/**
-  Convert temporal real types as retuned by field->real_type()
-  to field type as returned by field->type().
-  
-  @param real_type  Real type.
-  @retval           Field type.
-*/
-inline enum_field_types real_type_to_type(enum_field_types real_type)
-{
-  switch (real_type)
-  {
-  case MYSQL_TYPE_TIME2:
-    return MYSQL_TYPE_TIME;
-  case MYSQL_TYPE_DATETIME2:
-    return MYSQL_TYPE_DATETIME;
-  case MYSQL_TYPE_TIMESTAMP2:
-    return MYSQL_TYPE_TIMESTAMP;
-  case MYSQL_TYPE_NEWDATE:
-    return MYSQL_TYPE_DATE;
-  /* Note: NEWDECIMAL is a type, not only a real_type */
-  default: return real_type;
-  }
-}
-
-
 enum enum_vcol_info_type
 {
   VCOL_GENERATED_VIRTUAL, VCOL_GENERATED_STORED,
@@ -873,7 +848,6 @@ public:
   {
     return type_handler()->cmp_type();
   }
-  static enum_field_types field_type_merge(enum_field_types, enum_field_types);
   virtual bool eq(Field *field)
   {
     return (ptr == field->ptr && null_ptr == field->null_ptr &&

sql/sp_head.cc

@@ -75,9 +75,8 @@ sp_map_item_type(const Type_handler *handler)
 {
   if (handler == &type_handler_row)
     return Item::ROW_ITEM;
-  enum_field_types type= real_type_to_type(handler->real_field_type());
 
-  switch (type) {
+  switch (handler->real_field_type()) {
   case MYSQL_TYPE_BIT:
   case MYSQL_TYPE_TINY:
   case MYSQL_TYPE_SHORT:

sql/sql_type.h

@@ -1021,6 +1021,28 @@ public:
   virtual const Name name() const= 0;
   virtual enum_field_types field_type() const= 0;
   virtual enum_field_types real_field_type() const { return field_type(); }
+  /**
+    Type code which is used for merging of traditional data types for result
+    (for UNION and for hybrid functions such as COALESCE).
+    Mapping can be done both ways: old->new, new->old, depending
+    on the particular data type implementation:
+    - type_handler_var_string (MySQL-4.1 old VARCHAR) is converted to
+      new VARCHAR before merging.
+      field_type_merge_rules[][] returns new VARCHAR.
+    - type_handler_newdate is converted to old DATE before merging.
+      field_type_merge_rules[][] returns NEWDATE.
+    - Temporal type_handler_xxx2 (new MySQL-5.6 types) are converted to
+      corresponding old type codes before merging (e.g. TIME2->TIME).
+      field_type_merge_rules[][] returns old type codes (e.g. TIME).
+      Then old types codes are supposed to convert to new type codes somehow,
+      but they do not. So UNION and COALESCE create old columns.
+      This is a bug and should be fixed eventually.
+  */
+  virtual enum_field_types traditional_merge_field_type() const
+  {
+    DBUG_ASSERT(is_traditional_type());
+    return field_type();
+  }
   virtual Item_result result_type() const= 0;
   virtual Item_result cmp_type() const= 0;
   virtual enum_mysql_timestamp_type mysql_timestamp_type() const
@@ -3123,6 +3145,10 @@ public:
   const Name name() const { return m_name_var_string; }
   enum_field_types field_type() const { return MYSQL_TYPE_VAR_STRING; }
   enum_field_types real_field_type() const { return MYSQL_TYPE_STRING; }
+  enum_field_types traditional_merge_field_type() const
+  {
+    return MYSQL_TYPE_VARCHAR;
+  }
   const Type_handler *type_handler_for_tmp_table(const Item *item) const
   {
     return varstring_type_handler(item);
@@ -3427,7 +3453,11 @@ class Type_handler_enum: public Type_handler_typelib
 public:
   virtual ~Type_handler_enum() {}
   const Name name() const { return m_name_enum; }
-  virtual enum_field_types real_field_type() const { return MYSQL_TYPE_ENUM; }
+  enum_field_types real_field_type() const { return MYSQL_TYPE_ENUM; }
+  enum_field_types traditional_merge_field_type() const
+  {
+    return MYSQL_TYPE_ENUM;
+  }
   uint32 calc_pack_length(uint32 length) const;
   Field *make_conversion_table_field(TABLE *, uint metadata,
                                      const Field *target) const;
@@ -3455,7 +3485,11 @@ class Type_handler_set: public Type_handler_typelib
 public:
   virtual ~Type_handler_set() {}
   const Name name() const { return m_name_set; }
-  virtual enum_field_types real_field_type() const { return MYSQL_TYPE_SET; }
+  enum_field_types real_field_type() const { return MYSQL_TYPE_SET; }
+  enum_field_types traditional_merge_field_type() const
+  {
+    return MYSQL_TYPE_SET;
+  }
   uint32 calc_pack_length(uint32 length) const;
   Field *make_conversion_table_field(TABLE *, uint metadata,
                                      const Field *target) const;

storage/tokudb/mysql-test/tokudb/r/type_varchar.result

@@ -13,7 +13,7 @@ t1	CREATE TABLE `t1` (
 show create table vchar;
 Table	Create Table
 vchar	CREATE TABLE `vchar` (
-  `v` varchar(30) DEFAULT NULL,
+  `v` varchar(30)/*old*/ DEFAULT NULL,
   `c` char(3) DEFAULT NULL,
   `e` enum('abc','def','ghi') DEFAULT NULL,
   `t` text DEFAULT NULL