MDEV-33658 2/2 Cannot add a foreign key on a table with a matching long UNIQUE

Cannot add a foreign key on a table with a long UNIQUE multi-column index, that
contains a foreign key as a prefix.

Check that index algorithms match during the "generated" keys duplicate removal.

mysql-test/main/long_unique_bugs.result

@@ -741,6 +741,27 @@ insert into t1 select seq from seq_1_to_100;
 alter table t1 add partition (partition p3 values less than (maxvalue));
 alter table t1 force;
 drop table t1;
+#
+# MDEV-33658 cannot add a foreign key on a table with a long UNIQUE
+# multi-column index, that contains a foreign key as a prefix.
+#
+create table a (id int primary key) engine = innodb;
+create table b (id int,
+long_text varchar(1000),
+constraint unique_b unique key (id, long_text)
+) engine=innodb default charset utf8mb4;
+alter table b add constraint b_fk_id foreign key (id) references a (id);
+show create table b;
+Table	Create Table
+b	CREATE TABLE `b` (
+  `id` int(11) DEFAULT NULL,
+  `long_text` varchar(1000) DEFAULT NULL,
+  UNIQUE KEY `unique_b` (`id`,`long_text`) USING HASH,
+  KEY `b_fk_id` (`id`),
+  CONSTRAINT `b_fk_id` FOREIGN KEY (`id`) REFERENCES `a` (`id`)
+) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_general_ci
+drop table b;
+drop table a;
 # veirfy that duplicate has unique is detected
 create table t1 (a blob unique);
 alter table t1 add constraint constraint_1 unique (a);

mysql-test/main/long_unique_bugs.test

@@ -721,6 +721,21 @@ alter table t1 force;
 
 drop table t1;
 
+--echo #
+--echo # MDEV-33658 cannot add a foreign key on a table with a long UNIQUE
+--echo # multi-column index, that contains a foreign key as a prefix.
+--echo #
+create table a (id int primary key) engine = innodb;
+create table b (id int,
+                long_text varchar(1000),
+                constraint unique_b unique key (id, long_text)
+                ) engine=innodb default charset utf8mb4;
+
+alter table b add constraint b_fk_id foreign key (id) references a (id);
+show create table b;
+drop table b;
+drop table a;
+
 --echo # veirfy that duplicate has unique is detected
 create table t1 (a blob unique);
 alter table t1 add constraint constraint_1 unique (a);

sql/sql_class.cc

@@ -204,7 +204,7 @@ Foreign_key::Foreign_key(const Foreign_key &rhs, MEM_ROOT *mem_root)
 
 /*
   Test if a foreign key (= generated key) is a prefix of the given key
-  (ignoring key name, key type and order of columns)
+  (ignoring key name and type, but minding the algorithm)
 
   NOTES:
     This is only used to test if an index for a FOREIGN KEY exists
@@ -219,6 +219,16 @@ Foreign_key::Foreign_key(const Foreign_key &rhs, MEM_ROOT *mem_root)
 
 bool is_foreign_key_prefix(Key *a, Key *b)
 {
+  ha_key_alg a_alg= a->key_create_info.algorithm;
+  ha_key_alg b_alg= b->key_create_info.algorithm;
+
+  // The real algorithm in InnoDB will be BTREE if none was given by user.
+  a_alg= a_alg == HA_KEY_ALG_UNDEF ? HA_KEY_ALG_BTREE : a_alg;
+  b_alg= b_alg == HA_KEY_ALG_UNDEF ? HA_KEY_ALG_BTREE : b_alg;
+
+  if (a_alg != b_alg)
+    return false;
+
   /* Ensure that 'a' is the generated key */
   if (a->generated)
   {