Bug#11764487: myisam corruption with insert ignore and invalid spatial data

Problem: in case of wrong data insert into indexed GEOMETRY fields 
(e.g. NULL value for a not NULL field) MyISAM reported 
"ERROR 126 (HY000): Incorrect key file for table; try to repair it"
due to misuse of the key deletion function.

Fix: always use R-tree key functions for R-tree based indexes
and B-tree key functions for B-tree based indexes.

mysql-test/r/gis-rtree.result

@@ -1549,3 +1549,30 @@ HANDLER t1 READ a NEXT;
 HANDLER t1 CLOSE;
 DROP TABLE t1;
 End of 5.0 tests.
+#
+# Bug #57323/11764487: myisam corruption with insert ignore 
+# and invalid spatial data
+#
+CREATE TABLE t1(a LINESTRING NOT NULL, b GEOMETRY NOT NULL,
+SPATIAL KEY(a), SPATIAL KEY(b)) ENGINE=MyISAM;
+INSERT INTO t1 VALUES(GEOMFROMTEXT("point (0 0)"), GEOMFROMTEXT("point (1 1)"));
+INSERT IGNORE INTO t1 SET a=GEOMFROMTEXT("point (-6 0)"), b=GEOMFROMTEXT("error");
+ERROR 22003: Cannot get geometry object from data you send to the GEOMETRY field
+INSERT IGNORE INTO t1 SET a=GEOMFROMTEXT("point (-6 0)"), b=NULL;
+ERROR 22003: Cannot get geometry object from data you send to the GEOMETRY field
+SELECT ASTEXT(a), ASTEXT(b) FROM t1;
+ASTEXT(a)	ASTEXT(b)
+POINT(0 0)	POINT(1 1)
+DROP TABLE t1;
+CREATE TABLE t1(a INT NOT NULL, b GEOMETRY NOT NULL,
+KEY(a), SPATIAL KEY(b)) ENGINE=MyISAM;
+INSERT INTO t1 VALUES(0, GEOMFROMTEXT("point (1 1)"));
+INSERT IGNORE INTO t1 SET a=0, b=GEOMFROMTEXT("error");
+ERROR 22003: Cannot get geometry object from data you send to the GEOMETRY field
+INSERT IGNORE INTO t1 SET a=1, b=NULL;
+ERROR 22003: Cannot get geometry object from data you send to the GEOMETRY field
+SELECT a, ASTEXT(b) FROM t1;
+a	ASTEXT(b)
+0	POINT(1 1)
+DROP TABLE t1;
+End of 5.1 tests

mysql-test/t/gis-rtree.test

@@ -928,3 +928,31 @@ DROP TABLE t1;
 
 
 --echo End of 5.0 tests.
+
+
+--echo #
+--echo # Bug #57323/11764487: myisam corruption with insert ignore 
+--echo # and invalid spatial data
+--echo #
+
+CREATE TABLE t1(a LINESTRING NOT NULL, b GEOMETRY NOT NULL,
+  SPATIAL KEY(a), SPATIAL KEY(b)) ENGINE=MyISAM;
+INSERT INTO t1 VALUES(GEOMFROMTEXT("point (0 0)"), GEOMFROMTEXT("point (1 1)"));
+--error ER_CANT_CREATE_GEOMETRY_OBJECT
+INSERT IGNORE INTO t1 SET a=GEOMFROMTEXT("point (-6 0)"), b=GEOMFROMTEXT("error");
+--error ER_CANT_CREATE_GEOMETRY_OBJECT
+INSERT IGNORE INTO t1 SET a=GEOMFROMTEXT("point (-6 0)"), b=NULL;
+SELECT ASTEXT(a), ASTEXT(b) FROM t1;
+DROP TABLE t1;
+
+CREATE TABLE t1(a INT NOT NULL, b GEOMETRY NOT NULL,
+  KEY(a), SPATIAL KEY(b)) ENGINE=MyISAM;
+INSERT INTO t1 VALUES(0, GEOMFROMTEXT("point (1 1)"));
+--error ER_CANT_CREATE_GEOMETRY_OBJECT
+INSERT IGNORE INTO t1 SET a=0, b=GEOMFROMTEXT("error");
+--error ER_CANT_CREATE_GEOMETRY_OBJECT
+INSERT IGNORE INTO t1 SET a=1, b=NULL;
+SELECT a, ASTEXT(b) FROM t1;
+DROP TABLE t1;
+
+--echo End of 5.1 tests

storage/myisam/mi_update.c

@@ -193,8 +193,8 @@ err:
   save_errno=my_errno;
   if (changed)
     key_changed|= HA_STATE_CHANGED;
-  if (my_errno == HA_ERR_FOUND_DUPP_KEY || my_errno == HA_ERR_OUT_OF_MEM ||
+  if (my_errno == HA_ERR_FOUND_DUPP_KEY || my_errno == HA_ERR_RECORD_FILE_FULL ||
-      my_errno == HA_ERR_RECORD_FILE_FULL)
+      my_errno == HA_ERR_NULL_IN_SPATIAL || my_errno == HA_ERR_OUT_OF_MEM)
   {
     info->errkey= (int) i;
     flag=0;
@@ -212,8 +212,9 @@ err:
 	{
 	  uint new_length=_mi_make_key(info,i,new_key,newrec,pos);
 	  uint old_length= _mi_make_key(info,i,old_key,oldrec,pos);
-	  if ((flag++ && _mi_ck_delete(info,i,new_key,new_length)) ||
+	  if ((flag++ && 
-	      _mi_ck_write(info,i,old_key,old_length))
+	       share->keyinfo[i].ck_delete(info, i, new_key, new_length)) ||
+	      share->keyinfo[i].ck_insert(info, i, old_key, old_length))
 	    break;
 	}
       }

storage/myisam/mi_write.c

@@ -211,7 +211,7 @@ err:
         else
 	{
 	  uint key_length=_mi_make_key(info,i,buff,record,filepos);
-	  if (_mi_ck_delete(info,i,buff,key_length))
+	  if (share->keyinfo[i].ck_delete(info, i, buff, key_length))
 	  {
 	    if (local_lock_tree)
 	      rw_unlock(&share->key_root_lock[i]);