MDEV-35182 crash in online_alter_end_trans with XA over vector indexes

ONLINE ALTER didn't expect XA PREPARE to fail.
Mark rollback on failed prepare with the XA_ROLLBACK_ONLY state,
detect that in ONLINE ALTER

mysql-test/main/vector2_notembedded.result

@@ -71,3 +71,22 @@ create table t (a int primary key, v vector(10) not null, vector index(v));
 alter table t modify a int auto_increment, lock=none;
 ERROR 0A000: LOCK=NONE is not supported. Reason: CHANGE COLUMN ... AUTO_INCREMENT. Try LOCK=SHARED
 drop table t;
+#
+# MDEV-35061 XA PREPARE "not supported by the engine" from storage engine mhnsw, memory leak
+#
+create table t1 (v vector(1) not null, vector(v)) engine=innodb;
+insert into t1 select 0x00000000 as v from seq_1_to_1000;
+connect con1,localhost,root;
+alter table t1 add column x int, algorithm=copy;;
+connection default;
+xa start 'x';
+select * from non_existing_table;
+ERROR 42S02: Table 'test.non_existing_table' doesn't exist
+delete from t1;
+xa end 'x';
+xa prepare 'x';
+ERROR HY000: Got error 138 "Unsupported extension used for table" from storage engine mhnsw
+connection con1;
+disconnect con1;
+connection default;
+drop table t1;

mysql-test/main/vector2_notembedded.test

@@ -1,4 +1,6 @@
 --source include/not_embedded.inc
+--source include/have_innodb.inc
+--source include/have_sequence.inc
 
 --echo #
 --echo # mysqldump
@@ -32,3 +34,25 @@ create table t (a int primary key, v vector(10) not null, vector index(v));
 --error ER_ALTER_OPERATION_NOT_SUPPORTED_REASON
 alter table t modify a int auto_increment, lock=none;
 drop table t;
+
+--echo #
+--echo # MDEV-35061 XA PREPARE "not supported by the engine" from storage engine mhnsw, memory leak
+--echo #
+create table t1 (v vector(1) not null, vector(v)) engine=innodb;
+insert into t1 select 0x00000000 as v from seq_1_to_1000;
+
+--connect con1,localhost,root
+--send alter table t1 add column x int, algorithm=copy;
+--connection default
+xa start 'x';
+--error ER_NO_SUCH_TABLE
+select * from non_existing_table;
+delete from t1;
+xa end 'x';
+--error ER_GET_ERRNO
+xa prepare 'x';
+--connection con1
+--reap
+--disconnect con1
+--connection default
+drop table t1;

sql/handler.cc

@@ -1543,6 +1543,7 @@ int ha_prepare(THD *thd)
       {
         if (unlikely(prepare_or_error(ht, thd, all)))
         {
+          thd->transaction->xid_state.set_rollback_only();
           ha_rollback_trans(thd, all);
           error=1;
           break;

sql/online_alter.cc

@@ -385,8 +385,9 @@ static int online_alter_rollback(THD *thd, bool all)
 {
   int res;
   bool is_ending_transaction= ending_trans(thd, all);
-  if (is_ending_transaction
+  if (is_ending_transaction &&
-      && thd->transaction->xid_state.get_state_code() == XA_PREPARED)
+      (thd->transaction->xid_state.get_state_code() == XA_PREPARED ||
+       thd->transaction->xid_state.get_state_code() == XA_ROLLBACK_ONLY))
   {
     res= online_alter_rollback_by_xid(thd->transaction->xid_state.get_xid());
     // cleanup was already done by prepare()