diff --git a/mysql-test/main/vector2.result b/mysql-test/main/vector2.result
index 253ef8aecf7..798d09dccad 100644
--- a/mysql-test/main/vector2.result
+++ b/mysql-test/main/vector2.result
@@ -292,3 +292,15 @@ d	count(*)
 1	2
 2	1
 drop table t;
+#
+# MDEV-35219 Unexpected ER_DUP_KEY after OPTIMIZE on MyISAM table with vector key
+#
+create table t (v vector(1) not null default 0x30303030, vector(v)) engine=myisam;
+insert into t () values (),(),(),();
+delete from t limit 1;
+optimize table t;
+Table	Op	Msg_type	Msg_text
+test.t	optimize	note	Table does not support optimize, doing recreate + analyze instead
+test.t	optimize	status	OK
+insert into t select * from t;
+drop table t;
diff --git a/mysql-test/main/vector2.test b/mysql-test/main/vector2.test
index 72089f8f6f1..c1c70205bb1 100644
--- a/mysql-test/main/vector2.test
+++ b/mysql-test/main/vector2.test
@@ -226,3 +226,13 @@ create table t (a int, v vector(1) not null, primary key (a), vector(v));
 insert into t values (1,vec_fromtext('[-1]')),(2,vec_fromtext('[1]')),(3,vec_fromtext('[2]'));
 select vec_distance_euclidean(v,vec_fromtext('[0]')) d, count(*) from t group by d order by d limit 2;
 drop table t;
+
+--echo #
+--echo # MDEV-35219 Unexpected ER_DUP_KEY after OPTIMIZE on MyISAM table with vector key
+--echo #
+create table t (v vector(1) not null default 0x30303030, vector(v)) engine=myisam;
+insert into t () values (),(),(),();
+delete from t limit 1;
+optimize table t;
+insert into t select * from t;
+drop table t;
diff --git a/sql/handler.cc b/sql/handler.cc
index 123a1db9baa..942e856dea8 100644
--- a/sql/handler.cc
+++ b/sql/handler.cc
@@ -5534,7 +5534,8 @@ handler::ha_optimize(THD* thd, HA_CHECK_OPT* check_opt)
               m_lock_type == F_WRLCK);
   mark_trx_read_write();
 
-  return optimize(thd, check_opt);
+  // in-engine optimize can modify rowids, which will break hlindexes
+  return table->s->hlindexes() ? HA_ADMIN_TRY_ALTER : optimize(thd, check_opt);
 }