Fixed the bug mdev-12429 and its duplicates mdev-12145 and mdev-9886.

Also fixed a wrong result for a test case for mdev-7691
(the alternative one).
The test  cases for all these bug have materialized semi-joins used
inside dependent sub-queries.

The patch actually reverts the change inroduced by Monty in 2003.
It looks like this change is not valid anymore after the implementation
of semi-joins.
Adjusted output from EXPLAIN for many other test cases.

mysql-test/r/innodb_icp,innodb_plugin.rdiff

@@ -1,5 +1,5 @@
---- r/innodb_icp.result	2013-07-16 17:01:00.000000000 +0400
+--- innodb_icp.result	2017-04-02 23:39:45.063149325 -0700
-+++ r/innodb_icp,innodb_plugin.reject	2013-07-16 17:16:53.000000000 +0400
++++ innodb_icp,innodb_plugin.result	2017-04-02 23:36:33.279164925 -0700
 @@ -213,7 +213,7 @@
  EXPLAIN
  SELECT c1 FROM t3 WHERE c1 >= 'c-1004=w' and c1 <= 'c-1006=w' and i1 > 2;

mysql-test/r/innodb_icp.result

@@ -432,7 +432,7 @@ WHERE pk IN (SELECT it.pk FROM t2 JOIN t2 AS it ON it.i=it.i WHERE it.pk-t1.i<10
 id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
 1	PRIMARY	t1	ALL	NULL	NULL	NULL	NULL	3	Using where
 2	DEPENDENT SUBQUERY	it	eq_ref	PRIMARY	PRIMARY	4	func	1	Using where
-2	DEPENDENT SUBQUERY	t2	index	NULL	PRIMARY	4	NULL	3	Using where; Using index; Using join buffer (flat, BNL join)
+2	DEPENDENT SUBQUERY	t2	index	NULL	PRIMARY	4	NULL	3	Using index; Using join buffer (flat, BNL join)
 SELECT * FROM t1
 WHERE pk IN (SELECT it.pk FROM t2 JOIN t2 AS it ON it.i=it.i WHERE it.pk-t1.i<10);
 pk	i

mysql-test/r/join_outer.result

@@ -1744,7 +1744,7 @@ id	select_type	table	type	possible_keys	key	key_len	ref	rows	filtered	Extra
 1	PRIMARY	t1	index	PRIMARY	PRIMARY	4	NULL	2	100.00	Using where; Using index; Using temporary; Using filesort
 1	PRIMARY	t2	eq_ref	PRIMARY	PRIMARY	4	func	1	100.00	Using where; Using index
 2	DEPENDENT SUBQUERY	t3	eq_ref	PRIMARY	PRIMARY	4	func	1	100.00	Using where; Using index
-2	DEPENDENT SUBQUERY	t4	eq_ref	PRIMARY	PRIMARY	4	test.t3.pk	1	100.00	Using where; Using index
+2	DEPENDENT SUBQUERY	t4	eq_ref	PRIMARY	PRIMARY	4	test.t3.pk	1	100.00	Using index
 Warnings:
 Note	1276	Field or reference 'test.t2.pk' of SELECT #2 was resolved in SELECT #1
 Note	1003	select `test`.`t2`.`pk` AS `pk`,<expr_cache><`test`.`t2`.`pk`>((select (`test`.`t3`.`pk` + if(isnull(`test`.`t4`.`pk`),0,`test`.`t4`.`pk`)) from `test`.`t3` left join `test`.`t4` on((`test`.`t4`.`pk` = `test`.`t3`.`pk`)) where (`test`.`t3`.`pk` = (`test`.`t2`.`pk` + 1000)) limit 1)) AS `t` from `test`.`t1` join `test`.`t2` where ((`test`.`t2`.`pk` = (`test`.`t1`.`pk` + 1000)) and (`test`.`t1`.`pk` > 1000)) group by `test`.`t2`.`pk`

mysql-test/r/join_outer_jcl6.result

@@ -1755,7 +1755,7 @@ id	select_type	table	type	possible_keys	key	key_len	ref	rows	filtered	Extra
 1	PRIMARY	t1	index	PRIMARY	PRIMARY	4	NULL	2	100.00	Using where; Using index; Using temporary; Using filesort
 1	PRIMARY	t2	eq_ref	PRIMARY	PRIMARY	4	func	1	100.00	Using where; Using index
 2	DEPENDENT SUBQUERY	t3	eq_ref	PRIMARY	PRIMARY	4	func	1	100.00	Using where; Using index
-2	DEPENDENT SUBQUERY	t4	eq_ref	PRIMARY	PRIMARY	4	test.t3.pk	1	100.00	Using where; Using index
+2	DEPENDENT SUBQUERY	t4	eq_ref	PRIMARY	PRIMARY	4	test.t3.pk	1	100.00	Using index
 Warnings:
 Note	1276	Field or reference 'test.t2.pk' of SELECT #2 was resolved in SELECT #1
 Note	1003	select `test`.`t2`.`pk` AS `pk`,<expr_cache><`test`.`t2`.`pk`>((select (`test`.`t3`.`pk` + if(isnull(`test`.`t4`.`pk`),0,`test`.`t4`.`pk`)) from `test`.`t3` left join `test`.`t4` on((`test`.`t4`.`pk` = `test`.`t3`.`pk`)) where (`test`.`t3`.`pk` = (`test`.`t2`.`pk` + 1000)) limit 1)) AS `t` from `test`.`t1` join `test`.`t2` where ((`test`.`t2`.`pk` = (`test`.`t1`.`pk` + 1000)) and (`test`.`t1`.`pk` > 1000)) group by `test`.`t2`.`pk`

mysql-test/r/myisam_icp.result

@@ -436,7 +436,7 @@ WHERE pk IN (SELECT it.pk FROM t2 JOIN t2 AS it ON it.i=it.i WHERE it.pk-t1.i<10
 id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
 1	PRIMARY	t1	ALL	NULL	NULL	NULL	NULL	3	Using where
 2	DEPENDENT SUBQUERY	it	eq_ref	PRIMARY	PRIMARY	4	func	1	Using index condition
-2	DEPENDENT SUBQUERY	t2	index	NULL	PRIMARY	4	NULL	3	Using where; Using index; Using join buffer (flat, BNL join)
+2	DEPENDENT SUBQUERY	t2	index	NULL	PRIMARY	4	NULL	3	Using index; Using join buffer (flat, BNL join)
 SELECT * FROM t1
 WHERE pk IN (SELECT it.pk FROM t2 JOIN t2 AS it ON it.i=it.i WHERE it.pk-t1.i<10);
 pk	i
@@ -797,7 +797,7 @@ id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
 1	PRIMARY	t	ALL	PRIMARY,c	NULL	NULL	NULL	64	Using where
 1	PRIMARY	t2	ref	g	g	5	test.t.c	19	Using where
 2	DEPENDENT SUBQUERY	t1	index	PRIMARY	PRIMARY	4	NULL	64	Using where; Using index
-2	DEPENDENT SUBQUERY	t2	eq_ref	PRIMARY	PRIMARY	4	test.t1.a	1	Using index condition; Using where
+2	DEPENDENT SUBQUERY	t2	eq_ref	PRIMARY	PRIMARY	4	test.t1.a	1	Using where
 SELECT COUNT(*) FROM t1 AS t, t2
 WHERE c = g
 AND (EXISTS (SELECT * FROM t1, t2 WHERE a = f  AND h <= t.e AND a > t.b)

mysql-test/r/subselect3.result

@@ -266,7 +266,7 @@ from t2;
 id	select_type	table	type	possible_keys	key	key_len	ref	rows	filtered	Extra
 1	PRIMARY	t2	ALL	NULL	NULL	NULL	NULL	2	100.00	
 2	DEPENDENT SUBQUERY	t1	ref_or_null	a	a	5	func	2	100.00	Using where; Full scan on NULL key
-2	DEPENDENT SUBQUERY	t4	ALL	NULL	NULL	NULL	NULL	100	100.00	Using where; Using join buffer (flat, BNL join)
+2	DEPENDENT SUBQUERY	t4	ALL	NULL	NULL	NULL	NULL	100	100.00	Using join buffer (flat, BNL join)
 Warnings:
 Note	1276	Field or reference 'test.t2.oref' of SELECT #2 was resolved in SELECT #1
 Note	1003	select `test`.`t2`.`a` AS `a`,`test`.`t2`.`b` AS `b`,`test`.`t2`.`oref` AS `oref`,<expr_cache><`test`.`t2`.`a`,`test`.`t2`.`b`,`test`.`t2`.`oref`>(<in_optimizer>((`test`.`t2`.`a`,`test`.`t2`.`b`),<exists>(select `test`.`t1`.`a`,`test`.`t1`.`b` from `test`.`t1` join `test`.`t4` where ((`test`.`t1`.`c` = `test`.`t2`.`oref`) and trigcond(((<cache>(`test`.`t2`.`a`) = `test`.`t1`.`a`) or isnull(`test`.`t1`.`a`))) and trigcond(((<cache>(`test`.`t2`.`b`) = `test`.`t1`.`b`) or isnull(`test`.`t1`.`b`)))) having (trigcond(<is_not_null_test>(`test`.`t1`.`a`)) and trigcond(<is_not_null_test>(`test`.`t1`.`b`)))))) AS `Z` from `test`.`t2`
@@ -1240,19 +1240,19 @@ explain select * from t2 where a in (select straight_join A.a from t1 A, t1 B);
 id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
 1	PRIMARY	t2	ALL	NULL	NULL	NULL	NULL	10	Using where
 2	DEPENDENT SUBQUERY	A	ALL	NULL	NULL	NULL	NULL	10	Using where
-2	DEPENDENT SUBQUERY	B	ALL	NULL	NULL	NULL	NULL	10	Using where; Using join buffer (flat, BNL join)
+2	DEPENDENT SUBQUERY	B	ALL	NULL	NULL	NULL	NULL	10	Using join buffer (flat, BNL join)
 explain select * from t2 where a in (select straight_join A.a from t1 A, t1 B);
 id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
 1	PRIMARY	t2	ALL	NULL	NULL	NULL	NULL	10	Using where
 2	DEPENDENT SUBQUERY	A	ALL	NULL	NULL	NULL	NULL	10	Using where
-2	DEPENDENT SUBQUERY	B	ALL	NULL	NULL	NULL	NULL	10	Using where; Using join buffer (flat, BNL join)
+2	DEPENDENT SUBQUERY	B	ALL	NULL	NULL	NULL	NULL	10	Using join buffer (flat, BNL join)
 explain select straight_join * from t2 X, t2 Y 
 where X.a in (select straight_join A.a from t1 A, t1 B);
 id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
 1	PRIMARY	X	ALL	NULL	NULL	NULL	NULL	10	Using where
 1	PRIMARY	Y	ALL	NULL	NULL	NULL	NULL	10	Using join buffer (flat, BNL join)
 2	DEPENDENT SUBQUERY	A	ALL	NULL	NULL	NULL	NULL	10	Using where
-2	DEPENDENT SUBQUERY	B	ALL	NULL	NULL	NULL	NULL	10	Using where; Using join buffer (flat, BNL join)
+2	DEPENDENT SUBQUERY	B	ALL	NULL	NULL	NULL	NULL	10	Using join buffer (flat, BNL join)
 create table t0 (a int, b int);
 insert into t0 values(1,1);
 explain select * from t0, t3 where t3.a in (select a from t2) and (t3.a < 10 or t3.a >30);

mysql-test/r/subselect3_jcl6.result

@@ -276,7 +276,7 @@ from t2;
 id	select_type	table	type	possible_keys	key	key_len	ref	rows	filtered	Extra
 1	PRIMARY	t2	ALL	NULL	NULL	NULL	NULL	2	100.00	
 2	DEPENDENT SUBQUERY	t1	ref_or_null	a	a	5	func	2	100.00	Using where; Full scan on NULL key
-2	DEPENDENT SUBQUERY	t4	ALL	NULL	NULL	NULL	NULL	100	100.00	Using where; Using join buffer (flat, BNL join)
+2	DEPENDENT SUBQUERY	t4	ALL	NULL	NULL	NULL	NULL	100	100.00	Using join buffer (flat, BNL join)
 Warnings:
 Note	1276	Field or reference 'test.t2.oref' of SELECT #2 was resolved in SELECT #1
 Note	1003	select `test`.`t2`.`a` AS `a`,`test`.`t2`.`b` AS `b`,`test`.`t2`.`oref` AS `oref`,<expr_cache><`test`.`t2`.`a`,`test`.`t2`.`b`,`test`.`t2`.`oref`>(<in_optimizer>((`test`.`t2`.`a`,`test`.`t2`.`b`),<exists>(select `test`.`t1`.`a`,`test`.`t1`.`b` from `test`.`t1` join `test`.`t4` where ((`test`.`t1`.`c` = `test`.`t2`.`oref`) and trigcond(((<cache>(`test`.`t2`.`a`) = `test`.`t1`.`a`) or isnull(`test`.`t1`.`a`))) and trigcond(((<cache>(`test`.`t2`.`b`) = `test`.`t1`.`b`) or isnull(`test`.`t1`.`b`)))) having (trigcond(<is_not_null_test>(`test`.`t1`.`a`)) and trigcond(<is_not_null_test>(`test`.`t1`.`b`)))))) AS `Z` from `test`.`t2`
@@ -1250,19 +1250,19 @@ explain select * from t2 where a in (select straight_join A.a from t1 A, t1 B);
 id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
 1	PRIMARY	t2	ALL	NULL	NULL	NULL	NULL	10	Using where
 2	DEPENDENT SUBQUERY	A	ALL	NULL	NULL	NULL	NULL	10	Using where
-2	DEPENDENT SUBQUERY	B	ALL	NULL	NULL	NULL	NULL	10	Using where; Using join buffer (flat, BNL join)
+2	DEPENDENT SUBQUERY	B	ALL	NULL	NULL	NULL	NULL	10	Using join buffer (flat, BNL join)
 explain select * from t2 where a in (select straight_join A.a from t1 A, t1 B);
 id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
 1	PRIMARY	t2	ALL	NULL	NULL	NULL	NULL	10	Using where
 2	DEPENDENT SUBQUERY	A	ALL	NULL	NULL	NULL	NULL	10	Using where
-2	DEPENDENT SUBQUERY	B	ALL	NULL	NULL	NULL	NULL	10	Using where; Using join buffer (flat, BNL join)
+2	DEPENDENT SUBQUERY	B	ALL	NULL	NULL	NULL	NULL	10	Using join buffer (flat, BNL join)
 explain select straight_join * from t2 X, t2 Y 
 where X.a in (select straight_join A.a from t1 A, t1 B);
 id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
 1	PRIMARY	X	ALL	NULL	NULL	NULL	NULL	10	Using where
 1	PRIMARY	Y	ALL	NULL	NULL	NULL	NULL	10	Using join buffer (flat, BNL join)
 2	DEPENDENT SUBQUERY	A	ALL	NULL	NULL	NULL	NULL	10	Using where
-2	DEPENDENT SUBQUERY	B	ALL	NULL	NULL	NULL	NULL	10	Using where; Using join buffer (flat, BNL join)
+2	DEPENDENT SUBQUERY	B	ALL	NULL	NULL	NULL	NULL	10	Using join buffer (flat, BNL join)
 create table t0 (a int, b int);
 insert into t0 values(1,1);
 explain select * from t0, t3 where t3.a in (select a from t2) and (t3.a < 10 or t3.a >30);

mysql-test/r/subselect4.result

@@ -19,7 +19,7 @@ SELECT 1 FROM t1
 WHERE NOT EXISTS (SELECT 1 FROM t2 WHERE 1 = (SELECT MIN(t2.b) FROM t3))
 ORDER BY count(*);
 id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
-1	PRIMARY	t1	index	NULL	a	5	NULL	2	Using where; Using index; Using temporary
+1	PRIMARY	t1	index	NULL	a	5	NULL	2	Using index; Using temporary
 2	DEPENDENT SUBQUERY	t2	ALL	NULL	NULL	NULL	NULL	2	Using where
 3	DEPENDENT SUBQUERY	t3	system	NULL	NULL	NULL	NULL	0	const row not found
 # should not crash the next statement
@@ -1418,7 +1418,7 @@ GROUP BY f9;
 id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
 1	PRIMARY	NULL	NULL	NULL	NULL	NULL	NULL	NULL	Impossible WHERE
 3	SUBQUERY	NULL	NULL	NULL	NULL	NULL	NULL	NULL	Impossible WHERE noticed after reading const tables
-2	DEPENDENT SUBQUERY	t1	ALL	NULL	NULL	NULL	NULL	2	Using where
+2	DEPENDENT SUBQUERY	t1	ALL	NULL	NULL	NULL	NULL	2	
 SELECT COUNT(t2.f3),
 (SELECT COUNT(f3) FROM t1 WHERE t2.f1) AS f9
 FROM t2 JOIN t1 ON t1.f3
@@ -1434,7 +1434,7 @@ ORDER BY f9;
 id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
 1	PRIMARY	NULL	NULL	NULL	NULL	NULL	NULL	NULL	Impossible WHERE
 3	SUBQUERY	NULL	NULL	NULL	NULL	NULL	NULL	NULL	Impossible WHERE noticed after reading const tables
-2	DEPENDENT SUBQUERY	t1	ALL	NULL	NULL	NULL	NULL	2	Using where
+2	DEPENDENT SUBQUERY	t1	ALL	NULL	NULL	NULL	NULL	2	
 SELECT COUNT(t2.f3),
 (SELECT COUNT(f3) FROM t1 WHERE t2.f1) AS f9
 FROM t2 JOIN t1 ON t1.f3
@@ -2444,9 +2444,11 @@ WHERE EXISTS (
 )";
 EXECUTE stmt;
 i
+4
 6
 EXECUTE stmt;
 i
+4
 6
 drop table t1, t2, t3;
 #

mysql-test/r/subselect_innodb.result

@@ -273,7 +273,7 @@ EXPLAIN SELECT 1 FROM t1 WHERE NOT EXISTS
 id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
 1	PRIMARY	t1	ALL	NULL	NULL	NULL	NULL	1	Using where
 2	DEPENDENT SUBQUERY	t2	eq_ref	PRIMARY,d	d	2	func	1	Using where
-3	DEPENDENT SUBQUERY	t2	index	NULL	d	2	NULL	1	Using where; Using index
+3	DEPENDENT SUBQUERY	t2	index	NULL	d	2	NULL	1	Using index
 DROP TABLE t2;
 CREATE TABLE t2 (b INT, c INT, UNIQUE KEY (b), UNIQUE KEY (b, c )) ENGINE=INNODB;
 INSERT INTO t2 VALUES (1, 1);

mysql-test/r/subselect_mat.result

@@ -2163,6 +2163,115 @@ execute stmt;
 a
 0
 drop table t1;
+#
+# MDEV-12429: IN subquery used in WHERE of EXISTS subquery
+#
+CREATE TABLE t1 (
+pk INT, f1 INT NOT NULL, f2 VARCHAR(3), f3 INT NULL, PRIMARY KEY(pk)) ENGINE=MyISAM;
+INSERT INTO t1 VALUES (1,1,'foo',8), (2,5,'bar',7);
+SELECT sq1.f2    FROM t1 AS sq1 
+WHERE EXISTS ( SELECT * FROM t1 AS sq2                      
+WHERE sq1.`pk` IN ( SELECT f1 FROM t1 ) AND sq2.f1 = sq1.f1 );
+f2
+foo
+EXPLAIN
+SELECT sq1.f2    FROM t1 AS sq1 
+WHERE EXISTS ( SELECT * FROM t1 AS sq2                      
+WHERE sq1.`pk` IN ( SELECT f1 FROM t1 ) AND sq2.f1 = sq1.f1 );
+id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
+1	PRIMARY	sq1	ALL	NULL	NULL	NULL	NULL	2	Using where
+2	DEPENDENT SUBQUERY	<subquery3>	eq_ref	distinct_key	distinct_key	4	func	1	
+2	DEPENDENT SUBQUERY	sq2	ALL	NULL	NULL	NULL	NULL	2	Using where; Using join buffer (flat, BNL join)
+3	MATERIALIZED	t1	ALL	NULL	NULL	NULL	NULL	2	
+# this checks the result set above
+set @save_optimizer_switch= @@optimizer_switch;
+set optimizer_switch= 'materialization=off,semijoin=off';
+SELECT sq1.f2    FROM t1 AS sq1 
+WHERE EXISTS ( SELECT * FROM t1 AS sq2                      
+WHERE sq1.`pk` IN ( SELECT f1 FROM t1 ) AND sq2.f1 = sq1.f1 );
+f2
+foo
+set optimizer_switch= @save_optimizer_switch;
+DROP TABLE t1;
+#
+# MDEV-12145: IN subquery used in WHERE of EXISTS subquery
+#
+CREATE TABLE t1 (f1 INT) ENGINE=MyISAM;
+INSERT INTO t1 VALUES (4),(6);
+CREATE TABLE t2 (i2 INT, KEY(i2)) ENGINE=MyISAM;
+INSERT INTO t2 VALUES (8),(7),(1);
+CREATE TABLE t3 (f3 INT, i3 INT, KEY(i3)) ENGINE=MyISAM;
+INSERT INTO t3 VALUES (8,0),(6,3),(2,8),(3,8),(1,6),(0,0),(1,0),(1,5);
+SELECT * FROM t1 
+WHERE EXISTS ( SELECT * FROM t2, t3
+WHERE i3 = i2 AND f1 IN ( SELECT f3 FROM t3 ) );
+f1
+6
+EXPLAIN EXTENDED
+SELECT * FROM t1 
+WHERE EXISTS ( SELECT * FROM t2, t3
+WHERE i3 = i2 AND f1 IN ( SELECT f3 FROM t3 ) );
+id	select_type	table	type	possible_keys	key	key_len	ref	rows	filtered	Extra
+1	PRIMARY	t1	ALL	NULL	NULL	NULL	NULL	2	100.00	Using where
+2	DEPENDENT SUBQUERY	<subquery3>	eq_ref	distinct_key	distinct_key	4	func	1	100.00	
+2	DEPENDENT SUBQUERY	t2	index	i2	i2	5	NULL	3	100.00	Using where; Using index; Using join buffer (flat, BNL join)
+2	DEPENDENT SUBQUERY	t3	ref	i3	i3	5	test.t2.i2	2	100.00	Using index
+3	MATERIALIZED	t3	ALL	NULL	NULL	NULL	NULL	8	100.00	
+Warnings:
+Note	1276	Field or reference 'test.t1.f1' of SELECT #2 was resolved in SELECT #1
+Note	1003	select `test`.`t1`.`f1` AS `f1` from `test`.`t1` where <expr_cache><`test`.`t1`.`f1`>(exists(select 1 from `test`.`t2` semi join (`test`.`t3`) join `test`.`t3` where ((`test`.`t3`.`i3` = `test`.`t2`.`i2`) and (`test`.`t1`.`f1` = `test`.`t3`.`f3`))))
+# this checks the result set above
+set @save_optimizer_switch= @@optimizer_switch;
+set optimizer_switch= 'materialization=off,semijoin=off';
+SELECT * FROM t1 
+WHERE EXISTS ( SELECT * FROM t2, t3
+WHERE i3 = i2 AND f1 IN ( SELECT f3 FROM t3 ) );
+f1
+6
+set optimizer_switch= @save_optimizer_switch;
+DROP TABLE t1,t2,t3;
+#
+# MDEV-9686: IN subquery used in WHERE of a subquery from select list
+#
+CREATE TABLE t1 (pk INT PRIMARY KEY, f1 INT);
+INSERT INTO t1 VALUES (1, 4),(2, 3),(3, 3),(4, 6),(5, 3);
+CREATE TABLE t2 (f2 INT);
+INSERT INTO t2 VALUES (1),(2),(3),(4),(5);
+# t1.pk is always IN ( SELECT f2 FROM t2 ), 
+# so the IN condition should be true for every row, 
+# and thus COUNT(*) should always return 5
+SELECT pk, f1, ( SELECT COUNT(*) FROM t2 
+WHERE t1.pk IN ( SELECT f2 FROM t2 ) ) AS sq FROM t1;
+pk	f1	sq
+1	4	5
+2	3	5
+3	3	5
+4	6	5
+5	3	5
+EXPLAIN EXTENDED
+SELECT pk, f1, ( SELECT COUNT(*) FROM t2 
+WHERE t1.pk IN ( SELECT f2 FROM t2 ) ) AS sq FROM t1;
+id	select_type	table	type	possible_keys	key	key_len	ref	rows	filtered	Extra
+1	PRIMARY	t1	ALL	NULL	NULL	NULL	NULL	5	100.00	
+2	DEPENDENT SUBQUERY	<subquery3>	eq_ref	distinct_key	distinct_key	4	func	1	100.00	
+2	DEPENDENT SUBQUERY	t2	ALL	NULL	NULL	NULL	NULL	5	100.00	Using join buffer (flat, BNL join)
+3	MATERIALIZED	t2	ALL	NULL	NULL	NULL	NULL	5	100.00	
+Warnings:
+Note	1276	Field or reference 'test.t1.pk' of SELECT #2 was resolved in SELECT #1
+Note	1003	select `test`.`t1`.`pk` AS `pk`,`test`.`t1`.`f1` AS `f1`,<expr_cache><`test`.`t1`.`pk`>((select count(0) from `test`.`t2` semi join (`test`.`t2`) where (`test`.`t1`.`pk` = `test`.`t2`.`f2`))) AS `sq` from `test`.`t1`
+# this checks the result set above
+set @save_optimizer_switch= @@optimizer_switch;
+set optimizer_switch= 'materialization=off,semijoin=off';
+SELECT pk, f1, ( SELECT COUNT(*) FROM t2 
+WHERE t1.pk IN ( SELECT f2 FROM t2 ) ) AS sq FROM t1;
+pk	f1	sq
+1	4	5
+2	3	5
+3	3	5
+4	6	5
+5	3	5
+set optimizer_switch= @save_optimizer_switch;
+DROP TABLE t1,t2;
 # End of 5.5 tests
 set @subselect_mat_test_optimizer_switch_value=null;
 set @@optimizer_switch='materialization=on,in_to_exists=off,semijoin=off';

mysql-test/r/subselect_sj.result

@@ -177,7 +177,7 @@ id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
 2	DEPENDENT SUBQUERY	m16	ALL	NULL	NULL	NULL	NULL	3	Using join buffer (flat, BNL join)
 2	DEPENDENT SUBQUERY	m17	ALL	NULL	NULL	NULL	NULL	3	Using join buffer (flat, BNL join)
 2	DEPENDENT SUBQUERY	m18	ALL	NULL	NULL	NULL	NULL	3	Using join buffer (flat, BNL join)
-2	DEPENDENT SUBQUERY	m19	ALL	NULL	NULL	NULL	NULL	3	Using where; Using join buffer (flat, BNL join)
+2	DEPENDENT SUBQUERY	m19	ALL	NULL	NULL	NULL	NULL	3	Using join buffer (flat, BNL join)
 select * from
 t1 left join t2 on (t2.a= t1.a and t2.a in (select pk from t10)) 
 where t1.a < 5;

mysql-test/r/subselect_sj_jcl6.result

@@ -190,7 +190,7 @@ id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
 2	DEPENDENT SUBQUERY	m16	ALL	NULL	NULL	NULL	NULL	3	Using join buffer (incremental, BNL join)
 2	DEPENDENT SUBQUERY	m17	ALL	NULL	NULL	NULL	NULL	3	Using join buffer (incremental, BNL join)
 2	DEPENDENT SUBQUERY	m18	ALL	NULL	NULL	NULL	NULL	3	Using join buffer (incremental, BNL join)
-2	DEPENDENT SUBQUERY	m19	ALL	NULL	NULL	NULL	NULL	3	Using where; Using join buffer (incremental, BNL join)
+2	DEPENDENT SUBQUERY	m19	ALL	NULL	NULL	NULL	NULL	3	Using join buffer (incremental, BNL join)
 select * from
 t1 left join t2 on (t2.a= t1.a and t2.a in (select pk from t10)) 
 where t1.a < 5;

mysql-test/r/subselect_sj_mat.result

@@ -2203,4 +2203,113 @@ execute stmt;
 a
 0
 drop table t1;
+#
+# MDEV-12429: IN subquery used in WHERE of EXISTS subquery
+#
+CREATE TABLE t1 (
+pk INT, f1 INT NOT NULL, f2 VARCHAR(3), f3 INT NULL, PRIMARY KEY(pk)) ENGINE=MyISAM;
+INSERT INTO t1 VALUES (1,1,'foo',8), (2,5,'bar',7);
+SELECT sq1.f2    FROM t1 AS sq1 
+WHERE EXISTS ( SELECT * FROM t1 AS sq2                      
+WHERE sq1.`pk` IN ( SELECT f1 FROM t1 ) AND sq2.f1 = sq1.f1 );
+f2
+foo
+EXPLAIN
+SELECT sq1.f2    FROM t1 AS sq1 
+WHERE EXISTS ( SELECT * FROM t1 AS sq2                      
+WHERE sq1.`pk` IN ( SELECT f1 FROM t1 ) AND sq2.f1 = sq1.f1 );
+id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
+1	PRIMARY	sq1	ALL	NULL	NULL	NULL	NULL	2	Using where
+2	DEPENDENT SUBQUERY	<subquery3>	eq_ref	distinct_key	distinct_key	4	func	1	
+2	DEPENDENT SUBQUERY	sq2	ALL	NULL	NULL	NULL	NULL	2	Using where; Using join buffer (flat, BNL join)
+3	MATERIALIZED	t1	ALL	NULL	NULL	NULL	NULL	2	
+# this checks the result set above
+set @save_optimizer_switch= @@optimizer_switch;
+set optimizer_switch= 'materialization=off,semijoin=off';
+SELECT sq1.f2    FROM t1 AS sq1 
+WHERE EXISTS ( SELECT * FROM t1 AS sq2                      
+WHERE sq1.`pk` IN ( SELECT f1 FROM t1 ) AND sq2.f1 = sq1.f1 );
+f2
+foo
+set optimizer_switch= @save_optimizer_switch;
+DROP TABLE t1;
+#
+# MDEV-12145: IN subquery used in WHERE of EXISTS subquery
+#
+CREATE TABLE t1 (f1 INT) ENGINE=MyISAM;
+INSERT INTO t1 VALUES (4),(6);
+CREATE TABLE t2 (i2 INT, KEY(i2)) ENGINE=MyISAM;
+INSERT INTO t2 VALUES (8),(7),(1);
+CREATE TABLE t3 (f3 INT, i3 INT, KEY(i3)) ENGINE=MyISAM;
+INSERT INTO t3 VALUES (8,0),(6,3),(2,8),(3,8),(1,6),(0,0),(1,0),(1,5);
+SELECT * FROM t1 
+WHERE EXISTS ( SELECT * FROM t2, t3
+WHERE i3 = i2 AND f1 IN ( SELECT f3 FROM t3 ) );
+f1
+6
+EXPLAIN EXTENDED
+SELECT * FROM t1 
+WHERE EXISTS ( SELECT * FROM t2, t3
+WHERE i3 = i2 AND f1 IN ( SELECT f3 FROM t3 ) );
+id	select_type	table	type	possible_keys	key	key_len	ref	rows	filtered	Extra
+1	PRIMARY	t1	ALL	NULL	NULL	NULL	NULL	2	100.00	Using where
+2	DEPENDENT SUBQUERY	<subquery3>	eq_ref	distinct_key	distinct_key	4	func	1	100.00	
+2	DEPENDENT SUBQUERY	t2	index	i2	i2	5	NULL	3	100.00	Using where; Using index; Using join buffer (flat, BNL join)
+2	DEPENDENT SUBQUERY	t3	ref	i3	i3	5	test.t2.i2	2	100.00	Using index
+3	MATERIALIZED	t3	ALL	NULL	NULL	NULL	NULL	8	100.00	
+Warnings:
+Note	1276	Field or reference 'test.t1.f1' of SELECT #2 was resolved in SELECT #1
+Note	1003	select `test`.`t1`.`f1` AS `f1` from `test`.`t1` where <expr_cache><`test`.`t1`.`f1`>(exists(select 1 from `test`.`t2` semi join (`test`.`t3`) join `test`.`t3` where ((`test`.`t3`.`i3` = `test`.`t2`.`i2`) and (`test`.`t1`.`f1` = `test`.`t3`.`f3`))))
+# this checks the result set above
+set @save_optimizer_switch= @@optimizer_switch;
+set optimizer_switch= 'materialization=off,semijoin=off';
+SELECT * FROM t1 
+WHERE EXISTS ( SELECT * FROM t2, t3
+WHERE i3 = i2 AND f1 IN ( SELECT f3 FROM t3 ) );
+f1
+6
+set optimizer_switch= @save_optimizer_switch;
+DROP TABLE t1,t2,t3;
+#
+# MDEV-9686: IN subquery used in WHERE of a subquery from select list
+#
+CREATE TABLE t1 (pk INT PRIMARY KEY, f1 INT);
+INSERT INTO t1 VALUES (1, 4),(2, 3),(3, 3),(4, 6),(5, 3);
+CREATE TABLE t2 (f2 INT);
+INSERT INTO t2 VALUES (1),(2),(3),(4),(5);
+# t1.pk is always IN ( SELECT f2 FROM t2 ), 
+# so the IN condition should be true for every row, 
+# and thus COUNT(*) should always return 5
+SELECT pk, f1, ( SELECT COUNT(*) FROM t2 
+WHERE t1.pk IN ( SELECT f2 FROM t2 ) ) AS sq FROM t1;
+pk	f1	sq
+1	4	5
+2	3	5
+3	3	5
+4	6	5
+5	3	5
+EXPLAIN EXTENDED
+SELECT pk, f1, ( SELECT COUNT(*) FROM t2 
+WHERE t1.pk IN ( SELECT f2 FROM t2 ) ) AS sq FROM t1;
+id	select_type	table	type	possible_keys	key	key_len	ref	rows	filtered	Extra
+1	PRIMARY	t1	ALL	NULL	NULL	NULL	NULL	5	100.00	
+2	DEPENDENT SUBQUERY	<subquery3>	eq_ref	distinct_key	distinct_key	4	func	1	100.00	
+2	DEPENDENT SUBQUERY	t2	ALL	NULL	NULL	NULL	NULL	5	100.00	Using join buffer (flat, BNL join)
+3	MATERIALIZED	t2	ALL	NULL	NULL	NULL	NULL	5	100.00	
+Warnings:
+Note	1276	Field or reference 'test.t1.pk' of SELECT #2 was resolved in SELECT #1
+Note	1003	select `test`.`t1`.`pk` AS `pk`,`test`.`t1`.`f1` AS `f1`,<expr_cache><`test`.`t1`.`pk`>((select count(0) from `test`.`t2` semi join (`test`.`t2`) where (`test`.`t1`.`pk` = `test`.`t2`.`f2`))) AS `sq` from `test`.`t1`
+# this checks the result set above
+set @save_optimizer_switch= @@optimizer_switch;
+set optimizer_switch= 'materialization=off,semijoin=off';
+SELECT pk, f1, ( SELECT COUNT(*) FROM t2 
+WHERE t1.pk IN ( SELECT f2 FROM t2 ) ) AS sq FROM t1;
+pk	f1	sq
+1	4	5
+2	3	5
+3	3	5
+4	6	5
+5	3	5
+set optimizer_switch= @save_optimizer_switch;
+DROP TABLE t1,t2;
 # End of 5.5 tests

mysql-test/t/subselect_sj_mat.test

@@ -1857,4 +1857,91 @@ execute stmt;
 
 drop table t1;
 
+--echo #
+--echo # MDEV-12429: IN subquery used in WHERE of EXISTS subquery
+--echo #
+
+CREATE TABLE t1 (
+  pk INT, f1 INT NOT NULL, f2 VARCHAR(3), f3 INT NULL, PRIMARY KEY(pk)) ENGINE=MyISAM;
+INSERT INTO t1 VALUES (1,1,'foo',8), (2,5,'bar',7);
+ 
+SELECT sq1.f2    FROM t1 AS sq1 
+  WHERE EXISTS ( SELECT * FROM t1 AS sq2                      
+                   WHERE sq1.`pk` IN ( SELECT f1 FROM t1 ) AND sq2.f1 = sq1.f1 );
+
+EXPLAIN
+SELECT sq1.f2    FROM t1 AS sq1 
+  WHERE EXISTS ( SELECT * FROM t1 AS sq2                      
+                   WHERE sq1.`pk` IN ( SELECT f1 FROM t1 ) AND sq2.f1 = sq1.f1 );
+
+--echo # this checks the result set above
+set @save_optimizer_switch= @@optimizer_switch;
+set optimizer_switch= 'materialization=off,semijoin=off';
+SELECT sq1.f2    FROM t1 AS sq1 
+  WHERE EXISTS ( SELECT * FROM t1 AS sq2                      
+                   WHERE sq1.`pk` IN ( SELECT f1 FROM t1 ) AND sq2.f1 = sq1.f1 );
+set optimizer_switch= @save_optimizer_switch;
+
+DROP TABLE t1;
+
+--echo #
+--echo # MDEV-12145: IN subquery used in WHERE of EXISTS subquery
+--echo #
+
+CREATE TABLE t1 (f1 INT) ENGINE=MyISAM;
+INSERT INTO t1 VALUES (4),(6);
+
+CREATE TABLE t2 (i2 INT, KEY(i2)) ENGINE=MyISAM;
+INSERT INTO t2 VALUES (8),(7),(1);
+
+CREATE TABLE t3 (f3 INT, i3 INT, KEY(i3)) ENGINE=MyISAM;
+INSERT INTO t3 VALUES (8,0),(6,3),(2,8),(3,8),(1,6),(0,0),(1,0),(1,5);
+
+SELECT * FROM t1 
+  WHERE EXISTS ( SELECT * FROM t2, t3
+                   WHERE i3 = i2 AND f1 IN ( SELECT f3 FROM t3 ) );
+EXPLAIN EXTENDED
+SELECT * FROM t1 
+  WHERE EXISTS ( SELECT * FROM t2, t3
+                   WHERE i3 = i2 AND f1 IN ( SELECT f3 FROM t3 ) );
+
+--echo # this checks the result set above
+set @save_optimizer_switch= @@optimizer_switch;
+set optimizer_switch= 'materialization=off,semijoin=off';
+SELECT * FROM t1 
+  WHERE EXISTS ( SELECT * FROM t2, t3
+                   WHERE i3 = i2 AND f1 IN ( SELECT f3 FROM t3 ) );
+set optimizer_switch= @save_optimizer_switch;
+
+DROP TABLE t1,t2,t3;
+
+--echo #
+--echo # MDEV-9686: IN subquery used in WHERE of a subquery from select list
+--echo #
+
+CREATE TABLE t1 (pk INT PRIMARY KEY, f1 INT);
+INSERT INTO t1 VALUES (1, 4),(2, 3),(3, 3),(4, 6),(5, 3);
+ 
+CREATE TABLE t2 (f2 INT);
+INSERT INTO t2 VALUES (1),(2),(3),(4),(5);
+ 
+--echo # t1.pk is always IN ( SELECT f2 FROM t2 ), 
+--echo # so the IN condition should be true for every row, 
+--echo # and thus COUNT(*) should always return 5
+ 
+SELECT pk, f1, ( SELECT COUNT(*) FROM t2 
+                   WHERE t1.pk IN ( SELECT f2 FROM t2 ) ) AS sq FROM t1;
+EXPLAIN EXTENDED
+SELECT pk, f1, ( SELECT COUNT(*) FROM t2 
+                   WHERE t1.pk IN ( SELECT f2 FROM t2 ) ) AS sq FROM t1;
+
+--echo # this checks the result set above
+set @save_optimizer_switch= @@optimizer_switch;
+set optimizer_switch= 'materialization=off,semijoin=off';
+SELECT pk, f1, ( SELECT COUNT(*) FROM t2 
+                   WHERE t1.pk IN ( SELECT f2 FROM t2 ) ) AS sq FROM t1;
+set optimizer_switch= @save_optimizer_switch;
+
+DROP TABLE t1,t2;
+
 --echo # End of 5.5 tests

sql/sql_select.cc

@@ -8794,7 +8794,7 @@ make_join_select(JOIN *join,SQL_SELECT *select,COND *cond)
 	It solve problem with select like SELECT * FROM t1 WHERE rand() > 0.5
       */
       if (tab == join->join_tab + join->top_join_tab_count - 1)
-	current_map|= OUTER_REF_TABLE_BIT | RAND_TABLE_BIT;
+        current_map|= RAND_TABLE_BIT;
       used_tables|=current_map;
 
       if (tab->type == JT_REF && tab->quick &&