MDEV-31225 Crash converting IN with a nested ROW into IN subquery

What was wrong:
With in_predicate_conversion_threshold set low enough, an IN predicate
with a nested ROW, e.g.

  ROW(a,(a,a)) IN ((1,(1,1)),(2,(2,NULL)))

crashed the server. Before an IN list is converted into an IN subquery
over a table value constructor, cmp_row_types() checks every column
with subquery_type_allows_materialization(). With a nested ROW the
column is a ROW itself, and for a ROW that function must not be called.

How it is fixed:
A table value constructor cannot have a ROW as a column, so such a
predicate cannot be converted anyway. cmp_row_types() now reports the
columns as not comparable when either of them is a ROW, and the IN
predicate is evaluated without the conversion.

How it is tested:
Added a test to main.opt_tvc with IN and NOT IN over nested ROWs. It
crashes the server without the fix.

Author: GoldenEmperor1177

mysql-test/main/opt_tvc.result

@@ -791,4 +791,24 @@ select * from t1 where a in (1, 2) and b = 2 order by a, b;
 a	b	c
 1	2	NULL
 drop table t1;
+#
+# MDEV-31225: Signal 11 after select ..WHERE ROW(a,(a,a)) IN
+# ((1,(1,1)),(2,(2,null))) with in_predicate_conversion_threshold=1
+#
+create table t1 (a int);
+insert into t1 values (1), (2), (3);
+set in_predicate_conversion_threshold=1;
+explain extended select a from t1 where row(a,(a,a)) in ((1,(1,1)),(2,(2,null)));
+id	select_type	table	type	possible_keys	key	key_len	ref	rows	filtered	Extra
+1	SIMPLE	t1	ALL	NULL	NULL	NULL	NULL	3	100.00	Using where
+Warnings:
+Note	1003	select `test`.`t1`.`a` AS `a` from `test`.`t1` where (`test`.`t1`.`a`,(`test`.`t1`.`a`,`test`.`t1`.`a`)) in (<cache>((1,(1,1))),<cache>((2,(2,NULL))))
+select a from t1 where row(a,(a,a)) in ((1,(1,1)),(2,(2,null)));
+a
+1
+select a from t1 where row(a,(a,a)) not in ((1,(1,1)),(2,(2,2)));
+a
+3
+drop table t1;
+set in_predicate_conversion_threshold=default;
 # End of 11.4 tests

mysql-test/main/opt_tvc.test

@@ -488,4 +488,25 @@ select * from t1 where a in (1, 2) and b = 2 order by a, b;
 
 drop table t1;
 
+--echo #
+--echo # MDEV-31225: Signal 11 after select ..WHERE ROW(a,(a,a)) IN
+--echo # ((1,(1,1)),(2,(2,null))) with in_predicate_conversion_threshold=1
+--echo #
+
+create table t1 (a int);
+insert into t1 values (1), (2), (3);
+
+set in_predicate_conversion_threshold=1;
+
+let $query=
+select a from t1 where row(a,(a,a)) in ((1,(1,1)),(2,(2,null)));
+eval explain extended $query;
+eval $query;
+
+select a from t1 where row(a,(a,a)) not in ((1,(1,1)),(2,(2,2)));
+
+drop table t1;
+
+set in_predicate_conversion_threshold=default;
+
 --echo # End of 11.4 tests

sql/sql_tvc.cc

@@ -894,6 +894,12 @@ static bool cmp_row_types(Item* item1, Item* item2)
   {
     Item *inner= item1->element_index(i);
     Item *outer= item2->element_index(i);
+    /*
+      A table value constructor cannot have a ROW as a column, so an IN
+      predicate with a nested ROW cannot be converted.
+    */
+    if (inner->cols() != 1 || outer->cols() != 1)
+      return true;
     if (!inner->type_handler()->subquery_type_allows_materialization(inner,
                                                                      outer,
                                                                      true))