MDEV-41252: partial join cost Assertion failure in recompute_join_cost_with_limit()

recompute_join_cost_with_limit() computes the cost of the first table's
partial join as best_read*fraction - pos->read_time*fraction. When the
two costs are nearly equal and large (e.g. with a huge
optimizer_scan_setup_cost, or when fraction is close to 1), the two
products are rounded independently. The difference can then be a small
negative number whose magnitude exceeds the absolute DBL_EPSILON
tolerance used by the debug assertion, even though it is only a
floating-point rounding artifact.

Fix: scale the assertion tolerance with the magnitude of the operands
(DBL_EPSILON * pos->read_time * fraction). Negative values are still
clamped to 0.0 as before.

Add test cases to optimizer_crash.test, one with a very large
optimizer_scan_setup_cost and optimizer_join_limit_pref_ratio=1, and one
with a large LIMIT on a 30000-row table.

Author: bsrikanth-mariadb

mysql-test/main/order_by_limit_join.result

@@ -684,4 +684,39 @@ SELECT * FROM t1  ORDER BY c1 LIMIT 1;
 c1
 1
 DROP TABLE t1;
+#
+# MDEV-41252: Assertion partial_join_cost >= -2.2204460492503131e-16
+# when optimizer_scan_setup_cost=100000000, and
+# optimizer_join_limit_pref_ratio=1
+#
+CREATE TABLE t (c INT NOT NULL UNIQUE) ENGINE=InnoDB;
+INSERT INTO t VALUES (0),(1),(2),(3),(4),(5);
+ANALYZE TABLE t;
+Table	Op	Msg_type	Msg_text
+test.t	analyze	status	Engine-independent statistics collected
+test.t	analyze	status	OK
+SET optimizer_scan_setup_cost=100000000;
+SET optimizer_join_limit_pref_ratio=1;
+EXPLAIN SELECT c FROM t ORDER BY c LIMIT 1;
+id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
+1	SIMPLE	t	index	NULL	c	4	NULL	1	
+SET optimizer_join_limit_pref_ratio= DEFAULT;
+SET optimizer_scan_setup_cost= DEFAULT;
+DROP TABLE t;
+# another variant, with only optimizer_join_limit_pref_ratio=1
+CREATE TABLE t (c INT NOT NULL UNIQUE) ENGINE=InnoDB;
+INSERT INTO t SELECT seq FROM seq_1_to_30000;
+ANALYZE TABLE t;
+Table	Op	Msg_type	Msg_text
+test.t	analyze	status	Engine-independent statistics collected
+test.t	analyze	status	OK
+SET optimizer_join_limit_pref_ratio=1;
+EXPLAIN SELECT c FROM t ORDER BY c LIMIT 28000;
+id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
+1	SIMPLE	t	index	NULL	c	4	NULL	28000	
+SET optimizer_join_limit_pref_ratio= DEFAULT;
+DROP TABLE t;
+#
+# End of 11.4 tests
+#
 set optimizer_join_limit_pref_ratio=default;

mysql-test/main/order_by_limit_join.test

@@ -344,4 +344,41 @@ INSERT INTO t1  VALUES (1),(2);
 SELECT * FROM t1  ORDER BY c1 LIMIT 1;
 DROP TABLE t1;
 
+--echo #
+--echo # MDEV-41252: Assertion partial_join_cost >= -2.2204460492503131e-16
+--echo # when optimizer_scan_setup_cost=100000000, and
+--echo # optimizer_join_limit_pref_ratio=1
+--echo #
+CREATE TABLE t (c INT NOT NULL UNIQUE) ENGINE=InnoDB;
+INSERT INTO t VALUES (0),(1),(2),(3),(4),(5);
+ANALYZE TABLE t;
+
+SET optimizer_scan_setup_cost=100000000;
+SET optimizer_join_limit_pref_ratio=1;
+
+EXPLAIN SELECT c FROM t ORDER BY c LIMIT 1;
+
+SET optimizer_join_limit_pref_ratio= DEFAULT;
+SET optimizer_scan_setup_cost= DEFAULT;
+
+DROP TABLE t;
+
+--echo # another variant, with only optimizer_join_limit_pref_ratio=1
+
+CREATE TABLE t (c INT NOT NULL UNIQUE) ENGINE=InnoDB;
+INSERT INTO t SELECT seq FROM seq_1_to_30000;
+ANALYZE TABLE t;
+
+SET optimizer_join_limit_pref_ratio=1;
+
+EXPLAIN SELECT c FROM t ORDER BY c LIMIT 28000;
+
+SET optimizer_join_limit_pref_ratio= DEFAULT;
+
+DROP TABLE t;
+
+--echo #
+--echo # End of 11.4 tests
+--echo #
+
 set optimizer_join_limit_pref_ratio=default;

sql/sql_select.cc

@@ -12005,9 +12005,11 @@ double recompute_join_cost_with_limit(const JOIN *join, bool skip_sorting,
       /*
         Handle possible rounding errors when
         partial_join_cost==pos->read_time*fraction:
+        The rounding error grows with the cost, so the tolerance is taken
+        in proportion to it.
         Catch larger negatives, proceed with non-negative cost.
       */
-      DBUG_ASSERT(partial_join_cost >= -DBL_EPSILON);
+      DBUG_ASSERT(partial_join_cost >= -DBL_EPSILON*pos->read_time*fraction);
       if (partial_join_cost < 0.0)
         partial_join_cost= 0.0;