Refactor SQL expression cleaning to preserve charset introducers for functional index compatibility (#599)

Author: rakhiagr

dao-impl/ebean-dao/src/main/java/com/linkedin/metadata/dao/utils/SchemaValidatorUtil.java

@@ -119,22 +119,24 @@ public boolean indexExists(@Nonnull String tableName, @Nonnull String indexName)
 
 
   /**
-   * Cleans SQL expression by removing MySQL-specific encoding artifacts that otherwise result in unrecognized syntax.
-   * Removes _utf8mb4 and _utf8mb3 charset prefix, unescapes quotes, and removes newlines.
-   * MySQL team is the POC for questions about this since there is preprocessing needed to transform the as-is
+   * Cleans SQL expression by unescaping quotes and removing newlines while preserving the original structure.
+   * IMPORTANT: Preserves charset introducers (e.g., _utf8mb4) exactly as they appear in the index definition
+   * to ensure byte-for-byte matching with functional index expressions. MySQL requires exact expression matching
+   * for functional indexes - any mismatch (even just a missing charset introducer) causes MySQL to ignore the
+   * index and perform a full table scan instead.
+   *
+   * <p>MySQL team is the POC for questions about this since there is preprocessing needed to transform the as-is
    * index expression from the index table to a (string) expression that is usable directly in an indexed query.
    *
-   * @param expression Raw SQL expression from database
-   * @return Cleaned expression string, with enclosing parentheses
+   * @param expression Raw SQL expression from database (e.g., _utf8mb4\'$.aspect.model_group_urn\')
+   * @return Cleaned expression string with unescaped quotes and enclosing parentheses (e.g., (_utf8mb4'$.aspect.model_group_urn'))
    */
   public static String cleanIndexExpression(@Nullable String expression) {
     if (expression == null) {
       return null;
     }
 
     return "(" + expression
-        .replace("_utf8mb4\\'", "'")
-        .replace("_utf8mb3\\'", "'")
         .replace("\\'", "'")
         .replace("\\\"", "\"")
         .replace("\n", "") + ")";

dao-impl/ebean-dao/src/test/java/com/linkedin/metadata/dao/utils/SchemaValidatorUtilTest.java

@@ -89,20 +89,20 @@ public void testCheckIndexExists() {
   @Test
   @SuppressWarnings("checkstyle:LineLength")
   public void testCleanIndexExpression() {
-    assertEquals("(cast(json_unquote(json_extract(`a_azkabanjobinfo`,'$.aspect.project.clusterInfo.hadoopCluster')) as char(255) charset utf8mb4))",
+    assertEquals("(cast(json_unquote(json_extract(`a_azkabanjobinfo`,_utf8mb4'$.aspect.project.clusterInfo.hadoopCluster')) as char(255) charset utf8mb4))",
         SchemaValidatorUtil.cleanIndexExpression("cast(json_unquote(json_extract(`a_azkabanjobinfo`,_utf8mb4\\'$.aspect.project.clusterInfo.hadoopCluster\\')) as char(255) charset utf8mb4)"));
 
-    assertEquals("(cast(json_unquote(json_extract(`a_urn`,'$.\"\\\\\\\\/azkabanFlowUrn\"')) as char(255) charset utf8mb4))",
+    assertEquals("(cast(json_unquote(json_extract(`a_urn`,_utf8mb4'$.\"\\\\\\\\/azkabanFlowUrn\"')) as char(255) charset utf8mb4))",
         SchemaValidatorUtil.cleanIndexExpression("cast(json_unquote(json_extract(`a_urn`,_utf8mb4\\'$.\"\\\\\\\\/azkabanFlowUrn\"\\')) as char(255) charset utf8mb4)"));
 
-    assertEquals("(cast(replace(json_unquote(json_extract(`a_datapolicyinfo`,'$.aspect.annotation.ontologyIris[*]')),'\"','') as char(255) charset utf8mb4))",
+    assertEquals("(cast(replace(json_unquote(json_extract(`a_datapolicyinfo`,_utf8mb3'$.aspect.annotation.ontologyIris[*]')),_utf8mb4'\"',_utf8mb3'') as char(255) charset utf8mb4))",
         SchemaValidatorUtil.cleanIndexExpression("cast(replace(json_unquote(json_extract(`a_datapolicyinfo`,_utf8mb3\\'$.aspect.annotation.ontologyIris[*]\\')),_utf8mb4\\'\"\\',_utf8mb3\\'\\') as char(255) charset utf8mb4)"));
 
-    assertEquals("(cast(json_unquote(json_extract(`a_urn`,'$.\"\\\\\\\\/dataset\\\\\\\\/platform\\\\\\\\/platformName\"')) as char(255) charset utf8mb4))",
+    assertEquals("(cast(json_unquote(json_extract(`a_urn`,_utf8mb3'$.\"\\\\\\\\/dataset\\\\\\\\/platform\\\\\\\\/platformName\"')) as char(255) charset utf8mb4))",
         SchemaValidatorUtil.cleanIndexExpression("cast(json_unquote(json_extract(`a_urn`,_utf8mb3\\'$.\"\\\\\\\\/dataset\\\\\\\\/platform\\\\\\\\/platformName\"\\')) as char(255) charset utf8mb4)"));
 
     // crazy AIM use case lol
-    assertEquals("(cast(concat(json_unquote(json_extract(`a_model_instance_info`,'$.aspect.multi_product_version.major')),'.',json_unquote(json_extract(`a_model_instance_info`,'$.aspect.multi_product_version.minor')),'.',json_unquote(json_extract(`a_model_instance_info`,'$.aspect.multi_product_version.patch'))) as char(255) charset utf8mb4))",
+    assertEquals("(cast(concat(json_unquote(json_extract(`a_model_instance_info`,_utf8mb3'$.aspect.multi_product_version.major')),_utf8mb4'.',json_unquote(json_extract(`a_model_instance_info`,_utf8mb3'$.aspect.multi_product_version.minor')),_utf8mb4'.',json_unquote(json_extract(`a_model_instance_info`,_utf8mb3'$.aspect.multi_product_version.patch'))) as char(255) charset utf8mb4))",
         SchemaValidatorUtil.cleanIndexExpression("cast(concat(json_unquote(json_extract(`a_model_instance_info`,_utf8mb3\\'$.aspect.multi_product_version.major\\')),_utf8mb4\\'.\\',json_unquote(json_extract(`a_model_instance_info`,_utf8mb3\\'$.aspect.multi_product_version.minor\\')),_utf8mb4\\'.\\',json_unquote(json_extract(`a_model_instance_info`,_utf8mb3\\'$.aspect.multi_product_version.patch\\'))) as char(255) charset utf8mb4)"));
 
     assertNull(SchemaValidatorUtil.cleanIndexExpression(null));