|
| 1 | +/* |
| 2 | + * Wrap the "composer run wp-tests-phpunit" command to process tests |
| 3 | + * that are expected to error and fail at the moment. |
| 4 | + * |
| 5 | + * This makes sure that the CI job passes, while explicitly tracking |
| 6 | + * the issues that need to be addressed. Ideally, over time this script |
| 7 | + * will become obsolete when all errors and failures are resolved. |
| 8 | + */ |
| 9 | +const { execSync } = require( 'child_process' ); |
| 10 | +const fs = require( 'fs' ); |
| 11 | +const path = require( 'path' ); |
| 12 | + |
| 13 | +const expectedErrors = [ |
| 14 | + 'Tests_Admin_wpSiteHealth::test_object_cache_default_thresholds_non_multisite', |
| 15 | + 'Tests_Admin_wpSiteHealth::test_object_cache_thresholds with data set #0', |
| 16 | + 'Tests_Admin_wpSiteHealth::test_object_cache_thresholds with data set #1', |
| 17 | + 'Tests_Admin_wpSiteHealth::test_object_cache_thresholds with data set #2', |
| 18 | + 'Tests_Admin_wpSiteHealth::test_object_cache_thresholds with data set #3', |
| 19 | + 'Tests_Admin_wpSiteHealth::test_object_cache_thresholds with data set #4', |
| 20 | + 'Tests_Comment_WpComment::test_get_instance_should_succeed_for_float_that_is_equal_to_post_id', |
| 21 | + 'Tests_Cron_getCronArray::test_get_cron_array_output_validation with data set "null"', |
| 22 | + 'Tests_DB_Charset::test_strip_invalid_text', |
| 23 | + 'Tests_DB::test_db_reconnect', |
| 24 | + 'Tests_DB::test_get_col_info', |
| 25 | + 'Tests_DB::test_prepare_should_respect_the_allow_unsafe_unquoted_parameters_property with data set "escaped-false-1"', |
| 26 | + 'Tests_DB::test_prepare_should_respect_the_allow_unsafe_unquoted_parameters_property with data set "escaped-false-2"', |
| 27 | + 'Tests_DB::test_prepare_should_respect_the_allow_unsafe_unquoted_parameters_property with data set "escaped-true-1"', |
| 28 | + 'Tests_DB::test_prepare_should_respect_the_allow_unsafe_unquoted_parameters_property with data set "escaped-true-2"', |
| 29 | + 'Tests_DB::test_prepare_should_respect_the_allow_unsafe_unquoted_parameters_property with data set "format-false-1"', |
| 30 | + 'Tests_DB::test_prepare_should_respect_the_allow_unsafe_unquoted_parameters_property with data set "format-false-2"', |
| 31 | + 'Tests_DB::test_prepare_should_respect_the_allow_unsafe_unquoted_parameters_property with data set "format-true-1"', |
| 32 | + 'Tests_DB::test_prepare_should_respect_the_allow_unsafe_unquoted_parameters_property with data set "format-true-2"', |
| 33 | + 'Tests_DB::test_prepare_should_respect_the_allow_unsafe_unquoted_parameters_property with data set "numbered-false-1"', |
| 34 | + 'Tests_DB::test_prepare_should_respect_the_allow_unsafe_unquoted_parameters_property with data set "numbered-false-2"', |
| 35 | + 'Tests_DB::test_prepare_should_respect_the_allow_unsafe_unquoted_parameters_property with data set "numbered-true-1"', |
| 36 | + 'Tests_DB::test_prepare_should_respect_the_allow_unsafe_unquoted_parameters_property with data set "numbered-true-2"', |
| 37 | + 'Tests_DB::test_process_fields_value_too_long_for_field with data set "invalid chars"', |
| 38 | + 'Tests_DB::test_process_fields_value_too_long_for_field with data set "too long"', |
| 39 | + 'Tests_DB::test_process_fields', |
| 40 | + 'Tests_DB::test_set_allowed_incompatible_sql_mode', |
| 41 | + 'Tests_DB::test_set_incompatible_sql_mode', |
| 42 | + 'Tests_DB::test_set_sql_mode', |
| 43 | + 'Tests_Post_wpPost::test_get_instance_should_succeed_for_float_that_is_equal_to_post_id', |
| 44 | + 'Tests_Post::test_stick_post_with_unexpected_sticky_posts_option with data set "null"', |
| 45 | +]; |
| 46 | + |
| 47 | +const expectedFailures = [ |
| 48 | + 'Tests_Comment::test_wp_new_comment_respects_comment_field_lengths', |
| 49 | + 'Tests_Comment::test_wp_update_comment', |
| 50 | + 'Tests_DB_dbDelta::test_column_type_change_with_hyphens_in_name', |
| 51 | + 'Tests_DB_dbDelta::test_query_with_backticks_does_not_cause_a_query_to_alter_all_columns_and_indices_to_run_even_if_none_have_changed', |
| 52 | + 'Tests_DB_dbDelta::test_query_with_backticks_does_not_throw_an_undefined_index_warning', |
| 53 | + 'Tests_DB_dbDelta::test_spatial_indices', |
| 54 | + 'Tests_DB::test_charset_switched_to_utf8mb4', |
| 55 | + 'Tests_DB::test_close', |
| 56 | + 'Tests_DB::test_delete_value_too_long_for_field with data set "too long"', |
| 57 | + 'Tests_DB::test_has_cap', |
| 58 | + 'Tests_DB::test_insert_value_too_long_for_field with data set "too long"', |
| 59 | + 'Tests_DB::test_mysqli_flush_sync', |
| 60 | + 'Tests_DB::test_non_unicode_collations', |
| 61 | + 'Tests_DB::test_query_value_contains_invalid_chars', |
| 62 | + 'Tests_DB::test_replace_value_too_long_for_field with data set "too long"', |
| 63 | + 'Tests_DB::test_replace', |
| 64 | + 'Tests_DB::test_supports_collation', |
| 65 | + 'Tests_DB::test_update_value_too_long_for_field with data set "too long"', |
| 66 | + 'Tests_Menu_Walker_Nav_Menu::test_start_el_with_empty_attributes with data set #1', |
| 67 | + 'Tests_Menu_Walker_Nav_Menu::test_start_el_with_empty_attributes with data set #2', |
| 68 | + 'Tests_Menu_Walker_Nav_Menu::test_start_el_with_empty_attributes with data set #3', |
| 69 | + 'Tests_Menu_Walker_Nav_Menu::test_start_el_with_empty_attributes with data set #4', |
| 70 | + 'Tests_Menu_Walker_Nav_Menu::test_start_el_with_empty_attributes with data set #5', |
| 71 | + 'Tests_Menu_Walker_Nav_Menu::test_start_el_with_empty_attributes with data set #6', |
| 72 | + 'Tests_Menu_Walker_Nav_Menu::test_start_el_with_empty_attributes with data set #7', |
| 73 | + 'Tests_Menu_wpNavMenu::test_wp_nav_menu_should_not_have_has_children_class_with_custom_depth', |
| 74 | +]; |
| 75 | + |
| 76 | +console.log( 'Running WordPress PHPUnit tests with expected failures tracking...' ); |
| 77 | +console.log( 'Expected errors:', expectedErrors ); |
| 78 | +console.log( 'Expected failures:', expectedFailures ); |
| 79 | + |
| 80 | +try { |
| 81 | + try { |
| 82 | + execSync( |
| 83 | + `composer run wp-test-phpunit -- --log-junit=phpunit-results.xml --verbose`, |
| 84 | + { stdio: 'inherit' } |
| 85 | + ); |
| 86 | + console.log( '\n⚠️ All tests passed, checking if expected errors/failures occurred...' ); |
| 87 | + } catch ( error ) { |
| 88 | + console.log( '\n⚠️ Some tests errored/failed (expected). Analyzing results...' ); |
| 89 | + } |
| 90 | + |
| 91 | + // Read the JUnit XML test output: |
| 92 | + const junitOutputFile = path.join( __dirname, '..', '..', 'wordpress', 'phpunit-results.xml' ); |
| 93 | + if ( ! fs.existsSync( junitOutputFile ) ) { |
| 94 | + console.error( 'Error: JUnit output file not found!' ); |
| 95 | + process.exit( 1 ); |
| 96 | + } |
| 97 | + const junitXml = fs.readFileSync( junitOutputFile, 'utf8' ); |
| 98 | + |
| 99 | + // Extract test info from the XML: |
| 100 | + const actualErrors = []; |
| 101 | + const actualFailures = []; |
| 102 | + for ( const testcase of junitXml.matchAll( /<testcase([^>]*)\/>|<testcase([^>]*)>([\s\S]*?)<\/testcase>/g ) ) { |
| 103 | + const attributes = {}; |
| 104 | + const attributesString = testcase[2] ?? testcase[1]; |
| 105 | + for ( const attribute of attributesString.matchAll( /(\w+)="([^"]*)"/g ) ) { |
| 106 | + attributes[attribute[1]] = attribute[2]; |
| 107 | + } |
| 108 | + |
| 109 | + const content = testcase[3] ?? ''; |
| 110 | + const fqn = attributes.class ? `${attributes.class}::${attributes.name}` : attributes.name; |
| 111 | + const hasError = content.includes( '<error' ); |
| 112 | + const hasFailure = content.includes( '<failure' ); |
| 113 | + |
| 114 | + if ( hasError ) { |
| 115 | + actualErrors.push( fqn ); |
| 116 | + } |
| 117 | + |
| 118 | + if ( hasFailure ) { |
| 119 | + actualFailures.push( fqn ); |
| 120 | + } |
| 121 | + } |
| 122 | + |
| 123 | + let isSuccess = true; |
| 124 | + |
| 125 | + // Check if all expected errors actually errored |
| 126 | + const unexpectedNonErrors = expectedErrors.filter( test => ! actualErrors.includes( test ) ); |
| 127 | + if ( unexpectedNonErrors.length > 0 ) { |
| 128 | + console.error( '\n❌ The following tests were expected to error but did not:' ); |
| 129 | + unexpectedNonErrors.forEach( test => console.error( ` - ${test}` ) ); |
| 130 | + isSuccess = false; |
| 131 | + } |
| 132 | + |
| 133 | + // Check if all expected failures actually failed |
| 134 | + const unexpectedPasses = expectedFailures.filter( test => ! actualFailures.includes( test ) ); |
| 135 | + if ( unexpectedPasses.length > 0 ) { |
| 136 | + console.error( '\n❌ The following tests were expected to fail but passed:' ); |
| 137 | + unexpectedPasses.forEach( test => console.error( ` - ${test}` ) ); |
| 138 | + isSuccess = false; |
| 139 | + } |
| 140 | + |
| 141 | + // Check for unexpected errors |
| 142 | + const unexpectedErrors = actualErrors.filter( test => ! expectedErrors.includes( test ) ); |
| 143 | + if ( unexpectedErrors.length > 0 ) { |
| 144 | + console.error( '\n❌ The following tests errored unexpectedly:' ); |
| 145 | + unexpectedErrors.forEach( test => console.error( ` - ${test}` ) ); |
| 146 | + isSuccess = false; |
| 147 | + } |
| 148 | + |
| 149 | + // Check for unexpected failures |
| 150 | + const unexpectedFailures = actualFailures.filter( test => ! expectedFailures.includes( test ) ); |
| 151 | + if ( unexpectedFailures.length > 0 ) { |
| 152 | + console.error( '\n❌ The following tests failed unexpectedly:' ); |
| 153 | + unexpectedFailures.forEach( test => console.error( ` - ${test}` ) ); |
| 154 | + isSuccess = false; |
| 155 | + } |
| 156 | + |
| 157 | + if ( isSuccess ) { |
| 158 | + console.log( '\n✅ All tests behaved as expected!' ); |
| 159 | + process.exit( 0 ); |
| 160 | + } else { |
| 161 | + console.log( '\n❌ Some tests did not behave as expected!' ); |
| 162 | + process.exit( 1 ); |
| 163 | + } |
| 164 | +} catch ( error ) { |
| 165 | + console.error( '\n❌ Script execution error:', error.message ); |
| 166 | + process.exit( 1 ); |
| 167 | +} |
0 commit comments