Add lexer and parser generator script (#28)

This enables generating lexer and parser files with a command-line script:

    cmake -P cmake/scripts/GenerateLexersParsers.cmake

Instead of doing the entire CMake configuration phase and then executing some
target to generate files this simplifies step to generate these files when
needed.

* Two new utility modules: PHP/Bison.cmake and PHP/Re2c.cmake. They provide
  using Bison and re2c in CMake normal workflow and command-line. They rely on
  their find modules. Find modules are not meant to provide functions. This is
  a bad practice learned from some existing modules out there for convenience
  but it is more proper to have such functions as separate modules. Find modules
  should ideally only deal with finding packages and providing the imported
  targets. Ideally, how the package is used should be done in some wrapper
  module, which provides these functions. Here, the php_bison() and php_re2c().
  Also, downloading is moved to these modules for now.
* ExternalProject is for now used to download and build Bison and re2c if they
  are not found on the system.
* They also download these two tools if they are not found on the system. Bison
  download and custom build is for now only supported through its
  Autotools-based build system.
* More common naming for parser and lexer files generation functionality used in
  filenames: "grammar"
* Long Bison and re2c options used to find them in the docs more easily
* The --conditions option is available since re2c 1.1.
* Bison removed from CI

Author: petk

.github/workflows/ci.yaml

@@ -63,7 +63,6 @@ jobs:
         run: |
           sudo apt-get -y install \
             build-essential \
-            bison \
             libssl-dev \
             libpcre2-dev \
             libsqlite3-dev \

bin/check-cmake.php

@@ -191,6 +191,7 @@ function getProjectModules(): array
         'FindPackageMessage' => ['find_package_message'],
         'ProcessorCount' => ['processorcount'],
         'PHP/AddCustomCommand' => ['php_add_custom_command'],
+        'PHP/Bison' => ['php_bison', '/find_package\([\n ]*BISON/'],
         'PHP/CheckAttribute' => [
             'php_check_function_attribute',
             'php_check_variable_attribute',
@@ -199,6 +200,7 @@ function getProjectModules(): array
         'PHP/ConfigureFile' => ['php_configure_file'],
         'PHP/Install' => ['php_install'],
         'PHP/PkgConfigGenerator' => ['pkgconfig_generate_pc'],
+        'PHP/Re2c' => ['php_re2c', '/find_package\([\n ]*RE2C/'],
         'PHP/SearchLibraries' => ['php_search_libraries'],
         'PHP/Set' => ['php_set'],
         'PHP/SystemExtensions' => ['PHP::SystemExtensions'],
@@ -273,24 +275,29 @@ function checkCMakeInclude(Iterator $files, array $modules): int
         $content = getCMakeCode($file);
 
         // Check for redundant includes.
-        foreach ($modules as $module => $commands) {
+        foreach ($modules as $module => $patterns) {
             $hasModule = false;
             $moduleEscaped = str_replace('/', '\/', $module);
 
             if (1 === preg_match('/^[ \t]*include[ \t]*\(' . $moduleEscaped . '[ \t]*\)/m', $content)) {
                 $hasModule = true;
             }
 
-            $hasCommand = false;
-            foreach ($commands as $command) {
-                if (
+            $hasPattern = false;
+            foreach ($patterns as $pattern) {
+                if (isRegularExpression($pattern)) {
+                    if (1 === preg_match($pattern, $content)) {
+                        $hasPattern = true;
+                        break;
+                    }
+                } elseif (
                     (
-                        1 === preg_match('/::/', $command)
-                        && 1 === preg_match('/[^A-Za-z0-9_]' . $command . '[^A-Za-z0-9_]/m', $content)
+                        1 === preg_match('/::/', $pattern)
+                        && 1 === preg_match('/[^A-Za-z0-9_]' . $pattern . '[^A-Za-z0-9_]/m', $content)
                     )
-                    || 1 === preg_match('/^[ \t]*' . $command . '[ \t]*\(/m', $content)
+                    || 1 === preg_match('/^[ \t]*' . $pattern . '[ \t]*\(/m', $content)
                 ) {
-                    $hasCommand = true;
+                    $hasPattern = true;
                     break;
                 }
             }
@@ -304,12 +311,12 @@ function checkCMakeInclude(Iterator $files, array $modules): int
                 continue;
             }
 
-            if ($hasModule && !$hasCommand) {
+            if ($hasModule && !$hasPattern) {
                 $status = 1;
                 output("E: redundant include($module) in $file");
             }
 
-            if (!$hasModule && $hasCommand) {
+            if (!$hasModule && $hasPattern) {
                 $status = 1;
                 output("E: missing include($module) in $file");
             }
@@ -319,6 +326,18 @@ function checkCMakeInclude(Iterator $files, array $modules): int
     return $status;
 };
 
+/**
+ * Check if given string is regular expression.
+ */
+function isRegularExpression(string $string): bool
+{
+    set_error_handler(static function () {}, E_WARNING);
+    $isRegularExpression = false !== preg_match($string, '');
+    restore_error_handler();
+
+    return $isRegularExpression;
+}
+
 /**
  * Check for set(<variable>) usages with only one argument. These should be
  * replaced with set(<variable> "").

cmake/Zend/CMakeLists.txt

@@ -523,108 +523,10 @@ if(TARGET Zend::MaxExecutionTimers)
 endif()
 
 ################################################################################
-# Generate lexers and parsers.
+# Generate parser and lexer files.
 ################################################################################
 
-if(BISON_FOUND)
-  bison_target(
-    zend_ini_parser
-    zend_ini_parser.y
-    ${CMAKE_CURRENT_SOURCE_DIR}/zend_ini_parser.c
-    COMPILE_FLAGS "${PHP_DEFAULT_BISON_FLAGS}"
-    VERBOSE REPORT_FILE zend_ini_parser.output
-    DEFINES_FILE ${CMAKE_CURRENT_SOURCE_DIR}/zend_ini_parser.h
-  )
-
-  add_custom_target(zend_ini_parser DEPENDS ${BISON_TARGET_outputs})
-  add_dependencies(php_generate_files zend_ini_parser)
-
-  bison_target(
-    zend_language_parser
-    zend_language_parser.y
-    ${CMAKE_CURRENT_SOURCE_DIR}/zend_language_parser.c
-    COMPILE_FLAGS "${PHP_DEFAULT_BISON_FLAGS}"
-    VERBOSE REPORT_FILE zend_language_parser.output
-    DEFINES_FILE ${CMAKE_CURRENT_SOURCE_DIR}/zend_language_parser.h
-  )
-
-  # Tweak zendparse to be exported through ZEND_API. This has to be revisited
-  # once bison supports foreign skeletons and that bison version is used. Read
-  # https://git.savannah.gnu.org/cgit/bison.git/tree/data/README.md for more.
-  file(
-    GENERATE
-    OUTPUT CMakeFiles/PatchLanguageParser.cmake
-    CONTENT [[
-      file(READ "${SOURCE_DIR}/zend_language_parser.h" content)
-      string(
-        REPLACE
-        "int zendparse"
-        "ZEND_API int zendparse"
-        content_2
-        "${content}"
-      )
-      if(
-        NOT content MATCHES "ZEND_API int zendparse"
-        AND NOT content STREQUAL "${content_2}"
-      )
-        execute_process(
-          COMMAND ${CMAKE_COMMAND} -E cmake_echo_color --blue --bold
-            "       [Zend] Patching Zend/zend_language_parser.h"
-        )
-        file(WRITE "${SOURCE_DIR}/zend_language_parser.h" "${content_2}")
-      endif()
-
-      file(READ "${SOURCE_DIR}/zend_language_parser.c" content)
-      string(
-        REPLACE
-        "int zendparse"
-        "ZEND_API int zendparse"
-        content_2
-        "${content}"
-      )
-      if(
-        NOT content MATCHES "ZEND_API int zendparse"
-        AND NOT content STREQUAL "${content_2}"
-      )
-        execute_process(
-          COMMAND ${CMAKE_COMMAND} -E cmake_echo_color --blue --bold
-            "       [Zend] Patching Zend/zend_language_parser.c"
-        )
-        file(WRITE "${SOURCE_DIR}/zend_language_parser.c" "${content_2}")
-      endif()
-    ]]
-  )
-
-  add_custom_target(
-    zend_patch_language_parser
-    COMMAND ${CMAKE_COMMAND}
-      -D SOURCE_DIR=${CMAKE_CURRENT_SOURCE_DIR}
-      -P "CMakeFiles/PatchLanguageParser.cmake"
-    DEPENDS ${BISON_zend_language_parser_OUTPUTS}
-    VERBATIM
-  )
-
-  add_dependencies(zend zend_patch_language_parser)
-  add_dependencies(php_generate_files zend_patch_language_parser)
-endif()
-
-if(RE2C_FOUND)
-  re2c_target(
-    zend_language_scanner
-    zend_language_scanner.l
-    ${CMAKE_CURRENT_SOURCE_DIR}/zend_language_scanner.c
-    HEADER ${CMAKE_CURRENT_SOURCE_DIR}/zend_language_scanner_defs.h
-    OPTIONS --case-inverted -cbdF
-  )
-
-  re2c_target(
-    zend_ini_scanner
-    zend_ini_scanner.l
-    ${CMAKE_CURRENT_SOURCE_DIR}/zend_ini_scanner.c
-    HEADER ${CMAKE_CURRENT_SOURCE_DIR}/zend_ini_scanner_defs.h
-    OPTIONS --case-inverted -cbdF
-  )
-endif()
+include(cmake/GenerateGrammar.cmake)
 
 ################################################################################
 # Configure fibers.

cmake/Zend/cmake/GenerateGrammar.cmake

@@ -0,0 +1,131 @@
+# Generate parser and lexer files.
+
+if(CMAKE_SCRIPT_MODE_FILE STREQUAL CMAKE_CURRENT_LIST_FILE)
+  message(FATAL_ERROR "This file should be used with include().")
+endif()
+
+include(PHP/Bison)
+
+if(CMAKE_SCRIPT_MODE_FILE)
+  set(verbose "")
+else()
+  set(verbose VERBOSE)
+endif()
+
+php_bison(
+  zend_ini_parser
+  zend_ini_parser.y
+  ${CMAKE_CURRENT_SOURCE_DIR}/zend_ini_parser.c
+  HEADER
+  ADD_DEFAULT_OPTIONS
+  ${verbose}
+  CODEGEN
+)
+
+php_bison(
+  zend_language_parser
+  zend_language_parser.y
+  ${CMAKE_CURRENT_SOURCE_DIR}/zend_language_parser.c
+  HEADER
+  ADD_DEFAULT_OPTIONS
+  ${verbose}
+  CODEGEN
+)
+
+# Tweak zendparse to be exported through ZEND_API. This has to be revisited if
+# Bison will support foreign skeletons.
+# See: https://git.savannah.gnu.org/cgit/bison.git/tree/data/README.md
+block()
+  string(
+    CONCAT patch
+    "cmake_path(SET SOURCE_DIR NORMALIZE ${CMAKE_CURRENT_SOURCE_DIR})\n"
+    [[
+    cmake_path(
+      RELATIVE_PATH
+      SOURCE_DIR
+      BASE_DIRECTORY ${CMAKE_SOURCE_DIR}
+      OUTPUT_VARIABLE relativeDir
+    )
+    file(READ "${SOURCE_DIR}/zend_language_parser.h" content)
+    string(
+      REPLACE
+      "int zendparse"
+      "ZEND_API int zendparse"
+      content_2
+      "${content}"
+    )
+    if(
+      NOT content MATCHES "ZEND_API int zendparse"
+      AND NOT content STREQUAL "${content_2}"
+    )
+      message(STATUS "Patching ${relativeDir}/zend_language_parser.h")
+      file(WRITE "${SOURCE_DIR}/zend_language_parser.h" "${content_2}")
+    endif()
+
+    file(READ "${SOURCE_DIR}/zend_language_parser.c" content)
+    string(
+      REPLACE
+      "int zendparse"
+      "ZEND_API int zendparse"
+      content_2
+      "${content}"
+    )
+    if(
+      NOT content MATCHES "ZEND_API int zendparse"
+      AND NOT content STREQUAL "${content_2}"
+    )
+      message(STATUS "Patching ${relativeDir}/zend_language_parser.c")
+      file(WRITE "${SOURCE_DIR}/zend_language_parser.c" "${content_2}")
+    endif()
+  ]])
+
+  # Run patch based on whether building or running inside a script.
+  if(CMAKE_SCRIPT_MODE_FILE)
+    cmake_language(EVAL CODE "${patch}")
+  else()
+    file(
+      GENERATE
+      OUTPUT CMakeFiles/Zend/PatchLanguageParser.cmake
+      CONTENT "${patch}"
+    )
+    add_custom_target(
+      zend_language_parser_patch
+      COMMAND ${CMAKE_COMMAND} -P CMakeFiles/Zend/PatchLanguageParser.cmake
+      DEPENDS zend_language_parser
+      VERBATIM
+    )
+    add_dependencies(zend zend_language_parser_patch)
+  endif()
+endblock()
+
+include(PHP/Re2c)
+
+php_re2c(
+  zend_ini_scanner
+  zend_ini_scanner.l
+  ${CMAKE_CURRENT_SOURCE_DIR}/zend_ini_scanner.c
+  HEADER ${CMAKE_CURRENT_SOURCE_DIR}/zend_ini_scanner_defs.h
+  ADD_DEFAULT_OPTIONS
+  OPTIONS
+    --bit-vectors
+    --case-inverted
+    --conditions
+    --debug-output
+    --flex-syntax
+  CODEGEN
+)
+
+php_re2c(
+  zend_language_scanner
+  zend_language_scanner.l
+  ${CMAKE_CURRENT_SOURCE_DIR}/zend_language_scanner.c
+  HEADER ${CMAKE_CURRENT_SOURCE_DIR}/zend_language_scanner_defs.h
+  ADD_DEFAULT_OPTIONS
+  OPTIONS
+    --bit-vectors
+    --case-inverted
+    --conditions
+    --debug-output
+    --flex-syntax
+  CODEGEN
+)

cmake/cmake/Bootstrap.cmake

@@ -48,10 +48,6 @@ add_library(php_sapi INTERFACE)
 add_library(PHP::sapi ALIAS php_sapi)
 target_link_libraries(php_sapi INTERFACE PHP::config)
 
-# Create a custom target for generating files (parsers, lexers, etc.) manually:
-#   cmake --build <dir> -t php_generate_files
-add_custom_target(php_generate_files)
-
 # Configure build types.
 include(cmake/BuildTypes.cmake)
 

cmake/cmake/Configuration.cmake

@@ -92,9 +92,6 @@ mark_as_advanced(PHP_PROGRAM_PREFIX)
 set(PHP_PROGRAM_SUFFIX "" CACHE STRING "Append suffix to the program names")
 mark_as_advanced(PHP_PROGRAM_SUFFIX)
 
-option(PHP_RE2C_CGOTO "Enable computed goto GCC extension with re2c")
-mark_as_advanced(PHP_RE2C_CGOTO)
-
 option(PHP_THREAD_SAFETY "Enable thread safety (ZTS)")
 
 cmake_dependent_option(
@@ -213,13 +210,6 @@ set(PHP_ZLIB_MIN_VERSION 1.2.0.4)
 set(PHP_BZIP2_MIN_VERSION 1.0.0)
 
 # Additional metadata for external packages to avoid duplication.
-set_package_properties(
-  BISON
-  PROPERTIES
-    URL "https://www.gnu.org/software/bison/"
-    DESCRIPTION "General-purpose parser generator"
-)
-
 set_package_properties(
   BZip2
   PROPERTIES
@@ -268,3 +258,8 @@ set_package_properties(
     URL "https://zlib.net/"
     DESCRIPTION "Compression library"
 )
+
+# Set base directory for ExternalProject CMake module.
+set_directory_properties(
+  PROPERTIES EP_BASE ${PHP_BINARY_DIR}/CMakeFiles/PHP/ExternalProject
+)