Merge pull request github#16623 from github/criemen/new-pkg

Bazel: Use new packaging rules

Author: criemen

.github/workflows/csharp-qltest.yml

@@ -65,7 +65,7 @@ jobs:
           key: csharp-qltest-${{ matrix.slice }}
       - name: Run QL tests
         run: |
-          codeql test run --threads=0 --ram 50000 --slice ${{ matrix.slice }} --search-path extractor-pack --check-databases --check-undefined-labels --check-repeated-labels --check-redefined-labels --consistency-queries ql/consistency-queries ql/test --compilation-cache "${{ steps.query-cache.outputs.cache-dir }}"
+          codeql test run --threads=0 --ram 50000 --slice ${{ matrix.slice }} --search-path "${{ github.workspace }}" --check-databases --check-undefined-labels --check-repeated-labels --check-redefined-labels --consistency-queries ql/consistency-queries ql/test --compilation-cache "${{ steps.query-cache.outputs.cache-dir }}"
         env:
           GITHUB_TOKEN: ${{ github.token }}
   unit-tests:
@@ -101,6 +101,6 @@ jobs:
           # Update existing stubs in the repo with the freshly generated ones
           mv "$STUBS_PATH/output/stubs/_frameworks" ql/test/resources/stubs/
           git status
-          codeql test run --threads=0 --search-path extractor-pack --check-databases --check-undefined-labels --check-repeated-labels --check-redefined-labels --consistency-queries ql/consistency-queries -- ql/test/library-tests/dataflow/flowsources/aspremote
+          codeql test run --threads=0 --search-path "${{ github.workspace }}" --check-databases --check-undefined-labels --check-repeated-labels --check-redefined-labels --consistency-queries ql/consistency-queries -- ql/test/library-tests/dataflow/flowsources/aspremote
         env:
           GITHUB_TOKEN: ${{ github.token }}

.github/workflows/ql-for-ql-build.yml

@@ -49,20 +49,20 @@ jobs:
           key: ${{ runner.os }}-${{ steps.os_version.outputs.version }}-rust-cargo-${{ hashFiles('ql/**/Cargo.lock') }}
       - name: Release build
         if: steps.cache-extractor.outputs.cache-hit != 'true'
-        run: cd ql; ./scripts/create-extractor-pack.sh       
+        run: cd ql; ./scripts/create-extractor-pack.sh
         env:
-          GH_TOKEN: ${{ github.token }}   
+          GH_TOKEN: ${{ github.token }}
       - name: Cache compilation cache
         id: query-cache
         uses: ./.github/actions/cache-query-compilation
-        with: 
+        with:
           key: run-ql-for-ql
       - name: Make database and analyze
         run: |
           ./ql/target/release/buramu | tee deprecated.blame # Add a blame file for the extractor to parse.
-          ${CODEQL} database create -l=ql --search-path ql/extractor-pack ${DB}
+          ${CODEQL} database create -l=ql ${DB} --search-path "${{ github.workspace }}"
           ${CODEQL} database analyze -j0 --format=sarif-latest --output=ql-for-ql.sarif ${DB} ql/ql/src/codeql-suites/ql-code-scanning.qls  --compilation-cache "${{ steps.query-cache.outputs.cache-dir }}"
-        env: 
+        env:
           CODEQL: ${{ steps.find-codeql.outputs.codeql-path }}
           DB: ${{ runner.temp }}/DB
           LGTM_INDEX_FILTERS: |

.github/workflows/ql-for-ql-dataset_measure.yml

@@ -53,8 +53,8 @@ jobs:
       - name: Create database
         run: |
           "${CODEQL}" database create \
-            --search-path "ql/extractor-pack" \
-            --threads 4 \
+          --search-path "${{ github.workspace }}"
+          --threads 4 \
             --language ql --source-root "${{ github.workspace }}/repo" \
             "${{ runner.temp }}/database"
         env:

.github/workflows/ql-for-ql-tests.yml

@@ -49,23 +49,23 @@ jobs:
       - name: Cache compilation cache
         id: query-cache
         uses: ./.github/actions/cache-query-compilation
-        with: 
+        with:
           key: ql-for-ql-tests
       - name: Run QL tests
         run: |
-          "${CODEQL}" test run --check-databases --check-unused-labels --check-repeated-labels --check-redefined-labels --check-use-before-definition --search-path "${{ github.workspace }}/ql/extractor-pack" --consistency-queries ql/ql/consistency-queries --compilation-cache "${{ steps.query-cache.outputs.cache-dir }}" ql/ql/test
+          "${CODEQL}" test run --check-databases --check-unused-labels --check-repeated-labels --check-redefined-labels --check-use-before-definition --search-path "${{ github.workspace }}" --consistency-queries ql/ql/consistency-queries --compilation-cache "${{ steps.query-cache.outputs.cache-dir }}" ql/ql/test
         env:
           CODEQL: ${{ steps.find-codeql.outputs.codeql-path }}
 
-  other-os: 
+  other-os:
     strategy:
       matrix:
         os: [macos-latest, windows-latest]
     needs: [qltest]
     runs-on: ${{ matrix.os }}
     steps:
       - uses: actions/checkout@v4
-      - name: Install GNU tar 
+      - name: Install GNU tar
         if: runner.os == 'macOS'
         run: |
           brew install gnu-tar
@@ -100,13 +100,12 @@ jobs:
       - name: Run a single QL tests - Unix
         if: runner.os != 'Windows'
         run: |
-          "${CODEQL}" test run --check-databases --search-path "${{ github.workspace }}/ql/extractor-pack" ql/ql/test/queries/style/DeadCode/DeadCode.qlref
+          "${CODEQL}" test run --check-databases --search-path "${{ github.workspace }}" ql/ql/test/queries/style/DeadCode/DeadCode.qlref
         env:
           CODEQL: ${{ steps.find-codeql.outputs.codeql-path }}
       - name: Run a single QL tests - Windows
         if: runner.os == 'Windows'
         shell: pwsh
         run: |
           $Env:PATH += ";$(dirname ${{ steps.find-codeql.outputs.codeql-path }})"
-          codeql test run --check-databases --search-path "${{ github.workspace }}/ql/extractor-pack" ql/ql/test/queries/style/DeadCode/DeadCode.qlref
-      
+          codeql test run --check-databases --search-path "${{ github.workspace }}" ql/ql/test/queries/style/DeadCode/DeadCode.qlref

.github/workflows/ruby-dataset-measure.yml

@@ -44,7 +44,7 @@ jobs:
       - name: Create database
         run: |
           codeql database create \
-            --search-path "${{ github.workspace }}/ruby/extractor-pack" \
+            --search-path "${{ github.workspace }}" \
             --threads 4 \
             --language ruby --source-root "${{ github.workspace }}/repo" \
             "${{ runner.temp }}/database"

.github/workflows/ruby-qltest.yml

@@ -64,10 +64,10 @@ jobs:
       - name: Cache compilation cache
         id: query-cache
         uses: ./.github/actions/cache-query-compilation
-        with: 
+        with:
           key: ruby-qltest
       - name: Run QL tests
         run: |
-          codeql test run --threads=0 --ram 50000 --search-path "${{ github.workspace }}/ruby/extractor-pack" --check-databases --check-undefined-labels --check-unused-labels --check-repeated-labels --check-redefined-labels --check-use-before-definition --consistency-queries ql/consistency-queries ql/test --compilation-cache "${{ steps.query-cache.outputs.cache-dir }}"
+          codeql test run --threads=0 --ram 50000 --search-path "${{ github.workspace }}" --check-databases --check-undefined-labels --check-unused-labels --check-repeated-labels --check-redefined-labels --check-use-before-definition --consistency-queries ql/consistency-queries ql/test --compilation-cache "${{ steps.query-cache.outputs.cache-dir }}"
         env:
           GITHUB_TOKEN: ${{ github.token }}

.gitignore

@@ -62,3 +62,6 @@ node_modules/
 
 # Temporary folders for working with generated models
 .model-temp
+
+# bazel-built in-tree extractor packs
+/*/extractor-pack

codeql-workspace.yml

@@ -6,19 +6,16 @@ provide:
   - "*/ql/consistency-queries/qlpack.yml"
   - "*/ql/automodel/src/qlpack.yml"
   - "*/ql/automodel/test/qlpack.yml"
+  - "*/extractor-pack/codeql-extractor.yml"
   - "python/extractor/qlpack.yml"
   - "shared/**/qlpack.yml"
   - "cpp/ql/test/query-tests/Security/CWE/CWE-190/semmle/tainted/qlpack.yml"
   - "go/ql/config/legacy-support/qlpack.yml"
-  - "go/build/codeql-extractor-go/codeql-extractor.yml"
   - "csharp/ql/campaigns/Solorigate/lib/qlpack.yml"
   - "csharp/ql/campaigns/Solorigate/src/qlpack.yml"
   - "csharp/ql/campaigns/Solorigate/test/qlpack.yml"
   - "misc/legacy-support/*/qlpack.yml"
   - "misc/suite-helpers/qlpack.yml"
-  - "ruby/extractor-pack/codeql-extractor.yml"
-  - "swift/extractor-pack/codeql-extractor.yml"
-  - "ql/extractor-pack/codeql-extractor.yml"
   - ".github/codeql/extensions/**/codeql-pack.yml"
 
 versionPolicies:

cpp/downgrades/BUILD.bazel

@@ -6,7 +6,7 @@ pkg_files(
         ["**"],
         exclude = ["BUILD.bazel"],
     ),
-    prefix = "cpp/downgrades",
+    prefix = "downgrades",
     strip_prefix = strip_prefix.from_pkg(),
     visibility = ["//cpp:__pkg__"],
 )

cpp/ql/lib/BUILD.bazel

@@ -5,11 +5,9 @@ package(default_visibility = ["//cpp:__pkg__"])
 pkg_files(
     name = "dbscheme",
     srcs = ["semmlecode.cpp.dbscheme"],
-    prefix = "cpp",
 )
 
 pkg_files(
     name = "dbscheme-stats",
     srcs = ["semmlecode.cpp.dbscheme.stats"],
-    prefix = "cpp",
 )

csharp/BUILD.bazel

@@ -1,6 +1,5 @@
 load("@rules_pkg//pkg:mappings.bzl", "pkg_filegroup", "pkg_files")
-load("@semmle_code//:dist.bzl", "dist")
-load("//misc/bazel:pkg.bzl", "codeql_pkg_files_overlay")
+load("//misc/bazel:pkg.bzl", "codeql_pack", "codeql_pkg_files_overlay")
 
 package(default_visibility = ["//visibility:public"])
 
@@ -59,22 +58,16 @@ codeql_pkg_files_overlay(
     ],
 )
 
-dist(
-    name = "extractor-arch",
-    srcs = [":extractor-arch-overlay"],
-)
-
-dist(
-    name = "extractor-generic",
+codeql_pack(
+    name = "csharp",
     srcs = [
         ":dbscheme-group",
         ":extra-files",
+        ":extractor-arch-overlay",
         ":extractor-asp",
         "//csharp/downgrades",
         "//csharp/tools",
     ],
-    prefix = "csharp",
-    visibility = ["//visibility:public"],
 )
 
 test_suite(

csharp/autobuilder/Semmle.Autobuild.Cpp/BUILD.bazel

@@ -9,7 +9,6 @@ codeql_csharp_binary(
         "*.cs",
         "Properties/*.cs",
     ]),
-    language_prefix = "cpp",
     visibility = ["//visibility:public"],
     deps = [
         "//csharp/autobuilder/Semmle.Autobuild.Shared",

go/BUILD.bazel

@@ -1,7 +1,6 @@
 load("@bazel_skylib//rules:native_binary.bzl", "native_binary")
-load("@rules_pkg//pkg:install.bzl", "pkg_install")
-load("@rules_pkg//pkg:mappings.bzl", "pkg_attributes", "pkg_filegroup", "pkg_files")
-load("//:defs.bzl", "codeql_platform")
+load("@rules_pkg//pkg:mappings.bzl", "pkg_files")
+load("//misc/bazel:pkg.bzl", "codeql_pack", "codeql_pkg_files")
 
 native_binary(
     name = "gazelle",
@@ -44,50 +43,26 @@ pkg_files(
     ],
 )
 
-pkg_filegroup(
-    name = "extractor-pack-generic",
-    srcs = [
-        ":resources",
-        "//go/codeql-tools",
-        "//go/downgrades",
-        "//go/extractor:tokenizer",
-    ],
-    visibility = ["//visibility:public"],
-)
-
-pkg_files(
+codeql_pkg_files(
     name = "extractor-pack-arch",
-    srcs = [
+    exes = [
         "//go/extractor/cli/go-autobuilder",
         "//go/extractor/cli/go-bootstrap",
         "//go/extractor/cli/go-build-runner",
         "//go/extractor/cli/go-extractor",
         "//go/extractor/cli/go-gen-dbscheme",
         "//go/extractor/cli/go-tokenizer",
     ],
-    attributes = pkg_attributes(mode = "0755"),
-    prefix = "tools/" + codeql_platform,
-    visibility = ["//visibility:public"],
+    prefix = "tools/{CODEQL_PLATFORM}",
 )
 
-pkg_filegroup(
-    name = "extractor-pack",
+codeql_pack(
+    name = "go",
     srcs = [
         ":extractor-pack-arch",
-        ":extractor-pack-generic",
+        ":resources",
+        "//go/codeql-tools",
+        "//go/downgrades",
+        "//go/extractor:tokenizer",
     ],
-    visibility = ["//visibility:public"],
-)
-
-pkg_install(
-    name = "_extractor_pack",
-    srcs = [":extractor-pack"],
-)
-
-py_binary(
-    name = "create-extractor-pack",
-    srcs = ["create_extractor_pack.py"],
-    env = {"REPO_NAME": repo_name()},
-    main = "create_extractor_pack.py",
-    deps = ["_extractor_pack"],
 )

go/Makefile

@@ -1,6 +1,6 @@
 all: gen extractor
 
-EXTRACTOR_PACK_OUT = build/codeql-extractor-go
+EXTRACTOR_PACK_OUT = extractor-pack
 
 .PHONY: extractor gen clean autoformat check-formatting
 
@@ -28,7 +28,7 @@ qhelp-to-markdown:
 	scripts/qhelp-to-markdown.sh ql/src "$(QHELP_OUT_DIR)"
 
 extractor:
-	bazel run :create-extractor-pack
+	bazel run :go-installer
 
 gen:
 	bazel run :gen
@@ -45,9 +45,9 @@ ql/lib/go.dbscheme.stats: ql/lib/go.dbscheme build/stats/src.stamp extractor
 	codeql dataset measure -o $@ build/stats/database/db-go
 
 test: all build/testdb/check-upgrade-path
-	codeql test run -j0 ql/test --search-path build/codeql-extractor-go --consistency-queries ql/test/consistency --compilation-cache=$(cache)
+	codeql test run -j0 ql/test --search-path .. --consistency-queries ql/test/consistency --compilation-cache=$(cache)
 #	use GOOS=linux because GOOS=darwin GOARCH=386 is no longer supported
-	env GOOS=linux GOARCH=386 codeql$(EXE) test run -j0 ql/test/query-tests/Security/CWE-681 --search-path build/codeql-extractor-go --consistency-queries ql/test/consistency --compilation-cache=$(cache)
+	env GOOS=linux GOARCH=386 codeql$(EXE) test run -j0 ql/test/query-tests/Security/CWE-681 --search-path .. --consistency-queries ql/test/consistency --compilation-cache=$(cache)
 	cd extractor; bazel test ...
 	bash extractor-smoke-test/test.sh || (echo "Extractor smoke test FAILED"; exit 1)
 

go/actions/test/action.yml

@@ -40,7 +40,7 @@ runs:
     - name: Build
       shell: bash
       run: |
-        bazel run go:create-extractor-pack
+        bazel run go:go-installer
 
     - name: Check that all Go code is autoformatted
       if: inputs.run-code-checks == 'true' && !cancelled()

go/codeql-tools/BUILD.bazel

@@ -1,28 +1,15 @@
-load("@rules_pkg//pkg:mappings.bzl", "pkg_attributes", "pkg_filegroup", "pkg_files")
+load("//misc/bazel:pkg.bzl", "codeql_pkg_files")
 
-pkg_files(
-    name = "sh-files",
-    srcs = glob(["*.sh"]),
-    attributes = pkg_attributes(mode = "0755"),
-)
-
-pkg_files(
-    name = "non-sh-files",
+codeql_pkg_files(
+    name = "codeql-tools",
     srcs = glob(
         ["*"],
         exclude = [
             "*.sh",
             "BUILD.bazel",
         ],
     ),
-)
-
-pkg_filegroup(
-    name = "codeql-tools",
-    srcs = [
-        ":non-sh-files",
-        ":sh-files",
-    ],
+    exes = glob(["*.sh"]),
     prefix = "tools",
     visibility = ["//go:__pkg__"],
 )

go/create_extractor_pack.py

@@ -7,7 +7,7 @@ cd $DIR
 
 rm -rf testdb
 
-codeql database create --language=go testdb --search-path ../build/codeql-extractor-go
+codeql database create --language=go testdb --search-path ../..
 codeql dataset check testdb/db-go
 codeql query run ../ql/test/library-tests/semmle/go/controlflow/ControlFlowGraph/ControlFlowNode_getASuccessor.ql --database=testdb --output=notracing-out.bqrs --search-path ..
 codeql bqrs decode notracing-out.bqrs --format=csv --output=notracing-out.csv
@@ -19,7 +19,7 @@ export CODEQL_EXTRACTOR_GO_BUILD_TRACING=on
 
 rm -rf testdb
 
-codeql database create --language=go testdb --search-path ../build/codeql-extractor-go
+codeql database create --language=go testdb --search-path ../..
 codeql dataset check testdb/db-go
 codeql query run ../ql/test/library-tests/semmle/go/controlflow/ControlFlowGraph/ControlFlowNode_getASuccessor.ql --database=testdb --output=tracing-out.bqrs --search-path ..
 codeql bqrs decode tracing-out.bqrs --format=csv --output=tracing-out.csv