Skip to content

Commit 5b6f38f

Browse files
criemencriemen
authored
Merge pull request github#16623 from github/criemen/new-pkg
Bazel: Use new packaging rules
2 parents b0865c2 + 8783dfb commit 5b6f38f

File tree

27 files changed

+102
-184
lines changed

27 files changed

+102
-184
lines changed

‎.github/workflows/csharp-qltest.yml

+2-2
Original file line numberDiff line numberDiff line change
@@ -65,7 +65,7 @@ jobs:
6565
key: csharp-qltest-${{ matrix.slice }}
6666
- name: Run QL tests
6767
run: |
68-
codeql test run --threads=0 --ram 50000 --slice ${{ matrix.slice }} --search-path extractor-pack --check-databases --check-undefined-labels --check-repeated-labels --check-redefined-labels --consistency-queries ql/consistency-queries ql/test --compilation-cache "${{ steps.query-cache.outputs.cache-dir }}"
68+
codeql test run --threads=0 --ram 50000 --slice ${{ matrix.slice }} --search-path "${{ github.workspace }}" --check-databases --check-undefined-labels --check-repeated-labels --check-redefined-labels --consistency-queries ql/consistency-queries ql/test --compilation-cache "${{ steps.query-cache.outputs.cache-dir }}"
6969
env:
7070
GITHUB_TOKEN: ${{ github.token }}
7171
unit-tests:
@@ -101,6 +101,6 @@ jobs:
101101
# Update existing stubs in the repo with the freshly generated ones
102102
mv "$STUBS_PATH/output/stubs/_frameworks" ql/test/resources/stubs/
103103
git status
104-
codeql test run --threads=0 --search-path extractor-pack --check-databases --check-undefined-labels --check-repeated-labels --check-redefined-labels --consistency-queries ql/consistency-queries -- ql/test/library-tests/dataflow/flowsources/aspremote
104+
codeql test run --threads=0 --search-path "${{ github.workspace }}" --check-databases --check-undefined-labels --check-repeated-labels --check-redefined-labels --consistency-queries ql/consistency-queries -- ql/test/library-tests/dataflow/flowsources/aspremote
105105
env:
106106
GITHUB_TOKEN: ${{ github.token }}

‎.github/workflows/ql-for-ql-build.yml

+5-5
Original file line numberDiff line numberDiff line change
@@ -49,20 +49,20 @@ jobs:
4949
key: ${{ runner.os }}-${{ steps.os_version.outputs.version }}-rust-cargo-${{ hashFiles('ql/**/Cargo.lock') }}
5050
- name: Release build
5151
if: steps.cache-extractor.outputs.cache-hit != 'true'
52-
run: cd ql; ./scripts/create-extractor-pack.sh
52+
run: cd ql; ./scripts/create-extractor-pack.sh
5353
env:
54-
GH_TOKEN: ${{ github.token }}
54+
GH_TOKEN: ${{ github.token }}
5555
- name: Cache compilation cache
5656
id: query-cache
5757
uses: ./.github/actions/cache-query-compilation
58-
with:
58+
with:
5959
key: run-ql-for-ql
6060
- name: Make database and analyze
6161
run: |
6262
./ql/target/release/buramu | tee deprecated.blame # Add a blame file for the extractor to parse.
63-
${CODEQL} database create -l=ql --search-path ql/extractor-pack ${DB}
63+
${CODEQL} database create -l=ql ${DB} --search-path "${{ github.workspace }}"
6464
${CODEQL} database analyze -j0 --format=sarif-latest --output=ql-for-ql.sarif ${DB} ql/ql/src/codeql-suites/ql-code-scanning.qls --compilation-cache "${{ steps.query-cache.outputs.cache-dir }}"
65-
env:
65+
env:
6666
CODEQL: ${{ steps.find-codeql.outputs.codeql-path }}
6767
DB: ${{ runner.temp }}/DB
6868
LGTM_INDEX_FILTERS: |

‎.github/workflows/ql-for-ql-dataset_measure.yml

+2-2
Original file line numberDiff line numberDiff line change
@@ -53,8 +53,8 @@ jobs:
5353
- name: Create database
5454
run: |
5555
"${CODEQL}" database create \
56-
--search-path "ql/extractor-pack" \
57-
--threads 4 \
56+
--search-path "${{ github.workspace }}"
57+
--threads 4 \
5858
--language ql --source-root "${{ github.workspace }}/repo" \
5959
"${{ runner.temp }}/database"
6060
env:

‎.github/workflows/ql-for-ql-tests.yml

+6-7
Original file line numberDiff line numberDiff line change
@@ -49,23 +49,23 @@ jobs:
4949
- name: Cache compilation cache
5050
id: query-cache
5151
uses: ./.github/actions/cache-query-compilation
52-
with:
52+
with:
5353
key: ql-for-ql-tests
5454
- name: Run QL tests
5555
run: |
56-
"${CODEQL}" test run --check-databases --check-unused-labels --check-repeated-labels --check-redefined-labels --check-use-before-definition --search-path "${{ github.workspace }}/ql/extractor-pack" --consistency-queries ql/ql/consistency-queries --compilation-cache "${{ steps.query-cache.outputs.cache-dir }}" ql/ql/test
56+
"${CODEQL}" test run --check-databases --check-unused-labels --check-repeated-labels --check-redefined-labels --check-use-before-definition --search-path "${{ github.workspace }}" --consistency-queries ql/ql/consistency-queries --compilation-cache "${{ steps.query-cache.outputs.cache-dir }}" ql/ql/test
5757
env:
5858
CODEQL: ${{ steps.find-codeql.outputs.codeql-path }}
5959

60-
other-os:
60+
other-os:
6161
strategy:
6262
matrix:
6363
os: [macos-latest, windows-latest]
6464
needs: [qltest]
6565
runs-on: ${{ matrix.os }}
6666
steps:
6767
- uses: actions/checkout@v4
68-
- name: Install GNU tar
68+
- name: Install GNU tar
6969
if: runner.os == 'macOS'
7070
run: |
7171
brew install gnu-tar
@@ -100,13 +100,12 @@ jobs:
100100
- name: Run a single QL tests - Unix
101101
if: runner.os != 'Windows'
102102
run: |
103-
"${CODEQL}" test run --check-databases --search-path "${{ github.workspace }}/ql/extractor-pack" ql/ql/test/queries/style/DeadCode/DeadCode.qlref
103+
"${CODEQL}" test run --check-databases --search-path "${{ github.workspace }}" ql/ql/test/queries/style/DeadCode/DeadCode.qlref
104104
env:
105105
CODEQL: ${{ steps.find-codeql.outputs.codeql-path }}
106106
- name: Run a single QL tests - Windows
107107
if: runner.os == 'Windows'
108108
shell: pwsh
109109
run: |
110110
$Env:PATH += ";$(dirname ${{ steps.find-codeql.outputs.codeql-path }})"
111-
codeql test run --check-databases --search-path "${{ github.workspace }}/ql/extractor-pack" ql/ql/test/queries/style/DeadCode/DeadCode.qlref
112-
111+
codeql test run --check-databases --search-path "${{ github.workspace }}" ql/ql/test/queries/style/DeadCode/DeadCode.qlref

‎.github/workflows/ruby-dataset-measure.yml

+1-1
Original file line numberDiff line numberDiff line change
@@ -44,7 +44,7 @@ jobs:
4444
- name: Create database
4545
run: |
4646
codeql database create \
47-
--search-path "${{ github.workspace }}/ruby/extractor-pack" \
47+
--search-path "${{ github.workspace }}" \
4848
--threads 4 \
4949
--language ruby --source-root "${{ github.workspace }}/repo" \
5050
"${{ runner.temp }}/database"

‎.github/workflows/ruby-qltest.yml

+2-2
Original file line numberDiff line numberDiff line change
@@ -64,10 +64,10 @@ jobs:
6464
- name: Cache compilation cache
6565
id: query-cache
6666
uses: ./.github/actions/cache-query-compilation
67-
with:
67+
with:
6868
key: ruby-qltest
6969
- name: Run QL tests
7070
run: |
71-
codeql test run --threads=0 --ram 50000 --search-path "${{ github.workspace }}/ruby/extractor-pack" --check-databases --check-undefined-labels --check-unused-labels --check-repeated-labels --check-redefined-labels --check-use-before-definition --consistency-queries ql/consistency-queries ql/test --compilation-cache "${{ steps.query-cache.outputs.cache-dir }}"
71+
codeql test run --threads=0 --ram 50000 --search-path "${{ github.workspace }}" --check-databases --check-undefined-labels --check-unused-labels --check-repeated-labels --check-redefined-labels --check-use-before-definition --consistency-queries ql/consistency-queries ql/test --compilation-cache "${{ steps.query-cache.outputs.cache-dir }}"
7272
env:
7373
GITHUB_TOKEN: ${{ github.token }}

‎.gitignore

+3
Original file line numberDiff line numberDiff line change
@@ -62,3 +62,6 @@ node_modules/
6262

6363
# Temporary folders for working with generated models
6464
.model-temp
65+
66+
# bazel-built in-tree extractor packs
67+
/*/extractor-pack

‎codeql-workspace.yml

+1-4
Original file line numberDiff line numberDiff line change
@@ -6,19 +6,16 @@ provide:
66
- "*/ql/consistency-queries/qlpack.yml"
77
- "*/ql/automodel/src/qlpack.yml"
88
- "*/ql/automodel/test/qlpack.yml"
9+
- "*/extractor-pack/codeql-extractor.yml"
910
- "python/extractor/qlpack.yml"
1011
- "shared/**/qlpack.yml"
1112
- "cpp/ql/test/query-tests/Security/CWE/CWE-190/semmle/tainted/qlpack.yml"
1213
- "go/ql/config/legacy-support/qlpack.yml"
13-
- "go/build/codeql-extractor-go/codeql-extractor.yml"
1414
- "csharp/ql/campaigns/Solorigate/lib/qlpack.yml"
1515
- "csharp/ql/campaigns/Solorigate/src/qlpack.yml"
1616
- "csharp/ql/campaigns/Solorigate/test/qlpack.yml"
1717
- "misc/legacy-support/*/qlpack.yml"
1818
- "misc/suite-helpers/qlpack.yml"
19-
- "ruby/extractor-pack/codeql-extractor.yml"
20-
- "swift/extractor-pack/codeql-extractor.yml"
21-
- "ql/extractor-pack/codeql-extractor.yml"
2219
- ".github/codeql/extensions/**/codeql-pack.yml"
2320

2421
versionPolicies:

‎cpp/downgrades/BUILD.bazel

+1-1
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@ pkg_files(
66
["**"],
77
exclude = ["BUILD.bazel"],
88
),
9-
prefix = "cpp/downgrades",
9+
prefix = "downgrades",
1010
strip_prefix = strip_prefix.from_pkg(),
1111
visibility = ["//cpp:__pkg__"],
1212
)

‎cpp/ql/lib/BUILD.bazel

-2
Original file line numberDiff line numberDiff line change
@@ -5,11 +5,9 @@ package(default_visibility = ["//cpp:__pkg__"])
55
pkg_files(
66
name = "dbscheme",
77
srcs = ["semmlecode.cpp.dbscheme"],
8-
prefix = "cpp",
98
)
109

1110
pkg_files(
1211
name = "dbscheme-stats",
1312
srcs = ["semmlecode.cpp.dbscheme.stats"],
14-
prefix = "cpp",
1513
)

‎csharp/BUILD.bazel

+4-11
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,5 @@
11
load("@rules_pkg//pkg:mappings.bzl", "pkg_filegroup", "pkg_files")
2-
load("@semmle_code//:dist.bzl", "dist")
3-
load("//misc/bazel:pkg.bzl", "codeql_pkg_files_overlay")
2+
load("//misc/bazel:pkg.bzl", "codeql_pack", "codeql_pkg_files_overlay")
43

54
package(default_visibility = ["//visibility:public"])
65

@@ -59,22 +58,16 @@ codeql_pkg_files_overlay(
5958
],
6059
)
6160

62-
dist(
63-
name = "extractor-arch",
64-
srcs = [":extractor-arch-overlay"],
65-
)
66-
67-
dist(
68-
name = "extractor-generic",
61+
codeql_pack(
62+
name = "csharp",
6963
srcs = [
7064
":dbscheme-group",
7165
":extra-files",
66+
":extractor-arch-overlay",
7267
":extractor-asp",
7368
"//csharp/downgrades",
7469
"//csharp/tools",
7570
],
76-
prefix = "csharp",
77-
visibility = ["//visibility:public"],
7871
)
7972

8073
test_suite(

‎csharp/autobuilder/Semmle.Autobuild.Cpp/BUILD.bazel

-1
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,6 @@ codeql_csharp_binary(
99
"*.cs",
1010
"Properties/*.cs",
1111
]),
12-
language_prefix = "cpp",
1312
visibility = ["//visibility:public"],
1413
deps = [
1514
"//csharp/autobuilder/Semmle.Autobuild.Shared",

‎go/BUILD.bazel

+11-36
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,6 @@
11
load("@bazel_skylib//rules:native_binary.bzl", "native_binary")
2-
load("@rules_pkg//pkg:install.bzl", "pkg_install")
3-
load("@rules_pkg//pkg:mappings.bzl", "pkg_attributes", "pkg_filegroup", "pkg_files")
4-
load("//:defs.bzl", "codeql_platform")
2+
load("@rules_pkg//pkg:mappings.bzl", "pkg_files")
3+
load("//misc/bazel:pkg.bzl", "codeql_pack", "codeql_pkg_files")
54

65
native_binary(
76
name = "gazelle",
@@ -44,50 +43,26 @@ pkg_files(
4443
],
4544
)
4645

47-
pkg_filegroup(
48-
name = "extractor-pack-generic",
49-
srcs = [
50-
":resources",
51-
"//go/codeql-tools",
52-
"//go/downgrades",
53-
"//go/extractor:tokenizer",
54-
],
55-
visibility = ["//visibility:public"],
56-
)
57-
58-
pkg_files(
46+
codeql_pkg_files(
5947
name = "extractor-pack-arch",
60-
srcs = [
48+
exes = [
6149
"//go/extractor/cli/go-autobuilder",
6250
"//go/extractor/cli/go-bootstrap",
6351
"//go/extractor/cli/go-build-runner",
6452
"//go/extractor/cli/go-extractor",
6553
"//go/extractor/cli/go-gen-dbscheme",
6654
"//go/extractor/cli/go-tokenizer",
6755
],
68-
attributes = pkg_attributes(mode = "0755"),
69-
prefix = "tools/" + codeql_platform,
70-
visibility = ["//visibility:public"],
56+
prefix = "tools/{CODEQL_PLATFORM}",
7157
)
7258

73-
pkg_filegroup(
74-
name = "extractor-pack",
59+
codeql_pack(
60+
name = "go",
7561
srcs = [
7662
":extractor-pack-arch",
77-
":extractor-pack-generic",
63+
":resources",
64+
"//go/codeql-tools",
65+
"//go/downgrades",
66+
"//go/extractor:tokenizer",
7867
],
79-
visibility = ["//visibility:public"],
80-
)
81-
82-
pkg_install(
83-
name = "_extractor_pack",
84-
srcs = [":extractor-pack"],
85-
)
86-
87-
py_binary(
88-
name = "create-extractor-pack",
89-
srcs = ["create_extractor_pack.py"],
90-
env = {"REPO_NAME": repo_name()},
91-
main = "create_extractor_pack.py",
92-
deps = ["_extractor_pack"],
9368
)

‎go/Makefile

+4-4
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
all: gen extractor
22

3-
EXTRACTOR_PACK_OUT = build/codeql-extractor-go
3+
EXTRACTOR_PACK_OUT = extractor-pack
44

55
.PHONY: extractor gen clean autoformat check-formatting
66

@@ -28,7 +28,7 @@ qhelp-to-markdown:
2828
scripts/qhelp-to-markdown.sh ql/src "$(QHELP_OUT_DIR)"
2929

3030
extractor:
31-
bazel run :create-extractor-pack
31+
bazel run :go-installer
3232

3333
gen:
3434
bazel run :gen
@@ -45,9 +45,9 @@ ql/lib/go.dbscheme.stats: ql/lib/go.dbscheme build/stats/src.stamp extractor
4545
codeql dataset measure -o $@ build/stats/database/db-go
4646

4747
test: all build/testdb/check-upgrade-path
48-
codeql test run -j0 ql/test --search-path build/codeql-extractor-go --consistency-queries ql/test/consistency --compilation-cache=$(cache)
48+
codeql test run -j0 ql/test --search-path .. --consistency-queries ql/test/consistency --compilation-cache=$(cache)
4949
# use GOOS=linux because GOOS=darwin GOARCH=386 is no longer supported
50-
env GOOS=linux GOARCH=386 codeql$(EXE) test run -j0 ql/test/query-tests/Security/CWE-681 --search-path build/codeql-extractor-go --consistency-queries ql/test/consistency --compilation-cache=$(cache)
50+
env GOOS=linux GOARCH=386 codeql$(EXE) test run -j0 ql/test/query-tests/Security/CWE-681 --search-path .. --consistency-queries ql/test/consistency --compilation-cache=$(cache)
5151
cd extractor; bazel test ...
5252
bash extractor-smoke-test/test.sh || (echo "Extractor smoke test FAILED"; exit 1)
5353

‎go/actions/test/action.yml

+1-1
Original file line numberDiff line numberDiff line change
@@ -40,7 +40,7 @@ runs:
4040
- name: Build
4141
shell: bash
4242
run: |
43-
bazel run go:create-extractor-pack
43+
bazel run go:go-installer
4444
4545
- name: Check that all Go code is autoformatted
4646
if: inputs.run-code-checks == 'true' && !cancelled()

‎go/codeql-tools/BUILD.bazel

+4-17
Original file line numberDiff line numberDiff line change
@@ -1,28 +1,15 @@
1-
load("@rules_pkg//pkg:mappings.bzl", "pkg_attributes", "pkg_filegroup", "pkg_files")
1+
load("//misc/bazel:pkg.bzl", "codeql_pkg_files")
22

3-
pkg_files(
4-
name = "sh-files",
5-
srcs = glob(["*.sh"]),
6-
attributes = pkg_attributes(mode = "0755"),
7-
)
8-
9-
pkg_files(
10-
name = "non-sh-files",
3+
codeql_pkg_files(
4+
name = "codeql-tools",
115
srcs = glob(
126
["*"],
137
exclude = [
148
"*.sh",
159
"BUILD.bazel",
1610
],
1711
),
18-
)
19-
20-
pkg_filegroup(
21-
name = "codeql-tools",
22-
srcs = [
23-
":non-sh-files",
24-
":sh-files",
25-
],
12+
exes = glob(["*.sh"]),
2613
prefix = "tools",
2714
visibility = ["//go:__pkg__"],
2815
)

‎go/create_extractor_pack.py

-26
This file was deleted.

‎go/extractor-smoke-test/test.sh

+2-2
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,7 @@ cd $DIR
77

88
rm -rf testdb
99

10-
codeql database create --language=go testdb --search-path ../build/codeql-extractor-go
10+
codeql database create --language=go testdb --search-path ../..
1111
codeql dataset check testdb/db-go
1212
codeql query run ../ql/test/library-tests/semmle/go/controlflow/ControlFlowGraph/ControlFlowNode_getASuccessor.ql --database=testdb --output=notracing-out.bqrs --search-path ..
1313
codeql bqrs decode notracing-out.bqrs --format=csv --output=notracing-out.csv
@@ -19,7 +19,7 @@ export CODEQL_EXTRACTOR_GO_BUILD_TRACING=on
1919

2020
rm -rf testdb
2121

22-
codeql database create --language=go testdb --search-path ../build/codeql-extractor-go
22+
codeql database create --language=go testdb --search-path ../..
2323
codeql dataset check testdb/db-go
2424
codeql query run ../ql/test/library-tests/semmle/go/controlflow/ControlFlowGraph/ControlFlowNode_getASuccessor.ql --database=testdb --output=tracing-out.bqrs --search-path ..
2525
codeql bqrs decode tracing-out.bqrs --format=csv --output=tracing-out.csv

0 commit comments

Comments
 (0)