Skip to content

Commit 7b8c113

Browse files
redsun82criemen
redsun82
authored and
criemen
committed
Javascript: use codeql_pack for javascript extractor
1 parent fe8f13e commit 7b8c113

File tree

17 files changed

+219
-42
lines changed

17 files changed

+219
-42
lines changed

‎csharp/.gitignore

-1
Original file line numberDiff line numberDiff line change
@@ -14,5 +14,4 @@ csharp.log
1414
.vscode/launch.json
1515

1616
extractor/Semmle.Extraction.CSharp.Driver/Properties/launchSettings.json
17-
extractor-pack
1817
paket-files/

‎csharp/tools/BUILD.bazel

+3-3
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,11 @@
1-
load("@semmle_code//:dist.bzl", "pack_zip")
1+
load("//misc/bazel:pkg.bzl", "codeql_pkg_files")
22

3-
pack_zip(
3+
codeql_pkg_files(
44
name = "tools",
5-
srcs = glob(["**/*"]),
65
excludes = [
76
"BUILD.bazel",
87
],
8+
exes = glob(["**/*"]),
99
prefix = "tools",
1010
visibility = ["//csharp:__pkg__"],
1111
)

‎javascript/BUILD.bazel

+13-15
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,5 @@
11
load("@rules_pkg//pkg:mappings.bzl", "pkg_files")
2-
load("@semmle_code//:dist.bzl", "dist")
3-
load("@semmle_code//buildutils-internal:zipmerge.bzl", "zipmerge")
2+
load("//misc/bazel:pkg.bzl", "codeql_pack")
43

54
package(default_visibility = ["//visibility:public"])
65

@@ -23,26 +22,25 @@ pkg_files(
2322
strip_prefix = None,
2423
)
2524

26-
dist(
27-
name = "javascript-extractor-pack",
25+
# We have to use a zip of the typescript parser wrapper, as it's generated by a genrule
26+
# and we don't know a list of its output files.
27+
codeql_pack(
28+
name = "javascript",
2829
srcs = [
2930
":dbscheme-group",
3031
"//javascript/downgrades",
3132
"//javascript/externs",
3233
"//javascript/extractor:tools-extractor",
3334
"@semmle_code//language-packs/javascript:resources",
3435
],
35-
prefix = "javascript",
36+
visibility = ["//visibility:public"],
37+
zips = {"//javascript/extractor/lib/typescript": "tools"},
3638
)
3739

38-
# We have to zipmerge in the typescript parser wrapper, as it's generated by a genrule
39-
# and we don't know a list of its output files. Therefore, we sidestep the
40-
# rules_pkg tooling here, and generate the zip for the language pack manually.
41-
zipmerge(
42-
name = "javascript",
43-
srcs = [
44-
":javascript-extractor-pack.zip",
45-
"//javascript/extractor/lib/typescript",
46-
],
47-
out = "javascript.zip",
40+
# TODO copy for internal repository backward compatibility
41+
genrule(
42+
name = "javascript.zip",
43+
srcs = [":javascript-generic-zip"],
44+
outs = ["javascript.zip"],
45+
cmd = "cp $< $@",
4846
)

‎javascript/downgrades/BUILD.bazel

+4-3
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,12 @@
1-
load("@semmle_code//:dist.bzl", "pack_zip")
1+
load("//misc/bazel:pkg.bzl", "codeql_pkg_files", "strip_prefix")
22

3-
pack_zip(
3+
codeql_pkg_files(
44
name = "downgrades",
55
srcs = glob(
66
["**/*"],
77
exclude = ["BUILD.bazel"],
88
),
99
prefix = "downgrades",
10-
visibility = ["//visibility:public"],
10+
strip_prefix = strip_prefix.from_pkg(),
11+
visibility = ["//javascript:__pkg__"],
1112
)

‎javascript/externs/BUILD.bazel

+4-3
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,12 @@
1-
load("@semmle_code//:dist.bzl", "pack_zip")
1+
load("//misc/bazel:pkg.bzl", "codeql_pkg_files", "strip_prefix")
22

3-
pack_zip(
3+
codeql_pkg_files(
44
name = "externs",
55
srcs = glob(
66
["**/*"],
77
exclude = ["BUILD.bazel"],
88
),
99
prefix = "tools/data/externs",
10-
visibility = ["//visibility:public"],
10+
strip_prefix = strip_prefix.from_pkg(),
11+
visibility = ["//javascript:__pkg__"],
1112
)

‎javascript/extractor/lib/typescript/BUILD.bazel

+9-14
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,3 @@
1-
load("@semmle_code//:common.bzl", "on_windows")
2-
31
# Builds a zip file of the compiled typscript-parser-wrapper and its dependencies.
42
genrule(
53
name = "typescript",
@@ -33,19 +31,16 @@ genrule(
3331
# Install again with only runtime deps
3432
"$$NPM install --prod",
3533
"mv node_modules build/",
36-
"mkdir -p javascript/tools/typescript-parser-wrapper",
37-
"mv build/* javascript/tools/typescript-parser-wrapper",
34+
"mkdir -p typescript-parser-wrapper",
35+
"mv build/* typescript-parser-wrapper",
36+
"OUT=$$BAZEL_ROOT/$@",
37+
"case $$OSTYPE in",
38+
" cygwin|msys|win32) OUT=$$(cygpath -w $$OUT);;",
39+
"esac",
3840
"",
39-
]) + on_windows(
40-
" && ".join([
41-
"$$BAZEL_ROOT/$(execpath @bazel_tools//tools/zip:zipper) cC $$(cygpath -w $$BAZEL_ROOT/$@) $$(find javascript -name '*' -print)",
42-
"rm -rf $$TEMP",
43-
]),
44-
" && ".join([
45-
"$$BAZEL_ROOT/$(execpath @bazel_tools//tools/zip:zipper) cC $$BAZEL_ROOT/$@ $$(find javascript -name '*' -print)",
46-
"rm -rf $$TEMP",
47-
]),
48-
),
41+
"$$BAZEL_ROOT/$(execpath @bazel_tools//tools/zip:zipper) cC $$OUT $$(find typescript-parser-wrapper -name '*' -print)",
42+
"rm -rf $$TEMP",
43+
]),
4944
tools = [
5045
"@bazel_tools//tools/zip:zipper",
5146
"@nodejs//:node_bin",

‎javascript/extractor/test/com/semmle/js/extractor/test/AllTests.java

+1-1
Original file line numberDiff line numberDiff line change
@@ -50,7 +50,7 @@ public static void setUp() throws Exception {
5050
entry = zis.getNextEntry();
5151
}
5252
}
53-
Path tsWrapper = tempDir.resolve("javascript/tools/typescript-parser-wrapper/main.js");
53+
Path tsWrapper = tempDir.resolve("typescript-parser-wrapper/main.js");
5454
if (!Files.exists(tsWrapper)) {
5555
throw new RuntimeException("Could not find ts-wrapper at " + tsWrapper);
5656
}

‎javascript/resources/BUILD.bazel

+15
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
load("//misc/bazel:pkg.bzl", "codeql_pkg_files")
2+
3+
codeql_pkg_files(
4+
name = "resources",
5+
srcs = glob(
6+
["**/*"],
7+
exclude = [
8+
"tools/*.sh",
9+
"BUILD.bazel",
10+
],
11+
),
12+
exes = glob(["tools/*.sh"]),
13+
strip_prefix = "",
14+
visibility = ["//javascript:__pkg__"],
15+
)

‎javascript/resources/codeql-extractor.yml

+94
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,94 @@
1+
name: "javascript"
2+
aliases:
3+
- javascript-typescript
4+
- typescript
5+
display_name: "JavaScript/TypeScript"
6+
version: 1.22.1
7+
column_kind: "utf16"
8+
unicode_newlines: true
9+
build_modes:
10+
- none
11+
file_coverage_languages:
12+
- name: javascript
13+
display_name: JavaScript
14+
scc_languages:
15+
- JavaScript
16+
- name: typescript
17+
display_name: TypeScript
18+
scc_languages:
19+
- TypeScript
20+
- TypeScript Typings
21+
github_api_languages:
22+
- JavaScript
23+
- TypeScript
24+
scc_languages:
25+
- JavaScript
26+
- TypeScript
27+
- TypeScript Typings
28+
file_types:
29+
- name: javascript
30+
display_name: JavaScript
31+
extensions:
32+
- .js
33+
- .jsx
34+
- name: ecmascript
35+
display_name: ECMAScript
36+
extensions:
37+
- .es
38+
- .es6
39+
- .mjs
40+
- name: typescript
41+
display_name: TypeScript
42+
extensions:
43+
- .ts
44+
- .tsx
45+
- name: html
46+
display_name: HTML
47+
extensions:
48+
- .html
49+
- .htm
50+
- .xhtm
51+
- .xhtml
52+
- name: vue
53+
display_name: Vue.js component
54+
extensions:
55+
- .vue
56+
- name: data
57+
display_name: Data or configuration files
58+
extensions:
59+
- .json
60+
- .yml
61+
- .yaml
62+
- .raml
63+
legacy_qltest_extraction: true
64+
options:
65+
trap:
66+
title: TRAP options
67+
description: Options about how the extractor handles TRAP files
68+
type: object
69+
visibility: 3
70+
properties:
71+
cache:
72+
title: TRAP cache options
73+
description: Options about how the extractor handles its TRAP cache
74+
type: object
75+
properties:
76+
dir:
77+
title: TRAP cache directory
78+
description: The directory of the TRAP cache to use
79+
type: string
80+
bound:
81+
title: TRAP cache bound
82+
description: A soft limit (in MB) on the size of the TRAP cache
83+
type: string
84+
pattern: "[0-9]+"
85+
write:
86+
title: TRAP cache writeable
87+
description: Whether to write to the TRAP cache as well as reading it
88+
type: string
89+
pattern: "(true|TRUE|false|FALSE)"
90+
skip_types:
91+
title: Skip type extraction for TypeScript
92+
description: Whether to skip the extraction of types in a TypeScript application
93+
type: string
94+
pattern: "^(false|true)$"

‎javascript/resources/tools/autobuild.cmd

+30
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,30 @@
1+
@echo off
2+
SETLOCAL EnableDelayedExpansion
3+
4+
set jvm_args=-Xss16m
5+
6+
rem If CODEQL_RAM is set, use half for Java and half for TS.
7+
if NOT [%CODEQL_RAM%] == [] (
8+
set /a "half_ram=CODEQL_RAM/2"
9+
set LGTM_TYPESCRIPT_RAM=%half_ram%
10+
set jvm_args=!jvm_args! -Xmx!half_ram!m
11+
)
12+
13+
rem If CODEQL_THREADS is set, propagate via LGTM_THREADS.
14+
if NOT [%CODEQL_THREADS%] == [] (
15+
set LGTM_THREADS=%CODEQL_THREADS%
16+
)
17+
18+
rem The JS autobuilder expects to find typescript modules under SEMMLE_DIST/tools.
19+
rem They are included in the pack, but we need to set SEMMLE_DIST appropriately.
20+
set SEMMLE_DIST=%CODEQL_EXTRACTOR_JAVASCRIPT_ROOT%
21+
22+
rem The JS autobuilder expects LGTM_SRC to be set to the source root.
23+
set LGTM_SRC=%CD%
24+
25+
type NUL && "%CODEQL_JAVA_HOME%\bin\java.exe" %jvm_args% ^
26+
-cp "%CODEQL_EXTRACTOR_JAVASCRIPT_ROOT%\tools\extractor-javascript.jar" ^
27+
com.semmle.js.extractor.AutoBuild
28+
exit /b %ERRORLEVEL%
29+
30+
ENDLOCAL

‎javascript/resources/tools/autobuild.sh

+29
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,29 @@
1+
#!/bin/sh
2+
3+
set -eu
4+
5+
jvm_args=-Xss16m
6+
7+
# If CODEQL_RAM is set, use half for Java and half for TS.
8+
if [ -n "${CODEQL_RAM:-}" ] ; then
9+
half_ram="$(( CODEQL_RAM / 2 ))"
10+
LGTM_TYPESCRIPT_RAM="$half_ram"
11+
export LGTM_TYPESCRIPT_RAM
12+
jvm_args="$jvm_args -Xmx${half_ram}m"
13+
fi
14+
15+
# If CODEQL_THREADS is set, propagate via LGTM_THREADS.
16+
if [ -n "${CODEQL_THREADS:-}" ] ; then
17+
LGTM_THREADS="$CODEQL_THREADS"
18+
export LGTM_THREADS
19+
fi
20+
21+
# The JS autobuilder expects to find typescript modules under SEMMLE_DIST/tools.
22+
# They are included in the pack, but we need to set SEMMLE_DIST appropriately.
23+
# We want to word-split $jvm_args, so disable the shellcheck warning.
24+
# shellcheck disable=SC2086
25+
env SEMMLE_DIST="$CODEQL_EXTRACTOR_JAVASCRIPT_ROOT" \
26+
LGTM_SRC="$(pwd)" \
27+
"${CODEQL_JAVA_HOME}/bin/java" $jvm_args \
28+
-cp "$CODEQL_EXTRACTOR_JAVASCRIPT_ROOT/tools/extractor-javascript.jar" \
29+
com.semmle.js.extractor.AutoBuild

‎javascript/resources/tools/baseline-config.json

+8
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
{
2+
"paths-ignore": [
3+
"**/node_modules/**",
4+
"**/bower_components/**",
5+
"**/*.min.js",
6+
"**/*-min.js"
7+
]
8+
}

‎javascript/resources/tools/configure-baseline.cmd

+2
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
@echo off
2+
type "%CODEQL_EXTRACTOR_JAVASCRIPT_ROOT%\tools\baseline-config.json"

‎javascript/resources/tools/configure-baseline.sh

+3
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
#!/bin/sh
2+
3+
cat "$CODEQL_EXTRACTOR_JAVASCRIPT_ROOT/tools/baseline-config.json"

‎javascript/resources/tools/index-dir.sh

+4
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
#!/bin/sh
2+
3+
echo "Not implemented." 1>&2
4+
exit 1

‎ql/.gitignore

-1
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,4 @@
11
target
2-
extractor-pack
32
.vscode/launch.json
43
.cache
54
ql/test/**/*.testproj

‎ruby/.gitignore

-1
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,4 @@
11
extractor/target
2-
extractor-pack
32
.vscode/launch.json
43
.cache
54
ql/test/**/*.testproj

0 commit comments

Comments
 (0)