Skip to content

Commit ad7fb9c

Browse files
committed
Merge branch 'main' into redsun82/rust-has-implementation
2 parents 1110fea + e1b4dea commit ad7fb9c

File tree

1,277 files changed

+47404
-13891
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

1,277 files changed

+47404
-13891
lines changed

.github/workflows/validate-change-notes.yml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -31,4 +31,4 @@ jobs:
3131
- name: Fail if there are any errors with existing change notes
3232

3333
run: |
34-
codeql pack release --groups cpp,csharp,java,javascript,python,ruby,-examples,-test,-experimental
34+
codeql pack release --groups actions,cpp,csharp,go,java,javascript,python,ruby,shared,swift -examples,-test,-experimental

MODULE.bazel

Lines changed: 10 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -239,24 +239,24 @@ go_deps = use_extension("@gazelle//:extensions.bzl", "go_deps")
239239
go_deps.from_file(go_mod = "//go/extractor:go.mod")
240240
use_repo(go_deps, "org_golang_x_mod", "org_golang_x_tools")
241241

242-
lfs_files = use_repo_rule("//misc/bazel:lfs.bzl", "lfs_files")
242+
lfs_archive = use_repo_rule("//misc/bazel:lfs.bzl", "lfs_archive")
243243

244-
lfs_files(
244+
lfs_archive(
245245
name = "ripunzip-linux",
246-
srcs = ["//misc/ripunzip:ripunzip-linux"],
247-
executable = True,
246+
src = "//misc/ripunzip:ripunzip-Linux.zip",
247+
build_file = "//misc/ripunzip:BUILD.ripunzip.bazel",
248248
)
249249

250-
lfs_files(
250+
lfs_archive(
251251
name = "ripunzip-windows",
252-
srcs = ["//misc/ripunzip:ripunzip-windows.exe"],
253-
executable = True,
252+
src = "//misc/ripunzip:ripunzip-Windows.zip",
253+
build_file = "//misc/ripunzip:BUILD.ripunzip.bazel",
254254
)
255255

256-
lfs_files(
256+
lfs_archive(
257257
name = "ripunzip-macos",
258-
srcs = ["//misc/ripunzip:ripunzip-macos"],
259-
executable = True,
258+
src = "//misc/ripunzip:ripunzip-macOS.zip",
259+
build_file = "//misc/ripunzip:BUILD.ripunzip.bazel",
260260
)
261261

262262
register_toolchains(

actions/ql/lib/CHANGELOG.md

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,11 @@
1+
## 0.4.11
2+
3+
No user-facing changes.
4+
5+
## 0.4.10
6+
7+
No user-facing changes.
8+
19
## 0.4.9
210

311
No user-facing changes.
Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
---
2+
category: minorAnalysis
3+
---
4+
* Fixed performance issues in the parsing of Bash scripts in workflow files,
5+
which led to out-of-disk errors when analysing certain workflow files with
6+
complex interpolations of shell commands or quoted strings.
Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
## 0.4.10
2+
3+
No user-facing changes.
Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
## 0.4.11
2+
3+
No user-facing changes.
Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,2 +1,2 @@
11
---
2-
lastReleaseVersion: 0.4.9
2+
lastReleaseVersion: 0.4.11

actions/ql/lib/codeql/actions/Ast.qll

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -50,8 +50,8 @@ class Expression extends AstNode instanceof ExpressionImpl {
5050
string getNormalizedExpression() { result = normalizeExpr(expression) }
5151
}
5252

53-
/** A common class for `env` in workflow, job or step. */
54-
abstract class Env extends AstNode instanceof EnvImpl {
53+
/** An `env` in workflow, job or step. */
54+
class Env extends AstNode instanceof EnvImpl {
5555
/** Gets an environment variable value given its name. */
5656
ScalarValueImpl getEnvVarValue(string name) { result = super.getEnvVarValue(name) }
5757

actions/ql/lib/codeql/actions/Bash.qll

Lines changed: 98 additions & 44 deletions
Original file line numberDiff line numberDiff line change
@@ -8,35 +8,64 @@ class BashShellScript extends ShellScript {
88
)
99
}
1010

11-
private string lineProducer(int i) {
12-
result = this.getRawScript().regexpReplaceAll("\\\\\\s*\n", "").splitAt("\n", i)
13-
}
14-
15-
private predicate cmdSubstitutionReplacement(string cmdSubs, string id, int k) {
16-
exists(string line | line = this.lineProducer(k) |
17-
exists(int i, int j |
18-
cmdSubs =
19-
// $() cmd substitution
20-
line.regexpFind("\\$\\((?:[^()]+|\\((?:[^()]+|\\([^()]*\\))*\\))*\\)", i, j)
21-
.regexpReplaceAll("^\\$\\(", "")
22-
.regexpReplaceAll("\\)$", "") and
23-
id = "cmdsubs:" + k + ":" + i + ":" + j
24-
)
25-
or
26-
exists(int i, int j |
27-
// `...` cmd substitution
28-
cmdSubs =
29-
line.regexpFind("\\`[^\\`]+\\`", i, j)
30-
.regexpReplaceAll("^\\`", "")
31-
.regexpReplaceAll("\\`$", "") and
32-
id = "cmd:" + k + ":" + i + ":" + j
33-
)
11+
/**
12+
* Gets the line at 0-based index `lineIndex` within this shell script,
13+
* assuming newlines as separators.
14+
*/
15+
private string lineProducer(int lineIndex) {
16+
result = this.getRawScript().regexpReplaceAll("\\\\\\s*\n", "").splitAt("\n", lineIndex)
17+
}
18+
19+
private predicate cmdSubstitutionReplacement(string command, string id, int lineIndex) {
20+
this.commandInSubstitution(lineIndex, command, id)
21+
or
22+
this.commandInBackticks(lineIndex, command, id)
23+
}
24+
25+
/**
26+
* Holds if there is a command substitution `$(command)` in
27+
* the line at `lineIndex` in the shell script,
28+
* and `id` is a unique identifier for this command.
29+
*/
30+
private predicate commandInSubstitution(int lineIndex, string command, string id) {
31+
exists(int occurrenceIndex, int occurrenceOffset |
32+
command =
33+
// Look for the command inside a $(...) command substitution
34+
this.lineProducer(lineIndex)
35+
.regexpFind("\\$\\((?:[^()]+|\\((?:[^()]+|\\([^()]*\\))*\\))*\\)", occurrenceIndex,
36+
occurrenceOffset)
37+
// trim starting $( - TODO do this in first regex
38+
.regexpReplaceAll("^\\$\\(", "")
39+
// trim ending ) - TODO do this in first regex
40+
.regexpReplaceAll("\\)$", "") and
41+
id = "cmdsubs:" + lineIndex + ":" + occurrenceIndex + ":" + occurrenceOffset
3442
)
3543
}
3644

37-
private predicate rankedCmdSubstitutionReplacements(int i, string old, string new) {
38-
old = rank[i](string old2 | this.cmdSubstitutionReplacement(old2, _, _) | old2) and
39-
this.cmdSubstitutionReplacement(old, new, _)
45+
/**
46+
* Holds if `command` is a command in backticks `` `...` `` in
47+
* the line at `lineIndex` in the shell script,
48+
* and `id` is a unique identifier for this command.
49+
*/
50+
private predicate commandInBackticks(int lineIndex, string command, string id) {
51+
exists(int occurrenceIndex, int occurrenceOffset |
52+
command =
53+
this.lineProducer(lineIndex)
54+
.regexpFind("\\`[^\\`]+\\`", occurrenceIndex, occurrenceOffset)
55+
// trim leading backtick - TODO do this in first regex
56+
.regexpReplaceAll("^\\`", "")
57+
// trim trailing backtick - TODO do this in first regex
58+
.regexpReplaceAll("\\`$", "") and
59+
id = "cmd:" + lineIndex + ":" + occurrenceIndex + ":" + occurrenceOffset
60+
)
61+
}
62+
63+
private predicate rankedCmdSubstitutionReplacements(int i, string command, string commandId) {
64+
// rank commands by their unique IDs
65+
commandId = rank[i](string c, string id | this.cmdSubstitutionReplacement(c, id, _) | id) and
66+
// since we cannot output (command, ID) tuples from the rank operation,
67+
// we need to work out the specific command associated with the resulting ID
68+
this.cmdSubstitutionReplacement(command, commandId, _)
4069
}
4170

4271
private predicate doReplaceCmdSubstitutions(int line, int round, string old, string new) {
@@ -64,31 +93,56 @@ class BashShellScript extends ShellScript {
6493
this.cmdSubstitutionReplacement(result, _, i)
6594
}
6695

96+
/**
97+
* Holds if `quotedStr` is a string in double quotes in
98+
* the line at `lineIndex` in the shell script,
99+
* and `id` is a unique identifier for this quoted string.
100+
*/
101+
private predicate doubleQuotedString(int lineIndex, string quotedStr, string id) {
102+
exists(int occurrenceIndex, int occurrenceOffset |
103+
// double quoted string
104+
quotedStr =
105+
this.cmdSubstitutedLineProducer(lineIndex)
106+
.regexpFind("\"((?:[^\"\\\\]|\\\\.)*)\"", occurrenceIndex, occurrenceOffset) and
107+
id =
108+
"qstr:" + lineIndex + ":" + occurrenceIndex + ":" + occurrenceOffset + ":" +
109+
quotedStr.length() + ":" + quotedStr.regexpReplaceAll("[^a-zA-Z0-9]", "")
110+
)
111+
}
112+
113+
/**
114+
* Holds if `quotedStr` is a string in single quotes in
115+
* the line at `lineIndex` in the shell script,
116+
* and `id` is a unique identifier for this quoted string.
117+
*/
118+
private predicate singleQuotedString(int lineIndex, string quotedStr, string id) {
119+
exists(int occurrenceIndex, int occurrenceOffset |
120+
// single quoted string
121+
quotedStr =
122+
this.cmdSubstitutedLineProducer(lineIndex)
123+
.regexpFind("'((?:\\\\.|[^'\\\\])*)'", occurrenceIndex, occurrenceOffset) and
124+
id =
125+
"qstr:" + lineIndex + ":" + occurrenceIndex + ":" + occurrenceOffset + ":" +
126+
quotedStr.length() + ":" + quotedStr.regexpReplaceAll("[^a-zA-Z0-9]", "")
127+
)
128+
}
129+
67130
private predicate quotedStringReplacement(string quotedStr, string id) {
68-
exists(string line, int k | line = this.cmdSubstitutedLineProducer(k) |
69-
exists(int i, int j |
70-
// double quoted string
71-
quotedStr = line.regexpFind("\"((?:[^\"\\\\]|\\\\.)*)\"", i, j) and
72-
id =
73-
"qstr:" + k + ":" + i + ":" + j + ":" + quotedStr.length() + ":" +
74-
quotedStr.regexpReplaceAll("[^a-zA-Z0-9]", "")
75-
)
131+
exists(int lineIndex |
132+
this.doubleQuotedString(lineIndex, quotedStr, id)
76133
or
77-
exists(int i, int j |
78-
// single quoted string
79-
quotedStr = line.regexpFind("'((?:\\\\.|[^'\\\\])*)'", i, j) and
80-
id =
81-
"qstr:" + k + ":" + i + ":" + j + ":" + quotedStr.length() + ":" +
82-
quotedStr.regexpReplaceAll("[^a-zA-Z0-9]", "")
83-
)
134+
this.singleQuotedString(lineIndex, quotedStr, id)
84135
) and
85136
// Only do this for strings that might otherwise disrupt subsequent parsing
86137
quotedStr.regexpMatch("[\"'].*[$\n\r'\"" + Bash::separator() + "].*[\"']")
87138
}
88139

89-
private predicate rankedQuotedStringReplacements(int i, string old, string new) {
90-
old = rank[i](string old2 | this.quotedStringReplacement(old2, _) | old2) and
91-
this.quotedStringReplacement(old, new)
140+
private predicate rankedQuotedStringReplacements(int i, string quotedString, string quotedStringId) {
141+
// rank quoted strings by their nearly-unique IDs
142+
quotedStringId = rank[i](string s, string id | this.quotedStringReplacement(s, id) | id) and
143+
// since we cannot output (string, ID) tuples from the rank operation,
144+
// we need to work out the specific string associated with the resulting ID
145+
this.quotedStringReplacement(quotedString, quotedStringId)
92146
}
93147

94148
private predicate doReplaceQuotedStrings(int line, int round, string old, string new) {

actions/ql/lib/qlpack.yml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
name: codeql/actions-all
2-
version: 0.4.10-dev
2+
version: 0.4.12-dev
33
library: true
44
warnOnImplicitThis: true
55
dependencies:

0 commit comments

Comments
 (0)