Skip to content

Commit 74db0ab

Browse files
kdudkakdudka
committed
cstrans-df-run: propagate args of the RUN directive
... rather than passing them to a shell interpreter after the transformation: ``` [8/9] STEP 7/13: RUN ["env", "COV_HOST=cspodman", "COVERITY_POSIX_SPAWN_FALLBACK=1", "/opt/cov-sa-2024.3/bin/cov-build", "--dir=/cov", "--append-log", "sh", "-c", "--mount=type=bind,from=build,src=/workspace/dist,target=/workspace/dist --mount=type=cache,target=/root/.cache/pip pip install $(echo dist/*.whl)'[tensorizer]' --verbose"] Coverity Build Capture (64-bit) version 2024.3.0 on Linux 6.9.5-100.fc39.x86_64 x86_64 Internal version numbers: 3b0d073f2b p-2024.3-push-30 sh: --: invalid option ``` Reported-by: Steve Grubb Closes: csutils#190
1 parent 34bc96f commit 74db0ab

File tree

5 files changed

+106
-16
lines changed

5 files changed

+106
-16
lines changed

‎src/cstrans-df-run.cc

+22-16
Original file line numberDiff line numberDiff line change
@@ -55,12 +55,15 @@ class DockerFileTransformer {
5555
/// match ... in RUN ...
5656
const RE reLineRun_ = RE("^RUN (.*)$");
5757

58-
/// match ... in RUN [...]
59-
const RE reLineRunExec_ = RE("^RUN *\\[(.*)\\] *$");
60-
6158
/// match ... in ... BS-NL
6259
const RE reLineCont_ = RE("(^.*[^\\\\])\\\\ *$");
6360

61+
// split RUN directive with options from the actual command
62+
const RE reLineRunOpts_ = RE("^(RUN +(?:--[A-Za-z0-9_]+=[^ ]+ +)*)(.*)$");
63+
64+
/// match ... in RUN [...]
65+
const RE reLineRunExec_ = RE("^\\[(.*)\\] *$");
66+
6467
/// match in-line comments
6568
const RE reComment_ = RE("^\\s*#.*$");
6669
};
@@ -148,10 +151,10 @@ std::string runQuoteArg(std::string arg)
148151
return arg;
149152
}
150153

151-
std::string runLineFromExecList(const TStringList &execList)
154+
std::string runCmdFromExecList(const TStringList &execList)
152155
{
153156
// construct RUN ["cmd", "arg1", "arg2", ...] from execList
154-
std::string runLine = "RUN [";
157+
std::string runLine = "[";
155158
int i = 0;
156159
for (const std::string &arg : execList) {
157160
if (i++)
@@ -165,23 +168,26 @@ std::string runLineFromExecList(const TStringList &execList)
165168

166169
void DockerFileTransformer::transformRunLine(std::string *pRunLine)
167170
{
171+
// split RUN directive with options from the actual command
172+
boost::smatch sm;
173+
if (!boost::regex_match(*pRunLine, sm, reLineRunOpts_))
174+
// should never happen
175+
throw std::runtime_error("internal error");
176+
177+
std::string newRunLine = sm[1];
178+
const std::string cmd = sm[2];
179+
168180
// start with the prefix specified on cmd-line
169181
TStringList execList = prefixCmd_;
170182

171-
boost::smatch sm;
172-
if (boost::regex_match(*pRunLine, sm, reLineRunExec_))
173-
// RUN ["cmd", "arg1", "arg2", ...]
183+
if (boost::regex_match(cmd, sm, reLineRunExec_))
184+
// ["cmd", "arg1", "arg2", ...]
174185
appendExecArgs(&execList, sm[1]);
175-
176-
else if (boost::regex_match(*pRunLine, sm, reLineRun_))
177-
// RUN arbitrary shell code...
178-
appendShellExec(&execList, sm[1]);
179-
180186
else
181-
// should never happen
182-
throw std::runtime_error("internal error");
187+
// arbitrary shell code...
188+
appendShellExec(&execList, cmd);
183189

184-
const std::string newRunLine = runLineFromExecList(execList);
190+
newRunLine += runCmdFromExecList(execList);
185191
if (verbose_) {
186192
// diagnostic output printed with --verbose
187193
std::cerr << prog_name << " <<< " << *pRunLine << std::endl;

‎tests/cstrans-df-run/0011-stdin.txt

+38
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,38 @@
1+
## Python cuda base #################################################################
2+
FROM cuda-base AS python-cuda-base
3+
4+
ENV VIRTUAL_ENV=/opt/vllm
5+
ENV PATH="$VIRTUAL_ENV/bin:$PATH"
6+
7+
# install cuda and common dependencies
8+
RUN --mount=type=cache,target=/root/.cache/pip \
9+
--mount=type=bind,source=requirements-common.txt,target=requirements-common.txt \
10+
--mount=type=bind,source=requirements-cuda.txt,target=requirements-cuda.txt \
11+
pip install \
12+
-r requirements-cuda.txt
13+
14+
## Development #################################################################
15+
FROM python-cuda-base AS dev
16+
17+
# install build and runtime dependencies
18+
RUN --mount=type=cache,target=/root/.cache/pip \
19+
--mount=type=bind,source=requirements-common.txt,target=requirements-common.txt \
20+
--mount=type=bind,source=requirements-cuda.txt,target=requirements-cuda.txt \
21+
--mount=type=bind,source=requirements-dev.txt,target=requirements-dev.txt \
22+
--mount=type=bind,source=requirements-lint.txt,target=requirements-lint.txt \
23+
--mount=type=bind,source=requirements-test.txt,target=requirements-test.txt \
24+
pip3 install \
25+
-r requirements-cuda.txt \
26+
-r requirements-dev.txt
27+
28+
## Builder #####################################################################
29+
FROM dev AS build
30+
31+
# install build dependencies
32+
RUN --mount=type=cache,target=/root/.cache/pip \
33+
--mount=type=bind,source=requirements-build.txt,target=requirements-build.txt \
34+
pip install -r requirements-build.txt
35+
36+
# install compiler cache to speed up compilation leveraging local or remote caching
37+
# git is required for the cutlass kernels
38+
RUN rpm -ivh https://dl.fedoraproject.org/pub/epel/epel-release-latest-9.noarch.rpm && rpm -ql epel-release && microdnf install -y git ccache && microdnf clean all

‎tests/cstrans-df-run/0011-stdout.txt

+24
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,24 @@
1+
## Python cuda base #################################################################
2+
FROM cuda-base AS python-cuda-base
3+
4+
ENV VIRTUAL_ENV=/opt/vllm
5+
ENV PATH="$VIRTUAL_ENV/bin:$PATH"
6+
7+
# install cuda and common dependencies
8+
RUN --mount=type=cache,target=/root/.cache/pip --mount=type=bind,source=requirements-common.txt,target=requirements-common.txt --mount=type=bind,source=requirements-cuda.txt,target=requirements-cuda.txt ["/opt/cov-sa-2019.09/bin/cov-build", "--dir=/cov", "--append-log", "sh", "-c", "pip install -r requirements-cuda.txt"]
9+
10+
## Development #################################################################
11+
FROM python-cuda-base AS dev
12+
13+
# install build and runtime dependencies
14+
RUN --mount=type=cache,target=/root/.cache/pip --mount=type=bind,source=requirements-common.txt,target=requirements-common.txt --mount=type=bind,source=requirements-cuda.txt,target=requirements-cuda.txt --mount=type=bind,source=requirements-dev.txt,target=requirements-dev.txt --mount=type=bind,source=requirements-lint.txt,target=requirements-lint.txt --mount=type=bind,source=requirements-test.txt,target=requirements-test.txt ["/opt/cov-sa-2019.09/bin/cov-build", "--dir=/cov", "--append-log", "sh", "-c", "pip3 install -r requirements-cuda.txt -r requirements-dev.txt"]
15+
16+
## Builder #####################################################################
17+
FROM dev AS build
18+
19+
# install build dependencies
20+
RUN --mount=type=cache,target=/root/.cache/pip --mount=type=bind,source=requirements-build.txt,target=requirements-build.txt ["/opt/cov-sa-2019.09/bin/cov-build", "--dir=/cov", "--append-log", "sh", "-c", "pip install -r requirements-build.txt"]
21+
22+
# install compiler cache to speed up compilation leveraging local or remote caching
23+
# git is required for the cutlass kernels
24+
RUN ["/opt/cov-sa-2019.09/bin/cov-build", "--dir=/cov", "--append-log", "sh", "-c", "rpm -ivh https://dl.fedoraproject.org/pub/epel/epel-release-latest-9.noarch.rpm && rpm -ql epel-release && microdnf install -y git ccache && microdnf clean all"]

‎tests/cstrans-df-run/CMakeLists.txt

+1
Original file line numberDiff line numberDiff line change
@@ -36,3 +36,4 @@ tests_cstrans_df_run(0007)
3636
tests_cstrans_df_run(0008)
3737
tests_cstrans_df_run(0009)
3838
tests_cstrans_df_run(0010)
39+
tests_cstrans_df_run(0011)

‎tests/cstrans-df-run/sync.sh

+21
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,21 @@
1+
#!/bin/zsh
2+
set -exo pipefail
3+
4+
TEST_ARGS=(-- /opt/cov-sa-2019.09/bin/cov-build --dir=/cov --append-log)
5+
6+
# set path to project root
7+
PROJECT_ROOT="../.."
8+
9+
if [[ $# -eq 0 ]]; then
10+
tests=( *-stdin.txt )
11+
else
12+
tests=( "$@" )
13+
fi
14+
15+
for tst in "${tests[@]}"; do
16+
tst=${tst%-stdin.txt}
17+
${PROJECT_ROOT}/csdiff_build/src/cstrans-df-run \
18+
${TEST_ARGS[@]} \
19+
< ${tst}-stdin.txt \
20+
> ${tst}-stdout.txt
21+
done

0 commit comments

Comments
 (0)