Skip to content

Commit 1694b44

Browse files
cmortycmorty
authored
Bug 575057 Mirror files from docker images (#9)
This is a major redesign. It reduces the code duplication in CopyFromContainerCommandHandler and ContainerCommandProcess by introducing the CopyFromDockerJob. This CopyFromDockerJob can also mirror a folder, creating necessary symlinks and and following symlinks within that folder. This can be used by CDT to copy include paths, which often contain symlinks to library installation paths outside the actually included folder. The code was also extended in a way that should ensure that there are no stale containers left behind, even if something goes wrong. Even eclipse crashes the container should self-remove itself after an hour. Concurrency might be handled in a over-pessimistic way, which might result in more jobs being blocked then necessary, but should avoid issues caused by corner cases. Tests are added.
1 parent bb80bf8 commit 1694b44

File tree

14 files changed

+1959
-1120
lines changed

14 files changed

+1959
-1120
lines changed

Diff for: ‎containers/org.eclipse.linuxtools.docker.core/src/org/eclipse/linuxtools/internal/docker/core/CloseableContainer.java

+83
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,83 @@
1+
/********************************************************************************
2+
* Copyright (c) 2021 Eclipse Linux Tools project committers and others
3+
*
4+
* This program and the accompanying materials are made available under the
5+
* terms of the Eclipse Public License 2.0 which is available at
6+
* http://www.eclipse.org/legal/epl-2.0.
7+
*
8+
* SPDX-License-Identifier: EPL-2.0
9+
*
10+
********************************************************************************/
11+
12+
package org.eclipse.linuxtools.internal.docker.core;
13+
14+
import org.eclipse.linuxtools.docker.core.DockerException;
15+
import org.eclipse.linuxtools.docker.core.IDockerConnection;
16+
import org.eclipse.linuxtools.docker.core.IDockerContainerConfig;
17+
import org.eclipse.linuxtools.docker.core.IDockerHostConfig;
18+
19+
/**
20+
* This creates a container that implements AutoColosable. Thus allowing to use
21+
* it in a try statement. Special care is taken that the container is removed
22+
* eventually (currently 3600s), even if eclipse crashes.
23+
*/
24+
public class CloseableContainer implements AutoCloseable {
25+
26+
public final String containerId;
27+
private final IDockerConnection connection;
28+
29+
/**
30+
* Create a closable container, that is removed after leaving the try-block.
31+
*
32+
* @param connection
33+
* The docker-connection to use
34+
* @param image
35+
* The image name to use
36+
* @throws DockerException
37+
* if an docker related error occurs
38+
* @throws InterruptedException
39+
* if the thread was interrupted
40+
*/
41+
public CloseableContainer(IDockerConnection connection, String image)
42+
throws DockerException, InterruptedException {
43+
// Create base container to use for copying
44+
// It must be running, to execute ls & co
45+
46+
this.connection = connection;
47+
DockerContainerConfig.Builder builder = new DockerContainerConfig.Builder().cmd("sleep 3600") //$NON-NLS-1$
48+
.image(image);
49+
IDockerContainerConfig config = builder.build();
50+
DockerHostConfig.Builder hostBuilder = new DockerHostConfig.Builder();
51+
// Remove the container after usage.
52+
hostBuilder.autoRemove(true);
53+
IDockerHostConfig hostConfig = hostBuilder.build();
54+
containerId = connection.createContainer(config, hostConfig, null);
55+
56+
}
57+
58+
/**
59+
* Start the Container
60+
*
61+
* @throws DockerException
62+
* if an docker related error occurs
63+
* @throws InterruptedException
64+
* if the thread was interrupted
65+
*/
66+
public void start() throws DockerException, InterruptedException {
67+
connection.startContainer(containerId, null);
68+
}
69+
70+
/**
71+
* Close the container, by killing it. Remove it afterwards, just in case.
72+
* setting it to autoremove in the constructor should do the job, though.
73+
*/
74+
@Override
75+
public void close() throws DockerException, InterruptedException {
76+
try {
77+
connection.killContainer(containerId);
78+
} catch (Exception e) {
79+
}
80+
connection.removeContainer(containerId);
81+
}
82+
83+
}

Diff for: ‎containers/org.eclipse.linuxtools.docker.core/src/org/eclipse/linuxtools/internal/docker/core/DockerConnection.java

+13-25
Original file line numberDiff line numberDiff line change
@@ -2196,44 +2196,26 @@ public void attachContainerOutput(final Closeable token, final String id,
21962196
}
21972197
}
21982198

2199-
@SuppressWarnings("unused")
22002199
public List<ContainerFileProxy> readContainerDirectory(final String id,
22012200
final String path) throws DockerException {
22022201
List<ContainerFileProxy> childList = new ArrayList<>();
2202+
String dirListing = null;
22032203
try {
22042204
DockerClient copyClient = getClientCopy();
22052205
final ExecCreation execCreation = copyClient.execCreate(id,
2206-
new String[] { "/bin/sh", "-c", "ls -l -F -Q " + path }, //$NON-NLS-1$ //$NON-NLS-2$ //$NON-NLS-3$
2206+
new String[] { "/bin/sh", "-c", //$NON-NLS-1$ //$NON-NLS-2$
2207+
"ls -l -F -Q " + path + "/" }, //$NON-NLS-1$
22072208
ExecCreateParam.attachStdout(),
22082209
ExecCreateParam.attachStderr());
22092210
final String execId = execCreation.id();
22102211
final LogStream pty_stream = copyClient.execStart(execId);
2211-
String lastLine = ""; //$NON-NLS-1$
2212+
22122213
try {
2213-
while (pty_stream.hasNext()) {
2214-
ByteBuffer b = pty_stream.next().content();
2215-
byte[] buffer = new byte[b.remaining()];
2216-
b.get(buffer);
2217-
String s = lastLine + new String(buffer);
2218-
String[] lines = s.split("\\r?\\n"); //$NON-NLS-1$
2219-
if (lines.length > 0) {
2220-
lastLine = lines[lines.length - 1];
2221-
} else {
2222-
lastLine = ""; //$NON-NLS-1$
2223-
}
2224-
for (int i = 0; i < lines.length - 1; ++i) {
2225-
String line = lines[i];
2226-
processDirectoryLine(line, path, childList);
2227-
}
2228-
}
2229-
if (!lastLine.isEmpty()) {
2230-
processDirectoryLine(lastLine, path, childList);
2231-
}
2214+
dirListing = pty_stream.readFully();
22322215
} finally {
22332216
if (pty_stream != null)
22342217
pty_stream.close();
2235-
if (copyClient != null)
2236-
copyClient.close();
2218+
copyClient.close();
22372219
}
22382220
} catch (Exception e) {
22392221
if (e.getCause() instanceof IOException) {
@@ -2242,12 +2224,18 @@ public List<ContainerFileProxy> readContainerDirectory(final String id,
22422224
return readContainerDirectory(id, path);
22432225
}
22442226
// e.printStackTrace();
2227+
// TODO: Should probably rethrow
2228+
}
2229+
2230+
final String[] lines = dirListing.split("\\r?\\n");
2231+
for (String line : lines) {
2232+
processDirectoryLine(line, path, childList);
22452233
}
22462234
return childList;
22472235
}
22482236

22492237
private String[] tokenize(String line) {
2250-
String regex = "(\"[^\"]*\"/?)|(\\S+)";
2238+
String regex = "(\"[^\"]*\"\\S?)|(\\S+)";
22512239
List<String> result = new ArrayList<>();
22522240

22532241
Matcher m = Pattern.compile(regex).matcher(line);

Diff for: ‎containers/org.eclipse.linuxtools.docker.ui.tests/pom.xml

+3
Original file line numberDiff line numberDiff line change
@@ -62,6 +62,9 @@
6262
<excludes>
6363
<exclude>**/*SWTBotTest.class</exclude>
6464
</excludes>
65+
<excludedGroups>
66+
org.eclipse.linuxtools.docker.testCategory.NativeLinuxDocker
67+
</excludedGroups>
6568
</configuration>
6669
</plugin>
6770
</plugins>

Diff for: ‎containers/org.eclipse.linuxtools.docker.ui.tests/projects/CopyDeleteTest/Dockerfile

+19
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,19 @@
1+
# Copyright (c) 2022 Mathema.
2+
#
3+
# This program and the accompanying materials are made
4+
# available under the terms of the Eclipse Public License 2.0
5+
# which is available at https://www.eclipse.org/legal/epl-2.0/
6+
#
7+
# SPDX-License-Identifier: EPL-2.0
8+
9+
10+
11+
FROM bash:latest
12+
13+
# Default is still the alpine ash
14+
SHELL ["/usr/local/bin/bash", "-c"]
15+
16+
RUN mkdir -p \
17+
/test/deletetest \
18+
&& echo "Hello 1" > test/deletetest/hello1.txt \
19+
&& echo "Hello 2" > test/deletetest/hello2.txt

Diff for: ‎containers/org.eclipse.linuxtools.docker.ui.tests/projects/CopyTestEnv/Dockerfile

+51
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,51 @@
1+
FROM bash:latest
2+
3+
# Default is still the alpine ash
4+
SHELL ["/usr/local/bin/bash", "-c"]
5+
6+
RUN mkdir -p \
7+
/test/a/b/c \
8+
/test/a/b/d \
9+
&& echo "Hello 1" > test/a/b/c/h1.txt \
10+
&& echo "Hello 2" > test/a/b/d/h2.txt
11+
12+
RUN mkdir -p \
13+
/test/sl/ \
14+
&& ln -s /test/a/b/c /test/sl/link-c-abs \
15+
&& ln -s ../a/b/d/ /test/sl/link-d-rel
16+
17+
RUN mkdir -p \
18+
/test/sl-rec/ \
19+
/test/sl-rec/subdir \
20+
&& ln -s .. /test/sl-rec/subdir/link-up
21+
22+
# Test should terminate earlier, but more is better
23+
RUN mkdir -p /test/sl-depth/l30 \
24+
&& echo "Test" > /test/sl-depth/l30/F30 \
25+
&& for I in {0..29} ; do \
26+
mkdir -p /test/sl-depth/l$I ; \
27+
echo "Test" > /test/sl-depth/l$I/F$I; \
28+
ln -s "../l$(($I+1))" /test/sl-depth/l$I/L$I; \
29+
done;
30+
31+
32+
RUN mkdir -p \
33+
/test/manyfiles \
34+
&& for I in {1..10000} ; do \
35+
echo "File" > /test/manyfiles/F$I ;\
36+
done
37+
38+
39+
RUN mkdir -p /test/sl-back/l0 /test/sl-back/l30 \
40+
&& echo "Test" > /test/sl-back/l30/F30 \
41+
&& echo "Test" > /test/sl-back/l0/F0 \
42+
&& ln -s "../l1" /test/sl-back/l0/L0 \
43+
&& ln -s "../l29" /test/sl-back/l30/LB30 \
44+
&& for I in {1..29} ; do \
45+
mkdir -p /test/sl-back/l$I ; \
46+
echo "Test" > /test/sl-back/l$I/F$I; \
47+
ln -s "../l$(($I+1))" /test/sl-back/l$I/L$I; \
48+
ln -s "../l$(($I-1))" /test/sl-back/l$I/LB$I; \
49+
done
50+
51+

0 commit comments

Comments
 (0)