Bug 575057 Mirror files from docker images (#9)

This is a major redesign. It reduces the code duplication in
CopyFromContainerCommandHandler and ContainerCommandProcess by
introducing the CopyFromDockerJob. This CopyFromDockerJob can also
mirror a folder, creating necessary symlinks and and following symlinks
within that folder. This can be used by CDT to copy include paths, which
often contain symlinks to library installation paths outside the
actually included folder.
The code was also extended in a way that should ensure that there are no
stale containers left behind, even if something goes wrong. Even eclipse
crashes the container should self-remove itself after an hour.
Concurrency might be handled in a over-pessimistic way, which might
result in more jobs being blocked then necessary, but should avoid
issues caused by corner cases.  Tests are added.

Author: cmorty

containers/org.eclipse.linuxtools.docker.core/src/org/eclipse/linuxtools/internal/docker/core/CloseableContainer.java

@@ -0,0 +1,83 @@
+/********************************************************************************
+ * Copyright (c) 2021 Eclipse Linux Tools project committers and others
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Eclipse Public License 2.0 which is available at
+ * http://www.eclipse.org/legal/epl-2.0.
+ *
+ * SPDX-License-Identifier: EPL-2.0
+ *
+ ********************************************************************************/
+
+package org.eclipse.linuxtools.internal.docker.core;
+
+import org.eclipse.linuxtools.docker.core.DockerException;
+import org.eclipse.linuxtools.docker.core.IDockerConnection;
+import org.eclipse.linuxtools.docker.core.IDockerContainerConfig;
+import org.eclipse.linuxtools.docker.core.IDockerHostConfig;
+
+/**
+ * This creates a container that implements AutoColosable. Thus allowing to use
+ * it in a try statement. Special care is taken that the container is removed
+ * eventually (currently 3600s), even if eclipse crashes.
+ */
+public class CloseableContainer implements AutoCloseable {
+
+	public final String containerId;
+	private final IDockerConnection connection;
+
+	/**
+	 * Create a closable container, that is removed after leaving the try-block.
+	 *
+	 * @param connection
+	 *            The docker-connection to use
+	 * @param image
+	 *            The image name to use
+	 * @throws DockerException
+	 *             if an docker related error occurs
+	 * @throws InterruptedException
+	 *             if the thread was interrupted
+	 */
+	public CloseableContainer(IDockerConnection connection, String image)
+			throws DockerException, InterruptedException {
+		// Create base container to use for copying
+		// It must be running, to execute ls & co
+
+		this.connection = connection;
+		DockerContainerConfig.Builder builder = new DockerContainerConfig.Builder().cmd("sleep 3600") //$NON-NLS-1$
+				.image(image);
+		IDockerContainerConfig config = builder.build();
+		DockerHostConfig.Builder hostBuilder = new DockerHostConfig.Builder();
+		// Remove the container after usage.
+		hostBuilder.autoRemove(true);
+		IDockerHostConfig hostConfig = hostBuilder.build();
+		containerId = connection.createContainer(config, hostConfig, null);
+
+	}
+
+	/**
+	 * Start the Container
+	 *
+	 * @throws DockerException
+	 *             if an docker related error occurs
+	 * @throws InterruptedException
+	 *             if the thread was interrupted
+	 */
+	public void start() throws DockerException, InterruptedException {
+		connection.startContainer(containerId, null);
+	}
+
+	/**
+	 * Close the container, by killing it. Remove it afterwards, just in case.
+	 * setting it to autoremove in the constructor should do the job, though.
+	 */
+	@Override
+	public void close() throws DockerException, InterruptedException {
+		try {
+			connection.killContainer(containerId);
+		} catch (Exception e) {
+		}
+		connection.removeContainer(containerId);
+	}
+
+}

containers/org.eclipse.linuxtools.docker.core/src/org/eclipse/linuxtools/internal/docker/core/DockerConnection.java

@@ -2196,44 +2196,26 @@ public void attachContainerOutput(final Closeable token, final String id,
 		}
 	}
 
-	@SuppressWarnings("unused")
 	public List<ContainerFileProxy> readContainerDirectory(final String id,
 			final String path) throws DockerException {
 		List<ContainerFileProxy> childList = new ArrayList<>();
+		String dirListing = null;
 		try {
 			DockerClient copyClient = getClientCopy();
 			final ExecCreation execCreation = copyClient.execCreate(id,
-					new String[] { "/bin/sh", "-c", "ls -l -F -Q " + path }, //$NON-NLS-1$ //$NON-NLS-2$ //$NON-NLS-3$
+					new String[] { "/bin/sh", "-c", //$NON-NLS-1$ //$NON-NLS-2$
+							"ls -l -F -Q " + path + "/" }, //$NON-NLS-1$
 					ExecCreateParam.attachStdout(),
 					ExecCreateParam.attachStderr());
 			final String execId = execCreation.id();
 			final LogStream pty_stream = copyClient.execStart(execId);
-			String lastLine = ""; //$NON-NLS-1$
+
 			try {
-				while (pty_stream.hasNext()) {
-					ByteBuffer b = pty_stream.next().content();
-					byte[] buffer = new byte[b.remaining()];
-					b.get(buffer);
-					String s = lastLine + new String(buffer);
-					String[] lines = s.split("\\r?\\n"); //$NON-NLS-1$
-					if (lines.length > 0) {
-						lastLine = lines[lines.length - 1];
-					} else {
-						lastLine = ""; //$NON-NLS-1$
-					}
-					for (int i = 0; i < lines.length - 1; ++i) {
-						String line = lines[i];
-						processDirectoryLine(line, path, childList);
-					}
-				}
-				if (!lastLine.isEmpty()) {
-					processDirectoryLine(lastLine, path, childList);
-				}
+				dirListing = pty_stream.readFully();
 			} finally {
 				if (pty_stream != null)
 					pty_stream.close();
-				if (copyClient != null)
-					copyClient.close();
+				copyClient.close();
 			}
 		} catch (Exception e) {
 			if (e.getCause() instanceof IOException) {
@@ -2242,12 +2224,18 @@ public List<ContainerFileProxy> readContainerDirectory(final String id,
 				return readContainerDirectory(id, path);
 			}
 			// e.printStackTrace();
+			// TODO: Should probably rethrow
+		}
+
+		final String[] lines = dirListing.split("\\r?\\n");
+		for (String line : lines) {
+			processDirectoryLine(line, path, childList);
 		}
 		return childList;
 	}
 
 	private String[] tokenize(String line) {
-		String regex = "(\"[^\"]*\"/?)|(\\S+)";
+		String regex = "(\"[^\"]*\"\\S?)|(\\S+)";
 		List<String> result = new ArrayList<>();
 
 		Matcher m = Pattern.compile(regex).matcher(line);

containers/org.eclipse.linuxtools.docker.ui.tests/pom.xml

@@ -62,6 +62,9 @@
 							<excludes>
 								<exclude>**/*SWTBotTest.class</exclude>
 							</excludes>
+							<excludedGroups>
+								org.eclipse.linuxtools.docker.testCategory.NativeLinuxDocker
+							</excludedGroups>
 						</configuration>
 					</plugin>
 				</plugins>

containers/org.eclipse.linuxtools.docker.ui.tests/projects/CopyDeleteTest/Dockerfile

@@ -0,0 +1,19 @@
+ # Copyright (c) 2022 Mathema.
+ #
+ # This program and the accompanying materials are made
+ # available under the terms of the Eclipse Public License 2.0
+ # which is available at https://www.eclipse.org/legal/epl-2.0/
+ #
+ # SPDX-License-Identifier: EPL-2.0
+
+
+
+FROM bash:latest
+
+# Default is still the alpine ash
+SHELL ["/usr/local/bin/bash", "-c"]
+
+RUN mkdir -p \ 
+		/test/deletetest \
+	&& echo "Hello 1" > test/deletetest/hello1.txt \
+	&& echo "Hello 2" > test/deletetest/hello2.txt

containers/org.eclipse.linuxtools.docker.ui.tests/projects/CopyTestEnv/Dockerfile

@@ -0,0 +1,51 @@
+FROM bash:latest
+
+# Default is still the alpine ash
+SHELL ["/usr/local/bin/bash", "-c"]
+
+RUN mkdir -p \ 
+		/test/a/b/c \
+		/test/a/b/d \
+	&& echo "Hello 1" > test/a/b/c/h1.txt \
+	&& echo "Hello 2" > test/a/b/d/h2.txt
+	
+RUN mkdir -p \
+		/test/sl/ \
+	&& ln -s  /test/a/b/c /test/sl/link-c-abs \
+	&& ln -s ../a/b/d/ /test/sl/link-d-rel
+	
+RUN mkdir -p \
+		/test/sl-rec/ \
+		/test/sl-rec/subdir \
+	&& ln -s .. /test/sl-rec/subdir/link-up
+
+# Test should terminate earlier, but more is better
+RUN mkdir -p /test/sl-depth/l30 \
+	&& echo "Test" > /test/sl-depth/l30/F30 \
+	&& for I in {0..29} ; do \
+			mkdir -p /test/sl-depth/l$I ; \
+			echo "Test" > /test/sl-depth/l$I/F$I; \
+			ln -s "../l$(($I+1))"  /test/sl-depth/l$I/L$I; \
+	   done;
+		
+	
+RUN mkdir -p \
+		/test/manyfiles \
+	&&  for I in {1..10000} ; do \
+			echo "File" > /test/manyfiles/F$I ;\
+	    done
+	    
+	    
+RUN mkdir -p /test/sl-back/l0 /test/sl-back/l30 \
+	&& echo "Test" > /test/sl-back/l30/F30 \
+	&& echo "Test" > /test/sl-back/l0/F0 \
+	&& ln -s "../l1"  /test/sl-back/l0/L0 \
+	&& ln -s "../l29"  /test/sl-back/l30/LB30 \
+	&& for I in {1..29} ; do \
+			mkdir -p /test/sl-back/l$I ; \
+			echo "Test" > /test/sl-back/l$I/F$I; \
+			ln -s "../l$(($I+1))"  /test/sl-back/l$I/L$I; \
+			ln -s "../l$(($I-1))"  /test/sl-back/l$I/LB$I; \
+	   done
+	
+