From eb3f302840a1679a653cc96818704861b7c216d9 Mon Sep 17 00:00:00 2001 From: munish chouhan Date: Thu, 25 Jul 2024 01:02:05 +0200 Subject: [PATCH 01/15] Initial commit --- .../wave/configuration/BuildConfig.groovy | 7 ++ .../aws/ObjectStorageOperationsFactory.groovy | 22 ++++-- .../wave/service/builder/BuildRequest.groovy | 3 + .../wave/service/builder/BuildStrategy.groovy | 7 +- .../builder/ContainerBuildServiceImpl.groovy | 75 ++++++++---------- .../builder/DockerBuildStrategy.groovy | 77 ++++++++----------- .../io/seqera/wave/util/TarS3Utils.groovy | 57 ++++++++++++++ src/main/resources/application.yml | 3 +- 8 files changed, 154 insertions(+), 97 deletions(-) create mode 100644 src/main/groovy/io/seqera/wave/util/TarS3Utils.groovy diff --git a/src/main/groovy/io/seqera/wave/configuration/BuildConfig.groovy b/src/main/groovy/io/seqera/wave/configuration/BuildConfig.groovy index acc21afb8..4e007a598 100644 --- a/src/main/groovy/io/seqera/wave/configuration/BuildConfig.groovy +++ b/src/main/groovy/io/seqera/wave/configuration/BuildConfig.groovy @@ -56,6 +56,13 @@ class BuildConfig { @Value('${wave.build.public-repo}') String defaultPublicRepository + @Nullable + @Value('${wave.build.logs.bucket}') + String storageBucket + + @Value('${wave.build.workspace-bucket}') + String workspaceBucket + /** * File system path there the dockerfile is save */ diff --git a/src/main/groovy/io/seqera/wave/service/aws/ObjectStorageOperationsFactory.groovy b/src/main/groovy/io/seqera/wave/service/aws/ObjectStorageOperationsFactory.groovy index b0d649d16..d531c98a8 100644 --- a/src/main/groovy/io/seqera/wave/service/aws/ObjectStorageOperationsFactory.groovy +++ b/src/main/groovy/io/seqera/wave/service/aws/ObjectStorageOperationsFactory.groovy @@ -22,11 +22,12 @@ import groovy.transform.CompileStatic import groovy.util.logging.Slf4j import io.micronaut.context.annotation.Factory import io.micronaut.context.annotation.Requires -import io.micronaut.context.annotation.Value import io.micronaut.objectstorage.InputStreamMapper import io.micronaut.objectstorage.ObjectStorageOperations import io.micronaut.objectstorage.aws.AwsS3Configuration import io.micronaut.objectstorage.aws.AwsS3Operations +import io.seqera.wave.configuration.BuildConfig +import jakarta.inject.Inject import jakarta.inject.Named import jakarta.inject.Singleton import software.amazon.awssdk.services.s3.S3Client @@ -38,17 +39,26 @@ import software.amazon.awssdk.services.s3.S3Client @Factory @CompileStatic @Slf4j -@Requires(property = 'wave.build.logs.bucket') class ObjectStorageOperationsFactory { - @Value('${wave.build.logs.bucket}') - String storageBucket + @Inject + private BuildConfig buildConfig @Singleton @Named("build-logs") - ObjectStorageOperations awsStorageOperations(@Named("DefaultS3Client") S3Client s3Client, InputStreamMapper inputStreamMapper) { + @Requires(property = 'wave.build.logs.bucket') + ObjectStorageOperations awsStorageOperationsBuildLogs(@Named("DefaultS3Client") S3Client s3Client, InputStreamMapper inputStreamMapper) { AwsS3Configuration configuration = new AwsS3Configuration('build-logs') - configuration.setBucket(storageBucket) + configuration.setBucket(buildConfig.storageBucket) + return new AwsS3Operations(configuration, s3Client, inputStreamMapper) + } + + @Singleton + @Named("build-workspace") + @Requires(property = 'wave.build.workspace-bucket') + ObjectStorageOperations awsStorageOperationsBuildWorkspace(@Named("DefaultS3Client") S3Client s3Client, InputStreamMapper inputStreamMapper) { + AwsS3Configuration configuration = new AwsS3Configuration('build-workspace') + configuration.setBucket(buildConfig.workspaceBucket) return new AwsS3Operations(configuration, s3Client, inputStreamMapper) } } diff --git a/src/main/groovy/io/seqera/wave/service/builder/BuildRequest.groovy b/src/main/groovy/io/seqera/wave/service/builder/BuildRequest.groovy index 940c1dac6..bff973a88 100644 --- a/src/main/groovy/io/seqera/wave/service/builder/BuildRequest.groovy +++ b/src/main/groovy/io/seqera/wave/service/builder/BuildRequest.groovy @@ -138,6 +138,8 @@ class BuildRequest { volatile Path workDir + String s3Key + BuildRequest(String containerId, String containerFile, String condaFile, String spackFile, Path workspace, String targetImage, PlatformId identity, ContainerPlatform platform, String cacheRepository, String ip, String configJson, String offsetId, ContainerConfig containerConfig, String scanId, BuildContext buildContext, BuildFormat format) { this.containerId = containerId this.containerFile = containerFile @@ -255,6 +257,7 @@ class BuildRequest { BuildRequest withBuildId(String id) { this.buildId = containerId + SEP + id this.workDir = workspace.resolve(buildId).toAbsolutePath() + this.s3Key = "workspace/$buildId" return this } diff --git a/src/main/groovy/io/seqera/wave/service/builder/BuildStrategy.groovy b/src/main/groovy/io/seqera/wave/service/builder/BuildStrategy.groovy index 723d2611a..063524ef7 100644 --- a/src/main/groovy/io/seqera/wave/service/builder/BuildStrategy.groovy +++ b/src/main/groovy/io/seqera/wave/service/builder/BuildStrategy.groovy @@ -32,6 +32,8 @@ import jakarta.inject.Inject @CompileStatic abstract class BuildStrategy { + protected static final String FUSION_PREFIX = "/fusion/s3" + @Inject private BuildConfig buildConfig @@ -57,15 +59,16 @@ abstract class BuildStrategy { protected List dockerLaunchCmd(BuildRequest req) { final result = new ArrayList(10) result + << BUILDKIT_ENTRYPOINT << "build" << "--frontend" << "dockerfile.v0" << "--local" - << "dockerfile=$req.workDir".toString() + << "dockerfile=$FUSION_PREFIX/$buildConfig.workspaceBucket/$req.s3Key".toString() << "--opt" << "filename=Containerfile" << "--local" - << "context=$req.workDir/context".toString() + << "context=$FUSION_PREFIX/$buildConfig.workspaceBucket/$req.s3Key/context".toString() << "--output" << "type=image,name=$req.targetImage,push=true,oci-mediatypes=${buildConfig.ociMediatypes}".toString() << "--opt" diff --git a/src/main/groovy/io/seqera/wave/service/builder/ContainerBuildServiceImpl.groovy b/src/main/groovy/io/seqera/wave/service/builder/ContainerBuildServiceImpl.groovy index 7e57ae642..7d6ced448 100644 --- a/src/main/groovy/io/seqera/wave/service/builder/ContainerBuildServiceImpl.groovy +++ b/src/main/groovy/io/seqera/wave/service/builder/ContainerBuildServiceImpl.groovy @@ -18,10 +18,6 @@ package io.seqera.wave.service.builder -import java.nio.file.FileAlreadyExistsException -import java.nio.file.Files -import java.nio.file.Path -import java.nio.file.StandardCopyOption import java.util.concurrent.CompletableFuture import java.util.concurrent.ExecutorService @@ -29,6 +25,8 @@ import groovy.transform.CompileStatic import groovy.util.logging.Slf4j import io.micronaut.context.event.ApplicationEventPublisher import io.micronaut.core.annotation.Nullable +import io.micronaut.objectstorage.ObjectStorageOperations +import io.micronaut.objectstorage.request.UploadRequest import io.micronaut.scheduling.TaskExecutors import io.seqera.wave.api.BuildContext import io.seqera.wave.auth.RegistryCredentialsProvider @@ -48,17 +46,17 @@ import io.seqera.wave.service.stream.StreamService import io.seqera.wave.tower.PlatformId import io.seqera.wave.util.Retryable import io.seqera.wave.util.SpackHelper -import io.seqera.wave.util.TarUtils +import io.seqera.wave.util.TarS3Utils import io.seqera.wave.util.TemplateRenderer import jakarta.inject.Inject import jakarta.inject.Named import jakarta.inject.Singleton -import static io.seqera.wave.util.RegHelper.layerDir +import org.apache.commons.compress.archivers.tar.TarArchiveEntry +import org.apache.commons.compress.archivers.tar.TarArchiveInputStream +import org.apache.commons.compress.compressors.gzip.GzipCompressorInputStream +import org.apache.commons.io.IOUtils import static io.seqera.wave.util.RegHelper.layerName import static io.seqera.wave.util.StringUtils.indent -import static java.nio.file.StandardOpenOption.CREATE -import static java.nio.file.StandardOpenOption.TRUNCATE_EXISTING -import static java.nio.file.StandardOpenOption.WRITE /** * Implements container build service * @@ -118,6 +116,10 @@ class ContainerBuildServiceImpl implements ContainerBuildService { @Inject BuildRecordStore buildRecordStore + + @Inject + @Named('build-workspace') + private ObjectStorageOperations objectStorageOperations /** * Build a container image for the given {@link BuildRequest} @@ -148,10 +150,10 @@ class ContainerBuildServiceImpl implements ContainerBuildService { .awaitBuild(targetImage) } - protected String containerFile0(BuildRequest req, Path context, SpackConfig config) { + protected String containerFile0(BuildRequest req, String context, SpackConfig config) { // add the context dir for singularity builds final containerFile = req.formatSingularity() - ? req.containerFile.replace('{{wave_context_dir}}', context.toString()) + ? req.containerFile.replace('{{wave_context_dir}}', context) : req.containerFile // render the Spack template if needed @@ -170,35 +172,28 @@ class ContainerBuildServiceImpl implements ContainerBuildService { } protected BuildResult launch(BuildRequest req) { + //create context dir + objectStorageOperations.upload(UploadRequest.fromBytes(new byte[0] , "$req.s3Key/context/".toString())) // launch an external process to build the container BuildResult resp=null try { - // create the workdir path - Files.createDirectories(req.workDir) - // create context dir - final context = req.workDir.resolve('context') - try { Files.createDirectory(context) } - catch (FileAlreadyExistsException e) { /* ignore it */ } // save the dockerfile - final containerFile = req.workDir.resolve('Containerfile') - Files.write(containerFile, containerFile0(req, context, spackConfig).bytes, CREATE, WRITE, TRUNCATE_EXISTING) + objectStorageOperations.upload(UploadRequest.fromBytes(containerFile0(req, "$req.s3Key/Containerfile", spackConfig).bytes, "$req.s3Key/Containerfile".toString())) // save build context if( req.buildContext ) { - saveBuildContext(req.buildContext, context, req.identity) + saveBuildContext(req.buildContext, "$req.s3Key/context", req.identity) } // save the conda file if( req.condaFile ) { - final condaFile = context.resolve('conda.yml') - Files.write(condaFile, req.condaFile.bytes, CREATE, WRITE, TRUNCATE_EXISTING) + objectStorageOperations.upload(UploadRequest.fromBytes(req.condaFile.bytes, "$req.s3Key/conda.yml")) } // save the spack file if( req.spackFile ) { - final spackFile = context.resolve('spack.yaml') - Files.write(spackFile, req.spackFile.bytes, CREATE, WRITE, TRUNCATE_EXISTING) + objectStorageOperations.upload(UploadRequest.fromBytes(req.condaFile.bytes, "$req.s3Key/spack.yaml")) } // save layers provided via the container config if( req.containerConfig ) { - saveLayersToContext(req, context) + saveLayersToContext(req, "$req.s3Key/context") } resp = buildStrategy.build(req) def msg = "== Build request ${req.buildId} completed with status=$resp.exitStatus" @@ -281,59 +276,57 @@ class ContainerBuildServiceImpl implements ContainerBuildService { throw new IllegalStateException("Unable to determine build status for '$request.targetImage'") } - protected void saveLayersToContext(BuildRequest req, Path contextDir) { + protected void saveLayersToContext(BuildRequest req, String s3Key) { if(req.formatDocker()) { - saveLayersToDockerContext0(req, contextDir) + saveLayersToDockerContext0(req, s3Key) } else if(req.formatSingularity()) { - saveLayersToSingularityContext0(req, contextDir) + saveLayersToSingularityContext0(req, s3Key) } else throw new IllegalArgumentException("Unknown container format: $req.format") } - protected void saveLayersToDockerContext0(BuildRequest request, Path contextDir) { + protected void saveLayersToDockerContext0(BuildRequest request, String s3Key) { final layers = request.containerConfig.layers for(int i=0; i { try (InputStream stream = streamService.stream(it.location, request.identity)) { - Files.copy(stream, target, StandardCopyOption.REPLACE_EXISTING) + objectStorageOperations.upload(UploadRequest.fromBytes(IOUtils.toByteArray(stream), target)) } return }) } } - protected void saveLayersToSingularityContext0(BuildRequest request, Path contextDir) { + protected void saveLayersToSingularityContext0(BuildRequest request, String s3Key) { final layers = request.containerConfig.layers for(int i=0; i { try (InputStream stream = streamService.stream(it.location, request.identity)) { - TarUtils.untarGzip(stream, target) + TarS3Utils.untarGzipToS3(stream, s3Key) } return }) } } - protected void saveBuildContext(BuildContext buildContext, Path contextDir, PlatformId identity) { + protected void saveBuildContext(BuildContext buildContext, String s3Key, PlatformId identity) { // retry strategy - final retryable = retry0("Unable to copy '${buildContext.location} to build context '${contextDir}'") + final retryable = retry0("Unable to copy '${buildContext.location} to build context '${s3Key}'") // copy the layer to the build context retryable.apply(()-> { try (InputStream stream = streamService.stream(buildContext.location, identity)) { - TarUtils.untarGzip(stream, contextDir) + TarS3Utils.untarGzipToS3(stream, s3Key) } return }) diff --git a/src/main/groovy/io/seqera/wave/service/builder/DockerBuildStrategy.groovy b/src/main/groovy/io/seqera/wave/service/builder/DockerBuildStrategy.groovy index f0460f95d..6fcb82c92 100644 --- a/src/main/groovy/io/seqera/wave/service/builder/DockerBuildStrategy.groovy +++ b/src/main/groovy/io/seqera/wave/service/builder/DockerBuildStrategy.groovy @@ -18,26 +18,22 @@ package io.seqera.wave.service.builder -import java.nio.file.Files import java.nio.file.Path import java.util.concurrent.TimeUnit -import groovy.json.JsonOutput import groovy.transform.CompileStatic import groovy.util.logging.Slf4j import io.micronaut.context.annotation.Value +import io.micronaut.objectstorage.ObjectStorageOperations +import io.micronaut.objectstorage.request.UploadRequest import io.seqera.wave.configuration.BuildConfig import io.seqera.wave.configuration.SpackConfig import io.seqera.wave.core.ContainerPlatform import io.seqera.wave.core.RegistryProxyService import io.seqera.wave.util.RegHelper import jakarta.inject.Inject +import jakarta.inject.Named import jakarta.inject.Singleton -import static java.nio.file.StandardOpenOption.CREATE -import static java.nio.file.StandardOpenOption.TRUNCATE_EXISTING -import static java.nio.file.StandardOpenOption.WRITE -import static java.nio.file.attribute.PosixFilePermission.OWNER_READ -import static java.nio.file.attribute.PosixFilePermission.OWNER_WRITE /** * Build a container image using a Docker CLI tool * @@ -60,23 +56,23 @@ class DockerBuildStrategy extends BuildStrategy { @Inject RegistryProxyService proxyService + @Inject + @Named('build-workspace') + private ObjectStorageOperations objectStorageOperations + @Override BuildResult build(BuildRequest req) { - Path configFile = null + boolean configFile = false; // save docker config for creds if( req.configJson ) { - configFile = req.workDir.resolve('config.json') - Files.write(configFile, JsonOutput.prettyPrint(req.configJson).bytes, CREATE, WRITE, TRUNCATE_EXISTING) + objectStorageOperations.upload(UploadRequest.fromBytes(req.configJson.bytes, "$req.s3Key/config.json".toString())) + configFile = true } // save remote files for singularity if( req.configJson && req.formatSingularity()) { - final remoteFile = req.workDir.resolve('singularity-remote.yaml') - final content = RegHelper.singularityRemoteFile(req.targetImage) - Files.write(remoteFile, content.bytes, CREATE, WRITE, TRUNCATE_EXISTING) - // set permissions 600 as required by Singularity - Files.setPosixFilePermissions(configFile, Set.of(OWNER_READ, OWNER_WRITE)) - Files.setPosixFilePermissions(remoteFile, Set.of(OWNER_READ, OWNER_WRITE)) + objectStorageOperations.upload(UploadRequest.fromBytes(RegHelper.singularityRemoteFile(req.targetImage).bytes, "$req.s3Key/singularity-remote.yaml".toString())) + configFile = true } // command the docker build command @@ -84,16 +80,14 @@ class DockerBuildStrategy extends BuildStrategy { log.debug "Build run command: ${buildCmd.join(' ')}" // save docker cli for debugging purpose if( debug ) { - Files.write(req.workDir.resolve('docker.sh'), - buildCmd.join(' ').bytes, - CREATE, WRITE, TRUNCATE_EXISTING) + objectStorageOperations.upload(UploadRequest.fromBytes(buildCmd.join(' ').bytes, "$req.s3Key/docker.sh".toString())) } - final proc = new ProcessBuilder() + final builder = new ProcessBuilder() .command(buildCmd) - .directory(req.workDir.toFile()) .redirectErrorStream(true) - .start() + + def proc = builder.start() final completed = proc.waitFor(buildConfig.buildTimeout.toSeconds(), TimeUnit.SECONDS) final stdout = proc.inputStream.text @@ -106,35 +100,29 @@ class DockerBuildStrategy extends BuildStrategy { } } - protected List buildCmd(BuildRequest req, Path credsFile) { + protected List buildCmd(BuildRequest req, boolean credsFile) { final spack = req.isSpackBuild ? spackConfig : null final dockerCmd = req.formatDocker() - ? cmdForBuildkit( req.workDir, credsFile, spack, req.platform) + ? cmdForBuildkit( req, req.platform) : cmdForSingularity( req.workDir, credsFile, spack, req.platform) return dockerCmd + launchCmd(req) } - protected List cmdForBuildkit(Path workDir, Path credsFile, SpackConfig spackConfig, ContainerPlatform platform ) { + protected List cmdForBuildkit(BuildRequest req, ContainerPlatform platform ) { //checkout the documentation here to know more about these options https://github.com/moby/buildkit/blob/master/docs/rootless.md#docker final wrapper = ['docker', 'run', - '--rm', '--privileged', - '-v', "$workDir:$workDir".toString(), - '--entrypoint', - BUILDKIT_ENTRYPOINT] + '-e', + "AWS_ACCESS_KEY_ID=${System.getenv('AWS_ACCESS_KEY_ID')}".toString(), + '-e', + "AWS_SECRET_ACCESS_KEY=${System.getenv('AWS_SECRET_ACCESS_KEY')}".toString()] - if( credsFile ) { - wrapper.add('-v') - wrapper.add("$credsFile:/home/user/.docker/config.json:ro".toString()) - } - - if( spackConfig ) { - // secret file - wrapper.add('-v') - wrapper.add("${spackConfig.secretKeyFile}:${spackConfig.secretMountPath}:ro".toString()) + if( req.configJson ) { + wrapper.add('-e') + wrapper.add("DOCKER_CONFIG=$FUSION_PREFIX/$buildConfig.workspaceBucket/$req.s3Key".toString()) } if( platform ) { @@ -148,7 +136,7 @@ class DockerBuildStrategy extends BuildStrategy { return wrapper } - protected List cmdForSingularity(Path workDir, Path credsFile, SpackConfig spackConfig, ContainerPlatform platform) { + protected List cmdForSingularity(Path workDir, boolean credsFile, SpackConfig spackConfig, ContainerPlatform platform) { final wrapper = ['docker', 'run', '--rm', @@ -157,17 +145,12 @@ class DockerBuildStrategy extends BuildStrategy { '-v', "$workDir:$workDir".toString()] if( credsFile ) { + //todo for singularity remote file wrapper.add('-v') wrapper.add("$credsFile:/root/.singularity/docker-config.json:ro".toString()) - // - wrapper.add('-v') - wrapper.add("${credsFile.resolveSibling('singularity-remote.yaml')}:/root/.singularity/remote.yaml:ro".toString()) - } - if( spackConfig ) { - // secret file - wrapper.add('-v') - wrapper.add("${spackConfig.secretKeyFile}:${spackConfig.secretMountPath}:ro".toString()) + //wrapper.add('-v') + //wrapper.add("${credsFile.resolveSibling('singularity-remote.yaml')}:/root/.singularity/remote.yaml:ro".toString()) } if( platform ) { diff --git a/src/main/groovy/io/seqera/wave/util/TarS3Utils.groovy b/src/main/groovy/io/seqera/wave/util/TarS3Utils.groovy new file mode 100644 index 000000000..1c998ddf7 --- /dev/null +++ b/src/main/groovy/io/seqera/wave/util/TarS3Utils.groovy @@ -0,0 +1,57 @@ +/* + * Wave, containers provisioning service + * Copyright (c) 2024, Seqera Labs + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package io.seqera.wave.util + +import io.micronaut.objectstorage.request.UploadRequest +import org.apache.commons.compress.archivers.tar.TarArchiveEntry +import org.apache.commons.compress.archivers.tar.TarArchiveInputStream +import org.apache.commons.compress.compressors.gzip.GzipCompressorInputStream +/** + * Tar and Gzip utilities for S3 + * + * @author Munish Chouhan + */ +class TarS3Utils { + + static byte[] untarGzipToS3(final InputStream is, String s3Key) throws IOException { + try (GzipCompressorInputStream gzipStream = new GzipCompressorInputStream(is)) { + byte[] tarContent = untarToByteArray(gzipStream) + UploadRequest request = UploadRequest.fromBytes(tarContent, s3Key, "application/x-tar") + return tarContent; + } + } + + private static byte[] untarToByteArray(final InputStream is) throws IOException { + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + try (TarArchiveInputStream tarInputStream = new TarArchiveInputStream(is)) { + TarArchiveEntry entry; + byte[] buffer = new byte[1024]; + int count; + while ((entry = (TarArchiveEntry) tarInputStream.getNextEntry()) != null) { + if (!entry.isDirectory()) { + while ((count = tarInputStream.read(buffer)) != -1) { + baos.write(buffer, 0, count); + } + } + } + } + return baos.toByteArray(); + } + +} diff --git a/src/main/resources/application.yml b/src/main/resources/application.yml index 7c795d5f5..c8df6f244 100644 --- a/src/main/resources/application.yml +++ b/src/main/resources/application.yml @@ -66,7 +66,8 @@ wave: server: url: "${WAVE_SERVER_URL:`http://localhost:9090`}" build: - buildkit-image: "moby/buildkit:v0.14.1-rootless" + workspace-bucket: "${WAVE_WORKSPACE_BUCKET}" + buildkit-image: "cr.seqera.io/public/wave/buildkit:ef67f15426f36b72" singularity-image: "quay.io/singularity/singularity:v3.11.4-slim" singularity-image-arm64: "quay.io/singularity/singularity:v3.11.4-slim-arm64" repo: "195996028523.dkr.ecr.eu-west-1.amazonaws.com/wave/build/dev" From a426aa0a079db76eb3ee2db520216cc94cbbd14d Mon Sep 17 00:00:00 2001 From: munish chouhan Date: Thu, 25 Jul 2024 01:28:21 +0200 Subject: [PATCH 02/15] corrected tarutils --- .../builder/ContainerBuildServiceImpl.groovy | 15 ++++++--------- .../{TarS3Utils.groovy => TarGzipUtils.groovy} | 8 +++----- 2 files changed, 9 insertions(+), 14 deletions(-) rename src/main/groovy/io/seqera/wave/util/{TarS3Utils.groovy => TarGzipUtils.groovy} (86%) diff --git a/src/main/groovy/io/seqera/wave/service/builder/ContainerBuildServiceImpl.groovy b/src/main/groovy/io/seqera/wave/service/builder/ContainerBuildServiceImpl.groovy index 7d6ced448..858fd1f20 100644 --- a/src/main/groovy/io/seqera/wave/service/builder/ContainerBuildServiceImpl.groovy +++ b/src/main/groovy/io/seqera/wave/service/builder/ContainerBuildServiceImpl.groovy @@ -46,14 +46,11 @@ import io.seqera.wave.service.stream.StreamService import io.seqera.wave.tower.PlatformId import io.seqera.wave.util.Retryable import io.seqera.wave.util.SpackHelper -import io.seqera.wave.util.TarS3Utils +import io.seqera.wave.util.TarGzipUtils import io.seqera.wave.util.TemplateRenderer import jakarta.inject.Inject import jakarta.inject.Named import jakarta.inject.Singleton -import org.apache.commons.compress.archivers.tar.TarArchiveEntry -import org.apache.commons.compress.archivers.tar.TarArchiveInputStream -import org.apache.commons.compress.compressors.gzip.GzipCompressorInputStream import org.apache.commons.io.IOUtils import static io.seqera.wave.util.RegHelper.layerName import static io.seqera.wave.util.StringUtils.indent @@ -181,7 +178,7 @@ class ContainerBuildServiceImpl implements ContainerBuildService { objectStorageOperations.upload(UploadRequest.fromBytes(containerFile0(req, "$req.s3Key/Containerfile", spackConfig).bytes, "$req.s3Key/Containerfile".toString())) // save build context if( req.buildContext ) { - saveBuildContext(req.buildContext, "$req.s3Key/context", req.identity) + saveBuildContext(req.buildContext, "$req.s3Key/context/", req.identity) } // save the conda file if( req.condaFile ) { @@ -193,7 +190,7 @@ class ContainerBuildServiceImpl implements ContainerBuildService { } // save layers provided via the container config if( req.containerConfig ) { - saveLayersToContext(req, "$req.s3Key/context") + saveLayersToContext(req, "$req.s3Key/context/") } resp = buildStrategy.build(req) def msg = "== Build request ${req.buildId} completed with status=$resp.exitStatus" @@ -309,11 +306,11 @@ class ContainerBuildServiceImpl implements ContainerBuildService { final it = layers[i] final target = "$s3Key/${layerName(it)}".toString() // retry strategy - final retryable = retry0("Unable to copy '${it.location} to singularity context '$s3Key'") + final retryable = retry0("Unable to copy '${it.location} to singularity context '$target'") // copy the layer to the build context retryable.apply(()-> { try (InputStream stream = streamService.stream(it.location, request.identity)) { - TarS3Utils.untarGzipToS3(stream, s3Key) + objectStorageOperations.upload(UploadRequest.fromBytes(TarGzipUtils.untarGzip(stream), target, "application/x-tar")) } return }) @@ -326,7 +323,7 @@ class ContainerBuildServiceImpl implements ContainerBuildService { // copy the layer to the build context retryable.apply(()-> { try (InputStream stream = streamService.stream(buildContext.location, identity)) { - TarS3Utils.untarGzipToS3(stream, s3Key) + objectStorageOperations.upload(UploadRequest.fromBytes(TarGzipUtils.untarGzip(stream), s3Key, "application/x-tar")) } return }) diff --git a/src/main/groovy/io/seqera/wave/util/TarS3Utils.groovy b/src/main/groovy/io/seqera/wave/util/TarGzipUtils.groovy similarity index 86% rename from src/main/groovy/io/seqera/wave/util/TarS3Utils.groovy rename to src/main/groovy/io/seqera/wave/util/TarGzipUtils.groovy index 1c998ddf7..4ac28fad8 100644 --- a/src/main/groovy/io/seqera/wave/util/TarS3Utils.groovy +++ b/src/main/groovy/io/seqera/wave/util/TarGzipUtils.groovy @@ -18,21 +18,19 @@ package io.seqera.wave.util -import io.micronaut.objectstorage.request.UploadRequest import org.apache.commons.compress.archivers.tar.TarArchiveEntry import org.apache.commons.compress.archivers.tar.TarArchiveInputStream import org.apache.commons.compress.compressors.gzip.GzipCompressorInputStream /** - * Tar and Gzip utilities for S3 + * Tar and Gzip utilities * * @author Munish Chouhan */ -class TarS3Utils { +class TarGzipUtils { - static byte[] untarGzipToS3(final InputStream is, String s3Key) throws IOException { + static byte[] untarGzip(final InputStream is) throws IOException { try (GzipCompressorInputStream gzipStream = new GzipCompressorInputStream(is)) { byte[] tarContent = untarToByteArray(gzipStream) - UploadRequest request = UploadRequest.fromBytes(tarContent, s3Key, "application/x-tar") return tarContent; } } From 1edf22f7dd398f212451d41f1ebb6e9076bc5c18 Mon Sep 17 00:00:00 2001 From: munish chouhan Date: Wed, 7 Aug 2024 23:19:01 +0200 Subject: [PATCH 03/15] Changed singularity docker build --- .../service/builder/BuildConstants.groovy | 31 +++++++++++ .../wave/service/builder/BuildStrategy.groovy | 8 ++- .../builder/ContainerBuildServiceImpl.groovy | 35 ++++++------- .../builder/DockerBuildStrategy.groovy | 51 +++++++------------ .../wave/service/k8s/K8sServiceImpl.groovy | 2 +- 5 files changed, 68 insertions(+), 59 deletions(-) create mode 100644 src/main/groovy/io/seqera/wave/service/builder/BuildConstants.groovy diff --git a/src/main/groovy/io/seqera/wave/service/builder/BuildConstants.groovy b/src/main/groovy/io/seqera/wave/service/builder/BuildConstants.groovy new file mode 100644 index 000000000..d32f26385 --- /dev/null +++ b/src/main/groovy/io/seqera/wave/service/builder/BuildConstants.groovy @@ -0,0 +1,31 @@ +/* + * Wave, containers provisioning service + * Copyright (c) 2023-2024, Seqera Labs + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package io.seqera.wave.service.builder + +/** + * Constants for the build service + * + * @author Munish Chouhan + */ +class BuildConstants { + + protected static final String FUSION_PREFIX = "/fusion/s3" + + static final public String BUILDKIT_ENTRYPOINT = 'buildctl-daemonless.sh' +} diff --git a/src/main/groovy/io/seqera/wave/service/builder/BuildStrategy.groovy b/src/main/groovy/io/seqera/wave/service/builder/BuildStrategy.groovy index 063524ef7..04c1c306e 100644 --- a/src/main/groovy/io/seqera/wave/service/builder/BuildStrategy.groovy +++ b/src/main/groovy/io/seqera/wave/service/builder/BuildStrategy.groovy @@ -21,6 +21,8 @@ package io.seqera.wave.service.builder import groovy.transform.CompileStatic import io.seqera.wave.configuration.BuildConfig import jakarta.inject.Inject +import static io.seqera.wave.service.builder.BuildConstants.FUSION_PREFIX +import static io.seqera.wave.service.builder.BuildConstants.BUILDKIT_ENTRYPOINT /** * Defines an abstract container build strategy. * @@ -32,15 +34,11 @@ import jakarta.inject.Inject @CompileStatic abstract class BuildStrategy { - protected static final String FUSION_PREFIX = "/fusion/s3" - @Inject private BuildConfig buildConfig abstract BuildResult build(BuildRequest req) - static final public String BUILDKIT_ENTRYPOINT = 'buildctl-daemonless.sh' - void cleanup(BuildRequest req) { req.workDir?.deleteDir() } @@ -112,7 +110,7 @@ abstract class BuildStrategy { result << 'sh' << '-c' - << "singularity build image.sif ${req.workDir}/Containerfile && singularity push image.sif ${req.targetImage}".toString() + << "singularity build image.sif $FUSION_PREFIX/$buildConfig.workspaceBucket/$req.s3Key/Containerfile && singularity push image.sif ${req.targetImage}".toString() return result } diff --git a/src/main/groovy/io/seqera/wave/service/builder/ContainerBuildServiceImpl.groovy b/src/main/groovy/io/seqera/wave/service/builder/ContainerBuildServiceImpl.groovy index 858fd1f20..77e10eb65 100644 --- a/src/main/groovy/io/seqera/wave/service/builder/ContainerBuildServiceImpl.groovy +++ b/src/main/groovy/io/seqera/wave/service/builder/ContainerBuildServiceImpl.groovy @@ -45,15 +45,14 @@ import io.seqera.wave.service.persistence.WaveBuildRecord import io.seqera.wave.service.stream.StreamService import io.seqera.wave.tower.PlatformId import io.seqera.wave.util.Retryable -import io.seqera.wave.util.SpackHelper import io.seqera.wave.util.TarGzipUtils -import io.seqera.wave.util.TemplateRenderer import jakarta.inject.Inject import jakarta.inject.Named import jakarta.inject.Singleton import org.apache.commons.io.IOUtils import static io.seqera.wave.util.RegHelper.layerName import static io.seqera.wave.util.StringUtils.indent +import static io.seqera.wave.service.builder.BuildConstants.FUSION_PREFIX /** * Implements container build service * @@ -147,25 +146,11 @@ class ContainerBuildServiceImpl implements ContainerBuildService { .awaitBuild(targetImage) } - protected String containerFile0(BuildRequest req, String context, SpackConfig config) { + protected String containerFile0(BuildRequest req, String context) { // add the context dir for singularity builds - final containerFile = req.formatSingularity() - ? req.containerFile.replace('{{wave_context_dir}}', context) + return req.formatSingularity() + ? req.containerFile.replace('{{wave_context_dir}}', "$FUSION_PREFIX/$buildConfig.workspaceBucket/$req.s3Key/context".toString()) : req.containerFile - - // render the Spack template if needed - if( req.isSpackBuild ) { - final binding = new HashMap(2) - binding.spack_builder_image = config.builderImage - binding.spack_runner_image = config.runnerImage - binding.spack_arch = SpackHelper.toSpackArch(req.getPlatform()) - binding.spack_cache_bucket = config.cacheBucket - binding.spack_key_file = config.secretMountPath - return new TemplateRenderer().render(containerFile, binding) - } - else { - return containerFile - } } protected BuildResult launch(BuildRequest req) { @@ -175,7 +160,7 @@ class ContainerBuildServiceImpl implements ContainerBuildService { BuildResult resp=null try { // save the dockerfile - objectStorageOperations.upload(UploadRequest.fromBytes(containerFile0(req, "$req.s3Key/Containerfile", spackConfig).bytes, "$req.s3Key/Containerfile".toString())) + objectStorageOperations.upload(UploadRequest.fromBytes(containerFile0(req, "$req.s3Key/Containerfile").bytes, "$req.s3Key/Containerfile".toString())) // save build context if( req.buildContext ) { saveBuildContext(req.buildContext, "$req.s3Key/context/", req.identity) @@ -192,6 +177,16 @@ class ContainerBuildServiceImpl implements ContainerBuildService { if( req.containerConfig ) { saveLayersToContext(req, "$req.s3Key/context/") } + // save docker config for creds + if( req.configJson ) { + if (req.formatDocker()) { + objectStorageOperations.upload(UploadRequest.fromBytes(req.configJson.bytes, "$req.s3Key/config.json".toString())) + } + else { + objectStorageOperations.upload(UploadRequest.fromBytes(req.configJson.bytes, "$req.s3Key/.singularity/docker-config.json".toString())) + objectStorageOperations.upload(UploadRequest.fromBytes(req.configJson.bytes, "$req.s3Key/.singularity/remote.yaml".toString())) + } + } resp = buildStrategy.build(req) def msg = "== Build request ${req.buildId} completed with status=$resp.exitStatus" if( log.isTraceEnabled() ) diff --git a/src/main/groovy/io/seqera/wave/service/builder/DockerBuildStrategy.groovy b/src/main/groovy/io/seqera/wave/service/builder/DockerBuildStrategy.groovy index 6fcb82c92..3aebc08e9 100644 --- a/src/main/groovy/io/seqera/wave/service/builder/DockerBuildStrategy.groovy +++ b/src/main/groovy/io/seqera/wave/service/builder/DockerBuildStrategy.groovy @@ -18,7 +18,6 @@ package io.seqera.wave.service.builder -import java.nio.file.Path import java.util.concurrent.TimeUnit import groovy.transform.CompileStatic @@ -30,10 +29,10 @@ import io.seqera.wave.configuration.BuildConfig import io.seqera.wave.configuration.SpackConfig import io.seqera.wave.core.ContainerPlatform import io.seqera.wave.core.RegistryProxyService -import io.seqera.wave.util.RegHelper import jakarta.inject.Inject import jakarta.inject.Named import jakarta.inject.Singleton +import static io.seqera.wave.service.builder.BuildConstants.FUSION_PREFIX /** * Build a container image using a Docker CLI tool * @@ -63,20 +62,8 @@ class DockerBuildStrategy extends BuildStrategy { @Override BuildResult build(BuildRequest req) { - boolean configFile = false; - // save docker config for creds - if( req.configJson ) { - objectStorageOperations.upload(UploadRequest.fromBytes(req.configJson.bytes, "$req.s3Key/config.json".toString())) - configFile = true - } - // save remote files for singularity - if( req.configJson && req.formatSingularity()) { - objectStorageOperations.upload(UploadRequest.fromBytes(RegHelper.singularityRemoteFile(req.targetImage).bytes, "$req.s3Key/singularity-remote.yaml".toString())) - configFile = true - } - // command the docker build command - final buildCmd= buildCmd(req, configFile) + final buildCmd= buildCmd(req) log.debug "Build run command: ${buildCmd.join(' ')}" // save docker cli for debugging purpose if( debug ) { @@ -100,12 +87,10 @@ class DockerBuildStrategy extends BuildStrategy { } } - protected List buildCmd(BuildRequest req, boolean credsFile) { - final spack = req.isSpackBuild ? spackConfig : null - + protected List buildCmd(BuildRequest req) { final dockerCmd = req.formatDocker() ? cmdForBuildkit( req, req.platform) - : cmdForSingularity( req.workDir, credsFile, spack, req.platform) + : cmdForSingularity(req.platform) return dockerCmd + launchCmd(req) } @@ -121,8 +106,14 @@ class DockerBuildStrategy extends BuildStrategy { "AWS_SECRET_ACCESS_KEY=${System.getenv('AWS_SECRET_ACCESS_KEY')}".toString()] if( req.configJson ) { - wrapper.add('-e') - wrapper.add("DOCKER_CONFIG=$FUSION_PREFIX/$buildConfig.workspaceBucket/$req.s3Key".toString()) + if( req.formatDocker() ) { + wrapper.add('-e') + wrapper.add("DOCKER_CONFIG=$FUSION_PREFIX/$buildConfig.workspaceBucket/$req.s3Key".toString()) + } + else { + wrapper.add('-e') + wrapper.add("SINGULARITY_CACHEDIR=$FUSION_PREFIX/$buildConfig.workspaceBucket/$req.s3Key/.singularity".toString()) + } } if( platform ) { @@ -136,22 +127,16 @@ class DockerBuildStrategy extends BuildStrategy { return wrapper } - protected List cmdForSingularity(Path workDir, boolean credsFile, SpackConfig spackConfig, ContainerPlatform platform) { + protected List cmdForSingularity(ContainerPlatform platform) { final wrapper = ['docker', 'run', '--rm', '--privileged', - "--entrypoint", '', - '-v', "$workDir:$workDir".toString()] - - if( credsFile ) { - //todo for singularity remote file - wrapper.add('-v') - wrapper.add("$credsFile:/root/.singularity/docker-config.json:ro".toString()) - - //wrapper.add('-v') - //wrapper.add("${credsFile.resolveSibling('singularity-remote.yaml')}:/root/.singularity/remote.yaml:ro".toString()) - } + '-e', + "AWS_ACCESS_KEY_ID=${System.getenv('AWS_ACCESS_KEY_ID')}".toString(), + '-e', + "AWS_SECRET_ACCESS_KEY=${System.getenv('AWS_SECRET_ACCESS_KEY')}".toString(), + "--entrypoint", ''] if( platform ) { wrapper.add('--platform') diff --git a/src/main/groovy/io/seqera/wave/service/k8s/K8sServiceImpl.groovy b/src/main/groovy/io/seqera/wave/service/k8s/K8sServiceImpl.groovy index b7d96dafc..306c64873 100644 --- a/src/main/groovy/io/seqera/wave/service/k8s/K8sServiceImpl.groovy +++ b/src/main/groovy/io/seqera/wave/service/k8s/K8sServiceImpl.groovy @@ -50,7 +50,7 @@ import io.seqera.wave.core.ContainerPlatform import io.seqera.wave.service.scan.Trivy import jakarta.inject.Inject import jakarta.inject.Singleton -import static io.seqera.wave.service.builder.BuildStrategy.BUILDKIT_ENTRYPOINT +import static io.seqera.wave.service.builder.BuildConstants.BUILDKIT_ENTRYPOINT /** * implements the support for Kubernetes cluster * From ebcc16a9b43df81063ba03d7b3fb56f97a42e7b4 Mon Sep 17 00:00:00 2001 From: munish chouhan Date: Thu, 8 Aug 2024 15:30:37 +0200 Subject: [PATCH 04/15] working singularity build --- .../wave/service/builder/ContainerBuildServiceImpl.groovy | 6 +----- .../seqera/wave/service/builder/DockerBuildStrategy.groovy | 4 +--- src/main/resources/application.yml | 2 +- 3 files changed, 3 insertions(+), 9 deletions(-) diff --git a/src/main/groovy/io/seqera/wave/service/builder/ContainerBuildServiceImpl.groovy b/src/main/groovy/io/seqera/wave/service/builder/ContainerBuildServiceImpl.groovy index 77e10eb65..8f9597fee 100644 --- a/src/main/groovy/io/seqera/wave/service/builder/ContainerBuildServiceImpl.groovy +++ b/src/main/groovy/io/seqera/wave/service/builder/ContainerBuildServiceImpl.groovy @@ -167,11 +167,7 @@ class ContainerBuildServiceImpl implements ContainerBuildService { } // save the conda file if( req.condaFile ) { - objectStorageOperations.upload(UploadRequest.fromBytes(req.condaFile.bytes, "$req.s3Key/conda.yml")) - } - // save the spack file - if( req.spackFile ) { - objectStorageOperations.upload(UploadRequest.fromBytes(req.condaFile.bytes, "$req.s3Key/spack.yaml")) + objectStorageOperations.upload(UploadRequest.fromBytes(req.condaFile.bytes, "$req.s3Key/context/conda.yml")) } // save layers provided via the container config if( req.containerConfig ) { diff --git a/src/main/groovy/io/seqera/wave/service/builder/DockerBuildStrategy.groovy b/src/main/groovy/io/seqera/wave/service/builder/DockerBuildStrategy.groovy index 3aebc08e9..9e919aa87 100644 --- a/src/main/groovy/io/seqera/wave/service/builder/DockerBuildStrategy.groovy +++ b/src/main/groovy/io/seqera/wave/service/builder/DockerBuildStrategy.groovy @@ -130,13 +130,11 @@ class DockerBuildStrategy extends BuildStrategy { protected List cmdForSingularity(ContainerPlatform platform) { final wrapper = ['docker', 'run', - '--rm', '--privileged', '-e', "AWS_ACCESS_KEY_ID=${System.getenv('AWS_ACCESS_KEY_ID')}".toString(), '-e', - "AWS_SECRET_ACCESS_KEY=${System.getenv('AWS_SECRET_ACCESS_KEY')}".toString(), - "--entrypoint", ''] + "AWS_SECRET_ACCESS_KEY=${System.getenv('AWS_SECRET_ACCESS_KEY')}".toString()] if( platform ) { wrapper.add('--platform') diff --git a/src/main/resources/application.yml b/src/main/resources/application.yml index c8df6f244..84efd6992 100644 --- a/src/main/resources/application.yml +++ b/src/main/resources/application.yml @@ -68,7 +68,7 @@ wave: build: workspace-bucket: "${WAVE_WORKSPACE_BUCKET}" buildkit-image: "cr.seqera.io/public/wave/buildkit:ef67f15426f36b72" - singularity-image: "quay.io/singularity/singularity:v3.11.4-slim" + singularity-image: "cr.seqera.io/public/wave/singularity:f3a5accae865288f" singularity-image-arm64: "quay.io/singularity/singularity:v3.11.4-slim-arm64" repo: "195996028523.dkr.ecr.eu-west-1.amazonaws.com/wave/build/dev" cache: "195996028523.dkr.ecr.eu-west-1.amazonaws.com/wave/build/cache" From 9f2e2173fea9fea30a1aaa1954a6f404f9bb79ff Mon Sep 17 00:00:00 2001 From: munish chouhan Date: Tue, 13 Aug 2024 23:22:31 +0200 Subject: [PATCH 05/15] working singularity build and push --- .../wave/service/builder/BuildStrategy.groovy | 2 +- .../builder/DockerBuildStrategy.groovy | 24 +++++++------------ 2 files changed, 10 insertions(+), 16 deletions(-) diff --git a/src/main/groovy/io/seqera/wave/service/builder/BuildStrategy.groovy b/src/main/groovy/io/seqera/wave/service/builder/BuildStrategy.groovy index 04c1c306e..4b0c1f33e 100644 --- a/src/main/groovy/io/seqera/wave/service/builder/BuildStrategy.groovy +++ b/src/main/groovy/io/seqera/wave/service/builder/BuildStrategy.groovy @@ -110,7 +110,7 @@ abstract class BuildStrategy { result << 'sh' << '-c' - << "singularity build image.sif $FUSION_PREFIX/$buildConfig.workspaceBucket/$req.s3Key/Containerfile && singularity push image.sif ${req.targetImage}".toString() + << "ln -s $FUSION_PREFIX/$buildConfig.workspaceBucket/$req.s3Key/.singularity /root/.singularity && singularity build image.sif $FUSION_PREFIX/$buildConfig.workspaceBucket/$req.s3Key/Containerfile && singularity push image.sif ${req.targetImage}".toString() return result } diff --git a/src/main/groovy/io/seqera/wave/service/builder/DockerBuildStrategy.groovy b/src/main/groovy/io/seqera/wave/service/builder/DockerBuildStrategy.groovy index 9e919aa87..f209bff33 100644 --- a/src/main/groovy/io/seqera/wave/service/builder/DockerBuildStrategy.groovy +++ b/src/main/groovy/io/seqera/wave/service/builder/DockerBuildStrategy.groovy @@ -89,13 +89,13 @@ class DockerBuildStrategy extends BuildStrategy { protected List buildCmd(BuildRequest req) { final dockerCmd = req.formatDocker() - ? cmdForBuildkit( req, req.platform) - : cmdForSingularity(req.platform) + ? cmdForBuildkit( req) + : cmdForSingularity(req) return dockerCmd + launchCmd(req) } - protected List cmdForBuildkit(BuildRequest req, ContainerPlatform platform ) { + protected List cmdForBuildkit(BuildRequest req) { //checkout the documentation here to know more about these options https://github.com/moby/buildkit/blob/master/docs/rootless.md#docker final wrapper = ['docker', 'run', @@ -106,19 +106,13 @@ class DockerBuildStrategy extends BuildStrategy { "AWS_SECRET_ACCESS_KEY=${System.getenv('AWS_SECRET_ACCESS_KEY')}".toString()] if( req.configJson ) { - if( req.formatDocker() ) { wrapper.add('-e') wrapper.add("DOCKER_CONFIG=$FUSION_PREFIX/$buildConfig.workspaceBucket/$req.s3Key".toString()) - } - else { - wrapper.add('-e') - wrapper.add("SINGULARITY_CACHEDIR=$FUSION_PREFIX/$buildConfig.workspaceBucket/$req.s3Key/.singularity".toString()) - } } - if( platform ) { + if( req.platform ) { wrapper.add('--platform') - wrapper.add(platform.toString()) + wrapper.add(req.platform.toString()) } // the container image to be used to build @@ -127,7 +121,7 @@ class DockerBuildStrategy extends BuildStrategy { return wrapper } - protected List cmdForSingularity(ContainerPlatform platform) { + protected List cmdForSingularity(BuildRequest req) { final wrapper = ['docker', 'run', '--privileged', @@ -136,12 +130,12 @@ class DockerBuildStrategy extends BuildStrategy { '-e', "AWS_SECRET_ACCESS_KEY=${System.getenv('AWS_SECRET_ACCESS_KEY')}".toString()] - if( platform ) { + if( req.platform ) { wrapper.add('--platform') - wrapper.add(platform.toString()) + wrapper.add(req.platform.toString()) } - wrapper.add(buildConfig.singularityImage(platform)) + wrapper.add(buildConfig.singularityImage(req.platform)) return wrapper } } From 88d06c03066048335367f979de0da938c459d723 Mon Sep 17 00:00:00 2001 From: munish chouhan Date: Wed, 14 Aug 2024 16:11:07 +0200 Subject: [PATCH 06/15] refactored --- .../io/seqera/wave/service/builder/BuildStrategy.groovy | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/main/groovy/io/seqera/wave/service/builder/BuildStrategy.groovy b/src/main/groovy/io/seqera/wave/service/builder/BuildStrategy.groovy index 4b0c1f33e..d353693dc 100644 --- a/src/main/groovy/io/seqera/wave/service/builder/BuildStrategy.groovy +++ b/src/main/groovy/io/seqera/wave/service/builder/BuildStrategy.groovy @@ -106,11 +106,15 @@ abstract class BuildStrategy { } protected List singularityLaunchCmd(BuildRequest req) { + def symlinkSingularity = "" + if( req.configJson ){ + symlinkSingularity = "ln -s $FUSION_PREFIX/$buildConfig.workspaceBucket/$req.s3Key/.singularity /root/.singularity &&" + } final result = new ArrayList(10) result << 'sh' << '-c' - << "ln -s $FUSION_PREFIX/$buildConfig.workspaceBucket/$req.s3Key/.singularity /root/.singularity && singularity build image.sif $FUSION_PREFIX/$buildConfig.workspaceBucket/$req.s3Key/Containerfile && singularity push image.sif ${req.targetImage}".toString() + << "$symlinkSingularity singularity build image.sif $FUSION_PREFIX/$buildConfig.workspaceBucket/$req.s3Key/Containerfile && singularity push image.sif ${req.targetImage}".toString() return result } From fabe615e435f4af95e224fa751d9681c0eee4ccb Mon Sep 17 00:00:00 2001 From: munish chouhan Date: Fri, 16 Aug 2024 16:07:30 +0200 Subject: [PATCH 07/15] updated scan process --- .../service/builder/BuildConstants.groovy | 2 +- .../builder/DockerBuildStrategy.groovy | 1 - .../service/scan/DockerScanStrategy.groovy | 45 ++++++++++--------- .../wave/service/scan/ScanRequest.groovy | 2 + 4 files changed, 26 insertions(+), 24 deletions(-) diff --git a/src/main/groovy/io/seqera/wave/service/builder/BuildConstants.groovy b/src/main/groovy/io/seqera/wave/service/builder/BuildConstants.groovy index d32f26385..4245e3000 100644 --- a/src/main/groovy/io/seqera/wave/service/builder/BuildConstants.groovy +++ b/src/main/groovy/io/seqera/wave/service/builder/BuildConstants.groovy @@ -25,7 +25,7 @@ package io.seqera.wave.service.builder */ class BuildConstants { - protected static final String FUSION_PREFIX = "/fusion/s3" + public static final String FUSION_PREFIX = "/fusion/s3" static final public String BUILDKIT_ENTRYPOINT = 'buildctl-daemonless.sh' } diff --git a/src/main/groovy/io/seqera/wave/service/builder/DockerBuildStrategy.groovy b/src/main/groovy/io/seqera/wave/service/builder/DockerBuildStrategy.groovy index f209bff33..ea77e6e52 100644 --- a/src/main/groovy/io/seqera/wave/service/builder/DockerBuildStrategy.groovy +++ b/src/main/groovy/io/seqera/wave/service/builder/DockerBuildStrategy.groovy @@ -27,7 +27,6 @@ import io.micronaut.objectstorage.ObjectStorageOperations import io.micronaut.objectstorage.request.UploadRequest import io.seqera.wave.configuration.BuildConfig import io.seqera.wave.configuration.SpackConfig -import io.seqera.wave.core.ContainerPlatform import io.seqera.wave.core.RegistryProxyService import jakarta.inject.Inject import jakarta.inject.Named diff --git a/src/main/groovy/io/seqera/wave/service/scan/DockerScanStrategy.groovy b/src/main/groovy/io/seqera/wave/service/scan/DockerScanStrategy.groovy index 0a909f889..8216bb852 100644 --- a/src/main/groovy/io/seqera/wave/service/scan/DockerScanStrategy.groovy +++ b/src/main/groovy/io/seqera/wave/service/scan/DockerScanStrategy.groovy @@ -20,19 +20,19 @@ package io.seqera.wave.service.scan import java.nio.file.FileAlreadyExistsException import java.nio.file.Files -import java.nio.file.Path import java.time.Instant -import groovy.json.JsonOutput import groovy.transform.CompileStatic import groovy.util.logging.Slf4j import io.micronaut.context.annotation.Requires +import io.micronaut.objectstorage.ObjectStorageOperations +import io.micronaut.objectstorage.request.UploadRequest +import io.seqera.wave.configuration.BuildConfig import io.seqera.wave.configuration.ScanConfig import jakarta.inject.Inject +import jakarta.inject.Named import jakarta.inject.Singleton -import static java.nio.file.StandardOpenOption.CREATE -import static java.nio.file.StandardOpenOption.TRUNCATE_EXISTING -import static java.nio.file.StandardOpenOption.WRITE +import static io.seqera.wave.service.builder.BuildConstants.FUSION_PREFIX /** * Implements ScanStrategy for Docker * @@ -47,6 +47,13 @@ class DockerScanStrategy extends ScanStrategy { @Inject private ScanConfig scanConfig + @Inject + BuildConfig buildConfig + + @Inject + @Named('build-workspace') + private ObjectStorageOperations objectStorageOperations + DockerScanStrategy(ScanConfig scanConfig) { this.scanConfig = scanConfig } @@ -66,16 +73,14 @@ class DockerScanStrategy extends ScanStrategy { } // save the config file with docker auth credentials - Path configFile = null if( req.configJson ) { - configFile = req.workDir.resolve('config.json') - Files.write(configFile, JsonOutput.prettyPrint(req.configJson).bytes, CREATE, WRITE, TRUNCATE_EXISTING) + objectStorageOperations.upload(UploadRequest.fromBytes(req.configJson.bytes, "$req.s3Key/config.json".toString())) } // outfile file name final reportFile = req.workDir.resolve(Trivy.OUTPUT_FILE_NAME) // create the launch command - final dockerCommand = dockerWrapper(req.workDir, configFile) + final dockerCommand = dockerWrapper(req) final trivyCommand = List.of(scanConfig.scanImage) + scanCommand(req.targetImage, reportFile, scanConfig) final command = dockerCommand + trivyCommand @@ -102,27 +107,23 @@ class DockerScanStrategy extends ScanStrategy { } } - protected List dockerWrapper(Path scanDir, Path credsFile) { + protected List dockerWrapper(ScanRequest request) { final wrapper = ['docker','run', '--rm'] - - // scan work dir - wrapper.add('-w') - wrapper.add(scanDir.toString()) - wrapper.add('-v') - wrapper.add("$scanDir:$scanDir:rw".toString()) + // scan work dir + wrapper.add('-e') + wrapper.add("TRIVY_WORKSPACE_DIR=$FUSION_PREFIX/$buildConfig.workspaceBucket/$request.s3Key".toString()) // cache directory - wrapper.add('-v') - wrapper.add("${scanConfig.cacheDirectory}:${Trivy.CACHE_MOUNT_PATH}:rw".toString()) + wrapper.add('-e') + wrapper.add("TRIVY_CACHE_DIR=$FUSION_PREFIX/$buildConfig.workspaceBucket/.trivy-cache".toString()) - if(credsFile) { - wrapper.add('-v') - wrapper.add("${credsFile}:${Trivy.CONFIG_MOUNT_PATH}:ro".toString()) + if(request.configJson) { + wrapper.add('-e') + wrapper.add("DOCKER_CONFIG=$FUSION_PREFIX/$buildConfig.workspaceBucket/$request.s3Key".toString()) } - return wrapper } } diff --git a/src/main/groovy/io/seqera/wave/service/scan/ScanRequest.groovy b/src/main/groovy/io/seqera/wave/service/scan/ScanRequest.groovy index b00a8011d..e5901425b 100644 --- a/src/main/groovy/io/seqera/wave/service/scan/ScanRequest.groovy +++ b/src/main/groovy/io/seqera/wave/service/scan/ScanRequest.groovy @@ -38,10 +38,12 @@ class ScanRequest { final String targetImage final ContainerPlatform platform final Path workDir + String s3Key static ScanRequest fromBuild(BuildRequest request) { final id = request.scanId final workDir = request.workDir.resolveSibling("scan-${id}") + final s3Key = "workspace/scan-${id}" return new ScanRequest(id, request.buildId, request.configJson, request.targetImage, request.platform, workDir) } } From fe2b2f4b24103ebb6d874da78efdf43fb0fcc02d Mon Sep 17 00:00:00 2001 From: munish chouhan Date: Thu, 22 Aug 2024 13:28:23 +0200 Subject: [PATCH 08/15] working scan process --- .../builder/DockerBuildStrategy.groovy | 3 ++ .../scan/ContainerScanServiceImpl.groovy | 10 ++++-- .../service/scan/DockerScanStrategy.groovy | 35 +++++++++---------- .../wave/service/scan/KubeScanStrategy.groovy | 35 +++++++++++-------- .../wave/service/scan/ScanRequest.groovy | 4 +-- .../wave/service/scan/ScanStrategy.groovy | 7 ++-- 6 files changed, 53 insertions(+), 41 deletions(-) diff --git a/src/main/groovy/io/seqera/wave/service/builder/DockerBuildStrategy.groovy b/src/main/groovy/io/seqera/wave/service/builder/DockerBuildStrategy.groovy index 30e0fbc6e..37fd8da54 100644 --- a/src/main/groovy/io/seqera/wave/service/builder/DockerBuildStrategy.groovy +++ b/src/main/groovy/io/seqera/wave/service/builder/DockerBuildStrategy.groovy @@ -72,6 +72,8 @@ class DockerBuildStrategy extends BuildStrategy { final builder = new ProcessBuilder() .command(buildCmd) .redirectErrorStream(true) + .redirectError(ProcessBuilder.Redirect.INHERIT) + builder.redirectOutput(ProcessBuilder.Redirect.INHERIT) def proc = builder.start() @@ -99,6 +101,7 @@ class DockerBuildStrategy extends BuildStrategy { //checkout the documentation here to know more about these options https://github.com/moby/buildkit/blob/master/docs/rootless.md#docker final wrapper = ['docker', 'run', + '--rm', '--privileged', '-e', "AWS_ACCESS_KEY_ID=${System.getenv('AWS_ACCESS_KEY_ID')}".toString(), diff --git a/src/main/groovy/io/seqera/wave/service/scan/ContainerScanServiceImpl.groovy b/src/main/groovy/io/seqera/wave/service/scan/ContainerScanServiceImpl.groovy index a04ecc05c..0e40be47f 100644 --- a/src/main/groovy/io/seqera/wave/service/scan/ContainerScanServiceImpl.groovy +++ b/src/main/groovy/io/seqera/wave/service/scan/ContainerScanServiceImpl.groovy @@ -25,6 +25,7 @@ import java.util.concurrent.ExecutorService import groovy.transform.CompileStatic import groovy.util.logging.Slf4j import io.micronaut.context.annotation.Requires +import io.micronaut.objectstorage.ObjectStorageOperations import io.micronaut.runtime.event.annotation.EventListener import io.micronaut.scheduling.TaskExecutors import io.seqera.wave.configuration.ScanConfig @@ -67,6 +68,10 @@ class ContainerScanServiceImpl implements ContainerScanService { @Inject private CleanupStrategy cleanup + @Inject + @Named('build-workspace') + private ObjectStorageOperations objectStorageOperations + @EventListener void onBuildEvent(BuildEvent event) { try { @@ -109,9 +114,10 @@ class ContainerScanServiceImpl implements ContainerScanService { log.warn "Unable to launch the scan results for scan id: ${request.id} - cause: ${e.message}", e } finally{ - // cleanup build context + // cleanup scan workspace if( cleanup.shouldCleanup(scanResult?.isSucceeded() ? 0 : 1) ) - request.workDir?.deleteDir() + objectStorageOperations.delete(request.s3Key) + } return scanResult } diff --git a/src/main/groovy/io/seqera/wave/service/scan/DockerScanStrategy.groovy b/src/main/groovy/io/seqera/wave/service/scan/DockerScanStrategy.groovy index 8216bb852..ee9c5dd4a 100644 --- a/src/main/groovy/io/seqera/wave/service/scan/DockerScanStrategy.groovy +++ b/src/main/groovy/io/seqera/wave/service/scan/DockerScanStrategy.groovy @@ -18,8 +18,6 @@ package io.seqera.wave.service.scan -import java.nio.file.FileAlreadyExistsException -import java.nio.file.Files import java.time.Instant import groovy.transform.CompileStatic @@ -64,21 +62,14 @@ class DockerScanStrategy extends ScanStrategy { final startTime = Instant.now() try { - // create the scan dir - try { - Files.createDirectory(req.workDir) - } - catch (FileAlreadyExistsException e) { - log.warn("Container scan directory already exists: $e") - } - // save the config file with docker auth credentials if( req.configJson ) { objectStorageOperations.upload(UploadRequest.fromBytes(req.configJson.bytes, "$req.s3Key/config.json".toString())) } - // outfile file name - final reportFile = req.workDir.resolve(Trivy.OUTPUT_FILE_NAME) + // create outfile file + objectStorageOperations.upload(UploadRequest.fromBytes(new byte[0], "$req.s3Key/$Trivy.OUTPUT_FILE_NAME".toString())) + final reportFile = "$FUSION_PREFIX/$buildConfig.workspaceBucket/$req.s3Key/$Trivy.OUTPUT_FILE_NAME" // create the launch command final dockerCommand = dockerWrapper(req) final trivyCommand = List.of(scanConfig.scanImage) + scanCommand(req.targetImage, reportFile, scanConfig) @@ -98,7 +89,9 @@ class DockerScanStrategy extends ScanStrategy { } else{ log.info("Container scan completed with id: ${req.id}") - return ScanResult.success(req, startTime, TrivyResultProcessor.process(reportFile.text)) + def scanReportFile = objectStorageOperations.retrieve("$req.s3Key/$Trivy.OUTPUT_FILE_NAME".toString()) + .map { it.toStreamedFile().inputStream.text }.get() + return ScanResult.success(req, startTime, TrivyResultProcessor.process(scanReportFile)) } } catch (Throwable e){ @@ -107,21 +100,27 @@ class DockerScanStrategy extends ScanStrategy { } } - protected List dockerWrapper(ScanRequest request) { + protected List dockerWrapper(ScanRequest req) { - final wrapper = ['docker','run', '--rm'] + final wrapper = ['docker', + 'run', + '--privileged', + '-e', + "AWS_ACCESS_KEY_ID=${System.getenv('AWS_ACCESS_KEY_ID')}".toString(), + '-e', + "AWS_SECRET_ACCESS_KEY=${System.getenv('AWS_SECRET_ACCESS_KEY')}".toString()] // scan work dir wrapper.add('-e') - wrapper.add("TRIVY_WORKSPACE_DIR=$FUSION_PREFIX/$buildConfig.workspaceBucket/$request.s3Key".toString()) + wrapper.add("TRIVY_WORKSPACE_DIR=$FUSION_PREFIX/$buildConfig.workspaceBucket/$req.s3Key".toString()) // cache directory wrapper.add('-e') wrapper.add("TRIVY_CACHE_DIR=$FUSION_PREFIX/$buildConfig.workspaceBucket/.trivy-cache".toString()) - if(request.configJson) { + if(req.configJson) { wrapper.add('-e') - wrapper.add("DOCKER_CONFIG=$FUSION_PREFIX/$buildConfig.workspaceBucket/$request.s3Key".toString()) + wrapper.add("DOCKER_CONFIG=$FUSION_PREFIX/$buildConfig.workspaceBucket/$req.s3Key".toString()) } return wrapper diff --git a/src/main/groovy/io/seqera/wave/service/scan/KubeScanStrategy.groovy b/src/main/groovy/io/seqera/wave/service/scan/KubeScanStrategy.groovy index 2490f14ed..6338dd6b4 100644 --- a/src/main/groovy/io/seqera/wave/service/scan/KubeScanStrategy.groovy +++ b/src/main/groovy/io/seqera/wave/service/scan/KubeScanStrategy.groovy @@ -31,10 +31,16 @@ import io.kubernetes.client.openapi.ApiException import io.micronaut.context.annotation.Primary import io.micronaut.context.annotation.Property import io.micronaut.context.annotation.Requires +import io.micronaut.objectstorage.ObjectStorageOperations +import io.micronaut.objectstorage.request.UploadRequest +import io.seqera.wave.configuration.BuildConfig import io.seqera.wave.configuration.ScanConfig import io.seqera.wave.exception.BadRequestException import io.seqera.wave.service.k8s.K8sService +import jakarta.inject.Inject +import jakarta.inject.Named import jakarta.inject.Singleton +import static io.seqera.wave.service.builder.BuildConstants.FUSION_PREFIX import static io.seqera.wave.util.K8sHelper.getSelectorLabel import static java.nio.file.StandardOpenOption.CREATE import static java.nio.file.StandardOpenOption.TRUNCATE_EXISTING @@ -55,6 +61,13 @@ class KubeScanStrategy extends ScanStrategy { @Nullable private Map nodeSelectorMap + @Inject + BuildConfig buildConfig + + @Inject + @Named('build-workspace') + private ObjectStorageOperations objectStorageOperations + private final K8sService k8sService private final ScanConfig scanConfig @@ -71,30 +84,22 @@ class KubeScanStrategy extends ScanStrategy { final podName = "scan-${req.id}" try{ - // create the scan dir - try { - Files.createDirectory(req.workDir) - } - catch (FileAlreadyExistsException e) { - log.warn("Container scan directory already exists: $e") - } - // save the config file with docker auth credentials - Path configFile = null if( req.configJson ) { - configFile = req.workDir.resolve('config.json') - Files.write(configFile, JsonOutput.prettyPrint(req.configJson).bytes, CREATE, WRITE, TRUNCATE_EXISTING) + objectStorageOperations.upload(UploadRequest.fromBytes(req.configJson.bytes, "$req.s3Key/config.json".toString())) } - final reportFile = req.workDir.resolve(Trivy.OUTPUT_FILE_NAME) - + // outfile file name + final reportFile = "$FUSION_PREFIX/$buildConfig.workspaceBucket/$req.s3Key/$Trivy.OUTPUT_FILE_NAME" final trivyCommand = scanCommand(req.targetImage, reportFile, scanConfig) final selector= getSelectorLabel(req.platform, nodeSelectorMap) - final pod = k8sService.scanContainer(podName, scanConfig.scanImage, trivyCommand, req.workDir, configFile, scanConfig, selector) + final pod = k8sService.scanContainer(podName, scanConfig.scanImage, trivyCommand, req.workDir, null, scanConfig, selector) final exitCode = k8sService.waitPodCompletion(pod, scanConfig.timeout.toMillis()) if( exitCode==0 ) { log.info("Container scan completed for id: ${req.id}") - return ScanResult.success(req, startTime, TrivyResultProcessor.process(reportFile.text)) + def scanReportFile = objectStorageOperations.retrieve("$req.s3Key/$Trivy.OUTPUT_FILE_NAME".toString()).get() + scanReportFile = scanReportFile ? scanReportFile as String : null + return ScanResult.success(req, startTime, TrivyResultProcessor.process(scanReportFile)) } else{ final stdout = k8sService.logsPod(pod) diff --git a/src/main/groovy/io/seqera/wave/service/scan/ScanRequest.groovy b/src/main/groovy/io/seqera/wave/service/scan/ScanRequest.groovy index e5901425b..e8a1c3845 100644 --- a/src/main/groovy/io/seqera/wave/service/scan/ScanRequest.groovy +++ b/src/main/groovy/io/seqera/wave/service/scan/ScanRequest.groovy @@ -37,13 +37,11 @@ class ScanRequest { final String configJson final String targetImage final ContainerPlatform platform - final Path workDir String s3Key static ScanRequest fromBuild(BuildRequest request) { final id = request.scanId - final workDir = request.workDir.resolveSibling("scan-${id}") final s3Key = "workspace/scan-${id}" - return new ScanRequest(id, request.buildId, request.configJson, request.targetImage, request.platform, workDir) + return new ScanRequest(id, request.buildId, request.configJson, request.targetImage, request.platform, s3Key) } } diff --git a/src/main/groovy/io/seqera/wave/service/scan/ScanStrategy.groovy b/src/main/groovy/io/seqera/wave/service/scan/ScanStrategy.groovy index 3938af4ec..ffa883551 100644 --- a/src/main/groovy/io/seqera/wave/service/scan/ScanStrategy.groovy +++ b/src/main/groovy/io/seqera/wave/service/scan/ScanStrategy.groovy @@ -34,15 +34,16 @@ abstract class ScanStrategy { abstract ScanResult scanContainer(ScanRequest request) - protected List scanCommand(String targetImage, Path outputFile, ScanConfig config) { - def cmd = ['--quiet', + protected List scanCommand(String targetImage, String outputFile, ScanConfig config) { + def cmd = ['trivy', + '--quiet', 'image', '--timeout', "${config.timeout.toMinutes()}m".toString(), '--format', 'json', '--output', - outputFile.toString()] + outputFile] if( config.severity ) { cmd << '--severity' From ae16c5e88ec861794b7cf736e8c95b7ab23e6b40 Mon Sep 17 00:00:00 2001 From: munish chouhan Date: Thu, 22 Aug 2024 13:30:12 +0200 Subject: [PATCH 09/15] fixed compile error --- .../groovy/io/seqera/wave/service/scan/ScanRequest.groovy | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/main/groovy/io/seqera/wave/service/scan/ScanRequest.groovy b/src/main/groovy/io/seqera/wave/service/scan/ScanRequest.groovy index e8a1c3845..47b710aa8 100644 --- a/src/main/groovy/io/seqera/wave/service/scan/ScanRequest.groovy +++ b/src/main/groovy/io/seqera/wave/service/scan/ScanRequest.groovy @@ -37,11 +37,13 @@ class ScanRequest { final String configJson final String targetImage final ContainerPlatform platform + final Path workDir String s3Key static ScanRequest fromBuild(BuildRequest request) { final id = request.scanId + final workDir = request.workDir.resolveSibling("scan-${id}") final s3Key = "workspace/scan-${id}" - return new ScanRequest(id, request.buildId, request.configJson, request.targetImage, request.platform, s3Key) + return new ScanRequest(id, request.buildId, request.configJson, request.targetImage, request.platform, workDir, s3Key) } } From c3f5a9f0b154e822fae019b1478b758e52cc57b0 Mon Sep 17 00:00:00 2001 From: munish chouhan Date: Thu, 22 Aug 2024 15:31:53 +0200 Subject: [PATCH 10/15] update k8s services --- .../service/builder/KubeBuildStrategy.groovy | 29 +--------- .../seqera/wave/service/k8s/K8sService.groovy | 6 +- .../wave/service/k8s/K8sServiceImpl.groovy | 55 ++++++++----------- .../wave/service/scan/KubeScanStrategy.groovy | 2 +- .../wave/service/scan/ScanRequest.groovy | 4 +- 5 files changed, 28 insertions(+), 68 deletions(-) diff --git a/src/main/groovy/io/seqera/wave/service/builder/KubeBuildStrategy.groovy b/src/main/groovy/io/seqera/wave/service/builder/KubeBuildStrategy.groovy index 7ac6d20c5..a0e3f9288 100644 --- a/src/main/groovy/io/seqera/wave/service/builder/KubeBuildStrategy.groovy +++ b/src/main/groovy/io/seqera/wave/service/builder/KubeBuildStrategy.groovy @@ -18,10 +18,6 @@ package io.seqera.wave.service.builder -import java.nio.file.Files -import java.nio.file.Path - -import groovy.json.JsonOutput import groovy.transform.CompileStatic import groovy.util.logging.Slf4j import io.kubernetes.client.openapi.ApiException @@ -34,15 +30,9 @@ import io.seqera.wave.configuration.SpackConfig import io.seqera.wave.core.RegistryProxyService import io.seqera.wave.exception.BadRequestException import io.seqera.wave.service.k8s.K8sService -import io.seqera.wave.util.RegHelper import jakarta.inject.Inject import jakarta.inject.Singleton import static io.seqera.wave.util.K8sHelper.getSelectorLabel -import static java.nio.file.StandardOpenOption.CREATE -import static java.nio.file.StandardOpenOption.TRUNCATE_EXISTING -import static java.nio.file.StandardOpenOption.WRITE -import static java.nio.file.attribute.PosixFilePermission.OWNER_READ -import static java.nio.file.attribute.PosixFilePermission.OWNER_WRITE /** * Build a container image using running a K8s pod * @@ -77,30 +67,13 @@ class KubeBuildStrategy extends BuildStrategy { @Override BuildResult build(BuildRequest req) { - - Path configFile = null - if( req.configJson ) { - configFile = req.workDir.resolve('config.json') - Files.write(configFile, JsonOutput.prettyPrint(req.configJson).bytes, CREATE, WRITE, TRUNCATE_EXISTING) - } - // save remote files for singularity - if( req.configJson && req.formatSingularity()) { - final remoteFile = req.workDir.resolve('singularity-remote.yaml') - final content = RegHelper.singularityRemoteFile(req.targetImage) - Files.write(remoteFile, content.bytes, CREATE, WRITE, TRUNCATE_EXISTING) - // set permissions 600 as required by Singularity - Files.setPosixFilePermissions(configFile, Set.of(OWNER_READ, OWNER_WRITE)) - Files.setPosixFilePermissions(remoteFile, Set.of(OWNER_READ, OWNER_WRITE)) - } - try { final buildImage = getBuildImage(req) final buildCmd = launchCmd(req) final name = podName(req) final timeout = req.maxDuration ?: buildConfig.defaultTimeout final selector= getSelectorLabel(req.platform, nodeSelectorMap) - final spackCfg0 = req.isSpackBuild ? spackConfig : null - final pod = k8sService.buildContainer(name, buildImage, buildCmd, req.workDir, configFile, timeout, spackCfg0, selector) + final pod = k8sService.buildContainer(name, buildImage, buildCmd, req.s3Key, req.configJson, timeout, selector) final exitCode = k8sService.waitPodCompletion(pod, timeout.toMillis()) final stdout = k8sService.logsPod(pod) if( exitCode!=null ) { diff --git a/src/main/groovy/io/seqera/wave/service/k8s/K8sService.groovy b/src/main/groovy/io/seqera/wave/service/k8s/K8sService.groovy index d7005ab7b..994ed253f 100644 --- a/src/main/groovy/io/seqera/wave/service/k8s/K8sService.groovy +++ b/src/main/groovy/io/seqera/wave/service/k8s/K8sService.groovy @@ -18,7 +18,6 @@ package io.seqera.wave.service.k8s -import java.nio.file.Path import java.time.Duration import io.kubernetes.client.openapi.models.V1Job @@ -26,7 +25,6 @@ import io.kubernetes.client.openapi.models.V1Pod import io.kubernetes.client.openapi.models.V1PodList import io.seqera.wave.configuration.BlobCacheConfig import io.seqera.wave.configuration.ScanConfig -import io.seqera.wave.configuration.SpackConfig /** * Defines Kubernetes operations * @@ -42,9 +40,9 @@ interface K8sService { void deletePod(String name) - V1Pod buildContainer(String name, String containerImage, List args, Path workDir, Path creds, Duration timeout, SpackConfig spackConfig, Map nodeSelector) + V1Pod buildContainer(String name, String containerImage, List args, String s3Key, String creds, Duration timeout, Map nodeSelector) - V1Pod scanContainer(String name, String containerImage, List args, Path workDir, Path creds, ScanConfig scanConfig, Map nodeSelector) + V1Pod scanContainer(String name, String containerImage, List args, String s3Key, String creds, ScanConfig scanConfig, Map nodeSelector) Integer waitPodCompletion(V1Pod pod, long timeout) diff --git a/src/main/groovy/io/seqera/wave/service/k8s/K8sServiceImpl.groovy b/src/main/groovy/io/seqera/wave/service/k8s/K8sServiceImpl.groovy index fc13ac902..2f5e1468a 100644 --- a/src/main/groovy/io/seqera/wave/service/k8s/K8sServiceImpl.groovy +++ b/src/main/groovy/io/seqera/wave/service/k8s/K8sServiceImpl.groovy @@ -46,12 +46,12 @@ import io.micronaut.core.annotation.Nullable import io.seqera.wave.configuration.BlobCacheConfig import io.seqera.wave.configuration.BuildConfig import io.seqera.wave.configuration.ScanConfig -import io.seqera.wave.configuration.SpackConfig import io.seqera.wave.core.ContainerPlatform import io.seqera.wave.service.scan.Trivy import jakarta.inject.Inject import jakarta.inject.Singleton import static io.seqera.wave.service.builder.BuildConstants.BUILDKIT_ENTRYPOINT +import static io.seqera.wave.service.builder.BuildConstants.FUSION_PREFIX /** * implements the support for Kubernetes cluster * @@ -97,9 +97,6 @@ class K8sServiceImpl implements K8sService { @Nullable private String requestsMemory - @Inject - private SpackConfig spackConfig - @Inject private K8sClient k8sClient @@ -332,40 +329,31 @@ class K8sServiceImpl implements K8sService { * The {@link V1Pod} description the submitted pod */ @Override - V1Pod buildContainer(String name, String containerImage, List args, Path workDir, Path creds, Duration timeout, SpackConfig spackConfig, Map nodeSelector) { - final spec = buildSpec(name, containerImage, args, workDir, creds, timeout, spackConfig, nodeSelector) + V1Pod buildContainer(String name, String containerImage, List args, String workDir, String creds, Duration timeout, Map nodeSelector) { + final spec = buildSpec(name, containerImage, args, workDir, creds, timeout, nodeSelector) return k8sClient .coreV1Api() .createNamespacedPod(namespace, spec, null, null, null,null) } - V1Pod buildSpec(String name, String containerImage, List args, Path workDir, Path credsFile, Duration timeout, SpackConfig spackConfig, Map nodeSelector) { + V1Pod buildSpec(String name, String containerImage, List args, String s3Key, String credsFile, Duration timeout, Map nodeSelector) { // dirty dependency to avoid introducing another parameter final singularity = containerImage.contains('singularity') - // required volumes - final mounts = new ArrayList(5) - mounts.add(mountBuildStorage(workDir, storageMountPath, true)) - - final volumes = new ArrayList(5) - volumes.add(volumeBuildStorage(storageMountPath, storageClaimName)) + Map env = new HashMap() + addAWSCreds(env) if( credsFile ){ if( !singularity ) { - mounts.add(0, mountHostPath(credsFile, storageMountPath, '/home/user/.docker/config.json')) + env.put('DOCKER_CONFIG', "$FUSION_PREFIX/$buildConfig.workspaceBucket/$s3Key".toString()) } else { - final remoteFile = credsFile.resolveSibling('singularity-remote.yaml') - mounts.add(0, mountHostPath(credsFile, storageMountPath, '/root/.singularity/docker-config.json')) - mounts.add(1, mountHostPath(remoteFile, storageMountPath, '/root/.singularity/remote.yaml')) + env.put('DOCKER_CONFIG', "$FUSION_PREFIX/$buildConfig.workspaceBucket/$s3Key".toString()) + env.put('DOCKER_CONFIG', "$FUSION_PREFIX/$buildConfig.workspaceBucket/$s3Key".toString()) } } - if( spackConfig ) { - mounts.add(mountSpackSecretFile(spackConfig.secretKeyFile, storageMountPath, spackConfig.secretMountPath)) - } - V1PodBuilder builder = new V1PodBuilder() //metadata section @@ -383,7 +371,6 @@ class K8sServiceImpl implements K8sService { .withServiceAccount(serviceAccount) .withActiveDeadlineSeconds( timeout.toSeconds() ) .withRestartPolicy("Never") - .addAllToVolumes(volumes) final requests = new V1ResourceRequirements() @@ -396,7 +383,7 @@ class K8sServiceImpl implements K8sService { final container = new V1ContainerBuilder() .withName(name) .withImage(containerImage) - .withVolumeMounts(mounts) + .withEnv(toEnvList(env)) .withResources(requests) if( singularity ) { @@ -512,24 +499,23 @@ class K8sServiceImpl implements K8sService { } @Override - V1Pod scanContainer(String name, String containerImage, List args, Path workDir, Path creds, ScanConfig scanConfig, Map nodeSelector) { - final spec = scanSpec(name, containerImage, args, workDir, creds, scanConfig, nodeSelector) + V1Pod scanContainer(String name, String containerImage, List args, String s3Key, String creds, ScanConfig scanConfig, Map nodeSelector) { + final spec = scanSpec(name, containerImage, args, s3Key, creds, scanConfig, nodeSelector) return k8sClient .coreV1Api() .createNamespacedPod(namespace, spec, null, null, null,null) } - V1Pod scanSpec(String name, String containerImage, List args, Path workDir, Path credsFile, ScanConfig scanConfig, Map nodeSelector) { + V1Pod scanSpec(String name, String containerImage, List args, String s3Key, String creds, ScanConfig scanConfig, Map nodeSelector) { - final mounts = new ArrayList(5) - mounts.add(mountBuildStorage(workDir, storageMountPath, false)) - mounts.add(mountScanCacheDir(scanConfig.cacheDirectory, storageMountPath)) final volumes = new ArrayList(5) volumes.add(volumeBuildStorage(storageMountPath, storageClaimName)) - if( credsFile ){ - mounts.add(0, mountHostPath(credsFile, storageMountPath, Trivy.CONFIG_MOUNT_PATH)) + Map env = new HashMap() + addAWSCreds(env) + if( creds ){ + env.put('DOCKER_CONFIG', "$FUSION_PREFIX/$buildConfig.workspaceBucket/$s3Key".toString()) } V1PodBuilder builder = new V1PodBuilder() @@ -562,7 +548,7 @@ class K8sServiceImpl implements K8sService { .withName(name) .withImage(containerImage) .withArgs(args) - .withVolumeMounts(mounts) + .withEnv(toEnvList(env)) .withResources(requests) .endContainer() .endSpec() @@ -698,4 +684,9 @@ class K8sServiceImpl implements K8sService { } return latest } + + private void addAWSCreds(Map env) { + env.put('AWS_ACCESS_KEY_ID', "${System.getenv('AWS_ACCESS_KEY_ID')}".toString()) + env.put('AWS_SECRET_ACCESS_KEY', "${System.getenv('AWS_SECRET_ACCESS_KEY')}".toString()) + } } diff --git a/src/main/groovy/io/seqera/wave/service/scan/KubeScanStrategy.groovy b/src/main/groovy/io/seqera/wave/service/scan/KubeScanStrategy.groovy index 6338dd6b4..e73810e96 100644 --- a/src/main/groovy/io/seqera/wave/service/scan/KubeScanStrategy.groovy +++ b/src/main/groovy/io/seqera/wave/service/scan/KubeScanStrategy.groovy @@ -93,7 +93,7 @@ class KubeScanStrategy extends ScanStrategy { final reportFile = "$FUSION_PREFIX/$buildConfig.workspaceBucket/$req.s3Key/$Trivy.OUTPUT_FILE_NAME" final trivyCommand = scanCommand(req.targetImage, reportFile, scanConfig) final selector= getSelectorLabel(req.platform, nodeSelectorMap) - final pod = k8sService.scanContainer(podName, scanConfig.scanImage, trivyCommand, req.workDir, null, scanConfig, selector) + final pod = k8sService.scanContainer(podName, scanConfig.scanImage, trivyCommand, req.s3Key, req.configJson, scanConfig, selector) final exitCode = k8sService.waitPodCompletion(pod, scanConfig.timeout.toMillis()) if( exitCode==0 ) { log.info("Container scan completed for id: ${req.id}") diff --git a/src/main/groovy/io/seqera/wave/service/scan/ScanRequest.groovy b/src/main/groovy/io/seqera/wave/service/scan/ScanRequest.groovy index 47b710aa8..e8a1c3845 100644 --- a/src/main/groovy/io/seqera/wave/service/scan/ScanRequest.groovy +++ b/src/main/groovy/io/seqera/wave/service/scan/ScanRequest.groovy @@ -37,13 +37,11 @@ class ScanRequest { final String configJson final String targetImage final ContainerPlatform platform - final Path workDir String s3Key static ScanRequest fromBuild(BuildRequest request) { final id = request.scanId - final workDir = request.workDir.resolveSibling("scan-${id}") final s3Key = "workspace/scan-${id}" - return new ScanRequest(id, request.buildId, request.configJson, request.targetImage, request.platform, workDir, s3Key) + return new ScanRequest(id, request.buildId, request.configJson, request.targetImage, request.platform, s3Key) } } From fb60a0d6d0aa12e3027207546d39b794c5090975 Mon Sep 17 00:00:00 2001 From: munish chouhan Date: Thu, 22 Aug 2024 16:15:41 +0200 Subject: [PATCH 11/15] update trivy image --- src/main/resources/application.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/resources/application.yml b/src/main/resources/application.yml index 84efd6992..58a0bbd9e 100644 --- a/src/main/resources/application.yml +++ b/src/main/resources/application.yml @@ -84,7 +84,7 @@ wave: multiplier: '1.75' scan: image: - name: aquasec/trivy:0.53.0 + name: cr.seqera.io/public/wave/trivy:17135cecea328cbe blobCache: s5cmdImage: cr.seqera.io/public/wave/s5cmd:v2.2.2 --- From d18b668b284e486838185c0e242518bf0e0107a2 Mon Sep 17 00:00:00 2001 From: munish chouhan Date: Thu, 22 Aug 2024 17:43:03 +0200 Subject: [PATCH 12/15] removed workdir --- .../wave/configuration/BuildConfig.groovy | 7 ------- .../controller/ContainerController.groovy | 1 - .../wave/service/builder/BuildRequest.groovy | 16 -------------- .../wave/service/builder/BuildStrategy.groovy | 8 ++++++- .../wave/service/k8s/K8sServiceImpl.groovy | 21 +++---------------- 5 files changed, 10 insertions(+), 43 deletions(-) diff --git a/src/main/groovy/io/seqera/wave/configuration/BuildConfig.groovy b/src/main/groovy/io/seqera/wave/configuration/BuildConfig.groovy index 203892b88..993bc07e8 100644 --- a/src/main/groovy/io/seqera/wave/configuration/BuildConfig.groovy +++ b/src/main/groovy/io/seqera/wave/configuration/BuildConfig.groovy @@ -65,12 +65,6 @@ class BuildConfig { @Value('${wave.build.workspace-bucket}') String workspaceBucket - /** - * File system path there the dockerfile is save - */ - @Value('${wave.build.workspace}') - String buildWorkspace - @Value('${wave.build.status.delay}') Duration statusDelay @@ -126,7 +120,6 @@ class BuildConfig { "default-build-repository=${defaultBuildRepository}; " + "default-cache-repository=${defaultCacheRepository}; " + "default-public-repository=${defaultPublicRepository}; " + - "build-workspace=${buildWorkspace}; " + "build-timeout=${defaultTimeout}; " + "build-trusted-timeout=${trustedTimeout}; " + "status-delay=${statusDelay}; " + diff --git a/src/main/groovy/io/seqera/wave/controller/ContainerController.groovy b/src/main/groovy/io/seqera/wave/controller/ContainerController.groovy index be89c905a..fd63f9b65 100644 --- a/src/main/groovy/io/seqera/wave/controller/ContainerController.groovy +++ b/src/main/groovy/io/seqera/wave/controller/ContainerController.groovy @@ -335,7 +335,6 @@ class ContainerController { containerFile, condaContent, spackContent, - Path.of(buildConfig.buildWorkspace), targetImage, identity, platform, diff --git a/src/main/groovy/io/seqera/wave/service/builder/BuildRequest.groovy b/src/main/groovy/io/seqera/wave/service/builder/BuildRequest.groovy index cc65e026f..1dcf60f3c 100644 --- a/src/main/groovy/io/seqera/wave/service/builder/BuildRequest.groovy +++ b/src/main/groovy/io/seqera/wave/service/builder/BuildRequest.groovy @@ -18,7 +18,6 @@ package io.seqera.wave.service.builder -import java.nio.file.Path import java.time.Duration import java.time.Instant import java.time.OffsetDateTime @@ -65,11 +64,6 @@ class BuildRequest { */ final String spackFile - /** - * The build context work directory - */ - final Path workspace - /** * The target fully qualified image of the built container. It includes the target registry name */ @@ -142,7 +136,6 @@ class BuildRequest { volatile String buildId - volatile Path workDir String s3Key @@ -150,7 +143,6 @@ class BuildRequest { String containerFile, String condaFile, String spackFile, - Path workspace, String targetImage, PlatformId identity, ContainerPlatform platform, @@ -169,7 +161,6 @@ class BuildRequest { this.containerFile = containerFile this.condaFile = condaFile this.spackFile = spackFile - this.workspace = workspace this.targetImage = targetImage this.identity = identity this.platform = platform @@ -191,7 +182,6 @@ class BuildRequest { this.containerFile = opts.containerFile this.condaFile = opts.condaFile this.spackFile = opts.spackFile - this.workspace = opts.workspace as Path this.targetImage = opts.targetImage this.identity = opts.identity as PlatformId this.platform = opts.platform as ContainerPlatform @@ -205,7 +195,6 @@ class BuildRequest { this.scanId = opts.scanId this.buildContext = opts.buildContext as BuildContext this.format = opts.format as BuildFormat - this.workDir = opts.workDir as Path this.buildId = opts.buildId this.maxDuration = opts.maxDuration as Duration } @@ -236,10 +225,6 @@ class BuildRequest { return spackFile } - Path getWorkDir() { - return workDir - } - String getTargetImage() { return targetImage } @@ -286,7 +271,6 @@ class BuildRequest { BuildRequest withBuildId(String id) { this.buildId = containerId + SEP + id - this.workDir = workspace.resolve(buildId).toAbsolutePath() this.s3Key = "workspace/$buildId" return this } diff --git a/src/main/groovy/io/seqera/wave/service/builder/BuildStrategy.groovy b/src/main/groovy/io/seqera/wave/service/builder/BuildStrategy.groovy index d353693dc..9d6edc48a 100644 --- a/src/main/groovy/io/seqera/wave/service/builder/BuildStrategy.groovy +++ b/src/main/groovy/io/seqera/wave/service/builder/BuildStrategy.groovy @@ -19,8 +19,10 @@ package io.seqera.wave.service.builder import groovy.transform.CompileStatic +import io.micronaut.objectstorage.ObjectStorageOperations import io.seqera.wave.configuration.BuildConfig import jakarta.inject.Inject +import jakarta.inject.Named import static io.seqera.wave.service.builder.BuildConstants.FUSION_PREFIX import static io.seqera.wave.service.builder.BuildConstants.BUILDKIT_ENTRYPOINT /** @@ -37,10 +39,14 @@ abstract class BuildStrategy { @Inject private BuildConfig buildConfig + @Inject + @Named('build-workspace') + private ObjectStorageOperations objectStorageOperations + abstract BuildResult build(BuildRequest req) void cleanup(BuildRequest req) { - req.workDir?.deleteDir() + objectStorageOperations.delete(req.s3Key) } List launchCmd(BuildRequest req) { diff --git a/src/main/groovy/io/seqera/wave/service/k8s/K8sServiceImpl.groovy b/src/main/groovy/io/seqera/wave/service/k8s/K8sServiceImpl.groovy index 2f5e1468a..f97364f7b 100644 --- a/src/main/groovy/io/seqera/wave/service/k8s/K8sServiceImpl.groovy +++ b/src/main/groovy/io/seqera/wave/service/k8s/K8sServiceImpl.groovy @@ -73,10 +73,6 @@ class K8sServiceImpl implements K8sService { @Nullable private String storageClaimName - @Value('${wave.build.k8s.storage.mountPath}') - @Nullable - private String storageMountPath - @Property(name='wave.build.k8s.labels') @Nullable private Map labels @@ -118,15 +114,9 @@ class K8sServiceImpl implements K8sService { */ @PostConstruct private void init() { - log.info "K8s build config: namespace=$namespace; service-account=$serviceAccount; node-selector=$nodeSelectorMap; cpus=$requestsCpu; memory=$requestsMemory; buildWorkspace=$buildConfig.buildWorkspace; storageClaimName=$storageClaimName; storageMountPath=$storageMountPath; " - if( storageClaimName && !storageMountPath ) - throw new IllegalArgumentException("Missing 'wave.build.k8s.storage.mountPath' configuration attribute") - if( storageMountPath ) { - if( !buildConfig.buildWorkspace ) - throw new IllegalArgumentException("Missing 'wave.build.workspace' configuration attribute") - if( !Path.of(buildConfig.buildWorkspace).startsWith(storageMountPath) ) - throw new IllegalArgumentException("Build workspace should be a sub-directory of 'wave.build.k8s.storage.mountPath' - offending value: '$buildConfig.buildWorkspace' - expected value: '$storageMountPath'") - } + log.info "K8s build config: namespace=$namespace; service-account=$serviceAccount; node-selector=$nodeSelectorMap; cpus=$requestsCpu; memory=$requestsMemory; buildWorkspaceBucket=$buildConfig.workspaceBucket;" + if( !buildConfig.workspaceBucket ) + throw new IllegalArgumentException("Missing 'wave.build.workspaceBucket' configuration attribute") // validate node selectors final platforms = nodeSelectorMap ?: Collections.emptyMap() for( Map.Entry it : platforms ) { @@ -508,10 +498,6 @@ class K8sServiceImpl implements K8sService { V1Pod scanSpec(String name, String containerImage, List args, String s3Key, String creds, ScanConfig scanConfig, Map nodeSelector) { - - final volumes = new ArrayList(5) - volumes.add(volumeBuildStorage(storageMountPath, storageClaimName)) - Map env = new HashMap() addAWSCreds(env) if( creds ){ @@ -534,7 +520,6 @@ class K8sServiceImpl implements K8sService { .withServiceAccount(serviceAccount) .withActiveDeadlineSeconds( scanConfig.timeout.toSeconds() ) .withRestartPolicy("Never") - .addAllToVolumes(volumes) final requests = new V1ResourceRequirements() From 7751c8653b5daf77c149c1f5eae1d02d817278b9 Mon Sep 17 00:00:00 2001 From: munishchouhan Date: Thu, 22 Aug 2024 17:56:57 +0200 Subject: [PATCH 13/15] minor change [ci skip] Signed-off-by: munishchouhan --- .../io/seqera/wave/service/builder/DockerBuildStrategy.groovy | 1 + 1 file changed, 1 insertion(+) diff --git a/src/main/groovy/io/seqera/wave/service/builder/DockerBuildStrategy.groovy b/src/main/groovy/io/seqera/wave/service/builder/DockerBuildStrategy.groovy index 37fd8da54..060ca19a2 100644 --- a/src/main/groovy/io/seqera/wave/service/builder/DockerBuildStrategy.groovy +++ b/src/main/groovy/io/seqera/wave/service/builder/DockerBuildStrategy.groovy @@ -127,6 +127,7 @@ class DockerBuildStrategy extends BuildStrategy { protected List cmdForSingularity(BuildRequest req) { final wrapper = ['docker', 'run', + '--rm', '--privileged', '-e', "AWS_ACCESS_KEY_ID=${System.getenv('AWS_ACCESS_KEY_ID')}".toString(), From 88738ee06239d22ed17c3c7647d2fe6932f3b220 Mon Sep 17 00:00:00 2001 From: munishchouhan Date: Fri, 23 Aug 2024 16:17:40 +0200 Subject: [PATCH 14/15] added nextflow-io/k8s-fuse-plugin Signed-off-by: munishchouhan --- .../builder/DockerBuildStrategy.groovy | 3 +- .../wave/service/k8s/K8sServiceImpl.groovy | 106 +----------------- 2 files changed, 7 insertions(+), 102 deletions(-) diff --git a/src/main/groovy/io/seqera/wave/service/builder/DockerBuildStrategy.groovy b/src/main/groovy/io/seqera/wave/service/builder/DockerBuildStrategy.groovy index 060ca19a2..aef46c389 100644 --- a/src/main/groovy/io/seqera/wave/service/builder/DockerBuildStrategy.groovy +++ b/src/main/groovy/io/seqera/wave/service/builder/DockerBuildStrategy.groovy @@ -72,6 +72,7 @@ class DockerBuildStrategy extends BuildStrategy { final builder = new ProcessBuilder() .command(buildCmd) .redirectErrorStream(true) + //this is to run it in windows .redirectError(ProcessBuilder.Redirect.INHERIT) builder.redirectOutput(ProcessBuilder.Redirect.INHERIT) @@ -94,7 +95,7 @@ class DockerBuildStrategy extends BuildStrategy { ? cmdForBuildkit( req) : cmdForSingularity(req) - return dockerCmd + launchCmd(req) + return dockerCmd + "" +launchCmd(req) } protected List cmdForBuildkit(BuildRequest req) { diff --git a/src/main/groovy/io/seqera/wave/service/k8s/K8sServiceImpl.groovy b/src/main/groovy/io/seqera/wave/service/k8s/K8sServiceImpl.groovy index f97364f7b..ef15d1799 100644 --- a/src/main/groovy/io/seqera/wave/service/k8s/K8sServiceImpl.groovy +++ b/src/main/groovy/io/seqera/wave/service/k8s/K8sServiceImpl.groovy @@ -212,96 +212,6 @@ class K8sServiceImpl implements K8sService { .readNamespacedPod(name, namespace, null) } - /** - * Create a volume mount for the build storage. - * - * @param workDir The path representing a container build context - * @param storageMountPath - * @return A {@link V1VolumeMount} representing the mount path for the build config - */ - protected V1VolumeMount mountBuildStorage(Path workDir, String storageMountPath, boolean readOnly) { - assert workDir, "K8s mount build storage is mandatory" - - final vol = new V1VolumeMount() - .name('build-data') - .mountPath(workDir.toString()) - .readOnly(readOnly) - - if( storageMountPath ) { - // check sub-path - final rel = Path.of(storageMountPath).relativize(workDir).toString() - if (rel) - vol.subPath(rel) - } - return vol - } - - /** - * Defines the volume for the container building shared context - * - * @param workDir The path where the container image build context is located - * @param claimName The claim name of the corresponding storage - * @return An instance of {@link V1Volume} representing the build storage volume - */ - protected V1Volume volumeBuildStorage(String mountPath, @Nullable String claimName) { - final vol= new V1Volume() - .name('build-data') - if( claimName ) { - vol.persistentVolumeClaim( new V1PersistentVolumeClaimVolumeSource().claimName(claimName) ) - } - else { - vol.hostPath( new V1HostPathVolumeSource().path(mountPath) ) - } - - return vol - } - - /** - * Defines the volume mount for the docker config - * - * @return A {@link V1VolumeMount} representing the docker config - */ - protected V1VolumeMount mountHostPath(Path filePath, String storageMountPath, String mountPath) { - final rel = Path.of(storageMountPath).relativize(filePath).toString() - if( !rel ) throw new IllegalStateException("Mount relative path cannot be empty") - return new V1VolumeMount() - .name('build-data') - .mountPath(mountPath) - .subPath(rel) - .readOnly(true) - } - - protected V1VolumeMount mountSpackCacheDir(Path spackCacheDir, String storageMountPath, String containerPath) { - final rel = Path.of(storageMountPath).relativize(spackCacheDir).toString() - if( !rel || rel.startsWith('../') ) - throw new IllegalArgumentException("Spack cacheDirectory '$spackCacheDir' must be a sub-directory of storage path '$storageMountPath'") - return new V1VolumeMount() - .name('build-data') - .mountPath(containerPath) - .subPath(rel) - } - - protected V1VolumeMount mountSpackSecretFile(Path secretFile, String storageMountPath, String containerPath) { - final rel = Path.of(storageMountPath).relativize(secretFile).toString() - if( !rel || rel.startsWith('../') ) - throw new IllegalArgumentException("Spack secretKeyFile '$secretFile' must be a sub-directory of storage path '$storageMountPath'") - return new V1VolumeMount() - .name('build-data') - .readOnly(true) - .mountPath(containerPath) - .subPath(rel) - } - - protected V1VolumeMount mountScanCacheDir(Path scanCacheDir, String storageMountPath) { - final rel = Path.of(storageMountPath).relativize(scanCacheDir).toString() - if( !rel || rel.startsWith('../') ) - throw new IllegalArgumentException("Container scan cacheDirectory '$scanCacheDir' must be a sub-directory of storage path '$storageMountPath'") - return new V1VolumeMount() - .name('build-data') - .mountPath( Trivy.CACHE_MOUNT_PATH ) - .subPath(rel) - } - /** * Create a container for container image building via buildkit * @@ -369,25 +279,19 @@ class K8sServiceImpl implements K8sService { if( requestsMemory ) requests.putRequestsItem('memory', new Quantity(requestsMemory)) + //add https://github.com/nextflow-io/k8s-fuse-plugin + requests.limits(Map.of("nextflow.io/fuse", new Quantity("1"))) + // container section final container = new V1ContainerBuilder() .withName(name) .withImage(containerImage) .withEnv(toEnvList(env)) .withResources(requests) + .withArgs(args) if( singularity ) { - container - // use 'command' to override the entrypoint of the container - .withCommand(args) - .withNewSecurityContext().withPrivileged(true).endSecurityContext() - } else { - container - //required by buildkit rootless container - .withEnv(toEnvList(BUILDKIT_FLAGS)) - // buildCommand is to set entrypoint for buildkit - .withCommand(BUILDKIT_ENTRYPOINT) - .withArgs(args) + container.withNewSecurityContext().withPrivileged(true).endSecurityContext() } // spec section From f2ca9390c24727878122b91f677280adf2d0332c Mon Sep 17 00:00:00 2001 From: munish chouhan Date: Tue, 27 Aug 2024 12:16:29 +0200 Subject: [PATCH 15/15] minor change [ci skip] --- .../io/seqera/wave/service/builder/DockerBuildStrategy.groovy | 2 +- .../io/seqera/wave/service/scan/DockerScanStrategy.groovy | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/src/main/groovy/io/seqera/wave/service/builder/DockerBuildStrategy.groovy b/src/main/groovy/io/seqera/wave/service/builder/DockerBuildStrategy.groovy index aef46c389..fe025b90c 100644 --- a/src/main/groovy/io/seqera/wave/service/builder/DockerBuildStrategy.groovy +++ b/src/main/groovy/io/seqera/wave/service/builder/DockerBuildStrategy.groovy @@ -95,7 +95,7 @@ class DockerBuildStrategy extends BuildStrategy { ? cmdForBuildkit( req) : cmdForSingularity(req) - return dockerCmd + "" +launchCmd(req) + return dockerCmd + launchCmd(req) } protected List cmdForBuildkit(BuildRequest req) { diff --git a/src/main/groovy/io/seqera/wave/service/scan/DockerScanStrategy.groovy b/src/main/groovy/io/seqera/wave/service/scan/DockerScanStrategy.groovy index ee9c5dd4a..031529601 100644 --- a/src/main/groovy/io/seqera/wave/service/scan/DockerScanStrategy.groovy +++ b/src/main/groovy/io/seqera/wave/service/scan/DockerScanStrategy.groovy @@ -104,6 +104,7 @@ class DockerScanStrategy extends ScanStrategy { final wrapper = ['docker', 'run', + '--rm', '--privileged', '-e', "AWS_ACCESS_KEY_ID=${System.getenv('AWS_ACCESS_KEY_ID')}".toString(),