diff --git a/src/main/groovy/io/seqera/wave/configuration/BuildConfig.groovy b/src/main/groovy/io/seqera/wave/configuration/BuildConfig.groovy index 2ea625fde..993bc07e8 100644 --- a/src/main/groovy/io/seqera/wave/configuration/BuildConfig.groovy +++ b/src/main/groovy/io/seqera/wave/configuration/BuildConfig.groovy @@ -58,11 +58,12 @@ class BuildConfig { @Value('${wave.build.public-repo}') String defaultPublicRepository - /** - * File system path there the dockerfile is save - */ - @Value('${wave.build.workspace}') - String buildWorkspace + @Nullable + @Value('${wave.build.logs.bucket}') + String storageBucket + + @Value('${wave.build.workspace-bucket}') + String workspaceBucket @Value('${wave.build.status.delay}') Duration statusDelay @@ -119,7 +120,6 @@ class BuildConfig { "default-build-repository=${defaultBuildRepository}; " + "default-cache-repository=${defaultCacheRepository}; " + "default-public-repository=${defaultPublicRepository}; " + - "build-workspace=${buildWorkspace}; " + "build-timeout=${defaultTimeout}; " + "build-trusted-timeout=${trustedTimeout}; " + "status-delay=${statusDelay}; " + diff --git a/src/main/groovy/io/seqera/wave/controller/ContainerController.groovy b/src/main/groovy/io/seqera/wave/controller/ContainerController.groovy index 441234ece..9bfe1f2c3 100644 --- a/src/main/groovy/io/seqera/wave/controller/ContainerController.groovy +++ b/src/main/groovy/io/seqera/wave/controller/ContainerController.groovy @@ -335,7 +335,6 @@ class ContainerController { containerFile, condaContent, spackContent, - Path.of(buildConfig.buildWorkspace), targetImage, identity, platform, diff --git a/src/main/groovy/io/seqera/wave/service/aws/ObjectStorageOperationsFactory.groovy b/src/main/groovy/io/seqera/wave/service/aws/ObjectStorageOperationsFactory.groovy index b0d649d16..d531c98a8 100644 --- a/src/main/groovy/io/seqera/wave/service/aws/ObjectStorageOperationsFactory.groovy +++ b/src/main/groovy/io/seqera/wave/service/aws/ObjectStorageOperationsFactory.groovy @@ -22,11 +22,12 @@ import groovy.transform.CompileStatic import groovy.util.logging.Slf4j import io.micronaut.context.annotation.Factory import io.micronaut.context.annotation.Requires -import io.micronaut.context.annotation.Value import io.micronaut.objectstorage.InputStreamMapper import io.micronaut.objectstorage.ObjectStorageOperations import io.micronaut.objectstorage.aws.AwsS3Configuration import io.micronaut.objectstorage.aws.AwsS3Operations +import io.seqera.wave.configuration.BuildConfig +import jakarta.inject.Inject import jakarta.inject.Named import jakarta.inject.Singleton import software.amazon.awssdk.services.s3.S3Client @@ -38,17 +39,26 @@ import software.amazon.awssdk.services.s3.S3Client @Factory @CompileStatic @Slf4j -@Requires(property = 'wave.build.logs.bucket') class ObjectStorageOperationsFactory { - @Value('${wave.build.logs.bucket}') - String storageBucket + @Inject + private BuildConfig buildConfig @Singleton @Named("build-logs") - ObjectStorageOperations awsStorageOperations(@Named("DefaultS3Client") S3Client s3Client, InputStreamMapper inputStreamMapper) { + @Requires(property = 'wave.build.logs.bucket') + ObjectStorageOperations awsStorageOperationsBuildLogs(@Named("DefaultS3Client") S3Client s3Client, InputStreamMapper inputStreamMapper) { AwsS3Configuration configuration = new AwsS3Configuration('build-logs') - configuration.setBucket(storageBucket) + configuration.setBucket(buildConfig.storageBucket) + return new AwsS3Operations(configuration, s3Client, inputStreamMapper) + } + + @Singleton + @Named("build-workspace") + @Requires(property = 'wave.build.workspace-bucket') + ObjectStorageOperations awsStorageOperationsBuildWorkspace(@Named("DefaultS3Client") S3Client s3Client, InputStreamMapper inputStreamMapper) { + AwsS3Configuration configuration = new AwsS3Configuration('build-workspace') + configuration.setBucket(buildConfig.workspaceBucket) return new AwsS3Operations(configuration, s3Client, inputStreamMapper) } } diff --git a/src/main/groovy/io/seqera/wave/service/builder/BuildConstants.groovy b/src/main/groovy/io/seqera/wave/service/builder/BuildConstants.groovy new file mode 100644 index 000000000..4245e3000 --- /dev/null +++ b/src/main/groovy/io/seqera/wave/service/builder/BuildConstants.groovy @@ -0,0 +1,31 @@ +/* + * Wave, containers provisioning service + * Copyright (c) 2023-2024, Seqera Labs + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package io.seqera.wave.service.builder + +/** + * Constants for the build service + * + * @author Munish Chouhan + */ +class BuildConstants { + + public static final String FUSION_PREFIX = "/fusion/s3" + + static final public String BUILDKIT_ENTRYPOINT = 'buildctl-daemonless.sh' +} diff --git a/src/main/groovy/io/seqera/wave/service/builder/BuildRequest.groovy b/src/main/groovy/io/seqera/wave/service/builder/BuildRequest.groovy index 545a52ca7..1dcf60f3c 100644 --- a/src/main/groovy/io/seqera/wave/service/builder/BuildRequest.groovy +++ b/src/main/groovy/io/seqera/wave/service/builder/BuildRequest.groovy @@ -18,7 +18,6 @@ package io.seqera.wave.service.builder -import java.nio.file.Path import java.time.Duration import java.time.Instant import java.time.OffsetDateTime @@ -65,11 +64,6 @@ class BuildRequest { */ final String spackFile - /** - * The build context work directory - */ - final Path workspace - /** * The target fully qualified image of the built container. It includes the target registry name */ @@ -142,13 +136,13 @@ class BuildRequest { volatile String buildId - volatile Path workDir + + String s3Key BuildRequest(String containerId, String containerFile, String condaFile, String spackFile, - Path workspace, String targetImage, PlatformId identity, ContainerPlatform platform, @@ -167,7 +161,6 @@ class BuildRequest { this.containerFile = containerFile this.condaFile = condaFile this.spackFile = spackFile - this.workspace = workspace this.targetImage = targetImage this.identity = identity this.platform = platform @@ -189,7 +182,6 @@ class BuildRequest { this.containerFile = opts.containerFile this.condaFile = opts.condaFile this.spackFile = opts.spackFile - this.workspace = opts.workspace as Path this.targetImage = opts.targetImage this.identity = opts.identity as PlatformId this.platform = opts.platform as ContainerPlatform @@ -203,7 +195,6 @@ class BuildRequest { this.scanId = opts.scanId this.buildContext = opts.buildContext as BuildContext this.format = opts.format as BuildFormat - this.workDir = opts.workDir as Path this.buildId = opts.buildId this.maxDuration = opts.maxDuration as Duration } @@ -234,10 +225,6 @@ class BuildRequest { return spackFile } - Path getWorkDir() { - return workDir - } - String getTargetImage() { return targetImage } @@ -284,7 +271,7 @@ class BuildRequest { BuildRequest withBuildId(String id) { this.buildId = containerId + SEP + id - this.workDir = workspace.resolve(buildId).toAbsolutePath() + this.s3Key = "workspace/$buildId" return this } diff --git a/src/main/groovy/io/seqera/wave/service/builder/BuildStrategy.groovy b/src/main/groovy/io/seqera/wave/service/builder/BuildStrategy.groovy index 723d2611a..9d6edc48a 100644 --- a/src/main/groovy/io/seqera/wave/service/builder/BuildStrategy.groovy +++ b/src/main/groovy/io/seqera/wave/service/builder/BuildStrategy.groovy @@ -19,8 +19,12 @@ package io.seqera.wave.service.builder import groovy.transform.CompileStatic +import io.micronaut.objectstorage.ObjectStorageOperations import io.seqera.wave.configuration.BuildConfig import jakarta.inject.Inject +import jakarta.inject.Named +import static io.seqera.wave.service.builder.BuildConstants.FUSION_PREFIX +import static io.seqera.wave.service.builder.BuildConstants.BUILDKIT_ENTRYPOINT /** * Defines an abstract container build strategy. * @@ -35,12 +39,14 @@ abstract class BuildStrategy { @Inject private BuildConfig buildConfig - abstract BuildResult build(BuildRequest req) + @Inject + @Named('build-workspace') + private ObjectStorageOperations objectStorageOperations - static final public String BUILDKIT_ENTRYPOINT = 'buildctl-daemonless.sh' + abstract BuildResult build(BuildRequest req) void cleanup(BuildRequest req) { - req.workDir?.deleteDir() + objectStorageOperations.delete(req.s3Key) } List launchCmd(BuildRequest req) { @@ -57,15 +63,16 @@ abstract class BuildStrategy { protected List dockerLaunchCmd(BuildRequest req) { final result = new ArrayList(10) result + << BUILDKIT_ENTRYPOINT << "build" << "--frontend" << "dockerfile.v0" << "--local" - << "dockerfile=$req.workDir".toString() + << "dockerfile=$FUSION_PREFIX/$buildConfig.workspaceBucket/$req.s3Key".toString() << "--opt" << "filename=Containerfile" << "--local" - << "context=$req.workDir/context".toString() + << "context=$FUSION_PREFIX/$buildConfig.workspaceBucket/$req.s3Key/context".toString() << "--output" << "type=image,name=$req.targetImage,push=true,oci-mediatypes=${buildConfig.ociMediatypes}".toString() << "--opt" @@ -105,11 +112,15 @@ abstract class BuildStrategy { } protected List singularityLaunchCmd(BuildRequest req) { + def symlinkSingularity = "" + if( req.configJson ){ + symlinkSingularity = "ln -s $FUSION_PREFIX/$buildConfig.workspaceBucket/$req.s3Key/.singularity /root/.singularity &&" + } final result = new ArrayList(10) result << 'sh' << '-c' - << "singularity build image.sif ${req.workDir}/Containerfile && singularity push image.sif ${req.targetImage}".toString() + << "$symlinkSingularity singularity build image.sif $FUSION_PREFIX/$buildConfig.workspaceBucket/$req.s3Key/Containerfile && singularity push image.sif ${req.targetImage}".toString() return result } diff --git a/src/main/groovy/io/seqera/wave/service/builder/ContainerBuildServiceImpl.groovy b/src/main/groovy/io/seqera/wave/service/builder/ContainerBuildServiceImpl.groovy index 7e57ae642..8f9597fee 100644 --- a/src/main/groovy/io/seqera/wave/service/builder/ContainerBuildServiceImpl.groovy +++ b/src/main/groovy/io/seqera/wave/service/builder/ContainerBuildServiceImpl.groovy @@ -18,10 +18,6 @@ package io.seqera.wave.service.builder -import java.nio.file.FileAlreadyExistsException -import java.nio.file.Files -import java.nio.file.Path -import java.nio.file.StandardCopyOption import java.util.concurrent.CompletableFuture import java.util.concurrent.ExecutorService @@ -29,6 +25,8 @@ import groovy.transform.CompileStatic import groovy.util.logging.Slf4j import io.micronaut.context.event.ApplicationEventPublisher import io.micronaut.core.annotation.Nullable +import io.micronaut.objectstorage.ObjectStorageOperations +import io.micronaut.objectstorage.request.UploadRequest import io.micronaut.scheduling.TaskExecutors import io.seqera.wave.api.BuildContext import io.seqera.wave.auth.RegistryCredentialsProvider @@ -47,18 +45,14 @@ import io.seqera.wave.service.persistence.WaveBuildRecord import io.seqera.wave.service.stream.StreamService import io.seqera.wave.tower.PlatformId import io.seqera.wave.util.Retryable -import io.seqera.wave.util.SpackHelper -import io.seqera.wave.util.TarUtils -import io.seqera.wave.util.TemplateRenderer +import io.seqera.wave.util.TarGzipUtils import jakarta.inject.Inject import jakarta.inject.Named import jakarta.inject.Singleton -import static io.seqera.wave.util.RegHelper.layerDir +import org.apache.commons.io.IOUtils import static io.seqera.wave.util.RegHelper.layerName import static io.seqera.wave.util.StringUtils.indent -import static java.nio.file.StandardOpenOption.CREATE -import static java.nio.file.StandardOpenOption.TRUNCATE_EXISTING -import static java.nio.file.StandardOpenOption.WRITE +import static io.seqera.wave.service.builder.BuildConstants.FUSION_PREFIX /** * Implements container build service * @@ -118,6 +112,10 @@ class ContainerBuildServiceImpl implements ContainerBuildService { @Inject BuildRecordStore buildRecordStore + + @Inject + @Named('build-workspace') + private ObjectStorageOperations objectStorageOperations /** * Build a container image for the given {@link BuildRequest} @@ -148,57 +146,42 @@ class ContainerBuildServiceImpl implements ContainerBuildService { .awaitBuild(targetImage) } - protected String containerFile0(BuildRequest req, Path context, SpackConfig config) { + protected String containerFile0(BuildRequest req, String context) { // add the context dir for singularity builds - final containerFile = req.formatSingularity() - ? req.containerFile.replace('{{wave_context_dir}}', context.toString()) + return req.formatSingularity() + ? req.containerFile.replace('{{wave_context_dir}}', "$FUSION_PREFIX/$buildConfig.workspaceBucket/$req.s3Key/context".toString()) : req.containerFile - - // render the Spack template if needed - if( req.isSpackBuild ) { - final binding = new HashMap(2) - binding.spack_builder_image = config.builderImage - binding.spack_runner_image = config.runnerImage - binding.spack_arch = SpackHelper.toSpackArch(req.getPlatform()) - binding.spack_cache_bucket = config.cacheBucket - binding.spack_key_file = config.secretMountPath - return new TemplateRenderer().render(containerFile, binding) - } - else { - return containerFile - } } protected BuildResult launch(BuildRequest req) { + //create context dir + objectStorageOperations.upload(UploadRequest.fromBytes(new byte[0] , "$req.s3Key/context/".toString())) // launch an external process to build the container BuildResult resp=null try { - // create the workdir path - Files.createDirectories(req.workDir) - // create context dir - final context = req.workDir.resolve('context') - try { Files.createDirectory(context) } - catch (FileAlreadyExistsException e) { /* ignore it */ } // save the dockerfile - final containerFile = req.workDir.resolve('Containerfile') - Files.write(containerFile, containerFile0(req, context, spackConfig).bytes, CREATE, WRITE, TRUNCATE_EXISTING) + objectStorageOperations.upload(UploadRequest.fromBytes(containerFile0(req, "$req.s3Key/Containerfile").bytes, "$req.s3Key/Containerfile".toString())) // save build context if( req.buildContext ) { - saveBuildContext(req.buildContext, context, req.identity) + saveBuildContext(req.buildContext, "$req.s3Key/context/", req.identity) } // save the conda file if( req.condaFile ) { - final condaFile = context.resolve('conda.yml') - Files.write(condaFile, req.condaFile.bytes, CREATE, WRITE, TRUNCATE_EXISTING) - } - // save the spack file - if( req.spackFile ) { - final spackFile = context.resolve('spack.yaml') - Files.write(spackFile, req.spackFile.bytes, CREATE, WRITE, TRUNCATE_EXISTING) + objectStorageOperations.upload(UploadRequest.fromBytes(req.condaFile.bytes, "$req.s3Key/context/conda.yml")) } // save layers provided via the container config if( req.containerConfig ) { - saveLayersToContext(req, context) + saveLayersToContext(req, "$req.s3Key/context/") + } + // save docker config for creds + if( req.configJson ) { + if (req.formatDocker()) { + objectStorageOperations.upload(UploadRequest.fromBytes(req.configJson.bytes, "$req.s3Key/config.json".toString())) + } + else { + objectStorageOperations.upload(UploadRequest.fromBytes(req.configJson.bytes, "$req.s3Key/.singularity/docker-config.json".toString())) + objectStorageOperations.upload(UploadRequest.fromBytes(req.configJson.bytes, "$req.s3Key/.singularity/remote.yaml".toString())) + } } resp = buildStrategy.build(req) def msg = "== Build request ${req.buildId} completed with status=$resp.exitStatus" @@ -281,59 +264,57 @@ class ContainerBuildServiceImpl implements ContainerBuildService { throw new IllegalStateException("Unable to determine build status for '$request.targetImage'") } - protected void saveLayersToContext(BuildRequest req, Path contextDir) { + protected void saveLayersToContext(BuildRequest req, String s3Key) { if(req.formatDocker()) { - saveLayersToDockerContext0(req, contextDir) + saveLayersToDockerContext0(req, s3Key) } else if(req.formatSingularity()) { - saveLayersToSingularityContext0(req, contextDir) + saveLayersToSingularityContext0(req, s3Key) } else throw new IllegalArgumentException("Unknown container format: $req.format") } - protected void saveLayersToDockerContext0(BuildRequest request, Path contextDir) { + protected void saveLayersToDockerContext0(BuildRequest request, String s3Key) { final layers = request.containerConfig.layers for(int i=0; i { try (InputStream stream = streamService.stream(it.location, request.identity)) { - Files.copy(stream, target, StandardCopyOption.REPLACE_EXISTING) + objectStorageOperations.upload(UploadRequest.fromBytes(IOUtils.toByteArray(stream), target)) } return }) } } - protected void saveLayersToSingularityContext0(BuildRequest request, Path contextDir) { + protected void saveLayersToSingularityContext0(BuildRequest request, String s3Key) { final layers = request.containerConfig.layers for(int i=0; i { try (InputStream stream = streamService.stream(it.location, request.identity)) { - TarUtils.untarGzip(stream, target) + objectStorageOperations.upload(UploadRequest.fromBytes(TarGzipUtils.untarGzip(stream), target, "application/x-tar")) } return }) } } - protected void saveBuildContext(BuildContext buildContext, Path contextDir, PlatformId identity) { + protected void saveBuildContext(BuildContext buildContext, String s3Key, PlatformId identity) { // retry strategy - final retryable = retry0("Unable to copy '${buildContext.location} to build context '${contextDir}'") + final retryable = retry0("Unable to copy '${buildContext.location} to build context '${s3Key}'") // copy the layer to the build context retryable.apply(()-> { try (InputStream stream = streamService.stream(buildContext.location, identity)) { - TarUtils.untarGzip(stream, contextDir) + objectStorageOperations.upload(UploadRequest.fromBytes(TarGzipUtils.untarGzip(stream), s3Key, "application/x-tar")) } return }) diff --git a/src/main/groovy/io/seqera/wave/service/builder/DockerBuildStrategy.groovy b/src/main/groovy/io/seqera/wave/service/builder/DockerBuildStrategy.groovy index 966f3e2aa..fe025b90c 100644 --- a/src/main/groovy/io/seqera/wave/service/builder/DockerBuildStrategy.groovy +++ b/src/main/groovy/io/seqera/wave/service/builder/DockerBuildStrategy.groovy @@ -18,26 +18,20 @@ package io.seqera.wave.service.builder -import java.nio.file.Files -import java.nio.file.Path import java.util.concurrent.TimeUnit -import groovy.json.JsonOutput import groovy.transform.CompileStatic import groovy.util.logging.Slf4j import io.micronaut.context.annotation.Value +import io.micronaut.objectstorage.ObjectStorageOperations +import io.micronaut.objectstorage.request.UploadRequest import io.seqera.wave.configuration.BuildConfig import io.seqera.wave.configuration.SpackConfig -import io.seqera.wave.core.ContainerPlatform import io.seqera.wave.core.RegistryProxyService -import io.seqera.wave.util.RegHelper import jakarta.inject.Inject +import jakarta.inject.Named import jakarta.inject.Singleton -import static java.nio.file.StandardOpenOption.CREATE -import static java.nio.file.StandardOpenOption.TRUNCATE_EXISTING -import static java.nio.file.StandardOpenOption.WRITE -import static java.nio.file.attribute.PosixFilePermission.OWNER_READ -import static java.nio.file.attribute.PosixFilePermission.OWNER_WRITE +import static io.seqera.wave.service.builder.BuildConstants.FUSION_PREFIX /** * Build a container image using a Docker CLI tool * @@ -60,40 +54,29 @@ class DockerBuildStrategy extends BuildStrategy { @Inject RegistryProxyService proxyService + @Inject + @Named('build-workspace') + private ObjectStorageOperations objectStorageOperations + @Override BuildResult build(BuildRequest req) { - Path configFile = null - // save docker config for creds - if( req.configJson ) { - configFile = req.workDir.resolve('config.json') - Files.write(configFile, JsonOutput.prettyPrint(req.configJson).bytes, CREATE, WRITE, TRUNCATE_EXISTING) - } - // save remote files for singularity - if( req.configJson && req.formatSingularity()) { - final remoteFile = req.workDir.resolve('singularity-remote.yaml') - final content = RegHelper.singularityRemoteFile(req.targetImage) - Files.write(remoteFile, content.bytes, CREATE, WRITE, TRUNCATE_EXISTING) - // set permissions 600 as required by Singularity - Files.setPosixFilePermissions(configFile, Set.of(OWNER_READ, OWNER_WRITE)) - Files.setPosixFilePermissions(remoteFile, Set.of(OWNER_READ, OWNER_WRITE)) - } - // command the docker build command - final buildCmd= buildCmd(req, configFile) + final buildCmd= buildCmd(req) log.debug "Build run command: ${buildCmd.join(' ')}" // save docker cli for debugging purpose if( debug ) { - Files.write(req.workDir.resolve('docker.sh'), - buildCmd.join(' ').bytes, - CREATE, WRITE, TRUNCATE_EXISTING) + objectStorageOperations.upload(UploadRequest.fromBytes(buildCmd.join(' ').bytes, "$req.s3Key/docker.sh".toString())) } - final proc = new ProcessBuilder() + final builder = new ProcessBuilder() .command(buildCmd) - .directory(req.workDir.toFile()) .redirectErrorStream(true) - .start() + //this is to run it in windows + .redirectError(ProcessBuilder.Redirect.INHERIT) + builder.redirectOutput(ProcessBuilder.Redirect.INHERIT) + + def proc = builder.start() final timeout = req.maxDuration ?: buildConfig.defaultTimeout final completed = proc.waitFor(timeout.toSeconds(), TimeUnit.SECONDS) @@ -107,40 +90,33 @@ class DockerBuildStrategy extends BuildStrategy { } } - protected List buildCmd(BuildRequest req, Path credsFile) { - final spack = req.isSpackBuild ? spackConfig : null - + protected List buildCmd(BuildRequest req) { final dockerCmd = req.formatDocker() - ? cmdForBuildkit( req.workDir, credsFile, spack, req.platform) - : cmdForSingularity( req.workDir, credsFile, spack, req.platform) + ? cmdForBuildkit( req) + : cmdForSingularity(req) return dockerCmd + launchCmd(req) } - protected List cmdForBuildkit(Path workDir, Path credsFile, SpackConfig spackConfig, ContainerPlatform platform ) { + protected List cmdForBuildkit(BuildRequest req) { //checkout the documentation here to know more about these options https://github.com/moby/buildkit/blob/master/docs/rootless.md#docker final wrapper = ['docker', 'run', '--rm', '--privileged', - '-v', "$workDir:$workDir".toString(), - '--entrypoint', - BUILDKIT_ENTRYPOINT] - - if( credsFile ) { - wrapper.add('-v') - wrapper.add("$credsFile:/home/user/.docker/config.json:ro".toString()) - } + '-e', + "AWS_ACCESS_KEY_ID=${System.getenv('AWS_ACCESS_KEY_ID')}".toString(), + '-e', + "AWS_SECRET_ACCESS_KEY=${System.getenv('AWS_SECRET_ACCESS_KEY')}".toString()] - if( spackConfig ) { - // secret file - wrapper.add('-v') - wrapper.add("${spackConfig.secretKeyFile}:${spackConfig.secretMountPath}:ro".toString()) + if( req.configJson ) { + wrapper.add('-e') + wrapper.add("DOCKER_CONFIG=$FUSION_PREFIX/$buildConfig.workspaceBucket/$req.s3Key".toString()) } - if( platform ) { + if( req.platform ) { wrapper.add('--platform') - wrapper.add(platform.toString()) + wrapper.add(req.platform.toString()) } // the container image to be used to build @@ -149,34 +125,22 @@ class DockerBuildStrategy extends BuildStrategy { return wrapper } - protected List cmdForSingularity(Path workDir, Path credsFile, SpackConfig spackConfig, ContainerPlatform platform) { + protected List cmdForSingularity(BuildRequest req) { final wrapper = ['docker', 'run', '--rm', '--privileged', - "--entrypoint", '', - '-v', "$workDir:$workDir".toString()] - - if( credsFile ) { - wrapper.add('-v') - wrapper.add("$credsFile:/root/.singularity/docker-config.json:ro".toString()) - // - wrapper.add('-v') - wrapper.add("${credsFile.resolveSibling('singularity-remote.yaml')}:/root/.singularity/remote.yaml:ro".toString()) - } - - if( spackConfig ) { - // secret file - wrapper.add('-v') - wrapper.add("${spackConfig.secretKeyFile}:${spackConfig.secretMountPath}:ro".toString()) - } + '-e', + "AWS_ACCESS_KEY_ID=${System.getenv('AWS_ACCESS_KEY_ID')}".toString(), + '-e', + "AWS_SECRET_ACCESS_KEY=${System.getenv('AWS_SECRET_ACCESS_KEY')}".toString()] - if( platform ) { + if( req.platform ) { wrapper.add('--platform') - wrapper.add(platform.toString()) + wrapper.add(req.platform.toString()) } - wrapper.add(buildConfig.singularityImage(platform)) + wrapper.add(buildConfig.singularityImage(req.platform)) return wrapper } } diff --git a/src/main/groovy/io/seqera/wave/service/builder/KubeBuildStrategy.groovy b/src/main/groovy/io/seqera/wave/service/builder/KubeBuildStrategy.groovy index 7ac6d20c5..a0e3f9288 100644 --- a/src/main/groovy/io/seqera/wave/service/builder/KubeBuildStrategy.groovy +++ b/src/main/groovy/io/seqera/wave/service/builder/KubeBuildStrategy.groovy @@ -18,10 +18,6 @@ package io.seqera.wave.service.builder -import java.nio.file.Files -import java.nio.file.Path - -import groovy.json.JsonOutput import groovy.transform.CompileStatic import groovy.util.logging.Slf4j import io.kubernetes.client.openapi.ApiException @@ -34,15 +30,9 @@ import io.seqera.wave.configuration.SpackConfig import io.seqera.wave.core.RegistryProxyService import io.seqera.wave.exception.BadRequestException import io.seqera.wave.service.k8s.K8sService -import io.seqera.wave.util.RegHelper import jakarta.inject.Inject import jakarta.inject.Singleton import static io.seqera.wave.util.K8sHelper.getSelectorLabel -import static java.nio.file.StandardOpenOption.CREATE -import static java.nio.file.StandardOpenOption.TRUNCATE_EXISTING -import static java.nio.file.StandardOpenOption.WRITE -import static java.nio.file.attribute.PosixFilePermission.OWNER_READ -import static java.nio.file.attribute.PosixFilePermission.OWNER_WRITE /** * Build a container image using running a K8s pod * @@ -77,30 +67,13 @@ class KubeBuildStrategy extends BuildStrategy { @Override BuildResult build(BuildRequest req) { - - Path configFile = null - if( req.configJson ) { - configFile = req.workDir.resolve('config.json') - Files.write(configFile, JsonOutput.prettyPrint(req.configJson).bytes, CREATE, WRITE, TRUNCATE_EXISTING) - } - // save remote files for singularity - if( req.configJson && req.formatSingularity()) { - final remoteFile = req.workDir.resolve('singularity-remote.yaml') - final content = RegHelper.singularityRemoteFile(req.targetImage) - Files.write(remoteFile, content.bytes, CREATE, WRITE, TRUNCATE_EXISTING) - // set permissions 600 as required by Singularity - Files.setPosixFilePermissions(configFile, Set.of(OWNER_READ, OWNER_WRITE)) - Files.setPosixFilePermissions(remoteFile, Set.of(OWNER_READ, OWNER_WRITE)) - } - try { final buildImage = getBuildImage(req) final buildCmd = launchCmd(req) final name = podName(req) final timeout = req.maxDuration ?: buildConfig.defaultTimeout final selector= getSelectorLabel(req.platform, nodeSelectorMap) - final spackCfg0 = req.isSpackBuild ? spackConfig : null - final pod = k8sService.buildContainer(name, buildImage, buildCmd, req.workDir, configFile, timeout, spackCfg0, selector) + final pod = k8sService.buildContainer(name, buildImage, buildCmd, req.s3Key, req.configJson, timeout, selector) final exitCode = k8sService.waitPodCompletion(pod, timeout.toMillis()) final stdout = k8sService.logsPod(pod) if( exitCode!=null ) { diff --git a/src/main/groovy/io/seqera/wave/service/k8s/K8sService.groovy b/src/main/groovy/io/seqera/wave/service/k8s/K8sService.groovy index 7a292dad1..57c98ecef 100644 --- a/src/main/groovy/io/seqera/wave/service/k8s/K8sService.groovy +++ b/src/main/groovy/io/seqera/wave/service/k8s/K8sService.groovy @@ -18,7 +18,6 @@ package io.seqera.wave.service.k8s -import java.nio.file.Path import java.time.Duration import io.kubernetes.client.openapi.models.V1Job @@ -26,7 +25,6 @@ import io.kubernetes.client.openapi.models.V1Pod import io.kubernetes.client.openapi.models.V1PodList import io.seqera.wave.configuration.BlobCacheConfig import io.seqera.wave.configuration.ScanConfig -import io.seqera.wave.configuration.SpackConfig /** * Defines Kubernetes operations * @@ -42,9 +40,9 @@ interface K8sService { void deletePod(String name) - V1Pod buildContainer(String name, String containerImage, List args, Path workDir, Path creds, Duration timeout, SpackConfig spackConfig, Map nodeSelector) + V1Pod buildContainer(String name, String containerImage, List args, String s3Key, String creds, Duration timeout, Map nodeSelector) - V1Pod scanContainer(String name, String containerImage, List args, Path workDir, Path creds, ScanConfig scanConfig, Map nodeSelector) + V1Pod scanContainer(String name, String containerImage, List args, String s3Key, String creds, ScanConfig scanConfig, Map nodeSelector) Integer waitPodCompletion(V1Pod pod, long timeout) diff --git a/src/main/groovy/io/seqera/wave/service/k8s/K8sServiceImpl.groovy b/src/main/groovy/io/seqera/wave/service/k8s/K8sServiceImpl.groovy index 7872cc097..27979d3db 100644 --- a/src/main/groovy/io/seqera/wave/service/k8s/K8sServiceImpl.groovy +++ b/src/main/groovy/io/seqera/wave/service/k8s/K8sServiceImpl.groovy @@ -46,12 +46,12 @@ import io.micronaut.core.annotation.Nullable import io.seqera.wave.configuration.BlobCacheConfig import io.seqera.wave.configuration.BuildConfig import io.seqera.wave.configuration.ScanConfig -import io.seqera.wave.configuration.SpackConfig import io.seqera.wave.core.ContainerPlatform import io.seqera.wave.service.scan.Trivy import jakarta.inject.Inject import jakarta.inject.Singleton -import static io.seqera.wave.service.builder.BuildStrategy.BUILDKIT_ENTRYPOINT +import static io.seqera.wave.service.builder.BuildConstants.BUILDKIT_ENTRYPOINT +import static io.seqera.wave.service.builder.BuildConstants.FUSION_PREFIX /** * implements the support for Kubernetes cluster * @@ -73,10 +73,6 @@ class K8sServiceImpl implements K8sService { @Nullable private String storageClaimName - @Value('${wave.build.k8s.storage.mountPath}') - @Nullable - private String storageMountPath - @Property(name='wave.build.k8s.labels') @Nullable private Map labels @@ -97,9 +93,6 @@ class K8sServiceImpl implements K8sService { @Nullable private String requestsMemory - @Inject - private SpackConfig spackConfig - @Inject private K8sClient k8sClient @@ -121,15 +114,9 @@ class K8sServiceImpl implements K8sService { */ @PostConstruct private void init() { - log.info "K8s build config: namespace=$namespace; service-account=$serviceAccount; node-selector=$nodeSelectorMap; cpus=$requestsCpu; memory=$requestsMemory; buildWorkspace=$buildConfig.buildWorkspace; storageClaimName=$storageClaimName; storageMountPath=$storageMountPath; " - if( storageClaimName && !storageMountPath ) - throw new IllegalArgumentException("Missing 'wave.build.k8s.storage.mountPath' configuration attribute") - if( storageMountPath ) { - if( !buildConfig.buildWorkspace ) - throw new IllegalArgumentException("Missing 'wave.build.workspace' configuration attribute") - if( !Path.of(buildConfig.buildWorkspace).startsWith(storageMountPath) ) - throw new IllegalArgumentException("Build workspace should be a sub-directory of 'wave.build.k8s.storage.mountPath' - offending value: '$buildConfig.buildWorkspace' - expected value: '$storageMountPath'") - } + log.info "K8s build config: namespace=$namespace; service-account=$serviceAccount; node-selector=$nodeSelectorMap; cpus=$requestsCpu; memory=$requestsMemory; buildWorkspaceBucket=$buildConfig.workspaceBucket;" + if( !buildConfig.workspaceBucket ) + throw new IllegalArgumentException("Missing 'wave.build.workspaceBucket' configuration attribute") // validate node selectors final platforms = nodeSelectorMap ?: Collections.emptyMap() for( Map.Entry it : platforms ) { @@ -225,96 +212,6 @@ class K8sServiceImpl implements K8sService { .readNamespacedPod(name, namespace, null) } - /** - * Create a volume mount for the build storage. - * - * @param workDir The path representing a container build context - * @param storageMountPath - * @return A {@link V1VolumeMount} representing the mount path for the build config - */ - protected V1VolumeMount mountBuildStorage(Path workDir, String storageMountPath, boolean readOnly) { - assert workDir, "K8s mount build storage is mandatory" - - final vol = new V1VolumeMount() - .name('build-data') - .mountPath(workDir.toString()) - .readOnly(readOnly) - - if( storageMountPath ) { - // check sub-path - final rel = Path.of(storageMountPath).relativize(workDir).toString() - if (rel) - vol.subPath(rel) - } - return vol - } - - /** - * Defines the volume for the container building shared context - * - * @param workDir The path where the container image build context is located - * @param claimName The claim name of the corresponding storage - * @return An instance of {@link V1Volume} representing the build storage volume - */ - protected V1Volume volumeBuildStorage(String mountPath, @Nullable String claimName) { - final vol= new V1Volume() - .name('build-data') - if( claimName ) { - vol.persistentVolumeClaim( new V1PersistentVolumeClaimVolumeSource().claimName(claimName) ) - } - else { - vol.hostPath( new V1HostPathVolumeSource().path(mountPath) ) - } - - return vol - } - - /** - * Defines the volume mount for the docker config - * - * @return A {@link V1VolumeMount} representing the docker config - */ - protected V1VolumeMount mountHostPath(Path filePath, String storageMountPath, String mountPath) { - final rel = Path.of(storageMountPath).relativize(filePath).toString() - if( !rel ) throw new IllegalStateException("Mount relative path cannot be empty") - return new V1VolumeMount() - .name('build-data') - .mountPath(mountPath) - .subPath(rel) - .readOnly(true) - } - - protected V1VolumeMount mountSpackCacheDir(Path spackCacheDir, String storageMountPath, String containerPath) { - final rel = Path.of(storageMountPath).relativize(spackCacheDir).toString() - if( !rel || rel.startsWith('../') ) - throw new IllegalArgumentException("Spack cacheDirectory '$spackCacheDir' must be a sub-directory of storage path '$storageMountPath'") - return new V1VolumeMount() - .name('build-data') - .mountPath(containerPath) - .subPath(rel) - } - - protected V1VolumeMount mountSpackSecretFile(Path secretFile, String storageMountPath, String containerPath) { - final rel = Path.of(storageMountPath).relativize(secretFile).toString() - if( !rel || rel.startsWith('../') ) - throw new IllegalArgumentException("Spack secretKeyFile '$secretFile' must be a sub-directory of storage path '$storageMountPath'") - return new V1VolumeMount() - .name('build-data') - .readOnly(true) - .mountPath(containerPath) - .subPath(rel) - } - - protected V1VolumeMount mountScanCacheDir(Path scanCacheDir, String storageMountPath) { - final rel = Path.of(storageMountPath).relativize(scanCacheDir).toString() - if( !rel || rel.startsWith('../') ) - throw new IllegalArgumentException("Container scan cacheDirectory '$scanCacheDir' must be a sub-directory of storage path '$storageMountPath'") - return new V1VolumeMount() - .name('build-data') - .mountPath( Trivy.CACHE_MOUNT_PATH ) - .subPath(rel) - } - /** * Create a container for container image building via buildkit * @@ -332,40 +229,31 @@ class K8sServiceImpl implements K8sService { * The {@link V1Pod} description the submitted pod */ @Override - V1Pod buildContainer(String name, String containerImage, List args, Path workDir, Path creds, Duration timeout, SpackConfig spackConfig, Map nodeSelector) { - final spec = buildSpec(name, containerImage, args, workDir, creds, timeout, spackConfig, nodeSelector) + V1Pod buildContainer(String name, String containerImage, List args, String workDir, String creds, Duration timeout, Map nodeSelector) { + final spec = buildSpec(name, containerImage, args, workDir, creds, timeout, nodeSelector) return k8sClient .coreV1Api() .createNamespacedPod(namespace, spec, null, null, null,null) } - V1Pod buildSpec(String name, String containerImage, List args, Path workDir, Path credsFile, Duration timeout, SpackConfig spackConfig, Map nodeSelector) { + V1Pod buildSpec(String name, String containerImage, List args, String s3Key, String credsFile, Duration timeout, Map nodeSelector) { // dirty dependency to avoid introducing another parameter final singularity = containerImage.contains('singularity') - // required volumes - final mounts = new ArrayList(5) - mounts.add(mountBuildStorage(workDir, storageMountPath, true)) - - final volumes = new ArrayList(5) - volumes.add(volumeBuildStorage(storageMountPath, storageClaimName)) + Map env = new HashMap() + addAWSCreds(env) if( credsFile ){ if( !singularity ) { - mounts.add(0, mountHostPath(credsFile, storageMountPath, '/home/user/.docker/config.json')) + env.put('DOCKER_CONFIG', "$FUSION_PREFIX/$buildConfig.workspaceBucket/$s3Key".toString()) } else { - final remoteFile = credsFile.resolveSibling('singularity-remote.yaml') - mounts.add(0, mountHostPath(credsFile, storageMountPath, '/root/.singularity/docker-config.json')) - mounts.add(1, mountHostPath(remoteFile, storageMountPath, '/root/.singularity/remote.yaml')) + env.put('DOCKER_CONFIG', "$FUSION_PREFIX/$buildConfig.workspaceBucket/$s3Key".toString()) + env.put('DOCKER_CONFIG', "$FUSION_PREFIX/$buildConfig.workspaceBucket/$s3Key".toString()) } } - if( spackConfig ) { - mounts.add(mountSpackSecretFile(spackConfig.secretKeyFile, storageMountPath, spackConfig.secretMountPath)) - } - V1PodBuilder builder = new V1PodBuilder() //metadata section @@ -383,7 +271,6 @@ class K8sServiceImpl implements K8sService { .withServiceAccount(serviceAccount) .withActiveDeadlineSeconds( timeout.toSeconds() ) .withRestartPolicy("Never") - .addAllToVolumes(volumes) final requests = new V1ResourceRequirements() @@ -392,25 +279,19 @@ class K8sServiceImpl implements K8sService { if( requestsMemory ) requests.putRequestsItem('memory', new Quantity(requestsMemory)) + //add https://github.com/nextflow-io/k8s-fuse-plugin + requests.limits(Map.of("nextflow.io/fuse", new Quantity("1"))) + // container section final container = new V1ContainerBuilder() .withName(name) .withImage(containerImage) - .withVolumeMounts(mounts) + .withEnv(toEnvList(env)) .withResources(requests) + .withArgs(args) if( singularity ) { - container - // use 'command' to override the entrypoint of the container - .withCommand(args) - .withNewSecurityContext().withPrivileged(true).endSecurityContext() - } else { - container - //required by buildkit rootless container - .withEnv(toEnvList(BUILDKIT_FLAGS)) - // buildCommand is to set entrypoint for buildkit - .withCommand(BUILDKIT_ENTRYPOINT) - .withArgs(args) + container.withNewSecurityContext().withPrivileged(true).endSecurityContext() } // spec section @@ -512,24 +393,19 @@ class K8sServiceImpl implements K8sService { } @Override - V1Pod scanContainer(String name, String containerImage, List args, Path workDir, Path creds, ScanConfig scanConfig, Map nodeSelector) { - final spec = scanSpec(name, containerImage, args, workDir, creds, scanConfig, nodeSelector) + V1Pod scanContainer(String name, String containerImage, List args, String s3Key, String creds, ScanConfig scanConfig, Map nodeSelector) { + final spec = scanSpec(name, containerImage, args, s3Key, creds, scanConfig, nodeSelector) return k8sClient .coreV1Api() .createNamespacedPod(namespace, spec, null, null, null,null) } - V1Pod scanSpec(String name, String containerImage, List args, Path workDir, Path credsFile, ScanConfig scanConfig, Map nodeSelector) { + V1Pod scanSpec(String name, String containerImage, List args, String s3Key, String creds, ScanConfig scanConfig, Map nodeSelector) { - final mounts = new ArrayList(5) - mounts.add(mountBuildStorage(workDir, storageMountPath, false)) - mounts.add(mountScanCacheDir(scanConfig.cacheDirectory, storageMountPath)) - - final volumes = new ArrayList(5) - volumes.add(volumeBuildStorage(storageMountPath, storageClaimName)) - - if( credsFile ){ - mounts.add(0, mountHostPath(credsFile, storageMountPath, Trivy.CONFIG_MOUNT_PATH)) + Map env = new HashMap() + addAWSCreds(env) + if( creds ){ + env.put('DOCKER_CONFIG', "$FUSION_PREFIX/$buildConfig.workspaceBucket/$s3Key".toString()) } V1PodBuilder builder = new V1PodBuilder() @@ -548,7 +424,6 @@ class K8sServiceImpl implements K8sService { .withServiceAccount(serviceAccount) .withActiveDeadlineSeconds( scanConfig.timeout.toSeconds() ) .withRestartPolicy("Never") - .addAllToVolumes(volumes) final requests = new V1ResourceRequirements() @@ -562,7 +437,7 @@ class K8sServiceImpl implements K8sService { .withName(name) .withImage(containerImage) .withArgs(args) - .withVolumeMounts(mounts) + .withEnv(toEnvList(env)) .withResources(requests) .endContainer() .endSpec() @@ -698,4 +573,9 @@ class K8sServiceImpl implements K8sService { } return latest } + + private void addAWSCreds(Map env) { + env.put('AWS_ACCESS_KEY_ID', "${System.getenv('AWS_ACCESS_KEY_ID')}".toString()) + env.put('AWS_SECRET_ACCESS_KEY', "${System.getenv('AWS_SECRET_ACCESS_KEY')}".toString()) + } } diff --git a/src/main/groovy/io/seqera/wave/service/scan/ContainerScanServiceImpl.groovy b/src/main/groovy/io/seqera/wave/service/scan/ContainerScanServiceImpl.groovy index a04ecc05c..0e40be47f 100644 --- a/src/main/groovy/io/seqera/wave/service/scan/ContainerScanServiceImpl.groovy +++ b/src/main/groovy/io/seqera/wave/service/scan/ContainerScanServiceImpl.groovy @@ -25,6 +25,7 @@ import java.util.concurrent.ExecutorService import groovy.transform.CompileStatic import groovy.util.logging.Slf4j import io.micronaut.context.annotation.Requires +import io.micronaut.objectstorage.ObjectStorageOperations import io.micronaut.runtime.event.annotation.EventListener import io.micronaut.scheduling.TaskExecutors import io.seqera.wave.configuration.ScanConfig @@ -67,6 +68,10 @@ class ContainerScanServiceImpl implements ContainerScanService { @Inject private CleanupStrategy cleanup + @Inject + @Named('build-workspace') + private ObjectStorageOperations objectStorageOperations + @EventListener void onBuildEvent(BuildEvent event) { try { @@ -109,9 +114,10 @@ class ContainerScanServiceImpl implements ContainerScanService { log.warn "Unable to launch the scan results for scan id: ${request.id} - cause: ${e.message}", e } finally{ - // cleanup build context + // cleanup scan workspace if( cleanup.shouldCleanup(scanResult?.isSucceeded() ? 0 : 1) ) - request.workDir?.deleteDir() + objectStorageOperations.delete(request.s3Key) + } return scanResult } diff --git a/src/main/groovy/io/seqera/wave/service/scan/DockerScanStrategy.groovy b/src/main/groovy/io/seqera/wave/service/scan/DockerScanStrategy.groovy index 0a909f889..031529601 100644 --- a/src/main/groovy/io/seqera/wave/service/scan/DockerScanStrategy.groovy +++ b/src/main/groovy/io/seqera/wave/service/scan/DockerScanStrategy.groovy @@ -18,21 +18,19 @@ package io.seqera.wave.service.scan -import java.nio.file.FileAlreadyExistsException -import java.nio.file.Files -import java.nio.file.Path import java.time.Instant -import groovy.json.JsonOutput import groovy.transform.CompileStatic import groovy.util.logging.Slf4j import io.micronaut.context.annotation.Requires +import io.micronaut.objectstorage.ObjectStorageOperations +import io.micronaut.objectstorage.request.UploadRequest +import io.seqera.wave.configuration.BuildConfig import io.seqera.wave.configuration.ScanConfig import jakarta.inject.Inject +import jakarta.inject.Named import jakarta.inject.Singleton -import static java.nio.file.StandardOpenOption.CREATE -import static java.nio.file.StandardOpenOption.TRUNCATE_EXISTING -import static java.nio.file.StandardOpenOption.WRITE +import static io.seqera.wave.service.builder.BuildConstants.FUSION_PREFIX /** * Implements ScanStrategy for Docker * @@ -47,6 +45,13 @@ class DockerScanStrategy extends ScanStrategy { @Inject private ScanConfig scanConfig + @Inject + BuildConfig buildConfig + + @Inject + @Named('build-workspace') + private ObjectStorageOperations objectStorageOperations + DockerScanStrategy(ScanConfig scanConfig) { this.scanConfig = scanConfig } @@ -57,25 +62,16 @@ class DockerScanStrategy extends ScanStrategy { final startTime = Instant.now() try { - // create the scan dir - try { - Files.createDirectory(req.workDir) - } - catch (FileAlreadyExistsException e) { - log.warn("Container scan directory already exists: $e") - } - // save the config file with docker auth credentials - Path configFile = null if( req.configJson ) { - configFile = req.workDir.resolve('config.json') - Files.write(configFile, JsonOutput.prettyPrint(req.configJson).bytes, CREATE, WRITE, TRUNCATE_EXISTING) + objectStorageOperations.upload(UploadRequest.fromBytes(req.configJson.bytes, "$req.s3Key/config.json".toString())) } - // outfile file name - final reportFile = req.workDir.resolve(Trivy.OUTPUT_FILE_NAME) + // create outfile file + objectStorageOperations.upload(UploadRequest.fromBytes(new byte[0], "$req.s3Key/$Trivy.OUTPUT_FILE_NAME".toString())) + final reportFile = "$FUSION_PREFIX/$buildConfig.workspaceBucket/$req.s3Key/$Trivy.OUTPUT_FILE_NAME" // create the launch command - final dockerCommand = dockerWrapper(req.workDir, configFile) + final dockerCommand = dockerWrapper(req) final trivyCommand = List.of(scanConfig.scanImage) + scanCommand(req.targetImage, reportFile, scanConfig) final command = dockerCommand + trivyCommand @@ -93,7 +89,9 @@ class DockerScanStrategy extends ScanStrategy { } else{ log.info("Container scan completed with id: ${req.id}") - return ScanResult.success(req, startTime, TrivyResultProcessor.process(reportFile.text)) + def scanReportFile = objectStorageOperations.retrieve("$req.s3Key/$Trivy.OUTPUT_FILE_NAME".toString()) + .map { it.toStreamedFile().inputStream.text }.get() + return ScanResult.success(req, startTime, TrivyResultProcessor.process(scanReportFile)) } } catch (Throwable e){ @@ -102,27 +100,30 @@ class DockerScanStrategy extends ScanStrategy { } } - protected List dockerWrapper(Path scanDir, Path credsFile) { + protected List dockerWrapper(ScanRequest req) { - final wrapper = ['docker','run', '--rm'] - - // scan work dir - wrapper.add('-w') - wrapper.add(scanDir.toString()) + final wrapper = ['docker', + 'run', + '--rm', + '--privileged', + '-e', + "AWS_ACCESS_KEY_ID=${System.getenv('AWS_ACCESS_KEY_ID')}".toString(), + '-e', + "AWS_SECRET_ACCESS_KEY=${System.getenv('AWS_SECRET_ACCESS_KEY')}".toString()] - wrapper.add('-v') - wrapper.add("$scanDir:$scanDir:rw".toString()) + // scan work dir + wrapper.add('-e') + wrapper.add("TRIVY_WORKSPACE_DIR=$FUSION_PREFIX/$buildConfig.workspaceBucket/$req.s3Key".toString()) // cache directory - wrapper.add('-v') - wrapper.add("${scanConfig.cacheDirectory}:${Trivy.CACHE_MOUNT_PATH}:rw".toString()) + wrapper.add('-e') + wrapper.add("TRIVY_CACHE_DIR=$FUSION_PREFIX/$buildConfig.workspaceBucket/.trivy-cache".toString()) - if(credsFile) { - wrapper.add('-v') - wrapper.add("${credsFile}:${Trivy.CONFIG_MOUNT_PATH}:ro".toString()) + if(req.configJson) { + wrapper.add('-e') + wrapper.add("DOCKER_CONFIG=$FUSION_PREFIX/$buildConfig.workspaceBucket/$req.s3Key".toString()) } - return wrapper } } diff --git a/src/main/groovy/io/seqera/wave/service/scan/KubeScanStrategy.groovy b/src/main/groovy/io/seqera/wave/service/scan/KubeScanStrategy.groovy index 2490f14ed..e73810e96 100644 --- a/src/main/groovy/io/seqera/wave/service/scan/KubeScanStrategy.groovy +++ b/src/main/groovy/io/seqera/wave/service/scan/KubeScanStrategy.groovy @@ -31,10 +31,16 @@ import io.kubernetes.client.openapi.ApiException import io.micronaut.context.annotation.Primary import io.micronaut.context.annotation.Property import io.micronaut.context.annotation.Requires +import io.micronaut.objectstorage.ObjectStorageOperations +import io.micronaut.objectstorage.request.UploadRequest +import io.seqera.wave.configuration.BuildConfig import io.seqera.wave.configuration.ScanConfig import io.seqera.wave.exception.BadRequestException import io.seqera.wave.service.k8s.K8sService +import jakarta.inject.Inject +import jakarta.inject.Named import jakarta.inject.Singleton +import static io.seqera.wave.service.builder.BuildConstants.FUSION_PREFIX import static io.seqera.wave.util.K8sHelper.getSelectorLabel import static java.nio.file.StandardOpenOption.CREATE import static java.nio.file.StandardOpenOption.TRUNCATE_EXISTING @@ -55,6 +61,13 @@ class KubeScanStrategy extends ScanStrategy { @Nullable private Map nodeSelectorMap + @Inject + BuildConfig buildConfig + + @Inject + @Named('build-workspace') + private ObjectStorageOperations objectStorageOperations + private final K8sService k8sService private final ScanConfig scanConfig @@ -71,30 +84,22 @@ class KubeScanStrategy extends ScanStrategy { final podName = "scan-${req.id}" try{ - // create the scan dir - try { - Files.createDirectory(req.workDir) - } - catch (FileAlreadyExistsException e) { - log.warn("Container scan directory already exists: $e") - } - // save the config file with docker auth credentials - Path configFile = null if( req.configJson ) { - configFile = req.workDir.resolve('config.json') - Files.write(configFile, JsonOutput.prettyPrint(req.configJson).bytes, CREATE, WRITE, TRUNCATE_EXISTING) + objectStorageOperations.upload(UploadRequest.fromBytes(req.configJson.bytes, "$req.s3Key/config.json".toString())) } - final reportFile = req.workDir.resolve(Trivy.OUTPUT_FILE_NAME) - + // outfile file name + final reportFile = "$FUSION_PREFIX/$buildConfig.workspaceBucket/$req.s3Key/$Trivy.OUTPUT_FILE_NAME" final trivyCommand = scanCommand(req.targetImage, reportFile, scanConfig) final selector= getSelectorLabel(req.platform, nodeSelectorMap) - final pod = k8sService.scanContainer(podName, scanConfig.scanImage, trivyCommand, req.workDir, configFile, scanConfig, selector) + final pod = k8sService.scanContainer(podName, scanConfig.scanImage, trivyCommand, req.s3Key, req.configJson, scanConfig, selector) final exitCode = k8sService.waitPodCompletion(pod, scanConfig.timeout.toMillis()) if( exitCode==0 ) { log.info("Container scan completed for id: ${req.id}") - return ScanResult.success(req, startTime, TrivyResultProcessor.process(reportFile.text)) + def scanReportFile = objectStorageOperations.retrieve("$req.s3Key/$Trivy.OUTPUT_FILE_NAME".toString()).get() + scanReportFile = scanReportFile ? scanReportFile as String : null + return ScanResult.success(req, startTime, TrivyResultProcessor.process(scanReportFile)) } else{ final stdout = k8sService.logsPod(pod) diff --git a/src/main/groovy/io/seqera/wave/service/scan/ScanRequest.groovy b/src/main/groovy/io/seqera/wave/service/scan/ScanRequest.groovy index b00a8011d..e8a1c3845 100644 --- a/src/main/groovy/io/seqera/wave/service/scan/ScanRequest.groovy +++ b/src/main/groovy/io/seqera/wave/service/scan/ScanRequest.groovy @@ -37,11 +37,11 @@ class ScanRequest { final String configJson final String targetImage final ContainerPlatform platform - final Path workDir + String s3Key static ScanRequest fromBuild(BuildRequest request) { final id = request.scanId - final workDir = request.workDir.resolveSibling("scan-${id}") - return new ScanRequest(id, request.buildId, request.configJson, request.targetImage, request.platform, workDir) + final s3Key = "workspace/scan-${id}" + return new ScanRequest(id, request.buildId, request.configJson, request.targetImage, request.platform, s3Key) } } diff --git a/src/main/groovy/io/seqera/wave/service/scan/ScanStrategy.groovy b/src/main/groovy/io/seqera/wave/service/scan/ScanStrategy.groovy index 3938af4ec..ffa883551 100644 --- a/src/main/groovy/io/seqera/wave/service/scan/ScanStrategy.groovy +++ b/src/main/groovy/io/seqera/wave/service/scan/ScanStrategy.groovy @@ -34,15 +34,16 @@ abstract class ScanStrategy { abstract ScanResult scanContainer(ScanRequest request) - protected List scanCommand(String targetImage, Path outputFile, ScanConfig config) { - def cmd = ['--quiet', + protected List scanCommand(String targetImage, String outputFile, ScanConfig config) { + def cmd = ['trivy', + '--quiet', 'image', '--timeout', "${config.timeout.toMinutes()}m".toString(), '--format', 'json', '--output', - outputFile.toString()] + outputFile] if( config.severity ) { cmd << '--severity' diff --git a/src/main/groovy/io/seqera/wave/util/TarGzipUtils.groovy b/src/main/groovy/io/seqera/wave/util/TarGzipUtils.groovy new file mode 100644 index 000000000..4ac28fad8 --- /dev/null +++ b/src/main/groovy/io/seqera/wave/util/TarGzipUtils.groovy @@ -0,0 +1,55 @@ +/* + * Wave, containers provisioning service + * Copyright (c) 2024, Seqera Labs + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package io.seqera.wave.util + +import org.apache.commons.compress.archivers.tar.TarArchiveEntry +import org.apache.commons.compress.archivers.tar.TarArchiveInputStream +import org.apache.commons.compress.compressors.gzip.GzipCompressorInputStream +/** + * Tar and Gzip utilities + * + * @author Munish Chouhan + */ +class TarGzipUtils { + + static byte[] untarGzip(final InputStream is) throws IOException { + try (GzipCompressorInputStream gzipStream = new GzipCompressorInputStream(is)) { + byte[] tarContent = untarToByteArray(gzipStream) + return tarContent; + } + } + + private static byte[] untarToByteArray(final InputStream is) throws IOException { + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + try (TarArchiveInputStream tarInputStream = new TarArchiveInputStream(is)) { + TarArchiveEntry entry; + byte[] buffer = new byte[1024]; + int count; + while ((entry = (TarArchiveEntry) tarInputStream.getNextEntry()) != null) { + if (!entry.isDirectory()) { + while ((count = tarInputStream.read(buffer)) != -1) { + baos.write(buffer, 0, count); + } + } + } + } + return baos.toByteArray(); + } + +} diff --git a/src/main/resources/application.yml b/src/main/resources/application.yml index 7c795d5f5..58a0bbd9e 100644 --- a/src/main/resources/application.yml +++ b/src/main/resources/application.yml @@ -66,8 +66,9 @@ wave: server: url: "${WAVE_SERVER_URL:`http://localhost:9090`}" build: - buildkit-image: "moby/buildkit:v0.14.1-rootless" - singularity-image: "quay.io/singularity/singularity:v3.11.4-slim" + workspace-bucket: "${WAVE_WORKSPACE_BUCKET}" + buildkit-image: "cr.seqera.io/public/wave/buildkit:ef67f15426f36b72" + singularity-image: "cr.seqera.io/public/wave/singularity:f3a5accae865288f" singularity-image-arm64: "quay.io/singularity/singularity:v3.11.4-slim-arm64" repo: "195996028523.dkr.ecr.eu-west-1.amazonaws.com/wave/build/dev" cache: "195996028523.dkr.ecr.eu-west-1.amazonaws.com/wave/build/cache" @@ -83,7 +84,7 @@ wave: multiplier: '1.75' scan: image: - name: aquasec/trivy:0.53.0 + name: cr.seqera.io/public/wave/trivy:17135cecea328cbe blobCache: s5cmdImage: cr.seqera.io/public/wave/s5cmd:v2.2.2 ---