From 1e5fb7f8c86d51880efe4d934d2104e795935757 Mon Sep 17 00:00:00 2001 From: ericzzzzzzz <102683393+ericzzzzzzz@users.noreply.github.com> Date: Sat, 24 Feb 2024 20:43:44 -0500 Subject: [PATCH] kind/feat: passing artifacts between tasks --- docs/artifacts.md | 365 +++++++++++++++--- docs/pipeline-api.md | 17 + docs/variables.md | 109 +++--- .../alpha/consume-artifacts-from-task.yaml | 60 +++ .../alpha/produce-consume-artifacts.yaml | 3 +- .../v1/taskruns/alpha/task-artifacts.yaml | 44 +++ internal/artifactref/artifactref.go | 17 +- .../sidecarlogresults/sidecarlogresults.go | 3 + .../sidecarlogresults_test.go | 65 ++++ pkg/apis/pipeline/paths.go | 2 + pkg/apis/pipeline/v1/artifact_types.go | 89 +++++ pkg/apis/pipeline/v1/artifact_types_test.go | 188 +++++++++ pkg/apis/pipeline/v1/openapi_generated.go | 28 +- pkg/apis/pipeline/v1/pipeline_validation.go | 24 ++ .../pipeline/v1/pipeline_validation_test.go | 78 ++++ pkg/apis/pipeline/v1/swagger.json | 12 + pkg/apis/pipeline/v1/task_validation.go | 11 +- pkg/apis/pipeline/v1/task_validation_test.go | 4 +- pkg/apis/pipeline/v1/taskrun_types.go | 5 + pkg/apis/pipeline/v1/zz_generated.deepcopy.go | 1 + .../pipeline/v1beta1/pipeline_validation.go | 24 ++ .../v1beta1/pipeline_validation_test.go | 78 ++++ pkg/apis/pipeline/v1beta1/task_validation.go | 12 +- .../pipeline/v1beta1/task_validation_test.go | 4 +- pkg/entrypoint/entrypointer.go | 34 +- pkg/entrypoint/entrypointer_test.go | 111 ++---- pkg/pod/pod.go | 11 +- pkg/pod/status.go | 68 +++- pkg/pod/status_test.go | 25 ++ pkg/reconciler/pipelinerun/pipelinerun.go | 7 + pkg/reconciler/pipelinerun/resources/apply.go | 33 ++ .../pipelinerun/resources/apply_test.go | 156 ++++++++ .../pipelinerun/resources/pipelinerunstate.go | 24 ++ .../resources/pipelinerunstate_test.go | 294 ++++++++++++++ pkg/reconciler/taskrun/resources/apply.go | 10 +- pkg/reconciler/taskrun/taskrun_test.go | 12 + pkg/result/result.go | 7 +- pkg/result/result_test.go | 6 +- test/artifacts_test.go | 4 +- 39 files changed, 1799 insertions(+), 246 deletions(-) create mode 100644 examples/v1/pipelineruns/alpha/consume-artifacts-from-task.yaml create mode 100644 examples/v1/taskruns/alpha/task-artifacts.yaml create mode 100644 pkg/apis/pipeline/v1/artifact_types_test.go diff --git a/docs/artifacts.md b/docs/artifacts.md index 233dec1c53a..5b58d8b0471 100644 --- a/docs/artifacts.md +++ b/docs/artifacts.md @@ -8,8 +8,9 @@ weight: 201 # Artifacts - [Overview](#overview) -- [Artifact Provenance Data](#passing-artifacts-between-steps) +- [Artifact Provenance Data](#artifact-provenance-data) - [Passing Artifacts between Steps](#passing-artifacts-between-steps) + - [Passing Artifacts between Tasks](#passing-artifacts-between-tasks) @@ -104,13 +105,59 @@ spec: EOF ``` +The content is written by the `Step` to a file `$(artifacts.path)`: + +```yaml +apiVersion: tekton.dev/v1 +kind: TaskRun +metadata: + generateName: step-artifacts- +spec: + taskSpec: + description: | + A simple task that populates artifacts to TaskRun stepState + steps: + - name: artifacts-producer + image: bash:latest + script: | + cat > $(artifacts.path) << EOF + { + "inputs":[ + { + "name":"source", + "values":[ + { + "uri":"pkg:github/package-url/purl-spec@244fd47e07d1004f0aed9c", + "digest":{ + "sha256":"b35cacccfdb1e24dc497d15d553891345fd155713ffe647c281c583269eaaae0" + } + } + ] + } + ], + "outputs":[ + { + "name":"image", + "values":[ + { + "uri":"pkg:oci/nginx:stable-alpine3.17-slim?repository_url=docker.io/library", + "digest":{ + "sha256":"df85b9e3983fe2ce20ef76ad675ecf435cc99fc9350adc54fa230bae8c32ce48", + "sha1":"95588b8f34c31eb7d62c92aaa4e6506639b06ef2" + } + } + ] + } + ] + } + EOF +``` + It is recommended to use [purl format](https://github.com/package-url/purl-spec/blob/master/PURL-SPECIFICATION.rst) for artifacts uri as shown in the example. ### Passing Artifacts between Steps You can pass artifacts from one step to the next using: - - Specific Artifact: `$(steps..inputs.)` or `$(steps..outputs.)` -- Default (First) Artifact: `$(steps..inputs)` or `$(steps..outputs)` (if is omitted) The example below shows how to access the previous' step artifacts from another step in the same task @@ -162,12 +209,12 @@ spec: - name: artifacts-consumer image: bash:latest script: | - echo $(steps.artifacts-producer.outputs) + echo $(steps.artifacts-producer.outputs.image) ``` -The resolved value of `$(steps..outputs.)` or `$(steps..outputs)` is the values of an artifact. For this example, -`$(steps.artifacts-producer.outputs)` is resolved to +The resolved value of `$(steps..outputs.)` is the values of an artifact. For this example, +`$(steps.artifacts-producer.outputs.image)` is resolved to ```json [ { @@ -182,62 +229,278 @@ The resolved value of `$(steps..outputs.)` or Upon resolution and execution of the `TaskRun`, the `Status` will look something like: -```yaml -"steps": [ + +```json +{ + "artifacts": { + "inputs": [ { - "container": "step-artifacts-producer", - "imageID": "docker.io/library/bash@sha256:5353512b79d2963e92a2b97d9cb52df72d32f94661aa825fcfa0aede73304743", - "inputs": [ + "name": "source", + "values": [ { - "name": "source", - "values": [ - { - "digest": { - "sha256": "b35cacccfdb1e24dc497d15d553891345fd155713ffe647c281c583269eaaae0" - }, - "uri":"pkg:github/package-url/purl-spec@244fd47e07d1004f0aed9c", - } - ] + "digest": { + "sha256": "b35cacccfdb1e24dc497d15d553891345fd155713ffe647c281c583269eaaae0" + }, + "uri": "pkg:github/package-url/purl-spec@244fd47e07d1004f0aed9c" } - ], - "name": "artifacts-producer", - "outputs": [ + ] + } + ], + "outputs": [ + { + "name": "image", + "values": [ { - "name": "image", - "values": [ - { - "digest": { - "sha1": "95588b8f34c31eb7d62c92aaa4e6506639b06ef2", - "sha256": "df85b9e3983fe2ce20ef76ad675ecf435cc99fc9350adc54fa230bae8c32ce48" - }, - "uri":"pkg:oci/nginx:stable-alpine3.17-slim?repository_url=docker.io/library", - } - ] + "digest": { + "sha1": "95588b8f34c31eb7d62c92aaa4e6506639b06ef2", + "sha256": "df85b9e3983fe2ce20ef76ad675ecf435cc99fc9350adc54fa230bae8c32ce48" + }, + "uri": "pkg:oci/nginx:stable-alpine3.17-slim?repository_url=docker.io/library" } - ], - "terminated": { - "containerID": "containerd://010f02d103d1db48531327a1fe09797c87c1d50b6a216892319b3af93e0f56e7", - "exitCode": 0, - "finishedAt": "2024-03-18T17:05:06Z", - "message": "...", - "reason": "Completed", - "startedAt": "2024-03-18T17:05:06Z" - }, - "terminationReason": "Completed" + ] + } + ] + }, + "steps": [ + { + "container": "step-artifacts-producer", + "imageID": "docker.io/library/bash@sha256:5353512b79d2963e92a2b97d9cb52df72d32f94661aa825fcfa0aede73304743", + "inputs": [ + { + "name": "source", + "values": [ + { + "digest": { + "sha256": "b35cacccfdb1e24dc497d15d553891345fd155713ffe647c281c583269eaaae0" + }, + "uri": "pkg:github/package-url/purl-spec@244fd47e07d1004f0aed9c" + } + ] + } + ], + "name": "artifacts-producer", + "outputs": [ + { + "name": "image", + "values": [ + { + "digest": { + "sha1": "95588b8f34c31eb7d62c92aaa4e6506639b06ef2", + "sha256": "df85b9e3983fe2ce20ef76ad675ecf435cc99fc9350adc54fa230bae8c32ce48" + }, + "uri": "pkg:oci/nginx:stable-alpine3.17-slim?repository_url=docker.io/library" + } + ] + } + ], + "terminated": { + "containerID": "containerd://010f02d103d1db48531327a1fe09797c87c1d50b6a216892319b3af93e0f56e7", + "exitCode": 0, + "finishedAt": "2024-03-18T17:05:06Z", + "message": "...", + "reason": "Completed", + "startedAt": "2024-03-18T17:05:06Z" + }, + "terminationReason": "Completed" + }, + { + "container": "step-artifacts-consumer", + "imageID": "docker.io/library/bash@sha256:5353512b79d2963e92a2b97d9cb52df72d32f94661aa825fcfa0aede73304743", + "name": "artifacts-consumer", + "terminated": { + "containerID": "containerd://42428aa7e5a507eba924239f213d185dd4bc0882b6f217a79e6792f7fec3586e", + "exitCode": 0, + "finishedAt": "2024-03-18T17:05:06Z", + "reason": "Completed", + "startedAt": "2024-03-18T17:05:06Z" }, + "terminationReason": "Completed" + } + ] +} + +``` + +### Passing Artifacts between Tasks +You can pass artifacts from one task to the another using: + +- Specific Artifact: `$(tasks..inputs.)` or `$(tasks..outputs.)` + +The example below shows how to access the previous' task artifacts from another task in a pipeline + +```yaml +apiVersion: tekton.dev/v1 +kind: PipelineRun +metadata: + generateName: pipelinerun-consume-tasks-artifacts +spec: + pipelineSpec: + tasks: + - name: produce-artifacts-task + taskSpec: + description: | + A simple task that produces artifacts + steps: + - name: produce-artifacts + image: bash:latest + script: | + #!/usr/bin/env bash + cat > $(artifacts.path) << EOF + { + "inputs":[ + { + "name":"input-artifacts", + "values":[ + { + "uri":"pkg:example.github.com/inputs", + "digest":{ + "sha256":"b35cacccfdb1e24dc497d15d553891345fd155713ffe647c281c583269eaaae0" + } + } + ] + } + ], + "outputs":[ + { + "name":"image", + "values":[ + { + "uri":"pkg:github/package-url/purl-spec@244fd47e07d1004f0aed9c", + "digest":{ + "sha256":"df85b9e3983fe2ce20ef76ad675ecf435cc99fc9350adc54fa230bae8c32ce48", + "sha1":"95588b8f34c31eb7d62c92aaa4e6506639b06ef2" + } + } + ] + } + ] + } + EOF + - name: consume-artifacts + runAfter: + - produce-artifacts-task + taskSpec: + steps: + - name: artifacts-consumer-python + image: python:latest + script: | + #!/usr/bin/env python3 + import json + data = json.loads('$(tasks.produce-artifacts-task.outputs.image)') + if data[0]['uri'] != "pkg:github/package-url/purl-spec@244fd47e07d1004f0aed9c": + exit(1) +``` + + +Similar to Step Artifacts. The resolved value of `$(tasks..outputs.)` is the values of an artifact. For this example, +`$(tasks.produce-artifacts-task.outputs.image)` is resolved to +```json +[ + { + "uri":"pkg:example.github.com/inputs", + "digest":{ + "sha256":"b35cacccfdb1e24dc497d15d553891345fd155713ffe647c281c583269eaaae0" + } + } +] +``` +Upon resolution and execution of the `TaskRun`, the `Status` will look something like: +```json +{ + "artifacts": { + "inputs": [ + { + "name": "input-artifacts", + "values": [ + { + "digest": { + "sha256": "b35cacccfdb1e24dc497d15d553891345fd155713ffe647c281c583269eaaae0" + }, + "uri": "pkg:example.github.com/inputs" + } + ] + } + ], + "outputs": [ + { + "name": "image", + "values": [ + { + "digest": { + "sha1": "95588b8f34c31eb7d62c92aaa4e6506639b06ef2", + "sha256": "df85b9e3983fe2ce20ef76ad675ecf435cc99fc9350adc54fa230bae8c32ce48" + }, + "uri": "pkg:github/package-url/purl-spec@244fd47e07d1004f0aed9c" + } + ] + } + ] + }, + "completionTime": "2024-05-28T14:10:58Z", + "conditions": [ { - "container": "step-artifacts-consumer", - "imageID": "docker.io/library/bash@sha256:5353512b79d2963e92a2b97d9cb52df72d32f94661aa825fcfa0aede73304743", - "name": "artifacts-consumer", + "lastTransitionTime": "2024-05-28T14:10:58Z", + "message": "All Steps have completed executing", + "reason": "Succeeded", + "status": "True", + "type": "Succeeded" + } + ], + "podName": "pipelinerun-consume-tasks-a41ee44e4f964e95adfd3aea417d52f90-pod", + "provenance": { + "featureFlags": { + "AwaitSidecarReadiness": true, + "Coschedule": "workspaces", + "DisableAffinityAssistant": false, + "DisableCredsInit": false, + "DisableInlineSpec": "", + "EnableAPIFields": "beta", + "EnableArtifacts": true, + "EnableCELInWhenExpression": false, + "EnableConciseResolverSyntax": false, + "EnableKeepPodOnCancel": false, + "EnableParamEnum": false, + "EnableProvenanceInStatus": true, + "EnableStepActions": true, + "EnableTektonOCIBundles": false, + "EnforceNonfalsifiability": "none", + "MaxResultSize": 4096, + "RequireGitSSHSecretKnownHosts": false, + "ResultExtractionMethod": "termination-message", + "RunningInEnvWithInjectedSidecars": true, + "ScopeWhenExpressionsToTask": false, + "SendCloudEventsForRuns": false, + "SetSecurityContext": false, + "VerificationNoMatchPolicy": "ignore" + } + }, + "startTime": "2024-05-28T14:10:48Z", + "steps": [ + { + "container": "step-produce-artifacts", + "imageID": "docker.io/library/bash@sha256:23f90212fd89e4c292d7b41386ef1a6ac2b8a02bbc6947680bfe184cbc1a2899", + "name": "produce-artifacts", "terminated": { - "containerID": "containerd://42428aa7e5a507eba924239f213d185dd4bc0882b6f217a79e6792f7fec3586e", + "containerID": "containerd://1291ce07b175a7897beee6ba62eaa1528427bacb1f76b31435eeba68828c445a", "exitCode": 0, - "finishedAt": "2024-03-18T17:05:06Z", + "finishedAt": "2024-05-28T14:10:57Z", + "message": "...", "reason": "Completed", - "startedAt": "2024-03-18T17:05:06Z" + "startedAt": "2024-05-28T14:10:57Z" }, "terminationReason": "Completed" } ], - -``` \ No newline at end of file + "taskSpec": { + "description": "A simple task that produces artifacts\n", + "steps": [ + { + "computeResources": {}, + "image": "bash:latest", + "name": "produce-artifacts", + "script": "#!/usr/bin/env bash\ncat > /tekton/artifacts/provenance.json << EOF\n{\n \"inputs\":[\n {\n \"name\":\"input-artifacts\",\n \"values\":[\n {\n \"uri\":\"pkg:example.github.com/inputs\",\n \"digest\":{\n \"sha256\":\"b35cacccfdb1e24dc497d15d553891345fd155713ffe647c281c583269eaaae0\"\n }\n }\n ]\n }\n ],\n \"outputs\":[\n {\n \"name\":\"image\",\n \"values\":[\n {\n \"uri\":\"pkg:github/package-url/purl-spec@244fd47e07d1004f0aed9c\",\n \"digest\":{\n \"sha256\":\"df85b9e3983fe2ce20ef76ad675ecf435cc99fc9350adc54fa230bae8c32ce48\",\n \"sha1\":\"95588b8f34c31eb7d62c92aaa4e6506639b06ef2\"\n }\n }\n ]\n }\n ]\n}\nEOF\n" + } + ] + } +} +``` diff --git a/docs/pipeline-api.md b/docs/pipeline-api.md index bbe53542e1d..a2656ffaef2 100644 --- a/docs/pipeline-api.md +++ b/docs/pipeline-api.md @@ -1428,6 +1428,9 @@ string

Artifacts

+

+(Appears on:TaskRunStatusFields) +

Artifacts represents the collection of input and output artifacts associated with a task run or a similar process. Artifacts in this context are units of data or resources @@ -5902,6 +5905,20 @@ All TaskRunStatus stored in RetriesStatus will have no date within the RetriesSt +artifacts
+ + +Artifacts + + + + +(Optional) +

Artifacts are the list of artifacts written out by the task’s containers

+ + + + sidecars
diff --git a/docs/variables.md b/docs/variables.md index df3dadd818f..ce19e4acf12 100644 --- a/docs/variables.md +++ b/docs/variables.md @@ -15,63 +15,66 @@ For instructions on using variable substitutions see the relevant section of [th ## Variables available in a `Pipeline` -| Variable | Description | -| -------- | ----------- | -| `params.` | The value of the parameter at runtime. | -| `params['']` | (see above) | -| `params[""]` | (see above) | -| `params.[*]` | Get the whole param array or object.| -| `params[''][*]` | (see above) | -| `params[""][*]` | (see above) | -| `params.[i]` | Get the i-th element of param array. This is alpha feature, set `enable-api-fields` to `alpha` to use it.| -| `params[''][i]` | (see above) | -| `params[""][i]` | (see above) | -| `params.[*]` | Get the value of the whole object param. This is alpha feature, set `enable-api-fields` to `alpha` to use it.| -| `params..` | Get the value of an individual child of an object param. This is alpha feature, set `enable-api-fields` to `alpha` to use it. | -| `tasks..matrix.length` | The length of the `Matrix` combination count. | -| `tasks..results.` | The value of the `Task's` result. Can alter `Task` execution order within a `Pipeline`.) | -| `tasks..results.[i]` | The ith value of the `Task's` array result. Can alter `Task` execution order within a `Pipeline`.) | -| `tasks..results.[*]` | The array value of the `Task's` result. Can alter `Task` execution order within a `Pipeline`. Cannot be used in `script`.) | -| `tasks..results..key` | The `key` value of the `Task's` object result. Can alter `Task` execution order within a `Pipeline`.) | -| `tasks..matrix..length` | The length of the matrixed `Task's` results. (Can alter `Task` execution order within a `Pipeline`.) | -| `workspaces..bound` | Whether a `Workspace` has been bound or not. "false" if the `Workspace` declaration has `optional: true` and the Workspace binding was omitted by the PipelineRun. | -| `context.pipelineRun.name` | The name of the `PipelineRun` that this `Pipeline` is running in. | -| `context.pipelineRun.namespace` | The namespace of the `PipelineRun` that this `Pipeline` is running in. | -| `context.pipelineRun.uid` | The uid of the `PipelineRun` that this `Pipeline` is running in. | -| `context.pipeline.name` | The name of this `Pipeline` . | -| `tasks..status` | The execution status of the specified `pipelineTask`, only available in `finally` tasks. The execution status can be set to any one of the values (`Succeeded`, `Failed`, or `None`) described [here](pipelines.md#using-execution-status-of-pipelinetask)| -| `tasks.status` | An aggregate status of all the `pipelineTasks` under the `tasks` section (excluding the `finally` section). This variable is only available in the `finally` tasks and can have any one of the values (`Succeeded`, `Failed`, `Completed`, or `None`) described [here](pipelines.md#using-aggregate-execution-status-of-all-tasks). | -| `context.pipelineTask.retries` | The retries of this `PipelineTask`. | +| Variable | Description | +|----------------------------------------------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| `params.` | The value of the parameter at runtime. | +| `params['']` | (see above) | +| `params[""]` | (see above) | +| `params.[*]` | Get the whole param array or object. | +| `params[''][*]` | (see above) | +| `params[""][*]` | (see above) | +| `params.[i]` | Get the i-th element of param array. This is alpha feature, set `enable-api-fields` to `alpha` to use it. | +| `params[''][i]` | (see above) | +| `params[""][i]` | (see above) | +| `params.[*]` | Get the value of the whole object param. This is alpha feature, set `enable-api-fields` to `alpha` to use it. | +| `params..` | Get the value of an individual child of an object param. This is alpha feature, set `enable-api-fields` to `alpha` to use it. | +| `tasks..matrix.length` | The length of the `Matrix` combination count. | +| `tasks..results.` | The value of the `Task's` result. Can alter `Task` execution order within a `Pipeline`.) | +| `tasks..results.[i]` | The ith value of the `Task's` array result. Can alter `Task` execution order within a `Pipeline`.) | +| `tasks..results.[*]` | The array value of the `Task's` result. Can alter `Task` execution order within a `Pipeline`. Cannot be used in `script`.) | +| `tasks..results..key` | The `key` value of the `Task's` object result. Can alter `Task` execution order within a `Pipeline`.) | +| `tasks..matrix..length` | The length of the matrixed `Task's` results. (Can alter `Task` execution order within a `Pipeline`.) | +| `workspaces..bound` | Whether a `Workspace` has been bound or not. "false" if the `Workspace` declaration has `optional: true` and the Workspace binding was omitted by the PipelineRun. | +| `context.pipelineRun.name` | The name of the `PipelineRun` that this `Pipeline` is running in. | +| `context.pipelineRun.namespace` | The namespace of the `PipelineRun` that this `Pipeline` is running in. | +| `context.pipelineRun.uid` | The uid of the `PipelineRun` that this `Pipeline` is running in. | +| `context.pipeline.name` | The name of this `Pipeline` . | +| `tasks..status` | The execution status of the specified `pipelineTask`, only available in `finally` tasks. The execution status can be set to any one of the values (`Succeeded`, `Failed`, or `None`) described [here](pipelines.md#using-execution-status-of-pipelinetask) | +| `tasks.status` | An aggregate status of all the `pipelineTasks` under the `tasks` section (excluding the `finally` section). This variable is only available in the `finally` tasks and can have any one of the values (`Succeeded`, `Failed`, `Completed`, or `None`) described [here](pipelines.md#using-aggregate-execution-status-of-all-tasks). | +| `context.pipelineTask.retries` | The retries of this `PipelineTask`. | +| `tasks..outputs.` | The value of a specific output artifact of the `Task` | +| `tasks..inputs.` | The value of a specific input artifact of the `Task` | ## Variables available in a `Task` -| Variable | Description | -| -------- | ----------- | -| `params.` | The value of the parameter at runtime. | -| `params['']` | (see above) | -| `params[""]` | (see above) | -| `params.[*]` | Get the whole param array or object.| -| `params[''][*]` | (see above) | -| `params[""][*]` | (see above) | -| `params.[i]` | Get the i-th element of param array. This is alpha feature, set `enable-api-fields` to `alpha` to use it.| -| `params[''][i]` | (see above) | -| `params[""][i]` | (see above) | +| Variable | Description | +|----------------------------------------------------|--------------------------------------------------------------------------------------------------------------------------------| +| `params.` | The value of the parameter at runtime. | +| `params['']` | (see above) | +| `params[""]` | (see above) | +| `params.[*]` | Get the whole param array or object. | +| `params[''][*]` | (see above) | +| `params[""][*]` | (see above) | +| `params.[i]` | Get the i-th element of param array. This is alpha feature, set `enable-api-fields` to `alpha` to use it. | +| `params[''][i]` | (see above) | +| `params[""][i]` | (see above) | | `params..` | Get the value of an individual child of an object param. This is alpha feature, set `enable-api-fields` to `alpha` to use it. | -| `results..path` | The path to the file where the `Task` writes its results data. | -| `results[''].path` | (see above) | -| `results[""].path` | (see above) | -| `workspaces..path` | The path to the mounted `Workspace`. Empty string if an optional `Workspace` has not been provided by the TaskRun. | -| `workspaces..bound` | Whether a `Workspace` has been bound or not. "false" if an optional`Workspace` has not been provided by the TaskRun. | -| `workspaces..claim` | The name of the `PersistentVolumeClaim` specified as a volume source for the `Workspace`. Empty string for other volume types. | -| `workspaces..volume` | The name of the volume populating the `Workspace`. | -| `credentials.path` | The path to credentials injected from Secrets with matching annotations. | -| `context.taskRun.name` | The name of the `TaskRun` that this `Task` is running in. | -| `context.taskRun.namespace` | The namespace of the `TaskRun` that this `Task` is running in. | -| `context.taskRun.uid` | The uid of the `TaskRun` that this `Task` is running in. | -| `context.task.name` | The name of this `Task`. | -| `context.task.retry-count` | The current retry number of this `Task`. | -| `steps.step-.exitCode.path` | The path to the file where a Step's exit code is stored. | -| `steps.step-unnamed-.exitCode.path` | The path to the file where a Step's exit code is stored for a step without any name. | +| `results..path` | The path to the file where the `Task` writes its results data. | +| `results[''].path` | (see above) | +| `results[""].path` | (see above) | +| `workspaces..path` | The path to the mounted `Workspace`. Empty string if an optional `Workspace` has not been provided by the TaskRun. | +| `workspaces..bound` | Whether a `Workspace` has been bound or not. "false" if an optional`Workspace` has not been provided by the TaskRun. | +| `workspaces..claim` | The name of the `PersistentVolumeClaim` specified as a volume source for the `Workspace`. Empty string for other volume types. | +| `workspaces..volume` | The name of the volume populating the `Workspace`. | +| `credentials.path` | The path to credentials injected from Secrets with matching annotations. | +| `context.taskRun.name` | The name of the `TaskRun` that this `Task` is running in. | +| `context.taskRun.namespace` | The namespace of the `TaskRun` that this `Task` is running in. | +| `context.taskRun.uid` | The uid of the `TaskRun` that this `Task` is running in. | +| `context.task.name` | The name of this `Task`. | +| `context.task.retry-count` | The current retry number of this `Task`. | +| `steps.step-.exitCode.path` | The path to the file where a Step's exit code is stored. | +| `steps.step-unnamed-.exitCode.path` | The path to the file where a Step's exit code is stored for a step without any name. | +| `artifacts.path` | The path to the file where the `Task` writes its artifacts data. | ## Fields that accept variable substitutions diff --git a/examples/v1/pipelineruns/alpha/consume-artifacts-from-task.yaml b/examples/v1/pipelineruns/alpha/consume-artifacts-from-task.yaml new file mode 100644 index 00000000000..6ca60999c7f --- /dev/null +++ b/examples/v1/pipelineruns/alpha/consume-artifacts-from-task.yaml @@ -0,0 +1,60 @@ +apiVersion: tekton.dev/v1 +kind: PipelineRun +metadata: + generateName: pipelinerun-consume-tasks-artifacts +spec: + pipelineSpec: + tasks: + - name: produce-artifacts-task + taskSpec: + description: | + A simple task that produces artifacts + steps: + - name: produce-artifacts + image: bash:latest + script: | + #!/usr/bin/env bash + cat > $(artifacts.path) << EOF + { + "inputs":[ + { + "name":"input-artifacts", + "values":[ + { + "uri":"pkg:example.github.com/inputs", + "digest":{ + "sha256":"b35cacccfdb1e24dc497d15d553891345fd155713ffe647c281c583269eaaae0" + } + } + ] + } + ], + "outputs":[ + { + "name":"image", + "values":[ + { + "uri":"pkg:github/package-url/purl-spec@244fd47e07d1004f0aed9c", + "digest":{ + "sha256":"df85b9e3983fe2ce20ef76ad675ecf435cc99fc9350adc54fa230bae8c32ce48", + "sha1":"95588b8f34c31eb7d62c92aaa4e6506639b06ef2" + } + } + ] + } + ] + } + EOF + - name: consume-artifacts + runAfter: + - produce-artifacts-task + taskSpec: + steps: + - name: artifacts-consumer-python + image: python:latest + script: | + #!/usr/bin/env python3 + import json + data = json.loads('$(tasks.produce-artifacts-task.outputs.image)') + if data[0]['uri'] != "pkg:github/package-url/purl-spec@244fd47e07d1004f0aed9c": + exit(1) \ No newline at end of file diff --git a/examples/v1/taskruns/alpha/produce-consume-artifacts.yaml b/examples/v1/taskruns/alpha/produce-consume-artifacts.yaml index c0018f59484..cca1f84fd1e 100644 --- a/examples/v1/taskruns/alpha/produce-consume-artifacts.yaml +++ b/examples/v1/taskruns/alpha/produce-consume-artifacts.yaml @@ -44,12 +44,11 @@ spec: - name: artifacts-consumer image: docker.io/library/bash:latest script: | - echo $(steps.artifacts-producer.outputs) echo $(steps.artifacts-producer.inputs.input-artifacts) - name: artifacts-consumer-python image: docker.io/library/python:latest script: | #!/usr/bin/env python3 import json - data = json.loads('$(steps.artifacts-producer.outputs)') + data = json.loads('$(steps.artifacts-producer.outputs.image)') print(data[0]['uri']) diff --git a/examples/v1/taskruns/alpha/task-artifacts.yaml b/examples/v1/taskruns/alpha/task-artifacts.yaml new file mode 100644 index 00000000000..7357bdf245f --- /dev/null +++ b/examples/v1/taskruns/alpha/task-artifacts.yaml @@ -0,0 +1,44 @@ +apiVersion: tekton.dev/v1 +kind: TaskRun +metadata: + generateName: task-run-artifacts +spec: + taskSpec: + description: | + A simple task that produces artifacts + steps: + - name: produce-artifacts + image: bash:latest + script: | + #!/usr/bin/env bash + cat > $(artifacts.path) << EOF + { + "inputs":[ + { + "name":"input-artifacts", + "values":[ + { + "uri":"pkg:example.github.com/inputs", + "digest":{ + "sha256":"b35cacccfdb1e24dc497d15d553891345fd155713ffe647c281c583269eaaae0" + } + } + ] + } + ], + "outputs":[ + { + "name":"image", + "values":[ + { + "uri":"pkg:github/package-url/purl-spec@244fd47e07d1004f0aed9c", + "digest":{ + "sha256":"df85b9e3983fe2ce20ef76ad675ecf435cc99fc9350adc54fa230bae8c32ce48", + "sha1":"95588b8f34c31eb7d62c92aaa4e6506639b06ef2" + } + } + ] + } + ] + } + EOF \ No newline at end of file diff --git a/internal/artifactref/artifactref.go b/internal/artifactref/artifactref.go index b5a46a73023..cd8e1162ce3 100644 --- a/internal/artifactref/artifactref.go +++ b/internal/artifactref/artifactref.go @@ -2,10 +2,17 @@ package artifactref import "regexp" -// case 1: steps..inputs -// case 2: steps..outputs -// case 3: steps..inputs. -// case 4: steps..outputs. -const stepArtifactUsagePattern = `\$\(steps\.([^.]+)\.(?:inputs|outputs)(?:\.([^.^\)]+))?\)` +// case 1: steps..inputs. +// case 2: steps..outputs. +const stepArtifactUsagePattern = `\$\(steps\.([^.]+)\.(?:inputs|outputs)\.([^.)]+)\)` + +// case 1: tasks..inputs. +// case 2: tasks..outputs. +const taskArtifactUsagePattern = `\$\(tasks\.([^.]+)\.(?:inputs|outputs)\.([^.)]+)\)` + +const StepArtifactPathPattern = `step.artifacts.path` + +const TaskArtifactPathPattern = `artifacts.path` var StepArtifactRegex = regexp.MustCompile(stepArtifactUsagePattern) +var TaskArtifactRegex = regexp.MustCompile(taskArtifactUsagePattern) diff --git a/internal/sidecarlogresults/sidecarlogresults.go b/internal/sidecarlogresults/sidecarlogresults.go index 03b5d62fea1..823fe448075 100644 --- a/internal/sidecarlogresults/sidecarlogresults.go +++ b/internal/sidecarlogresults/sidecarlogresults.go @@ -49,6 +49,7 @@ const ( stepResultType SidecarLogResultType = "step" stepArtifactType SidecarLogResultType = "stepArtifact" + taskArtifactType SidecarLogResultType = "taskArtifact" sidecarResultNameSeparator string = "." ) @@ -285,6 +286,8 @@ func parseResults(resultBytes []byte, maxResultLimit int) (result.RunResult, err resultType = result.StepResultType case stepArtifactType: resultType = result.StepArtifactsResultType + case taskArtifactType: + resultType = result.TaskRunArtifactsResultType default: return result.RunResult{}, fmt.Errorf("invalid sidecar result type %v. Must be %v or %v or %v", res.Type, taskResultType, stepResultType, stepArtifactType) } diff --git a/internal/sidecarlogresults/sidecarlogresults_test.go b/internal/sidecarlogresults/sidecarlogresults_test.go index ffe2660c773..bad265cb35d 100644 --- a/internal/sidecarlogresults/sidecarlogresults_test.go +++ b/internal/sidecarlogresults/sidecarlogresults_test.go @@ -338,6 +338,39 @@ func TestParseResults(t *testing.T) { }`, Type: "stepArtifact", }, + { + Name: "task-run-artifacts-result", + Value: `{ + "inputs":[ + { + "name":"input-artifacts", + "values":[ + { + "uri":"pkg:example.github.com/inputs", + "digest":{ + "sha256":"b35cacccfdb1e24dc497d15d553891345fd155713ffe647c281c583269eaaae0" + } + } + ] + } + ], + "outputs":[ + { + "name":"image", + "values":[ + { + "uri":"pkg:github/package-url/purl-spec@244fd47e07d1004f0aed9c", + "digest":{ + "sha256":"df85b9e3983fe2ce20ef76ad675ecf435cc99fc9350adc54fa230bae8c32ce48", + "sha1":"95588b8f34c31eb7d62c92aaa4e6506639b06ef2" + } + } + ] + } + ] + }`, + Type: "taskArtifact", + }, } podLogs := []string{} for _, r := range results { @@ -400,6 +433,38 @@ func TestParseResults(t *testing.T) { ] }`, ResultType: result.StepArtifactsResultType, + }, { + Key: "task-run-artifacts-result", + Value: `{ + "inputs":[ + { + "name":"input-artifacts", + "values":[ + { + "uri":"pkg:example.github.com/inputs", + "digest":{ + "sha256":"b35cacccfdb1e24dc497d15d553891345fd155713ffe647c281c583269eaaae0" + } + } + ] + } + ], + "outputs":[ + { + "name":"image", + "values":[ + { + "uri":"pkg:github/package-url/purl-spec@244fd47e07d1004f0aed9c", + "digest":{ + "sha256":"df85b9e3983fe2ce20ef76ad675ecf435cc99fc9350adc54fa230bae8c32ce48", + "sha1":"95588b8f34c31eb7d62c92aaa4e6506639b06ef2" + } + } + ] + } + ] + }`, + ResultType: result.TaskRunArtifactsResultType, }} stepResults := []result.RunResult{} for _, plog := range podLogs { diff --git a/pkg/apis/pipeline/paths.go b/pkg/apis/pipeline/paths.go index fb2b3bcf87c..efb28ea9025 100644 --- a/pkg/apis/pipeline/paths.go +++ b/pkg/apis/pipeline/paths.go @@ -30,4 +30,6 @@ const ( StepsDir = "/tekton/steps" ScriptDir = "/tekton/scripts" + + ArtifactsDir = "/tekton/artifacts" ) diff --git a/pkg/apis/pipeline/v1/artifact_types.go b/pkg/apis/pipeline/v1/artifact_types.go index 07e43ebe171..1cd898d2ef5 100644 --- a/pkg/apis/pipeline/v1/artifact_types.go +++ b/pkg/apis/pipeline/v1/artifact_types.go @@ -1,5 +1,25 @@ +/* +Copyright 2024 The Tekton Authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + package v1 +import ( + "github.com/google/go-cmp/cmp" +) + // Algorithm Standard cryptographic hash algorithm type Algorithm string @@ -27,3 +47,72 @@ type Artifacts struct { Inputs []Artifact `json:"inputs,omitempty"` Outputs []Artifact `json:"outputs,omitempty"` } + +func (a *Artifacts) Merge(another Artifacts) { + inputMap := make(map[string][]ArtifactValue) + var newInputs []Artifact + + for _, v := range a.Inputs { + inputMap[v.Name] = v.Values + } + + for _, v := range another.Inputs { + _, ok := inputMap[v.Name] + if !ok { + inputMap[v.Name] = []ArtifactValue{} + } + for _, vv := range v.Values { + exists := false + for _, av := range inputMap[v.Name] { + if cmp.Equal(vv, av) { + exists = true + break + } + } + if !exists { + inputMap[v.Name] = append(inputMap[v.Name], vv) + } + } + } + + for k, v := range inputMap { + newInputs = append(newInputs, Artifact{ + Name: k, + Values: v, + }) + } + + outputMap := make(map[string][]ArtifactValue) + var newOutputs []Artifact + for _, v := range a.Outputs { + outputMap[v.Name] = v.Values + } + + for _, v := range another.Outputs { + _, ok := outputMap[v.Name] + if !ok { + outputMap[v.Name] = []ArtifactValue{} + } + for _, vv := range v.Values { + exists := false + for _, av := range outputMap[v.Name] { + if cmp.Equal(vv, av) { + exists = true + break + } + } + if !exists { + outputMap[v.Name] = append(outputMap[v.Name], vv) + } + } + } + + for k, v := range outputMap { + newOutputs = append(newOutputs, Artifact{ + Name: k, + Values: v, + }) + } + a.Inputs = newInputs + a.Outputs = newOutputs +} diff --git a/pkg/apis/pipeline/v1/artifact_types_test.go b/pkg/apis/pipeline/v1/artifact_types_test.go new file mode 100644 index 00000000000..895244d6363 --- /dev/null +++ b/pkg/apis/pipeline/v1/artifact_types_test.go @@ -0,0 +1,188 @@ +/* +Copyright 2024 The Tekton Authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1 + +import ( + "testing" + + "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" + "github.com/tektoncd/pipeline/test/diff" +) + +func TestArtifactsMerge(t *testing.T) { + type testCase struct { + name string + a1 Artifacts + a2 Artifacts + expected Artifacts + } + + testCases := []testCase{ + { + name: "Merges inputs and outputs with deduplication", + a1: Artifacts{ + Inputs: []Artifact{ + { + Name: "input1", + Values: []ArtifactValue{ + { + Digest: map[Algorithm]string{"sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"}, + Uri: "pkg:maven/org.apache.commons/commons-lang3/3.12.0", + }, + }, + }, + { + Name: "input2", + Values: []ArtifactValue{ + { + Digest: map[Algorithm]string{"sha256": "d596377f2d54b3f8b4619f137d08892989893b886742759144582c94157526f1"}, + Uri: "pkg:pypi/requests/2.28.2", + }, + }, + }, + }, + Outputs: []Artifact{ + { + Name: "output1", + Values: []ArtifactValue{ + { + Digest: map[Algorithm]string{"sha256": "47de7a85905970a45132f48a9247879a15c483477e23a637504694e611135b40e"}, + Uri: "pkg:npm/lodash/4.17.21", + }, + }, + }, + }, + }, + a2: Artifacts{ + Inputs: []Artifact{ + { + Name: "input1", + Values: []ArtifactValue{ + { + Digest: map[Algorithm]string{"sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"}, + Uri: "pkg:maven/org.apache.commons/commons-lang3/3.12.0", + }, + { + Digest: map[Algorithm]string{"sha256": "97c13e1812b666824266111701398e56e30d14418a2d9b35987f516a66e2129f"}, + Uri: "pkg:nuget/Microsoft.Extensions.Logging/7.0.0", + }, + }, + }, + { + Name: "input3", + Values: []ArtifactValue{ + { + Digest: map[Algorithm]string{"sha256": "13c2b709e3a100726680e53e19666656a89a2f2490e917ba15d6b15475ab7b79"}, + Uri: "pkg:debian/openssl/1.1.1", + }, + }, + }, + }, + Outputs: []Artifact{ + { + Name: "output1", + Values: []ArtifactValue{ + { + Digest: map[Algorithm]string{"sha256": "698c4539633943f7889f41605003d7fa63833722ebd2b37c7e75df1d3d06941a"}, + Uri: "pkg:nuget/Newtonsoft.Json/13.0.3", + }, + }, + }, + { + Name: "output2", + Values: []ArtifactValue{ + { + Digest: map[Algorithm]string{"sha256": "7e406d83706c7193df3e38b66d350e55df6f13d2a28a1d35917a043533a70f5c"}, + Uri: "pkg:pypi/pandas/2.0.1", + }, + }, + }, + }, + }, + expected: Artifacts{ + Inputs: []Artifact{ + { + Name: "input1", + Values: []ArtifactValue{ + { + Digest: map[Algorithm]string{"sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"}, + Uri: "pkg:maven/org.apache.commons/commons-lang3/3.12.0", + }, + { + Digest: map[Algorithm]string{"sha256": "97c13e1812b666824266111701398e56e30d14418a2d9b35987f516a66e2129f"}, + Uri: "pkg:nuget/Microsoft.Extensions.Logging/7.0.0", + }, + }, + }, + { + Name: "input2", + Values: []ArtifactValue{ + { + Digest: map[Algorithm]string{"sha256": "d596377f2d54b3f8b4619f137d08892989893b886742759144582c94157526f1"}, + Uri: "pkg:pypi/requests/2.28.2", + }, + }, + }, + { + Name: "input3", + Values: []ArtifactValue{ + { + Digest: map[Algorithm]string{"sha256": "13c2b709e3a100726680e53e19666656a89a2f2490e917ba15d6b15475ab7b79"}, + Uri: "pkg:debian/openssl/1.1.1", + }, + }, + }, + }, + Outputs: []Artifact{ + { + Name: "output1", + Values: []ArtifactValue{ + { + Digest: map[Algorithm]string{"sha256": "47de7a85905970a45132f48a9247879a15c483477e23a637504694e611135b40e"}, + Uri: "pkg:npm/lodash/4.17.21", + }, + { + Digest: map[Algorithm]string{"sha256": "698c4539633943f7889f41605003d7fa63833722ebd2b37c7e75df1d3d06941a"}, + Uri: "pkg:nuget/Newtonsoft.Json/13.0.3", + }, + }, + }, + { + Name: "output2", + Values: []ArtifactValue{ + { + Digest: map[Algorithm]string{"sha256": "7e406d83706c7193df3e38b66d350e55df6f13d2a28a1d35917a043533a70f5c"}, + Uri: "pkg:pypi/pandas/2.0.1", + }, + }, + }, + }, + }, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + tc.a1.Merge(tc.a2) + got := tc.a1 + if d := cmp.Diff(tc.expected, got, cmpopts.SortSlices(func(a, b Artifact) bool { return a.Name > b.Name })); d != "" { + t.Errorf("TestArtifactsMerge() did not produce expected artifacts for test %s: %s", tc.name, diff.PrintWantGot(d)) + } + }) + } +} diff --git a/pkg/apis/pipeline/v1/openapi_generated.go b/pkg/apis/pipeline/v1/openapi_generated.go index e9664a7d32e..928f446c294 100644 --- a/pkg/apis/pipeline/v1/openapi_generated.go +++ b/pkg/apis/pipeline/v1/openapi_generated.go @@ -4186,6 +4186,18 @@ func schema_pkg_apis_pipeline_v1_TaskRunStatus(ref common.ReferenceCallback) com }, }, }, + "artifacts": { + VendorExtensible: spec.VendorExtensible{ + Extensions: spec.Extensions{ + "x-kubernetes-list-type": "atomic", + }, + }, + SchemaProps: spec.SchemaProps{ + Description: "Artifacts are the list of artifacts written out by the task's containers", + Default: map[string]interface{}{}, + Ref: ref("github.com/tektoncd/pipeline/pkg/apis/pipeline/v1.Artifacts"), + }, + }, "sidecars": { VendorExtensible: spec.VendorExtensible{ Extensions: spec.Extensions{ @@ -4238,7 +4250,7 @@ func schema_pkg_apis_pipeline_v1_TaskRunStatus(ref common.ReferenceCallback) com }, }, Dependencies: []string{ - "github.com/tektoncd/pipeline/pkg/apis/pipeline/v1.Provenance", "github.com/tektoncd/pipeline/pkg/apis/pipeline/v1.SidecarState", "github.com/tektoncd/pipeline/pkg/apis/pipeline/v1.StepState", "github.com/tektoncd/pipeline/pkg/apis/pipeline/v1.TaskRunResult", "github.com/tektoncd/pipeline/pkg/apis/pipeline/v1.TaskRunStatus", "github.com/tektoncd/pipeline/pkg/apis/pipeline/v1.TaskSpec", "k8s.io/apimachinery/pkg/apis/meta/v1.Time", "knative.dev/pkg/apis.Condition"}, + "github.com/tektoncd/pipeline/pkg/apis/pipeline/v1.Artifacts", "github.com/tektoncd/pipeline/pkg/apis/pipeline/v1.Provenance", "github.com/tektoncd/pipeline/pkg/apis/pipeline/v1.SidecarState", "github.com/tektoncd/pipeline/pkg/apis/pipeline/v1.StepState", "github.com/tektoncd/pipeline/pkg/apis/pipeline/v1.TaskRunResult", "github.com/tektoncd/pipeline/pkg/apis/pipeline/v1.TaskRunStatus", "github.com/tektoncd/pipeline/pkg/apis/pipeline/v1.TaskSpec", "k8s.io/apimachinery/pkg/apis/meta/v1.Time", "knative.dev/pkg/apis.Condition"}, } } @@ -4326,6 +4338,18 @@ func schema_pkg_apis_pipeline_v1_TaskRunStatusFields(ref common.ReferenceCallbac }, }, }, + "artifacts": { + VendorExtensible: spec.VendorExtensible{ + Extensions: spec.Extensions{ + "x-kubernetes-list-type": "atomic", + }, + }, + SchemaProps: spec.SchemaProps{ + Description: "Artifacts are the list of artifacts written out by the task's containers", + Default: map[string]interface{}{}, + Ref: ref("github.com/tektoncd/pipeline/pkg/apis/pipeline/v1.Artifacts"), + }, + }, "sidecars": { VendorExtensible: spec.VendorExtensible{ Extensions: spec.Extensions{ @@ -4378,7 +4402,7 @@ func schema_pkg_apis_pipeline_v1_TaskRunStatusFields(ref common.ReferenceCallbac }, }, Dependencies: []string{ - "github.com/tektoncd/pipeline/pkg/apis/pipeline/v1.Provenance", "github.com/tektoncd/pipeline/pkg/apis/pipeline/v1.SidecarState", "github.com/tektoncd/pipeline/pkg/apis/pipeline/v1.StepState", "github.com/tektoncd/pipeline/pkg/apis/pipeline/v1.TaskRunResult", "github.com/tektoncd/pipeline/pkg/apis/pipeline/v1.TaskRunStatus", "github.com/tektoncd/pipeline/pkg/apis/pipeline/v1.TaskSpec", "k8s.io/apimachinery/pkg/apis/meta/v1.Time"}, + "github.com/tektoncd/pipeline/pkg/apis/pipeline/v1.Artifacts", "github.com/tektoncd/pipeline/pkg/apis/pipeline/v1.Provenance", "github.com/tektoncd/pipeline/pkg/apis/pipeline/v1.SidecarState", "github.com/tektoncd/pipeline/pkg/apis/pipeline/v1.StepState", "github.com/tektoncd/pipeline/pkg/apis/pipeline/v1.TaskRunResult", "github.com/tektoncd/pipeline/pkg/apis/pipeline/v1.TaskRunStatus", "github.com/tektoncd/pipeline/pkg/apis/pipeline/v1.TaskSpec", "k8s.io/apimachinery/pkg/apis/meta/v1.Time"}, } } diff --git a/pkg/apis/pipeline/v1/pipeline_validation.go b/pkg/apis/pipeline/v1/pipeline_validation.go index 406634d1967..55b3d8248df 100644 --- a/pkg/apis/pipeline/v1/pipeline_validation.go +++ b/pkg/apis/pipeline/v1/pipeline_validation.go @@ -22,6 +22,7 @@ import ( "slices" "strings" + "github.com/tektoncd/pipeline/internal/artifactref" "github.com/tektoncd/pipeline/pkg/apis/config" "github.com/tektoncd/pipeline/pkg/apis/validate" "github.com/tektoncd/pipeline/pkg/internal/resultref" @@ -89,6 +90,7 @@ func (ps *PipelineSpec) Validate(ctx context.Context) (errs *apis.FieldError) { errs = errs.Also(validateTasksAndFinallySection(ps)) errs = errs.Also(validateFinalTasks(ps.Tasks, ps.Finally)) errs = errs.Also(validateWhenExpressions(ctx, ps.Tasks, ps.Finally)) + errs = errs.Also(validateArtifactReference(ctx, ps.Tasks, ps.Finally)) errs = errs.Also(validateMatrix(ctx, ps.Tasks).ViaField("tasks")) errs = errs.Also(validateMatrix(ctx, ps.Finally).ViaField("finally")) return errs @@ -886,6 +888,28 @@ func validateStringResults(results []TaskResult, resultName string) (errs *apis. return errs } +// validateArtifactReference ensure that the feature flag enableArtifacts is set to true when using artifacts +func validateArtifactReference(ctx context.Context, tasks []PipelineTask, finalTasks []PipelineTask) (errs *apis.FieldError) { + if config.FromContextOrDefaults(ctx).FeatureFlags.EnableArtifacts { + return errs + } + for i, t := range tasks { + for _, v := range t.Params.extractValues() { + if len(artifactref.TaskArtifactRegex.FindAllStringSubmatch(v, -1)) > 0 { + return errs.Also(apis.ErrGeneric(fmt.Sprintf("feature flag %s should be set to true to use artifacts feature.", config.EnableArtifacts), "").ViaField("params").ViaFieldIndex("tasks", i)) + } + } + } + for i, t := range finalTasks { + for _, v := range t.Params.extractValues() { + if len(artifactref.TaskArtifactRegex.FindAllStringSubmatch(v, -1)) > 0 { + return errs.Also(apis.ErrGeneric(fmt.Sprintf("feature flag %s should be set to true to use artifacts feature.", config.EnableArtifacts), "").ViaField("params").ViaFieldIndex("finally", i)) + } + } + } + return errs +} + // GetIndexingReferencesToArrayParams returns all strings referencing indices of PipelineRun array parameters // from parameters, workspaces, and when expressions defined in the Pipeline's Tasks and Finally Tasks. // For example, if a Task in the Pipeline has a parameter with a value "$(params.array-param-name[1])", diff --git a/pkg/apis/pipeline/v1/pipeline_validation_test.go b/pkg/apis/pipeline/v1/pipeline_validation_test.go index ad32a969abd..339edf28f42 100644 --- a/pkg/apis/pipeline/v1/pipeline_validation_test.go +++ b/pkg/apis/pipeline/v1/pipeline_validation_test.go @@ -298,6 +298,31 @@ func TestPipeline_Validate_Success(t *testing.T) { }}, }, }, + }, { + name: "valid pipeline with pipeline task and final task referencing artifacts in task params with enable-artifacts flag true", + p: &Pipeline{ + ObjectMeta: metav1.ObjectMeta{Name: "pipeline"}, + Spec: PipelineSpec{ + Description: "this is an invalid pipeline referencing artifacts with enable-artifacts flag false", + Tasks: []PipelineTask{{ + Name: "pre-task", + TaskRef: &TaskRef{Name: "foo-task"}, + }, { + Name: "consume-artifacts-task", + Params: Params{{Name: "aaa", Value: ParamValue{ + Type: ParamTypeString, + StringVal: "$(tasks.produce-artifacts-task.outputs.image)", + }}}, + TaskSpec: &EmbeddedTask{TaskSpec: getTaskSpec()}, + }}, + }, + }, + wc: func(ctx context.Context) context.Context { + return cfgtesting.SetFeatureFlags(ctx, t, + map[string]string{ + "enable-artifacts": "true", + "enable-api-fields": "alpha"}) + }, }} for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { @@ -1281,6 +1306,59 @@ func TestPipelineSpec_Validate_Failure(t *testing.T) { Message: `missing field(s)`, Paths: []string{"tasks[1].when[0]", "finally[0].when[0]"}, }, + }, { + name: "invalid pipeline with one pipeline task referencing artifacts in task params with enable-artifacts flag false", + ps: &PipelineSpec{ + Description: "this is an invalid pipeline referencing artifacts with enable-artifacts flag false", + Tasks: []PipelineTask{{ + Name: "pre-task", + TaskRef: &TaskRef{Name: "foo-task"}, + }, { + Name: "consume-artifacts-task", + Params: Params{{Name: "aaa", Value: ParamValue{ + Type: ParamTypeString, + StringVal: "$(tasks.produce-artifacts-task.outputs.image)", + }}}, + TaskSpec: &EmbeddedTask{TaskSpec: getTaskSpec()}, + }}, + }, + expectedError: apis.FieldError{ + Message: `feature flag enable-artifacts should be set to true to use artifacts feature.`, + Paths: []string{"tasks[1].params"}, + }, + wc: func(ctx context.Context) context.Context { + return cfgtesting.SetFeatureFlags(ctx, t, + map[string]string{ + "enable-artifacts": "false", + "enable-api-fields": "alpha"}) + }, + }, { + name: "invalid pipeline with one final pipeline task referencing artifacts in params with enable-artifacts flag false", + ps: &PipelineSpec{ + Description: "this is an invalid pipeline referencing artifacts with enable-artifacts flag false", + Tasks: []PipelineTask{{ + Name: "pre-task", + TaskRef: &TaskRef{Name: "foo-task"}, + }}, + Finally: []PipelineTask{{ + Name: "consume-artifacts-task", + Params: Params{{Name: "aaa", Value: ParamValue{ + Type: ParamTypeString, + StringVal: "$(tasks.produce-artifacts-task.outputs.image)", + }}}, + TaskSpec: &EmbeddedTask{TaskSpec: getTaskSpec()}, + }}, + }, + wc: func(ctx context.Context) context.Context { + return cfgtesting.SetFeatureFlags(ctx, t, + map[string]string{ + "enable-artifacts": "false", + "enable-api-fields": "alpha"}) + }, + expectedError: apis.FieldError{ + Message: `feature flag enable-artifacts should be set to true to use artifacts feature.`, + Paths: []string{"finally[0].params"}, + }, }} for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { diff --git a/pkg/apis/pipeline/v1/swagger.json b/pkg/apis/pipeline/v1/swagger.json index f4abdedd2ba..4ece3c37804 100644 --- a/pkg/apis/pipeline/v1/swagger.json +++ b/pkg/apis/pipeline/v1/swagger.json @@ -2111,6 +2111,12 @@ "default": "" } }, + "artifacts": { + "description": "Artifacts are the list of artifacts written out by the task's containers", + "default": {}, + "$ref": "#/definitions/v1.Artifacts", + "x-kubernetes-list-type": "atomic" + }, "completionTime": { "description": "CompletionTime is the time the build completed.", "$ref": "#/definitions/v1.Time" @@ -2200,6 +2206,12 @@ "podName" ], "properties": { + "artifacts": { + "description": "Artifacts are the list of artifacts written out by the task's containers", + "default": {}, + "$ref": "#/definitions/v1.Artifacts", + "x-kubernetes-list-type": "atomic" + }, "completionTime": { "description": "CompletionTime is the time the build completed.", "$ref": "#/definitions/v1.Time" diff --git a/pkg/apis/pipeline/v1/task_validation.go b/pkg/apis/pipeline/v1/task_validation.go index fa364d13b54..4232d295d2b 100644 --- a/pkg/apis/pipeline/v1/task_validation.go +++ b/pkg/apis/pipeline/v1/task_validation.go @@ -315,7 +315,10 @@ func errorIfStepResultReferenceinField(value, fieldName string) (errs *apis.Fiel } func stepArtifactReferenceExists(src string) bool { - return len(artifactref.StepArtifactRegex.FindAllStringSubmatch(src, -1)) > 0 || strings.Contains(src, "$(step.artifacts.path)") + return len(artifactref.StepArtifactRegex.FindAllStringSubmatch(src, -1)) > 0 || strings.Contains(src, "$("+artifactref.StepArtifactPathPattern+")") +} +func taskArtifactReferenceExists(src string) bool { + return len(artifactref.TaskArtifactRegex.FindAllStringSubmatch(src, -1)) > 0 || strings.Contains(src, "$("+artifactref.TaskArtifactPathPattern+")") } func errorIfStepArtifactReferencedInField(value, fieldName string) (errs *apis.FieldError) { if stepArtifactReferenceExists(value) { @@ -381,7 +384,7 @@ func validateStepResultReference(s Step) (errs *apis.FieldError) { } func validateStep(ctx context.Context, s Step, names sets.String) (errs *apis.FieldError) { - if err := validateStepArtifactsReferences(ctx, s); err != nil { + if err := validateArtifactsReferencesInStep(ctx, s); err != nil { return err } @@ -537,7 +540,7 @@ func validateStep(ctx context.Context, s Step, names sets.String) (errs *apis.Fi return errs } -func validateStepArtifactsReferences(ctx context.Context, s Step) *apis.FieldError { +func validateArtifactsReferencesInStep(ctx context.Context, s Step) *apis.FieldError { if !config.FromContextOrDefaults(ctx).FeatureFlags.EnableArtifacts { var t []string t = append(t, s.Script) @@ -546,7 +549,7 @@ func validateStepArtifactsReferences(ctx context.Context, s Step) *apis.FieldErr for _, e := range s.Env { t = append(t, e.Value) } - if slices.ContainsFunc(t, stepArtifactReferenceExists) { + if slices.ContainsFunc(t, stepArtifactReferenceExists) || slices.ContainsFunc(t, taskArtifactReferenceExists) { return apis.ErrGeneric(fmt.Sprintf("feature flag %s should be set to true to use artifacts feature.", config.EnableArtifacts), "") } } diff --git a/pkg/apis/pipeline/v1/task_validation_test.go b/pkg/apis/pipeline/v1/task_validation_test.go index be4ffc40ec1..0ea6993648a 100644 --- a/pkg/apis/pipeline/v1/task_validation_test.go +++ b/pkg/apis/pipeline/v1/task_validation_test.go @@ -1787,7 +1787,7 @@ func TestTaskSpecValidateSuccessWithArtifactsRefFlagEnabled(t *testing.T) { name: "reference step artifacts in Env", Steps: []v1.Step{{ Image: "busybox", - Env: []corev1.EnvVar{{Name: "AAA", Value: "$(steps.aaa.outputs)"}}, + Env: []corev1.EnvVar{{Name: "AAA", Value: "$(steps.aaa.outputs.image)"}}, }}, }, { @@ -1869,7 +1869,7 @@ func TestTaskSpecValidateErrorWithArtifactsRefFlagNotEnabled(t *testing.T) { { name: "Cannot reference step artifacts in Env without setting enable-artifacts to true", Steps: []v1.Step{{ - Env: []corev1.EnvVar{{Name: "AAA", Value: "$(steps.aaa.outputs)"}}, + Env: []corev1.EnvVar{{Name: "AAA", Value: "$(steps.aaa.outputs.image)"}}, }}, expectedError: apis.FieldError{ Message: fmt.Sprintf("feature flag %s should be set to true to use artifacts feature.", config.EnableArtifacts), diff --git a/pkg/apis/pipeline/v1/taskrun_types.go b/pkg/apis/pipeline/v1/taskrun_types.go index 615eaaa788c..f2ae83a91fc 100644 --- a/pkg/apis/pipeline/v1/taskrun_types.go +++ b/pkg/apis/pipeline/v1/taskrun_types.go @@ -279,6 +279,11 @@ type TaskRunStatusFields struct { // +listType=atomic Results []TaskRunResult `json:"results,omitempty"` + // Artifacts are the list of artifacts written out by the task's containers + // +optional + // +listType=atomic + Artifacts Artifacts `json:"artifacts,omitempty"` + // The list has one entry per sidecar in the manifest. Each entry is // represents the imageid of the corresponding sidecar. // +listType=atomic diff --git a/pkg/apis/pipeline/v1/zz_generated.deepcopy.go b/pkg/apis/pipeline/v1/zz_generated.deepcopy.go index 506579fe024..5b8ca4bb07c 100644 --- a/pkg/apis/pipeline/v1/zz_generated.deepcopy.go +++ b/pkg/apis/pipeline/v1/zz_generated.deepcopy.go @@ -1924,6 +1924,7 @@ func (in *TaskRunStatusFields) DeepCopyInto(out *TaskRunStatusFields) { (*in)[i].DeepCopyInto(&(*out)[i]) } } + in.Artifacts.DeepCopyInto(&out.Artifacts) if in.Sidecars != nil { in, out := &in.Sidecars, &out.Sidecars *out = make([]SidecarState, len(*in)) diff --git a/pkg/apis/pipeline/v1beta1/pipeline_validation.go b/pkg/apis/pipeline/v1beta1/pipeline_validation.go index 79094d97c61..78bec89176d 100644 --- a/pkg/apis/pipeline/v1beta1/pipeline_validation.go +++ b/pkg/apis/pipeline/v1beta1/pipeline_validation.go @@ -21,6 +21,7 @@ import ( "fmt" "strings" + "github.com/tektoncd/pipeline/internal/artifactref" "github.com/tektoncd/pipeline/pkg/apis/config" "github.com/tektoncd/pipeline/pkg/apis/validate" "github.com/tektoncd/pipeline/pkg/internal/resultref" @@ -91,6 +92,7 @@ func (ps *PipelineSpec) Validate(ctx context.Context) (errs *apis.FieldError) { errs = errs.Also(validateTasksAndFinallySection(ps)) errs = errs.Also(validateFinalTasks(ps.Tasks, ps.Finally)) errs = errs.Also(validateWhenExpressions(ctx, ps.Tasks, ps.Finally)) + errs = errs.Also(validateArtifactReference(ctx, ps.Tasks, ps.Finally)) errs = errs.Also(validateMatrix(ctx, ps.Tasks).ViaField("tasks")) errs = errs.Also(validateMatrix(ctx, ps.Finally).ViaField("finally")) return errs @@ -866,6 +868,28 @@ func validateStringResults(results []TaskResult, resultName string) (errs *apis. return errs } +// validateArtifactReference ensure that the feature flag enableArtifacts is set to true when using artifacts +func validateArtifactReference(ctx context.Context, tasks []PipelineTask, finalTasks []PipelineTask) (errs *apis.FieldError) { + if config.FromContextOrDefaults(ctx).FeatureFlags.EnableArtifacts { + return errs + } + for i, t := range tasks { + for _, v := range t.Params.extractValues() { + if len(artifactref.TaskArtifactRegex.FindAllStringSubmatch(v, -1)) > 0 { + return errs.Also(apis.ErrGeneric(fmt.Sprintf("feature flag %s should be set to true to use artifacts feature.", config.EnableArtifacts), "").ViaField("params").ViaFieldIndex("tasks", i)) + } + } + } + for i, t := range finalTasks { + for _, v := range t.Params.extractValues() { + if len(artifactref.TaskArtifactRegex.FindAllStringSubmatch(v, -1)) > 0 { + return errs.Also(apis.ErrGeneric(fmt.Sprintf("feature flag %s should be set to true to use artifacts feature.", config.EnableArtifacts), "").ViaField("params").ViaFieldIndex("finally", i)) + } + } + } + return errs +} + // GetIndexingReferencesToArrayParams returns all strings referencing indices of PipelineRun array parameters // from parameters, workspaces, and when expressions defined in the Pipeline's Tasks and Finally Tasks. // For example, if a Task in the Pipeline has a parameter with a value "$(params.array-param-name[1])", diff --git a/pkg/apis/pipeline/v1beta1/pipeline_validation_test.go b/pkg/apis/pipeline/v1beta1/pipeline_validation_test.go index 0c4aaa9d7d2..b584239fac0 100644 --- a/pkg/apis/pipeline/v1beta1/pipeline_validation_test.go +++ b/pkg/apis/pipeline/v1beta1/pipeline_validation_test.go @@ -175,6 +175,31 @@ func TestPipeline_Validate_Success(t *testing.T) { }}, }, }, + }, { + name: "valid pipeline with pipeline task and final task referencing artifacts in task params with enable-artifacts flag true", + p: &Pipeline{ + ObjectMeta: metav1.ObjectMeta{Name: "pipeline"}, + Spec: PipelineSpec{ + Description: "this is an invalid pipeline referencing artifacts with enable-artifacts flag true", + Tasks: []PipelineTask{{ + Name: "pre-task", + TaskRef: &TaskRef{Name: "foo-task"}, + }, { + Name: "consume-artifacts-task", + Params: Params{{Name: "aaa", Value: ParamValue{ + Type: ParamTypeString, + StringVal: "$(tasks.produce-artifacts-task.outputs.image)", + }}}, + TaskSpec: &EmbeddedTask{TaskSpec: getTaskSpec()}, + }}, + }, + }, + wc: func(ctx context.Context) context.Context { + return cfgtesting.SetFeatureFlags(ctx, t, + map[string]string{ + "enable-artifacts": "true", + "enable-api-fields": "alpha"}) + }, }} for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { @@ -1199,6 +1224,59 @@ func TestPipelineSpec_Validate_Failure(t *testing.T) { Message: `must not set the field(s)`, Paths: []string{"finally[0].taskSpec.resources"}, }, + }, { + name: "invalid pipeline with one pipeline task referencing artifacts in task params with enable-artifacts flag false", + ps: &PipelineSpec{ + Description: "this is an invalid pipeline referencing artifacts with enable-artifacts flag false", + Tasks: []PipelineTask{{ + Name: "pre-task", + TaskRef: &TaskRef{Name: "foo-task"}, + }, { + Name: "consume-artifacts-task", + Params: Params{{Name: "aaa", Value: ParamValue{ + Type: ParamTypeString, + StringVal: "$(tasks.produce-artifacts-task.outputs.image)", + }}}, + TaskSpec: &EmbeddedTask{TaskSpec: getTaskSpec()}, + }}, + }, + expectedError: apis.FieldError{ + Message: `feature flag enable-artifacts should be set to true to use artifacts feature.`, + Paths: []string{"tasks[1].params"}, + }, + wc: func(ctx context.Context) context.Context { + return cfgtesting.SetFeatureFlags(ctx, t, + map[string]string{ + "enable-artifacts": "false", + "enable-api-fields": "alpha"}) + }, + }, { + name: "invalid pipeline with one final pipeline task referencing artifacts in params with enable-artifacts flag false", + ps: &PipelineSpec{ + Description: "this is an invalid pipeline referencing artifacts with enable-artifacts flag false", + Tasks: []PipelineTask{{ + Name: "pre-task", + TaskRef: &TaskRef{Name: "foo-task"}, + }}, + Finally: []PipelineTask{{ + Name: "consume-artifacts-task", + Params: Params{{Name: "aaa", Value: ParamValue{ + Type: ParamTypeString, + StringVal: "$(tasks.produce-artifacts-task.outputs.image)", + }}}, + TaskSpec: &EmbeddedTask{TaskSpec: getTaskSpec()}, + }}, + }, + wc: func(ctx context.Context) context.Context { + return cfgtesting.SetFeatureFlags(ctx, t, + map[string]string{ + "enable-artifacts": "false", + "enable-api-fields": "alpha"}) + }, + expectedError: apis.FieldError{ + Message: `feature flag enable-artifacts should be set to true to use artifacts feature.`, + Paths: []string{"finally[0].params"}, + }, }} for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { diff --git a/pkg/apis/pipeline/v1beta1/task_validation.go b/pkg/apis/pipeline/v1beta1/task_validation.go index ebff081b2b4..4d03a950125 100644 --- a/pkg/apis/pipeline/v1beta1/task_validation.go +++ b/pkg/apis/pipeline/v1beta1/task_validation.go @@ -304,7 +304,11 @@ func errorIfStepResultReferenceinField(value, fieldName string) (errs *apis.Fiel } func stepArtifactReferenceExists(src string) bool { - return len(artifactref.StepArtifactRegex.FindAllStringSubmatch(src, -1)) > 0 || strings.Contains(src, "$(step.artifacts.path)") + return len(artifactref.StepArtifactRegex.FindAllStringSubmatch(src, -1)) > 0 || strings.Contains(src, "$("+artifactref.StepArtifactPathPattern+")") +} + +func taskArtifactReferenceExists(src string) bool { + return len(artifactref.TaskArtifactRegex.FindAllStringSubmatch(src, -1)) > 0 || strings.Contains(src, "$("+artifactref.TaskArtifactPathPattern+")") } func errorIfStepArtifactReferencedInField(value, fieldName string) (errs *apis.FieldError) { @@ -371,7 +375,7 @@ func validateStepResultReference(s Step) (errs *apis.FieldError) { } func validateStep(ctx context.Context, s Step, names sets.String) (errs *apis.FieldError) { - if err := validateStepArtifactsReferences(ctx, s); err != nil { + if err := validateArtifactsReferencesInStep(ctx, s); err != nil { return err } @@ -528,7 +532,7 @@ func validateStep(ctx context.Context, s Step, names sets.String) (errs *apis.Fi return errs } -func validateStepArtifactsReferences(ctx context.Context, s Step) *apis.FieldError { +func validateArtifactsReferencesInStep(ctx context.Context, s Step) *apis.FieldError { if !config.FromContextOrDefaults(ctx).FeatureFlags.EnableArtifacts { var t []string t = append(t, s.Script) @@ -537,7 +541,7 @@ func validateStepArtifactsReferences(ctx context.Context, s Step) *apis.FieldErr for _, e := range s.Env { t = append(t, e.Value) } - if slices.ContainsFunc(t, stepArtifactReferenceExists) { + if slices.ContainsFunc(t, stepArtifactReferenceExists) || slices.ContainsFunc(t, taskArtifactReferenceExists) { return apis.ErrGeneric(fmt.Sprintf("feature flag %s should be set to true to use artifacts feature.", config.EnableArtifacts), "") } } diff --git a/pkg/apis/pipeline/v1beta1/task_validation_test.go b/pkg/apis/pipeline/v1beta1/task_validation_test.go index 181b444edc7..fec9351b99e 100644 --- a/pkg/apis/pipeline/v1beta1/task_validation_test.go +++ b/pkg/apis/pipeline/v1beta1/task_validation_test.go @@ -2623,7 +2623,7 @@ func TestTaskSpecValidateSuccessWithArtifactsRefFlagEnabled(t *testing.T) { name: "reference step artifacts in Env", Steps: []v1beta1.Step{{ Image: "busybox", - Env: []corev1.EnvVar{{Name: "AAA", Value: "$(steps.aaa.outputs)"}}, + Env: []corev1.EnvVar{{Name: "AAA", Value: "$(steps.aaa.outputs.image)"}}, }}, }, { @@ -2705,7 +2705,7 @@ func TestTaskSpecValidateErrorWithArtifactsRefFlagNotEnabled(t *testing.T) { { name: "Cannot reference step artifacts in Env without setting enable-artifacts to true", Steps: []v1beta1.Step{{ - Env: []corev1.EnvVar{{Name: "AAA", Value: "$(steps.aaa.outputs)"}}, + Env: []corev1.EnvVar{{Name: "AAA", Value: "$(steps.aaa.outputs.image)"}}, }}, expectedError: apis.FieldError{ Message: fmt.Sprintf("feature flag %s should be set to true to use artifacts feature.", config.EnableArtifacts), diff --git a/pkg/entrypoint/entrypointer.go b/pkg/entrypoint/entrypointer.go index 58665dd3d7d..f27ffc32c7c 100644 --- a/pkg/entrypoint/entrypointer.go +++ b/pkg/entrypoint/entrypointer.go @@ -137,6 +137,9 @@ type Entrypointer struct { // StepWhenExpressions a list of when expression to decide if the step should be skipped StepWhenExpressions v1.StepWhenExpressions + + // ArtifactsDirectory is the directory to find artifacts, defaults to pipeline.ArtifactsDir + ArtifactsDirectory string } // Waiter encapsulates waiting for files to exist. @@ -298,11 +301,23 @@ func (e Entrypointer) Go() error { } if e.ResultExtractionMethod == config.ResultExtractionMethodTerminationMessage { + // step artifacts fp := filepath.Join(e.StepMetadataDir, "artifacts", "provenance.json") + artifacts, err := readArtifacts(fp, result.StepArtifactsResultType) + if err != nil { + logger.Fatalf("Error while handling step artifacts: %s", err) + } + output = append(output, artifacts...) - artifacts, err := readArtifacts(fp) + artifactsDir := pipeline.ArtifactsDir + // task artifacts + if e.ArtifactsDirectory != "" { + artifactsDir = e.ArtifactsDirectory + } + fp = filepath.Join(artifactsDir, "provenance.json") + artifacts, err = readArtifacts(fp, result.TaskRunArtifactsResultType) if err != nil { - logger.Fatalf("Error while handling artifacts: %s", err) + logger.Fatalf("Error while handling task artifacts: %s", err) } output = append(output, artifacts...) } @@ -310,7 +325,7 @@ func (e Entrypointer) Go() error { return err } -func readArtifacts(fp string) ([]result.RunResult, error) { +func readArtifacts(fp string, resultType result.ResultType) ([]result.RunResult, error) { file, err := os.ReadFile(fp) if os.IsNotExist(err) { return []result.RunResult{}, nil @@ -318,7 +333,7 @@ func readArtifacts(fp string) ([]result.RunResult, error) { if err != nil { return nil, err } - return []result.RunResult{{Key: fp, Value: string(file), ResultType: result.StepArtifactsResultType}}, nil + return []result.RunResult{{Key: fp, Value: string(file), ResultType: resultType}}, nil } func (e Entrypointer) allowExec() (bool, error) { @@ -661,7 +676,6 @@ func getArtifactValues(dir string, template string) (string, error) { } // $(steps.stepName.outputs.artifactName) <- artifacts.Output[artifactName].Values - // $(steps.stepName.outputs) <- artifacts.Output[0].Values var t []v1.Artifact if artifactTemplate.Type == "outputs" { t = artifacts.Outputs @@ -669,13 +683,6 @@ func getArtifactValues(dir string, template string) (string, error) { t = artifacts.Inputs } - if artifactTemplate.ArtifactName == "" { - marshal, err := json.Marshal(t[0].Values) - if err != nil { - return "", err - } - return string(marshal), err - } for _, ar := range t { if ar.Name == artifactTemplate.ArtifactName { marshal, err := json.Marshal(ar.Values) @@ -689,8 +696,7 @@ func getArtifactValues(dir string, template string) (string, error) { } // parseArtifactTemplate parses an artifact template string and extracts relevant information into an ArtifactTemplate struct. -// -// The artifact template is expected to be in the format "$(steps.{step-name}.outputs.{artifact-name})" or "$(steps.{step-name}.outputs)". +// The artifact template is expected to be in the format "$(steps..outputs.)". func parseArtifactTemplate(template string) (ArtifactTemplate, error) { if template == "" { return ArtifactTemplate{}, errors.New("template is empty") diff --git a/pkg/entrypoint/entrypointer_test.go b/pkg/entrypoint/entrypointer_test.go index 48afc2e2873..78ad3a70046 100644 --- a/pkg/entrypoint/entrypointer_test.go +++ b/pkg/entrypoint/entrypointer_test.go @@ -1508,7 +1508,8 @@ func TestReadArtifactsFileDoesNotExist(t *testing.T) { t.Run("readArtifact file doesn't exist, empty result, no error.", func(t *testing.T) { dir := createTmpDir(t, "") fp := filepath.Join(dir, "provenance.json") - got, err := readArtifacts(fp) + got, err := readArtifacts(fp, result.StepArtifactsResultType) + if err != nil { t.Fatalf("Did not expect and error but got: %v", err) } @@ -1528,7 +1529,8 @@ func TestReadArtifactsFileExistNoError(t *testing.T) { if err != nil { t.Fatalf("Did not expect and error but got: %v", err) } - got, err := readArtifacts(fp) + got, err := readArtifacts(fp, result.StepArtifactsResultType) + if err != nil { t.Fatalf("Did not expect and error but got: %v", err) } @@ -1551,7 +1553,7 @@ func TestReadArtifactsFileExistReadError(t *testing.T) { if err != nil { t.Fatalf("Did not expect and error but got: %v", err) } - got, err := readArtifacts(fp) + got, err := readArtifacts(fp, result.StepArtifactsResultType) if err == nil { t.Fatalf("expecting error but got nil") @@ -1584,13 +1586,13 @@ func TestLoadStepArtifacts(t *testing.T) { }{ { desc: "read artifact success", - fileContent: `{"inputs":[{"name":"inputs","values":[{"digest":{"sha256":"cfc7749b96f63bd31c3c42b5c471bf756814053e847c10f3eb003417bc523d30"},"uri":"pkg:example.github.com/inputs"}]}],"outputs":[{"name":"output","values":[{"digest":{"sha256":"64d0b157fdf2d7f6548836dd82085fd8401c9481a9f59e554f1b337f134074b0"},"uri":"docker:example.registry.com/outputs"}]}]}`, + fileContent: `{"inputs":[{"name":"inputs","values":[{"digest":{"sha256":"cfc7749b96f63bd31c3c42b5c471bf756814053e847c10f3eb003417bc523d30"},"uri":"pkg:example.github.com/inputs"}]}],"outputs":[{"name":"image","values":[{"digest":{"sha256":"64d0b157fdf2d7f6548836dd82085fd8401c9481a9f59e554f1b337f134074b0"},"uri":"docker:example.registry.com/outputs"}]}]}`, want: v1.Artifacts{ Inputs: []v1.Artifact{{Name: "inputs", Values: []v1.ArtifactValue{{ Digest: map[v1.Algorithm]string{"sha256": "cfc7749b96f63bd31c3c42b5c471bf756814053e847c10f3eb003417bc523d30"}, Uri: "pkg:example.github.com/inputs", }}}}, - Outputs: []v1.Artifact{{Name: "output", Values: []v1.ArtifactValue{{ + Outputs: []v1.Artifact{{Name: "image", Values: []v1.ArtifactValue{{ Digest: map[v1.Algorithm]string{"sha256": "64d0b157fdf2d7f6548836dd82085fd8401c9481a9f59e554f1b337f134074b0"}, Uri: "docker:example.registry.com/outputs", }}}}, @@ -1610,7 +1612,7 @@ func TestLoadStepArtifacts(t *testing.T) { }, { desc: "read artifact, file cannot be read, error", - fileContent: `{"inputs":[{"name":"inputs","values":[{"digest":{"sha256":"cfc7749b96f63bd31c3c42b5c471bf756814053e847c10f3eb003417bc523d30"},"uri":"pkg:example.github.com/inputs"}]}],"outputs":[{"name":"output","values":[{"digest":{"sha256":"64d0b157fdf2d7f6548836dd82085fd8401c9481a9f59e554f1b337f134074b0"},"uri":"docker:example.registry.com/outputs"}]}]}`, + fileContent: `{"inputs":[{"name":"inputs","values":[{"digest":{"sha256":"cfc7749b96f63bd31c3c42b5c471bf756814053e847c10f3eb003417bc523d30"},"uri":"pkg:example.github.com/inputs"}]}],"outputs":[{"name":"image","values":[{"digest":{"sha256":"64d0b157fdf2d7f6548836dd82085fd8401c9481a9f59e554f1b337f134074b0"},"uri":"docker:example.registry.com/outputs"}]}]}`, mode: 0o000, wantErr: true, }, @@ -1653,7 +1655,7 @@ func TestParseArtifactTemplate(t *testing.T) { wantErr bool }{ { - desc: "valid outputs template with artifact name", + desc: "valid outputs template", input: "$(steps.name.outputs.aaa)", want: ArtifactTemplate{ ContainerName: "step-name", @@ -1662,15 +1664,7 @@ func TestParseArtifactTemplate(t *testing.T) { }, }, { - desc: "valid outputs template without artifact name", - input: "$(steps.name.outputs)", - want: ArtifactTemplate{ - Type: "outputs", - ContainerName: "step-name", - }, - }, - { - desc: "valid inputs template with artifact name", + desc: "valid inputs template", input: "$(steps.name.inputs.aaa)", want: ArtifactTemplate{ ContainerName: "step-name", @@ -1678,19 +1672,6 @@ func TestParseArtifactTemplate(t *testing.T) { ArtifactName: "aaa", }, }, - { - desc: "valid outputs template without artifact name", - input: "$(steps.name.inputs)", - want: ArtifactTemplate{ - Type: "inputs", - ContainerName: "step-name", - }, - }, - { - desc: "invalid template without artifact name, no prefix and suffix", - input: "steps.name.outputs", - wantErr: true, - }, { desc: "invalid template with artifact name, no prefix and suffix", input: "steps.name.outputs.aaa", @@ -1757,40 +1738,12 @@ func TestGetArtifactValues(t *testing.T) { mode os.FileMode template string }{ - { - desc: "read outputs artifact without artifact name, success", - fileContent: `{"inputs":[{"name":"inputs","values":[{"digest":{"sha256":"cfc7749b96f63bd31c3c42b5c471bf756814053e847c10f3eb003417bc523d30"},"uri":"pkg:example.github.com/inputs"}]}],"outputs":[{"name":"output","values":[{"digest":{"sha256":"64d0b157fdf2d7f6548836dd82085fd8401c9481a9f59e554f1b337f134074b0"},"uri":"docker:example.registry.com/outputs"}]}]}`, - want: `[{"digest":{"sha256":"64d0b157fdf2d7f6548836dd82085fd8401c9481a9f59e554f1b337f134074b0"},"uri":"docker:example.registry.com/outputs"}]`, - mode: 0o755, - template: fmt.Sprintf("$(steps.%s.outputs)", name), - }, - { - desc: "read inputs artifact without artifact name, success", - fileContent: `{"inputs":[{"name":"inputs","values":[{"digest":{"sha256":"cfc7749b96f63bd31c3c42b5c471bf756814053e847c10f3eb003417bc523d30"},"uri":"pkg:example.github.com/inputs"}]}],"outputs":[{"name":"output","values":[{"digest":{"sha256":"64d0b157fdf2d7f6548836dd82085fd8401c9481a9f59e554f1b337f134074b0"},"uri":"docker:example.registry.com/outputs"}]}]}`, - want: `[{"digest":{"sha256":"cfc7749b96f63bd31c3c42b5c471bf756814053e847c10f3eb003417bc523d30"},"uri":"pkg:example.github.com/inputs"}]`, - mode: 0o755, - template: fmt.Sprintf("$(steps.%s.inputs)", name), - }, - { - desc: "read outputs artifact without artifact name, multiple outputs, default to first", - fileContent: `{"inputs":[{"name":"inputs","values":[{"digest":{"sha256":"cfc7749b96f63bd31c3c42b5c471bf756814053e847c10f3eb003417bc523d30"},"uri":"pkg:example.github.com/inputs"}]}],"outputs":[{"name":"output","values":[{"digest":{"sha256":"64d0b157fdf2d7f6548836dd82085fd8401c9481a9f59e554f1b337f134074b0"},"uri":"docker:example.registry.com/outputs"}]},{"name":"output2","values":[{"digest":{"sha256":"22222157fdf2d7f6548836dd82085fd8401c9481a9f59e554f1b337f13402222"},"uri":"docker2:example.registry.com/outputs"}]}]}`, - want: `[{"digest":{"sha256":"64d0b157fdf2d7f6548836dd82085fd8401c9481a9f59e554f1b337f134074b0"},"uri":"docker:example.registry.com/outputs"}]`, - mode: 0o755, - template: fmt.Sprintf("$(steps.%s.outputs)", name), - }, - { - desc: "read inputs artifact without artifact name, multiple outputs, default to first", - fileContent: `{"outputs":[{"name":"out","values":[{"digest":{"sha256":"cfc7749b96f63bd31c3c42b5c471bf756814053e847c10f3eb003417bc523d30"},"uri":"pkg:example.github.com/inputs"}]}],"inputs":[{"name":"in","values":[{"digest":{"sha256":"64d0b157fdf2d7f6548836dd82085fd8401c9481a9f59e554f1b337f134074b0"},"uri":"docker:example.registry.com/inputs"}]},{"name":"in2","values":[{"digest":{"sha256":"22222157fdf2d7f6548836dd82085fd8401c9481a9f59e554f1b337f13402222"},"uri":"docker2:example.registry.com/inputs"}]}]}`, - want: `[{"digest":{"sha256":"64d0b157fdf2d7f6548836dd82085fd8401c9481a9f59e554f1b337f134074b0"},"uri":"docker:example.registry.com/inputs"}]`, - mode: 0o755, - template: fmt.Sprintf("$(steps.%s.inputs)", name), - }, { desc: "read outputs artifact with artifact name, success", - fileContent: `{"inputs":[{"name":"inputs","values":[{"digest":{"sha256":"cfc7749b96f63bd31c3c42b5c471bf756814053e847c10f3eb003417bc523d30"},"uri":"pkg:example.github.com/inputs"}]}],"outputs":[{"name":"output","values":[{"digest":{"sha256":"64d0b157fdf2d7f6548836dd82085fd8401c9481a9f59e554f1b337f134074b0"},"uri":"docker:example.registry.com/outputs"}]}]}`, + fileContent: `{"inputs":[{"name":"inputs","values":[{"digest":{"sha256":"cfc7749b96f63bd31c3c42b5c471bf756814053e847c10f3eb003417bc523d30"},"uri":"pkg:example.github.com/inputs"}]}],"outputs":[{"name":"image","values":[{"digest":{"sha256":"64d0b157fdf2d7f6548836dd82085fd8401c9481a9f59e554f1b337f134074b0"},"uri":"docker:example.registry.com/outputs"}]}]}`, want: `[{"digest":{"sha256":"64d0b157fdf2d7f6548836dd82085fd8401c9481a9f59e554f1b337f134074b0"},"uri":"docker:example.registry.com/outputs"}]`, mode: 0o755, - template: fmt.Sprintf("$(steps.%s.outputs.output)", name), + template: fmt.Sprintf("$(steps.%s.outputs.image)", name), }, { desc: "read inputs artifact with artifact name, success", @@ -1801,7 +1754,7 @@ func TestGetArtifactValues(t *testing.T) { }, { desc: "read outputs artifact with artifact name, multiple outputs, success", - fileContent: `{"inputs":[{"name":"inputs","values":[{"digest":{"sha256":"cfc7749b96f63bd31c3c42b5c471bf756814053e847c10f3eb003417bc523d30"},"uri":"pkg:example.github.com/inputs"}]}],"outputs":[{"name":"output","values":[{"digest":{"sha256":"64d0b157fdf2d7f6548836dd82085fd8401c9481a9f59e554f1b337f134074b0"},"uri":"docker:example.registry.com/outputs"}]},{"name":"output2","values":[{"digest":{"sha256":"22222157fdf2d7f6548836dd82085fd8401c9481a9f59e554f1b337f13402222"},"uri":"docker2:example.registry.com/outputs"}]}]}`, + fileContent: `{"inputs":[{"name":"inputs","values":[{"digest":{"sha256":"cfc7749b96f63bd31c3c42b5c471bf756814053e847c10f3eb003417bc523d30"},"uri":"pkg:example.github.com/inputs"}]}],"outputs":[{"name":"image","values":[{"digest":{"sha256":"64d0b157fdf2d7f6548836dd82085fd8401c9481a9f59e554f1b337f134074b0"},"uri":"docker:example.registry.com/outputs"}]},{"name":"output2","values":[{"digest":{"sha256":"22222157fdf2d7f6548836dd82085fd8401c9481a9f59e554f1b337f13402222"},"uri":"docker2:example.registry.com/outputs"}]}]}`, want: `[{"digest":{"sha256":"22222157fdf2d7f6548836dd82085fd8401c9481a9f59e554f1b337f13402222"},"uri":"docker2:example.registry.com/outputs"}]`, mode: 0o755, template: fmt.Sprintf("$(steps.%s.outputs.output2)", name), @@ -1815,21 +1768,21 @@ func TestGetArtifactValues(t *testing.T) { }, { desc: "invalid template", - fileContent: `{"inputs":[{"name":"inputs","values":[{"digest":{"sha256":"cfc7749b96f63bd31c3c42b5c471bf756814053e847c10f3eb003417bc523d30"},"uri":"pkg:example.github.com/inputs"}]}],"outputs":[{"name":"output","values":[{"digest":{"sha256":"64d0b157fdf2d7f6548836dd82085fd8401c9481a9f59e554f1b337f134074b0"},"uri":"docker:example.registry.com/outputs"}]},{"name":"output2","values":[{"digest":{"sha256":"22222157fdf2d7f6548836dd82085fd8401c9481a9f59e554f1b337f13402222"},"uri":"docker2:example.registry.com/outputs"}]}]}`, + fileContent: `{"inputs":[{"name":"inputs","values":[{"digest":{"sha256":"cfc7749b96f63bd31c3c42b5c471bf756814053e847c10f3eb003417bc523d30"},"uri":"pkg:example.github.com/inputs"}]}],"outputs":[{"name":"image","values":[{"digest":{"sha256":"64d0b157fdf2d7f6548836dd82085fd8401c9481a9f59e554f1b337f134074b0"},"uri":"docker:example.registry.com/outputs"}]},{"name":"output2","values":[{"digest":{"sha256":"22222157fdf2d7f6548836dd82085fd8401c9481a9f59e554f1b337f13402222"},"uri":"docker2:example.registry.com/outputs"}]}]}`, mode: 0o755, template: fmt.Sprintf("$(steps.%s.outputs.output2.333)", name), wantErr: true, }, { desc: "fail to load artifacts", - fileContent: `{"inputs":[{"name":"inputs","values":[{"digest":{"sha256":"cfc7749b96f63bd31c3c42b5c471bf756814053e847c10f3eb003417bc523d30"},"uri":"pkg:example.github.com/inputs"}]}],"outputs":[{"name":"output","values":[{"digest":{"sha256":"64d0b157fdf2d7f6548836dd82085fd8401c9481a9f59e554f1b337f134074b0"},"uri":"docker:example.registry.com/outputs"}]},{"name":"output2","values":[{"digest":{"sha256":"22222157fdf2d7f6548836dd82085fd8401c9481a9f59e554f1b337f13402222"},"uri":"docker2:example.registry.com/outputs"}]}]}`, + fileContent: `{"inputs":[{"name":"inputs","values":[{"digest":{"sha256":"cfc7749b96f63bd31c3c42b5c471bf756814053e847c10f3eb003417bc523d30"},"uri":"pkg:example.github.com/inputs"}]}],"outputs":[{"name":"image","values":[{"digest":{"sha256":"64d0b157fdf2d7f6548836dd82085fd8401c9481a9f59e554f1b337f134074b0"},"uri":"docker:example.registry.com/outputs"}]},{"name":"output2","values":[{"digest":{"sha256":"22222157fdf2d7f6548836dd82085fd8401c9481a9f59e554f1b337f13402222"},"uri":"docker2:example.registry.com/outputs"}]}]}`, mode: 0o000, template: fmt.Sprintf("$(steps.%s.outputs.output2.333)", name), wantErr: true, }, { desc: "template not found", - fileContent: `{"inputs":[{"name":"inputs","values":[{"digest":{"sha256":"cfc7749b96f63bd31c3c42b5c471bf756814053e847c10f3eb003417bc523d30"},"uri":"pkg:example.github.com/inputs"}]}],"outputs":[{"name":"output","values":[{"digest":{"sha256":"64d0b157fdf2d7f6548836dd82085fd8401c9481a9f59e554f1b337f134074b0"},"uri":"docker:example.registry.com/outputs"}]},{"name":"output2","values":[{"digest":{"sha256":"22222157fdf2d7f6548836dd82085fd8401c9481a9f59e554f1b337f13402222"},"uri":"docker2:example.registry.com/outputs"}]}]}`, + fileContent: `{"inputs":[{"name":"inputs","values":[{"digest":{"sha256":"cfc7749b96f63bd31c3c42b5c471bf756814053e847c10f3eb003417bc523d30"},"uri":"pkg:example.github.com/inputs"}]}],"outputs":[{"name":"image","values":[{"digest":{"sha256":"64d0b157fdf2d7f6548836dd82085fd8401c9481a9f59e554f1b337f134074b0"},"uri":"docker:example.registry.com/outputs"}]},{"name":"output2","values":[{"digest":{"sha256":"22222157fdf2d7f6548836dd82085fd8401c9481a9f59e554f1b337f13402222"},"uri":"docker2:example.registry.com/outputs"}]}]}`, mode: 0o755, template: fmt.Sprintf("$(steps.%s.outputs.output3)", name), wantErr: true, @@ -1887,10 +1840,10 @@ func TestApplyStepArtifactSubstitutionsCommandSuccess(t *testing.T) { }{ { desc: "apply substitution to command from script file, success", - fileContent: `{"inputs":[{"name":"inputs","values":[{"digest":{"sha256":"cfc7749b96f63bd31c3c42b5c471bf756814053e847c10f3eb003417bc523d30"},"uri":"pkg:example.github.com/inputs"}]}],"outputs":[{"name":"output","values":[{"digest":{"sha256":"64d0b157fdf2d7f6548836dd82085fd8401c9481a9f59e554f1b337f134074b0"},"uri":"docker:example.registry.com/outputs"}]}]}`, + fileContent: `{"inputs":[{"name":"inputs","values":[{"digest":{"sha256":"cfc7749b96f63bd31c3c42b5c471bf756814053e847c10f3eb003417bc523d30"},"uri":"pkg:example.github.com/inputs"}]}],"outputs":[{"name":"image","values":[{"digest":{"sha256":"64d0b157fdf2d7f6548836dd82085fd8401c9481a9f59e554f1b337f134074b0"},"uri":"docker:example.registry.com/outputs"}]}]}`, want: `echo [{"digest":{"sha256":"64d0b157fdf2d7f6548836dd82085fd8401c9481a9f59e554f1b337f134074b0"},"uri":"docker:example.registry.com/outputs"}]`, mode: 0o755, - scriptContent: fmt.Sprintf("echo $(steps.%s.outputs)", stepName), + scriptContent: fmt.Sprintf("echo $(steps.%s.outputs.image)", stepName), scriptFile: filepath.Join(scriptDir, "foo.sh"), command: []string{filepath.Join(scriptDir, "foo.sh")}, }, @@ -1953,17 +1906,17 @@ func TestApplyStepArtifactSubstitutionsCommand(t *testing.T) { }{ { desc: "apply substitution script, fail to read artifacts", - fileContent: `{"inputs":[{"name":"inputs","values":[{"digest":{"sha256":"cfc7749b96f63bd31c3c42b5c471bf756814053e847c10f3eb003417bc523d30"},"uri":"pkg:example.github.com/inputs"}]}],"outputs":[{"name":"output","values":[{"digest":{"sha256":"64d0b157fdf2d7f6548836dd82085fd8401c9481a9f59e554f1b337f134074b0"},"uri":"docker:example.registry.com/outputs"}]}]}`, + fileContent: `{"inputs":[{"name":"inputs","values":[{"digest":{"sha256":"cfc7749b96f63bd31c3c42b5c471bf756814053e847c10f3eb003417bc523d30"},"uri":"pkg:example.github.com/inputs"}]}],"outputs":[{"name":"image","values":[{"digest":{"sha256":"64d0b157fdf2d7f6548836dd82085fd8401c9481a9f59e554f1b337f134074b0"},"uri":"docker:example.registry.com/outputs"}]}]}`, want: []string{filepath.Join(scriptDir, "foo2.sh")}, mode: 0o000, wantErr: true, - scriptContent: fmt.Sprintf("echo $(steps.%s.outputs)", stepName), + scriptContent: fmt.Sprintf("echo $(steps.%s.outputs.image)", stepName), scriptFile: filepath.Join(scriptDir, "foo2.sh"), command: []string{filepath.Join(scriptDir, "foo2.sh")}, }, { desc: "apply substitution to command from script file , no matches success", - fileContent: `{"inputs":[{"name":"inputs","values":[{"digest":{"sha256":"cfc7749b96f63bd31c3c42b5c471bf756814053e847c10f3eb003417bc523d30"},"uri":"pkg:example.github.com/inputs"}]}],"outputs":[{"name":"output","values":[{"digest":{"sha256":"64d0b157fdf2d7f6548836dd82085fd8401c9481a9f59e554f1b337f134074b0"},"uri":"docker:example.registry.com/outputs"}]}]}`, + fileContent: `{"inputs":[{"name":"inputs","values":[{"digest":{"sha256":"cfc7749b96f63bd31c3c42b5c471bf756814053e847c10f3eb003417bc523d30"},"uri":"pkg:example.github.com/inputs"}]}],"outputs":[{"name":"image","values":[{"digest":{"sha256":"64d0b157fdf2d7f6548836dd82085fd8401c9481a9f59e554f1b337f134074b0"},"uri":"docker:example.registry.com/outputs"}]}]}`, want: []string{filepath.Join(scriptDir, "bar.sh")}, mode: 0o755, scriptContent: "echo 123", @@ -1972,18 +1925,18 @@ func TestApplyStepArtifactSubstitutionsCommand(t *testing.T) { }, { desc: "apply substitution to inline command, success", - fileContent: `{"inputs":[{"name":"inputs","values":[{"digest":{"sha256":"cfc7749b96f63bd31c3c42b5c471bf756814053e847c10f3eb003417bc523d30"},"uri":"pkg:example.github.com/inputs"}]}],"outputs":[{"name":"output","values":[{"digest":{"sha256":"64d0b157fdf2d7f6548836dd82085fd8401c9481a9f59e554f1b337f134074b0"},"uri":"docker:example.registry.com/outputs"}]}]}`, + fileContent: `{"inputs":[{"name":"inputs","values":[{"digest":{"sha256":"cfc7749b96f63bd31c3c42b5c471bf756814053e847c10f3eb003417bc523d30"},"uri":"pkg:example.github.com/inputs"}]}],"outputs":[{"name":"image","values":[{"digest":{"sha256":"64d0b157fdf2d7f6548836dd82085fd8401c9481a9f59e554f1b337f134074b0"},"uri":"docker:example.registry.com/outputs"}]}]}`, want: []string{"echo", `[{"digest":{"sha256":"64d0b157fdf2d7f6548836dd82085fd8401c9481a9f59e554f1b337f134074b0"},"uri":"docker:example.registry.com/outputs"}]`, "|", "jq", "."}, mode: 0o755, - command: []string{"echo", fmt.Sprintf("$(steps.%s.outputs)", stepName), "|", "jq", "."}, + command: []string{"echo", fmt.Sprintf("$(steps.%s.outputs.image)", stepName), "|", "jq", "."}, }, { desc: "apply substitution to inline command, fail to read, command no change", - fileContent: `{"inputs":[{"name":"inputs","values":[{"digest":{"sha256":"cfc7749b96f63bd31c3c42b5c471bf756814053e847c10f3eb003417bc523d30"},"uri":"pkg:example.github.com/inputs"}]}],"outputs":[{"name":"output","values":[{"digest":{"sha256":"64d0b157fdf2d7f6548836dd82085fd8401c9481a9f59e554f1b337f134074b0"},"uri":"docker:example.registry.com/outputs"}]}]}`, - want: []string{"echo", fmt.Sprintf("$(steps.%s.outputs)", stepName), "|", "jq", "."}, + fileContent: `{"inputs":[{"name":"inputs","values":[{"digest":{"sha256":"cfc7749b96f63bd31c3c42b5c471bf756814053e847c10f3eb003417bc523d30"},"uri":"pkg:example.github.com/inputs"}]}],"outputs":[{"name":"image","values":[{"digest":{"sha256":"64d0b157fdf2d7f6548836dd82085fd8401c9481a9f59e554f1b337f134074b0"},"uri":"docker:example.registry.com/outputs"}]}]}`, + want: []string{"echo", fmt.Sprintf("$(steps.%s.outputs.image)", stepName), "|", "jq", "."}, mode: 0o000, wantErr: true, - command: []string{"echo", fmt.Sprintf("$(steps.%s.outputs)", stepName), "|", "jq", "."}, + command: []string{"echo", fmt.Sprintf("$(steps.%s.outputs.image)", stepName), "|", "jq", "."}, }, } @@ -2043,7 +1996,7 @@ func TestApplyStepArtifactSubstitutionsEnv(t *testing.T) { }{ { desc: "apply substitution to env, no matches, no changes", - fileContent: `{"inputs":[{"name":"inputs","values":[{"digest":{"sha256":"cfc7749b96f63bd31c3c42b5c471bf756814053e847c10f3eb003417bc523d30"},"uri":"pkg:example.github.com/inputs"}]}],"outputs":[{"name":"output","values":[{"digest":{"sha256":"64d0b157fdf2d7f6548836dd82085fd8401c9481a9f59e554f1b337f134074b0"},"uri":"docker:example.registry.com/outputs"}]}]}`, + fileContent: `{"inputs":[{"name":"inputs","values":[{"digest":{"sha256":"cfc7749b96f63bd31c3c42b5c471bf756814053e847c10f3eb003417bc523d30"},"uri":"pkg:example.github.com/inputs"}]}],"outputs":[{"name":"image","values":[{"digest":{"sha256":"64d0b157fdf2d7f6548836dd82085fd8401c9481a9f59e554f1b337f134074b0"},"uri":"docker:example.registry.com/outputs"}]}]}`, mode: 0o755, envKey: "aaa", envValue: "bbb", @@ -2051,19 +2004,19 @@ func TestApplyStepArtifactSubstitutionsEnv(t *testing.T) { }, { desc: "apply substitution to env, matches found, has change", - fileContent: `{"inputs":[{"name":"inputs","values":[{"digest":{"sha256":"cfc7749b96f63bd31c3c42b5c471bf756814053e847c10f3eb003417bc523d30"},"uri":"pkg:example.github.com/inputs"}]}],"outputs":[{"name":"output","values":[{"digest":{"sha256":"64d0b157fdf2d7f6548836dd82085fd8401c9481a9f59e554f1b337f134074b0"},"uri":"docker:example.registry.com/outputs"}]}]}`, + fileContent: `{"inputs":[{"name":"inputs","values":[{"digest":{"sha256":"cfc7749b96f63bd31c3c42b5c471bf756814053e847c10f3eb003417bc523d30"},"uri":"pkg:example.github.com/inputs"}]}],"outputs":[{"name":"image","values":[{"digest":{"sha256":"64d0b157fdf2d7f6548836dd82085fd8401c9481a9f59e554f1b337f134074b0"},"uri":"docker:example.registry.com/outputs"}]}]}`, mode: 0o755, envKey: "aaa", - envValue: fmt.Sprintf("abc-$(steps.%s.outputs)", stepName), + envValue: fmt.Sprintf("abc-$(steps.%s.outputs.image)", stepName), want: `abc-[{"digest":{"sha256":"64d0b157fdf2d7f6548836dd82085fd8401c9481a9f59e554f1b337f134074b0"},"uri":"docker:example.registry.com/outputs"}]`, }, { desc: "apply substitution to env, matches found, read artifacts failed.", - fileContent: `{"inputs":[{"name":"inputs","values":[{"digest":{"sha256":"cfc7749b96f63bd31c3c42b5c471bf756814053e847c10f3eb003417bc523d30"},"uri":"pkg:example.github.com/inputs"}]}],"outputs":[{"name":"output","values":[{"digest":{"sha256":"64d0b157fdf2d7f6548836dd82085fd8401c9481a9f59e554f1b337f134074b0"},"uri":"docker:example.registry.com/outputs"}]}]}`, + fileContent: `{"inputs":[{"name":"inputs","values":[{"digest":{"sha256":"cfc7749b96f63bd31c3c42b5c471bf756814053e847c10f3eb003417bc523d30"},"uri":"pkg:example.github.com/inputs"}]}],"outputs":[{"name":"image","values":[{"digest":{"sha256":"64d0b157fdf2d7f6548836dd82085fd8401c9481a9f59e554f1b337f134074b0"},"uri":"docker:example.registry.com/outputs"}]}]}`, mode: 0o000, envKey: "aaa", - envValue: fmt.Sprintf("abc-$(steps.%s.outputs)", stepName), - want: fmt.Sprintf("abc-$(steps.%s.outputs)", stepName), + envValue: fmt.Sprintf("abc-$(steps.%s.outputs.image)", stepName), + want: fmt.Sprintf("abc-$(steps.%s.outputs.image)", stepName), wantErr: true, }, } diff --git a/pkg/pod/pod.go b/pkg/pod/pod.go index 4b7fd542d4f..8e808cb6abc 100644 --- a/pkg/pod/pod.go +++ b/pkg/pod/pod.go @@ -26,6 +26,7 @@ import ( "strconv" "strings" + "github.com/tektoncd/pipeline/internal/artifactref" "github.com/tektoncd/pipeline/pkg/apis/config" "github.com/tektoncd/pipeline/pkg/apis/pipeline" "github.com/tektoncd/pipeline/pkg/apis/pipeline/pod" @@ -73,8 +74,6 @@ const ( // TerminationReasonCancelled indicates a step was cancelled. TerminationReasonCancelled = "Cancelled" - - StepArtifactPathPattern = "step.artifacts.path" ) // These are effectively const, but Go doesn't have such an annotation. @@ -100,6 +99,9 @@ var ( Name: "tekton-internal-steps", MountPath: pipeline.StepsDir, ReadOnly: true, + }, { + Name: "tekton-internal-artifacts", + MountPath: pipeline.ArtifactsDir, }} implicitVolumes = []corev1.Volume{{ Name: "tekton-internal-workspace", @@ -113,6 +115,9 @@ var ( }, { Name: "tekton-internal-steps", VolumeSource: corev1.VolumeSource{EmptyDir: &corev1.EmptyDirVolumeSource{}}, + }, { + Name: "tekton-internal-artifacts", + VolumeSource: corev1.VolumeSource{EmptyDir: &corev1.EmptyDirVolumeSource{}}, }} // MaxActiveDeadlineSeconds is a maximum permitted value to be used for a task with no timeout @@ -768,7 +773,7 @@ func artifactsPathReferenced(steps []v1.Step) bool { func artifactPathReferencedInStep(step v1.Step) bool { // `$(step.artifacts.path)` in taskRun.Spec.TaskSpec.Steps and `taskSpec.steps` are substituted when building the pod while when setting status for taskRun // neither of them is substituted, so we need two forms to check if artifactsPath is referenced in steps. - unresolvedPath := "$(" + StepArtifactPathPattern + ")" + unresolvedPath := "$(" + artifactref.StepArtifactPathPattern + ")" path := filepath.Join(pipeline.StepsDir, GetContainerName(step.Name), "artifacts", "provenance.json") if strings.Contains(step.Script, path) || strings.Contains(step.Script, unresolvedPath) { diff --git a/pkg/pod/status.go b/pkg/pod/status.go index 59a18db7414..bee46ebcf90 100644 --- a/pkg/pod/status.go +++ b/pkg/pod/status.go @@ -180,9 +180,18 @@ func createTaskResultsFromStepResults(stepRunRes []v1.TaskRunStepResult, neededS return taskResults } -func getTaskResultsFromSidecarLogs(sidecarLogResults []result.RunResult) []result.RunResult { +func setTaskRunArtifactsFromRunResult(runResults []result.RunResult, artifacts *v1.Artifacts) error { + for _, slr := range runResults { + if slr.ResultType == result.TaskRunArtifactsResultType { + return json.Unmarshal([]byte(slr.Value), artifacts) + } + } + return nil +} + +func getTaskResultsFromSidecarLogs(runResults []result.RunResult) []result.RunResult { taskResultsFromSidecarLogs := []result.RunResult{} - for _, slr := range sidecarLogResults { + for _, slr := range runResults { if slr.ResultType == result.TaskRunResultType { taskResultsFromSidecarLogs = append(taskResultsFromSidecarLogs, slr) } @@ -241,6 +250,14 @@ func setTaskRunStatusBasedOnStepStatus(ctx context.Context, logger *zap.SugaredL taskResults, _, _ := filterResults(taskResultsFromSidecarLogs, specResults, nil) if tr.IsDone() { trs.Results = append(trs.Results, taskResults...) + var tras v1.Artifacts + err := setTaskRunArtifactsFromRunResult(sidecarLogResults, &tras) + if err != nil { + logger.Errorf("Failed to set artifacts value from sidecar logs: %v", err) + merr = multierror.Append(merr, err) + } else { + trs.Artifacts = tras + } } // Continue with extraction of termination messages @@ -275,9 +292,9 @@ func setTaskRunStatusBasedOnStepStatus(ctx context.Context, logger *zap.SugaredL // Set TaskResults from StepResults trs.Results = append(trs.Results, createTaskResultsFromStepResults(stepRunRes, neededStepResults)...) } - var as v1.Artifacts + var sas v1.Artifacts - err = setStepArtifactsValueFromSidecarLogs(sidecarLogResults, s.Name, &as) + err = setStepArtifactsValueFromSidecarLogResult(sidecarLogResults, s.Name, &sas) if err != nil { logger.Errorf("Failed to set artifacts value from sidecar logs: %v", err) merr = multierror.Append(merr, err) @@ -290,18 +307,13 @@ func setTaskRunStatusBasedOnStepStatus(ctx context.Context, logger *zap.SugaredL results, err := termination.ParseMessage(logger, msg) if err != nil { - logger.Errorf("termination message could not be parsed as JSON: %v", err) + logger.Errorf("termination message could not be parsed sas JSON: %v", err) merr = multierror.Append(merr, err) } else { - for _, r := range results { - if r.ResultType == result.StepArtifactsResultType { - if err := json.Unmarshal([]byte(r.Value), &as); err != nil { - logger.Errorf("result value could not be parsed as Artifacts: %v", err) - merr = multierror.Append(merr, err) - } - // there should be only one ArtifactsResult - break - } + err := setStepArtifactsValueFromTerminationMessageRunResult(results, &sas) + if err != nil { + logger.Errorf("error setting step artifacts of step %q in taskrun %q: %v", s.Name, tr.Name, err) + merr = multierror.Append(merr, err) } time, err := extractStartedAtTimeFromResults(results) if err != nil { @@ -320,6 +332,15 @@ func setTaskRunStatusBasedOnStepStatus(ctx context.Context, logger *zap.SugaredL // Set TaskResults from StepResults taskResults = append(taskResults, createTaskResultsFromStepResults(stepRunRes, neededStepResults)...) trs.Results = append(trs.Results, taskResults...) + + var tras v1.Artifacts + err := setTaskRunArtifactsFromRunResult(filteredResults, &tras) + if err != nil { + logger.Errorf("error setting step artifacts in taskrun %q: %v", tr.Name, err) + merr = multierror.Append(merr, err) + } + trs.Artifacts.Merge(tras) + trs.Artifacts.Merge(sas) } msg, err = createMessageFromResults(filteredResults) if err != nil { @@ -339,6 +360,7 @@ func setTaskRunStatusBasedOnStepStatus(ctx context.Context, logger *zap.SugaredL terminationReason = getTerminationReason(state.Terminated.Reason, terminationFromResults, exitCode) } } + trs.Steps = append(trs.Steps, v1.StepState{ ContainerState: *state, Name: trimStepPrefix(s.Name), @@ -346,15 +368,15 @@ func setTaskRunStatusBasedOnStepStatus(ctx context.Context, logger *zap.SugaredL ImageID: s.ImageID, Results: taskRunStepResults, TerminationReason: terminationReason, - Inputs: as.Inputs, - Outputs: as.Outputs, + Inputs: sas.Inputs, + Outputs: sas.Outputs, }) } return merr } -func setStepArtifactsValueFromSidecarLogs(results []result.RunResult, name string, artifacts *v1.Artifacts) error { +func setStepArtifactsValueFromSidecarLogResult(results []result.RunResult, name string, artifacts *v1.Artifacts) error { for _, r := range results { if r.Key == name && r.ResultType == result.StepArtifactsResultType { return json.Unmarshal([]byte(r.Value), artifacts) @@ -363,6 +385,15 @@ func setStepArtifactsValueFromSidecarLogs(results []result.RunResult, name strin return nil } +func setStepArtifactsValueFromTerminationMessageRunResult(results []result.RunResult, artifacts *v1.Artifacts) error { + for _, r := range results { + if r.ResultType == result.StepArtifactsResultType { + return json.Unmarshal([]byte(r.Value), artifacts) + } + } + return nil +} + func setTaskRunStatusBasedOnSidecarStatus(sidecarStatuses []corev1.ContainerStatus, trs *v1.TaskRunStatus) { for _, s := range sidecarStatuses { trs.Sidecars = append(trs.Sidecars, v1.SidecarState{ @@ -471,6 +502,9 @@ func filterResults(results []result.RunResult, specResults []v1.TaskResult, step case result.StepArtifactsResultType: filteredResults = append(filteredResults, r) continue + case result.TaskRunArtifactsResultType: + filteredResults = append(filteredResults, r) + continue case result.InternalTektonResultType: // Internal messages are ignored because they're not used as external result continue diff --git a/pkg/pod/status_test.go b/pkg/pod/status_test.go index b298aba1a54..12fb0c80809 100644 --- a/pkg/pod/status_test.go +++ b/pkg/pod/status_test.go @@ -559,6 +559,31 @@ func TestMakeTaskRunStatus_StepArtifacts(t *testing.T) { }, Results: []v1.TaskRunResult{}, }}, + Artifacts: v1.Artifacts{ + Inputs: []v1.Artifact{ + { + Name: "input-artifacts", + Values: []v1.ArtifactValue{{ + Digest: map[v1.Algorithm]string{"sha256": "b35cacccfdb1e24dc497d15d553891345fd155713ffe647c281c583269eaaae0"}, + Uri: "git:jjjsss", + }, + }, + }, + }, + Outputs: []v1.Artifact{ + { + Name: "build-results", + Values: []v1.ArtifactValue{{ + Digest: map[v1.Algorithm]string{ + "sha1": "95588b8f34c31eb7d62c92aaa4e6506639b06ef2", + "sha256": "df85b9e3983fe2ce20ef76ad675ecf435cc99fc9350adc54fa230bae8c32ce48", + }, + Uri: "pkg:balba", + }, + }, + }, + }, + }, Sidecars: []v1.SidecarState{}, // We don't actually care about the time, just that it's not nil CompletionTime: &metav1.Time{Time: time.Now()}, diff --git a/pkg/reconciler/pipelinerun/pipelinerun.go b/pkg/reconciler/pipelinerun/pipelinerun.go index 8a2c8ea4376..5e9162e04d9 100644 --- a/pkg/reconciler/pipelinerun/pipelinerun.go +++ b/pkg/reconciler/pipelinerun/pipelinerun.go @@ -894,6 +894,13 @@ func (c *Reconciler) runNextSchedulableTask(ctx context.Context, pr *v1.Pipeline // propagate previous task results resources.PropagateResults(rpt, pipelineRunFacts.State) + // propagate previous task artifacts + err = resources.PropagateArtifacts(rpt, pipelineRunFacts.State) + if err != nil { + logger.Errorf("Failed to propagate artifacts due to error: %v", err) + return controller.NewPermanentError(err) + } + // Validate parameter types in matrix after apply substitutions from Task Results if rpt.PipelineTask.IsMatrixed() { if err := resources.ValidateParameterTypesInMatrix(pipelineRunFacts.State); err != nil { diff --git a/pkg/reconciler/pipelinerun/resources/apply.go b/pkg/reconciler/pipelinerun/resources/apply.go index 046487c2637..d8de08dae5e 100644 --- a/pkg/reconciler/pipelinerun/resources/apply.go +++ b/pkg/reconciler/pipelinerun/resources/apply.go @@ -18,6 +18,7 @@ package resources import ( "context" + "encoding/json" "fmt" "strconv" "strings" @@ -442,6 +443,38 @@ func PropagateResults(rpt *ResolvedPipelineTask, runStates PipelineRunState) { rpt.ResolvedTask.TaskSpec = resources.ApplyReplacements(rpt.ResolvedTask.TaskSpec, stringReplacements, arrayReplacements, map[string]map[string]string{}) } +// PropagateArtifacts propagates artifact values from previous task runs into the TaskSpec of the current task. +func PropagateArtifacts(rpt *ResolvedPipelineTask, runStates PipelineRunState) error { + if rpt.ResolvedTask == nil || rpt.ResolvedTask.TaskSpec == nil { + return nil + } + stringReplacements := map[string]string{} + for taskName, artifacts := range runStates.GetTaskRunsArtifacts() { + for i, input := range artifacts.Inputs { + ib, err := json.Marshal(input.Values) + if err != nil { + return err + } + stringReplacements[fmt.Sprintf("tasks.%s.inputs.%s", taskName, input.Name)] = string(ib) + if i == 0 { + stringReplacements[fmt.Sprintf("tasks.%s.inputs", taskName)] = string(ib) + } + } + for i, output := range artifacts.Outputs { + ob, err := json.Marshal(output.Values) + if err != nil { + return err + } + stringReplacements[fmt.Sprintf("tasks.%s.outputs.%s", taskName, output.Name)] = string(ob) + if i == 0 { + stringReplacements[fmt.Sprintf("tasks.%s.outputs", taskName)] = string(ob) + } + } + } + rpt.ResolvedTask.TaskSpec = resources.ApplyReplacements(rpt.ResolvedTask.TaskSpec, stringReplacements, map[string][]string{}, map[string]map[string]string{}) + return nil +} + // ApplyTaskResultsToPipelineResults applies the results of completed TasksRuns and Runs to a Pipeline's // list of PipelineResults, returning the computed set of PipelineRunResults. References to // non-existent TaskResults or failed TaskRuns or Runs result in a PipelineResult being considered invalid diff --git a/pkg/reconciler/pipelinerun/resources/apply_test.go b/pkg/reconciler/pipelinerun/resources/apply_test.go index f0ec7ee9beb..507d53c17b3 100644 --- a/pkg/reconciler/pipelinerun/resources/apply_test.go +++ b/pkg/reconciler/pipelinerun/resources/apply_test.go @@ -4750,6 +4750,162 @@ func TestPropagateResults(t *testing.T) { } } +func TestPropagateArtifacts(t *testing.T) { + for _, tt := range []struct { + name string + resolvedTask *resources.ResolvedPipelineTask + runStates resources.PipelineRunState + expectedResolvedTask *resources.ResolvedPipelineTask + wantErr bool + }{ + { + name: "not propagate artifact when resolved task is nil", + resolvedTask: &resources.ResolvedPipelineTask{ + ResolvedTask: nil, + }, + runStates: resources.PipelineRunState{}, + expectedResolvedTask: &resources.ResolvedPipelineTask{ + ResolvedTask: nil, + }, + }, { + name: "not propagate artifact when taskSpec is nil", + resolvedTask: &resources.ResolvedPipelineTask{ + ResolvedTask: &taskresources.ResolvedTask{ + TaskSpec: nil, + }, + }, + runStates: resources.PipelineRunState{}, + expectedResolvedTask: &resources.ResolvedPipelineTask{ + ResolvedTask: &taskresources.ResolvedTask{ + TaskSpec: nil, + }, + }, + }, + { + name: "propagate artifacts inputs", + resolvedTask: &resources.ResolvedPipelineTask{ + ResolvedTask: &taskresources.ResolvedTask{ + TaskSpec: &v1.TaskSpec{ + Steps: []v1.Step{ + { + Name: "get-artifacts-inputs-from-pt1", + Command: []string{"$(tasks.pt1.inputs.source)"}, + Args: []string{"$(tasks.pt1.inputs.source)"}, + }, + }, + }, + }, + }, + runStates: resources.PipelineRunState{ + { + PipelineTask: &v1.PipelineTask{ + Name: "pt1", + }, + TaskRuns: []*v1.TaskRun{ + { + Status: v1.TaskRunStatus{ + Status: duckv1.Status{ + Conditions: duckv1.Conditions{ + { + Type: apis.ConditionSucceeded, + Status: corev1.ConditionTrue, + }, + }, + }, + TaskRunStatusFields: v1.TaskRunStatusFields{ + Artifacts: v1.Artifacts{ + Inputs: []v1.Artifact{{Name: "source", Values: []v1.ArtifactValue{{Digest: map[v1.Algorithm]string{"sha256": "b35cacccfdb1e24dc497d15d553891345fd155713ffe647c281c583269eaaae0"}, Uri: "pkg:example.github.com/inputs"}}}}, + Outputs: nil, + }, + }, + }, + }, + }, + }, + }, + expectedResolvedTask: &resources.ResolvedPipelineTask{ + ResolvedTask: &taskresources.ResolvedTask{ + TaskSpec: &v1.TaskSpec{ + Steps: []v1.Step{ + { + Name: "get-artifacts-inputs-from-pt1", + Command: []string{`[{"digest":{"sha256":"b35cacccfdb1e24dc497d15d553891345fd155713ffe647c281c583269eaaae0"},"uri":"pkg:example.github.com/inputs"}]`}, + Args: []string{`[{"digest":{"sha256":"b35cacccfdb1e24dc497d15d553891345fd155713ffe647c281c583269eaaae0"},"uri":"pkg:example.github.com/inputs"}]`}, + }, + }, + }, + }, + }, + }, + { + name: "propagate artifacts outputs", + resolvedTask: &resources.ResolvedPipelineTask{ + ResolvedTask: &taskresources.ResolvedTask{ + TaskSpec: &v1.TaskSpec{ + Steps: []v1.Step{ + { + Name: "get-artifacts-outputs-from-pt1", + Command: []string{"$(tasks.pt1.outputs.image)"}, + Args: []string{"$(tasks.pt1.outputs.image)"}, + }, + }, + }, + }, + }, + runStates: resources.PipelineRunState{ + { + PipelineTask: &v1.PipelineTask{ + Name: "pt1", + }, + TaskRuns: []*v1.TaskRun{ + { + Status: v1.TaskRunStatus{ + Status: duckv1.Status{ + Conditions: duckv1.Conditions{ + { + Type: apis.ConditionSucceeded, + Status: corev1.ConditionTrue, + }, + }, + }, + TaskRunStatusFields: v1.TaskRunStatusFields{ + Artifacts: v1.Artifacts{ + Inputs: []v1.Artifact{{Name: "source", Values: []v1.ArtifactValue{{Digest: map[v1.Algorithm]string{"sha256": "b35cacccfdb1e24dc497d15d553891345fd155713ffe647c281c583269eaaae0"}, Uri: "pkg:example.github.com/inputs"}}}}, + Outputs: []v1.Artifact{{Name: "image", Values: []v1.ArtifactValue{{Digest: map[v1.Algorithm]string{"sha1": "95588b8f34c31eb7d62c92aaa4e6506639b06ef2"}, Uri: "pkg:github/package-url/purl-spec@244fd47e07d1004f0aed9c"}}}}, + }, + }, + }, + }, + }, + }, + }, + expectedResolvedTask: &resources.ResolvedPipelineTask{ + ResolvedTask: &taskresources.ResolvedTask{ + TaskSpec: &v1.TaskSpec{ + Steps: []v1.Step{ + { + Name: "get-artifacts-outputs-from-pt1", + Command: []string{`[{"digest":{"sha1":"95588b8f34c31eb7d62c92aaa4e6506639b06ef2"},"uri":"pkg:github/package-url/purl-spec@244fd47e07d1004f0aed9c"}]`}, + Args: []string{`[{"digest":{"sha1":"95588b8f34c31eb7d62c92aaa4e6506639b06ef2"},"uri":"pkg:github/package-url/purl-spec@244fd47e07d1004f0aed9c"}]`}, + }, + }, + }, + }, + }, + }, + } { + t.Run(tt.name, func(t *testing.T) { + err := resources.PropagateArtifacts(tt.resolvedTask, tt.runStates) + if tt.wantErr != (err != nil) { + t.Fatalf("Failed to check err want %t, got %v", tt.wantErr, err) + } + if d := cmp.Diff(tt.expectedResolvedTask, tt.resolvedTask); d != "" { + t.Fatalf("TestPropagateArtifacts() %s", diff.PrintWantGot(d)) + } + }) + } +} + func TestApplyParametersToWorkspaceBindings(t *testing.T) { testCases := []struct { name string diff --git a/pkg/reconciler/pipelinerun/resources/pipelinerunstate.go b/pkg/reconciler/pipelinerun/resources/pipelinerunstate.go index 71537671cae..6d2905e4397 100644 --- a/pkg/reconciler/pipelinerun/resources/pipelinerunstate.go +++ b/pkg/reconciler/pipelinerun/resources/pipelinerunstate.go @@ -173,6 +173,30 @@ func (state PipelineRunState) GetTaskRunsResults() map[string][]v1.TaskRunResult return results } +// GetTaskRunsArtifacts returns a map of all successfully completed TaskRuns in the state, with the pipeline task name as +// the key and the artifacts from the corresponding TaskRun as the value. It only includes tasks which have completed successfully. +func (state PipelineRunState) GetTaskRunsArtifacts() map[string]v1.Artifacts { + results := make(map[string]v1.Artifacts) + for _, rpt := range state { + if rpt.IsCustomTask() { + continue + } + if !rpt.isSuccessful() { + continue + } + if rpt.PipelineTask.IsMatrixed() { + var ars v1.Artifacts + for _, tr := range rpt.TaskRuns { + ars.Merge(tr.Status.Artifacts) + } + results[rpt.PipelineTask.Name] = ars + } else { + results[rpt.PipelineTask.Name] = rpt.TaskRuns[0].Status.Artifacts + } + } + return results +} + // ConvertResultsMapToTaskRunResults converts the map of results from Matrixed PipelineTasks to a list // of TaskRunResults to standard the format func ConvertResultsMapToTaskRunResults(resultsMap map[string][]string) []v1.TaskRunResult { diff --git a/pkg/reconciler/pipelinerun/resources/pipelinerunstate_test.go b/pkg/reconciler/pipelinerun/resources/pipelinerunstate_test.go index 7c42fe300e9..06418d25157 100644 --- a/pkg/reconciler/pipelinerun/resources/pipelinerunstate_test.go +++ b/pkg/reconciler/pipelinerun/resources/pipelinerunstate_test.go @@ -3146,6 +3146,300 @@ func TestPipelineRunState_GetResultsFuncs(t *testing.T) { } } +func TestPipelineRunState_GetTaskRunsArtifacts(t *testing.T) { + testCases := []struct { + name string + state PipelineRunState + expectedArtifacts map[string]v1.Artifacts + }{ + { + name: "successful-task-with-artifacts", + state: PipelineRunState{{ + TaskRunNames: []string{"successful-task-with-artifacts"}, + PipelineTask: &v1.PipelineTask{ + Name: "successful-task-with-artifacts-1", + }, + TaskRuns: []*v1.TaskRun{{ + Status: v1.TaskRunStatus{ + Status: duckv1.Status{Conditions: []apis.Condition{{ + Type: apis.ConditionSucceeded, + Status: corev1.ConditionTrue, + }}}, + TaskRunStatusFields: v1.TaskRunStatusFields{ + Artifacts: v1.Artifacts{ + Inputs: []v1.Artifact{{Name: "source", Values: []v1.ArtifactValue{{Digest: map[v1.Algorithm]string{"sha256": "b35cacccfdb1e24dc497d15d553891345fd155713ffe647c281c583269eaaae0"}, Uri: "pkg:example.github.com/inputs"}}}}, + Outputs: []v1.Artifact{{Name: "image", Values: []v1.ArtifactValue{{Digest: map[v1.Algorithm]string{"sha1": "95588b8f34c31eb7d62c92aaa4e6506639b06ef2"}, Uri: "pkg:github/package-url/purl-spec@244fd47e07d1004f0aed9c"}}}}, + }, + }}, + }}, + }}, + expectedArtifacts: map[string]v1.Artifacts{"successful-task-with-artifacts-1": { + Inputs: []v1.Artifact{{Name: "source", Values: []v1.ArtifactValue{{Digest: map[v1.Algorithm]string{"sha256": "b35cacccfdb1e24dc497d15d553891345fd155713ffe647c281c583269eaaae0"}, Uri: "pkg:example.github.com/inputs"}}}}, + Outputs: []v1.Artifact{{Name: "image", Values: []v1.ArtifactValue{{Digest: map[v1.Algorithm]string{"sha1": "95588b8f34c31eb7d62c92aaa4e6506639b06ef2"}, Uri: "pkg:github/package-url/purl-spec@244fd47e07d1004f0aed9c"}}}}, + }}, + }, + { + name: "two-successful-tasks-with-artifacts", + state: PipelineRunState{{ + TaskRunNames: []string{"first-successful-task-with-artifacts"}, + PipelineTask: &v1.PipelineTask{ + Name: "successful-task-with-artifacts-1", + }, + TaskRuns: []*v1.TaskRun{{ + Status: v1.TaskRunStatus{ + Status: duckv1.Status{Conditions: []apis.Condition{{ + Type: apis.ConditionSucceeded, + Status: corev1.ConditionTrue, + }}}, + TaskRunStatusFields: v1.TaskRunStatusFields{ + Artifacts: v1.Artifacts{ + Inputs: []v1.Artifact{{Name: "source", Values: []v1.ArtifactValue{{Digest: map[v1.Algorithm]string{"sha256": "b35cacccfdb1e24dc497d15d553891345fd155713ffe647c281c583269eaaae0"}, Uri: "pkg:example.github.com/inputs"}}}}, + Outputs: []v1.Artifact{{Name: "image", Values: []v1.ArtifactValue{{Digest: map[v1.Algorithm]string{"sha1": "95588b8f34c31eb7d62c92aaa4e6506639b06ef2"}, Uri: "pkg:github/package-url/purl-spec@244fd47e07d1004f0aed9c"}}}}, + }, + }}, + }}, + }, { + TaskRunNames: []string{"second-successful-task-with-artifacts"}, + PipelineTask: &v1.PipelineTask{ + Name: "successful-task-with-artifacts-2", + }, + TaskRuns: []*v1.TaskRun{{ + Status: v1.TaskRunStatus{ + Status: duckv1.Status{Conditions: []apis.Condition{{ + Type: apis.ConditionSucceeded, + Status: corev1.ConditionTrue, + }}}, + TaskRunStatusFields: v1.TaskRunStatusFields{ + Artifacts: v1.Artifacts{ + Inputs: []v1.Artifact{{Name: "source2", Values: []v1.ArtifactValue{{Digest: map[v1.Algorithm]string{"sha256": "b35cacccfdb1e24dc497d15d553891345fd155713ffe647c281c583269eaaae0"}, Uri: "pkg:example.github.com/inputs"}}}}, + Outputs: []v1.Artifact{{Name: "image2", Values: []v1.ArtifactValue{{Digest: map[v1.Algorithm]string{"sha1": "95588b8f34c31eb7d62c92aaa4e6506639b06ef2"}, Uri: "pkg:github/package-url/purl-spec@244fd47e07d1004f0aed9c"}}}}, + }, + }}, + }}, + }}, + expectedArtifacts: map[string]v1.Artifacts{"successful-task-with-artifacts-1": { + Inputs: []v1.Artifact{{Name: "source", Values: []v1.ArtifactValue{{Digest: map[v1.Algorithm]string{"sha256": "b35cacccfdb1e24dc497d15d553891345fd155713ffe647c281c583269eaaae0"}, Uri: "pkg:example.github.com/inputs"}}}}, + Outputs: []v1.Artifact{{Name: "image", Values: []v1.ArtifactValue{{Digest: map[v1.Algorithm]string{"sha1": "95588b8f34c31eb7d62c92aaa4e6506639b06ef2"}, Uri: "pkg:github/package-url/purl-spec@244fd47e07d1004f0aed9c"}}}}, + }, "successful-task-with-artifacts-2": { + Inputs: []v1.Artifact{{Name: "source2", Values: []v1.ArtifactValue{{Digest: map[v1.Algorithm]string{"sha256": "b35cacccfdb1e24dc497d15d553891345fd155713ffe647c281c583269eaaae0"}, Uri: "pkg:example.github.com/inputs"}}}}, + Outputs: []v1.Artifact{{Name: "image2", Values: []v1.ArtifactValue{{Digest: map[v1.Algorithm]string{"sha1": "95588b8f34c31eb7d62c92aaa4e6506639b06ef2"}, Uri: "pkg:github/package-url/purl-spec@244fd47e07d1004f0aed9c"}}}}, + }}, + }, + { + name: "Skip retrieving artifacts from unsuccessful task", + state: PipelineRunState{{ + TaskRunNames: []string{"unsuccessful-task-with-artifacts"}, + PipelineTask: &v1.PipelineTask{ + Name: "unsuccessful-task-with-artifacts-1", + }, + TaskRuns: []*v1.TaskRun{{ + Status: v1.TaskRunStatus{ + Status: duckv1.Status{Conditions: []apis.Condition{{ + Type: apis.ConditionSucceeded, + Status: corev1.ConditionFalse, + }}}, + TaskRunStatusFields: v1.TaskRunStatusFields{ + Artifacts: v1.Artifacts{ + Inputs: []v1.Artifact{{Name: "source", Values: []v1.ArtifactValue{{Digest: map[v1.Algorithm]string{"sha256": "b35cacccfdb1e24dc497d15d553891345fd155713ffe647c281c583269eaaae0"}, Uri: "pkg:example.github.com/inputs"}}}}, + Outputs: []v1.Artifact{{Name: "image", Values: []v1.ArtifactValue{{Digest: map[v1.Algorithm]string{"sha1": "95588b8f34c31eb7d62c92aaa4e6506639b06ef2"}, Uri: "pkg:github/package-url/purl-spec@244fd47e07d1004f0aed9c"}}}}, + }, + }}, + }}, + }}, + expectedArtifacts: map[string]v1.Artifacts{}, + }, + { + name: "One successful task and one failed task, only retrieving artifacts from the successful one", + state: PipelineRunState{ + { + TaskRunNames: []string{"successful-task-with-artifacts"}, + PipelineTask: &v1.PipelineTask{ + Name: "successful-task-with-artifacts-1", + }, + TaskRuns: []*v1.TaskRun{{ + Status: v1.TaskRunStatus{ + Status: duckv1.Status{Conditions: []apis.Condition{{ + Type: apis.ConditionSucceeded, + Status: corev1.ConditionTrue, + }}}, + TaskRunStatusFields: v1.TaskRunStatusFields{ + Artifacts: v1.Artifacts{ + Inputs: []v1.Artifact{{Name: "source", Values: []v1.ArtifactValue{{Digest: map[v1.Algorithm]string{"sha256": "b35cacccfdb1e24dc497d15d553891345fd155713ffe647c281c583269eaaae0"}, Uri: "pkg:example.github.com/inputs"}}}}, + Outputs: []v1.Artifact{{Name: "image", Values: []v1.ArtifactValue{{Digest: map[v1.Algorithm]string{"sha1": "95588b8f34c31eb7d62c92aaa4e6506639b06ef2"}, Uri: "pkg:github/package-url/purl-spec@244fd47e07d1004f0aed9c"}}}}, + }, + }}, + }}, + }, + { + TaskRunNames: []string{"unsuccessful-task-with-artifacts"}, + PipelineTask: &v1.PipelineTask{ + Name: "unsuccessful-task-with-artifacts-1", + }, + TaskRuns: []*v1.TaskRun{{ + Status: v1.TaskRunStatus{ + Status: duckv1.Status{Conditions: []apis.Condition{{ + Type: apis.ConditionSucceeded, + Status: corev1.ConditionFalse, + }}}, + TaskRunStatusFields: v1.TaskRunStatusFields{ + Artifacts: v1.Artifacts{ + Inputs: []v1.Artifact{{Name: "source0", Values: []v1.ArtifactValue{{Digest: map[v1.Algorithm]string{"sha256": "b35cacccfdb1e24dc497d15d553891345fd155713ffe647c281c583269eaaae0"}, Uri: "pkg:example.github.com/inputs"}}}}, + Outputs: []v1.Artifact{{Name: "image0", Values: []v1.ArtifactValue{{Digest: map[v1.Algorithm]string{"sha1": "95588b8f34c31eb7d62c92aaa4e6506639b06ef2"}, Uri: "pkg:github/package-url/purl-spec@244fd47e07d1004f0aed9c"}}}}, + }, + }}, + }}, + }}, + expectedArtifacts: map[string]v1.Artifacts{"successful-task-with-artifacts-1": { + Inputs: []v1.Artifact{{Name: "source", Values: []v1.ArtifactValue{{Digest: map[v1.Algorithm]string{"sha256": "b35cacccfdb1e24dc497d15d553891345fd155713ffe647c281c583269eaaae0"}, Uri: "pkg:example.github.com/inputs"}}}}, + Outputs: []v1.Artifact{{Name: "image", Values: []v1.ArtifactValue{{Digest: map[v1.Algorithm]string{"sha1": "95588b8f34c31eb7d62c92aaa4e6506639b06ef2"}, Uri: "pkg:github/package-url/purl-spec@244fd47e07d1004f0aed9c"}}}}, + }}, + }, + { + name: "One standard successful taskRun with artifacts and one custom task, custom task has no effect", + state: PipelineRunState{{ + CustomRunNames: []string{"successful-run-without-results"}, + CustomTask: true, + PipelineTask: &v1.PipelineTask{ + Name: "successful-run-without-results-1", + }, + CustomRuns: []*v1beta1.CustomRun{ + { + Status: v1beta1.CustomRunStatus{ + Status: duckv1.Status{Conditions: []apis.Condition{{ + Type: apis.ConditionSucceeded, + Status: corev1.ConditionTrue, + }}}, + CustomRunStatusFields: v1beta1.CustomRunStatusFields{}, + }, + }}, + }, + { + TaskRunNames: []string{"successful-task-with-artifacts"}, + PipelineTask: &v1.PipelineTask{ + Name: "successful-task-with-artifacts-1", + }, + TaskRuns: []*v1.TaskRun{{ + Status: v1.TaskRunStatus{ + Status: duckv1.Status{Conditions: []apis.Condition{{ + Type: apis.ConditionSucceeded, + Status: corev1.ConditionTrue, + }}}, + TaskRunStatusFields: v1.TaskRunStatusFields{ + Artifacts: v1.Artifacts{ + Inputs: []v1.Artifact{{Name: "source", Values: []v1.ArtifactValue{{Digest: map[v1.Algorithm]string{"sha256": "b35cacccfdb1e24dc497d15d553891345fd155713ffe647c281c583269eaaae0"}, Uri: "pkg:example.github.com/inputs"}}}}, + Outputs: []v1.Artifact{{Name: "image", Values: []v1.ArtifactValue{{Digest: map[v1.Algorithm]string{"sha1": "95588b8f34c31eb7d62c92aaa4e6506639b06ef2"}, Uri: "pkg:github/package-url/purl-spec@244fd47e07d1004f0aed9c"}}}}, + }, + }}, + }}, + }, + }, + expectedArtifacts: map[string]v1.Artifacts{"successful-task-with-artifacts-1": { + Inputs: []v1.Artifact{{Name: "source", Values: []v1.ArtifactValue{{Digest: map[v1.Algorithm]string{"sha256": "b35cacccfdb1e24dc497d15d553891345fd155713ffe647c281c583269eaaae0"}, Uri: "pkg:example.github.com/inputs"}}}}, + Outputs: []v1.Artifact{{Name: "image", Values: []v1.ArtifactValue{{Digest: map[v1.Algorithm]string{"sha1": "95588b8f34c31eb7d62c92aaa4e6506639b06ef2"}, Uri: "pkg:github/package-url/purl-spec@244fd47e07d1004f0aed9c"}}}}, + }}, + }, + { + name: "matrixed tasks", + state: PipelineRunState{{ + TaskRunNames: []string{ + "matrixed-task-run-0", + "matrixed-task-run-1", + "matrixed-task-run-2", + "matrixed-task-run-3", + }, + PipelineTask: &v1.PipelineTask{ + Name: "matrixed-task-with-artifacts", + TaskRef: &v1.TaskRef{ + Name: "task", + Kind: "Task", + APIVersion: "v1", + }, + Matrix: &v1.Matrix{ + Params: v1.Params{{ + Name: "foobar", + Value: v1.ParamValue{Type: v1.ParamTypeArray, ArrayVal: []string{"foo", "bar"}}, + }, { + Name: "quxbaz", + Value: v1.ParamValue{Type: v1.ParamTypeArray, ArrayVal: []string{"qux", "baz"}}, + }}}, + }, + TaskRuns: []*v1.TaskRun{{ + TypeMeta: metav1.TypeMeta{APIVersion: "tekton.dev/v1"}, + ObjectMeta: metav1.ObjectMeta{Name: "matrixed-task-run-0"}, + Status: v1.TaskRunStatus{ + Status: duckv1.Status{Conditions: []apis.Condition{{ + Type: apis.ConditionSucceeded, + Status: corev1.ConditionTrue, + Reason: v1.TaskRunReasonSuccessful.String(), + }}}, + TaskRunStatusFields: v1.TaskRunStatusFields{ + Artifacts: v1.Artifacts{ + Inputs: []v1.Artifact{{Name: "source1", Values: []v1.ArtifactValue{{Digest: map[v1.Algorithm]string{"sha256": "b35cacccfdb1e24dc497d15d553891345fd155713ffe647c281c583269eaaae0"}, Uri: "pkg:example.github.com/inputs"}}}}, + Outputs: []v1.Artifact{{Name: "image1", Values: []v1.ArtifactValue{{Digest: map[v1.Algorithm]string{"sha1": "95588b8f34c31eb7d62c92aaa4e6506639b06ef2"}, Uri: "pkg:github/package-url/purl-spec@244fd47e07d1004f0aed9c"}}}}, + }, + }}, + }, { + TypeMeta: metav1.TypeMeta{APIVersion: "tekton.dev/v1"}, + ObjectMeta: metav1.ObjectMeta{Name: "matrixed-task-run-1"}, + Status: v1.TaskRunStatus{ + Status: duckv1.Status{Conditions: []apis.Condition{{ + Type: apis.ConditionSucceeded, + Status: corev1.ConditionTrue, + Reason: v1.TaskRunReasonSuccessful.String(), + }}}, + TaskRunStatusFields: v1.TaskRunStatusFields{ + Artifacts: v1.Artifacts{ + Inputs: []v1.Artifact{{Name: "source2", Values: []v1.ArtifactValue{{Digest: map[v1.Algorithm]string{"sha256": "b35cacccfdb1e24dc497d15d553891345fd155713ffe647c281c583269eaaae0"}, Uri: "pkg:example.github.com/inputs"}}}}, + Outputs: []v1.Artifact{{Name: "image2", Values: []v1.ArtifactValue{{Digest: map[v1.Algorithm]string{"sha1": "95588b8f34c31eb7d62c92aaa4e6506639b06ef2"}, Uri: "pkg:github/package-url/purl-spec@244fd47e07d1004f0aed9c"}}}}, + }, + }}, + }, { + TypeMeta: metav1.TypeMeta{APIVersion: "tekton.dev/v1"}, + ObjectMeta: metav1.ObjectMeta{Name: "matrixed-task-run-2"}, + Status: v1.TaskRunStatus{ + Status: duckv1.Status{Conditions: []apis.Condition{{ + Type: apis.ConditionSucceeded, + Status: corev1.ConditionTrue, + Reason: v1.TaskRunReasonSuccessful.String(), + }}}, + TaskRunStatusFields: v1.TaskRunStatusFields{ + Artifacts: v1.Artifacts{ + Inputs: []v1.Artifact{{Name: "source3", Values: []v1.ArtifactValue{{Digest: map[v1.Algorithm]string{"sha256": "b35cacccfdb1e24dc497d15d553891345fd155713ffe647c281c583269eaaae0"}, Uri: "pkg:example.github.com/inputs"}}}}, + Outputs: []v1.Artifact{{Name: "image3", Values: []v1.ArtifactValue{{Digest: map[v1.Algorithm]string{"sha1": "95588b8f34c31eb7d62c92aaa4e6506639b06ef2"}, Uri: "pkg:github/package-url/purl-spec@244fd47e07d1004f0aed9c"}}}}, + }, + }}, + }, { + TypeMeta: metav1.TypeMeta{APIVersion: "tekton.dev/v1"}, + ObjectMeta: metav1.ObjectMeta{Name: "matrixed-task-run-3"}, + Status: v1.TaskRunStatus{ + Status: duckv1.Status{Conditions: []apis.Condition{{ + Type: apis.ConditionSucceeded, + Status: corev1.ConditionTrue, + Reason: v1.TaskRunReasonSuccessful.String(), + }}}, + TaskRunStatusFields: v1.TaskRunStatusFields{ + Artifacts: v1.Artifacts{ + Inputs: []v1.Artifact{{Name: "source4", Values: []v1.ArtifactValue{{Digest: map[v1.Algorithm]string{"sha256": "b35cacccfdb1e24dc497d15d553891345fd155713ffe647c281c583269eaaae0"}, Uri: "pkg:example.github.com/inputs"}}}}, + Outputs: []v1.Artifact{{Name: "image4", Values: []v1.ArtifactValue{{Digest: map[v1.Algorithm]string{"sha1": "95588b8f34c31eb7d62c92aaa4e6506639b06ef2"}, Uri: "pkg:github/package-url/purl-spec@244fd47e07d1004f0aed9c"}}}}, + }, + }}, + }}, + }}, + expectedArtifacts: map[string]v1.Artifacts{"matrixed-task-with-artifacts": { + Inputs: []v1.Artifact{{Name: "source1", Values: []v1.ArtifactValue{{Digest: map[v1.Algorithm]string{"sha256": "b35cacccfdb1e24dc497d15d553891345fd155713ffe647c281c583269eaaae0"}, Uri: "pkg:example.github.com/inputs"}}}, {Name: "source2", Values: []v1.ArtifactValue{{Digest: map[v1.Algorithm]string{"sha256": "b35cacccfdb1e24dc497d15d553891345fd155713ffe647c281c583269eaaae0"}, Uri: "pkg:example.github.com/inputs"}}}, {Name: "source3", Values: []v1.ArtifactValue{{Digest: map[v1.Algorithm]string{"sha256": "b35cacccfdb1e24dc497d15d553891345fd155713ffe647c281c583269eaaae0"}, Uri: "pkg:example.github.com/inputs"}}}, {Name: "source4", Values: []v1.ArtifactValue{{Digest: map[v1.Algorithm]string{"sha256": "b35cacccfdb1e24dc497d15d553891345fd155713ffe647c281c583269eaaae0"}, Uri: "pkg:example.github.com/inputs"}}}}, + Outputs: []v1.Artifact{{Name: "image1", Values: []v1.ArtifactValue{{Digest: map[v1.Algorithm]string{"sha1": "95588b8f34c31eb7d62c92aaa4e6506639b06ef2"}, Uri: "pkg:github/package-url/purl-spec@244fd47e07d1004f0aed9c"}}}, {Name: "image2", Values: []v1.ArtifactValue{{Digest: map[v1.Algorithm]string{"sha1": "95588b8f34c31eb7d62c92aaa4e6506639b06ef2"}, Uri: "pkg:github/package-url/purl-spec@244fd47e07d1004f0aed9c"}}}, {Name: "image3", Values: []v1.ArtifactValue{{Digest: map[v1.Algorithm]string{"sha1": "95588b8f34c31eb7d62c92aaa4e6506639b06ef2"}, Uri: "pkg:github/package-url/purl-spec@244fd47e07d1004f0aed9c"}}}, {Name: "image4", Values: []v1.ArtifactValue{{Digest: map[v1.Algorithm]string{"sha1": "95588b8f34c31eb7d62c92aaa4e6506639b06ef2"}, Uri: "pkg:github/package-url/purl-spec@244fd47e07d1004f0aed9c"}}}}, + }}, + }, + } + + for _, tt := range testCases { + got := tt.state.GetTaskRunsArtifacts() + if d := cmp.Diff(tt.expectedArtifacts, got, cmpopts.SortSlices(func(a, b v1.Artifact) bool { return a.Name > b.Name })); d != "" { + t.Errorf("GetTaskRunsArtifacts() did not produce expected artifacts for test %s: %s", tt.name, diff.PrintWantGot(d)) + } + } +} + func TestPipelineRunState_GetChildReferences(t *testing.T) { testCases := []struct { name string diff --git a/pkg/reconciler/taskrun/resources/apply.go b/pkg/reconciler/taskrun/resources/apply.go index a76729c4994..2b40716b409 100644 --- a/pkg/reconciler/taskrun/resources/apply.go +++ b/pkg/reconciler/taskrun/resources/apply.go @@ -24,6 +24,7 @@ import ( "strconv" "strings" + "github.com/tektoncd/pipeline/internal/artifactref" "github.com/tektoncd/pipeline/pkg/apis/pipeline" v1 "github.com/tektoncd/pipeline/pkg/apis/pipeline/v1" "github.com/tektoncd/pipeline/pkg/container" @@ -399,19 +400,20 @@ func getTaskResultReplacements(spec *v1.TaskSpec) map[string]string { return stringReplacements } -// ApplyArtifacts replaces the occurrences of step.artifacts.path with the absolute tekton internal path +// ApplyArtifacts replaces the occurrences of artifacts.path and step.artifacts.path with the absolute tekton internal path func ApplyArtifacts(spec *v1.TaskSpec) *v1.TaskSpec { for i := range spec.Steps { - stringReplacements := getStepArtifactReplacements(spec.Steps[i], i) + stringReplacements := getArtifactReplacements(spec.Steps[i], i) container.ApplyStepReplacements(&spec.Steps[i], stringReplacements, map[string][]string{}) } return spec } -func getStepArtifactReplacements(step v1.Step, idx int) map[string]string { +func getArtifactReplacements(step v1.Step, idx int) map[string]string { stringReplacements := map[string]string{} stepName := pod.StepName(step.Name, idx) - stringReplacements[pod.StepArtifactPathPattern] = filepath.Join(pipeline.StepsDir, stepName, "artifacts", "provenance.json") + stringReplacements[artifactref.StepArtifactPathPattern] = filepath.Join(pipeline.StepsDir, stepName, "artifacts", "provenance.json") + stringReplacements[artifactref.TaskArtifactPathPattern] = filepath.Join(pipeline.ArtifactsDir, "provenance.json") return stringReplacements } diff --git a/pkg/reconciler/taskrun/taskrun_test.go b/pkg/reconciler/taskrun/taskrun_test.go index 5c3d9a456f7..282ac982151 100644 --- a/pkg/reconciler/taskrun/taskrun_test.go +++ b/pkg/reconciler/taskrun/taskrun_test.go @@ -313,6 +313,13 @@ var ( EmptyDir: &corev1.EmptyDirVolumeSource{}, }, } + + artifactsVolume = corev1.Volume{ + Name: "tekton-internal-artifacts", + VolumeSource: corev1.VolumeSource{ + EmptyDir: &corev1.EmptyDirVolumeSource{}, + }, + } downwardVolume = corev1.Volume{ Name: "tekton-internal-downward", VolumeSource: corev1.VolumeSource{ @@ -5892,6 +5899,10 @@ func podVolumeMounts(idx, totalSteps int) []corev1.VolumeMount { MountPath: "/tekton/steps", ReadOnly: true, }) + mnts = append(mnts, corev1.VolumeMount{ + Name: "tekton-internal-artifacts", + MountPath: "/tekton/artifacts", + }) return mnts } @@ -5986,6 +5997,7 @@ func expectedPod(podName, taskName, taskRunName, ns, saName string, isClusterTas workspaceVolume, homeVolume, resultsVolume, + artifactsVolume, stepsVolume, binVolume, downwardVolume, diff --git a/pkg/result/result.go b/pkg/result/result.go index ab52c6c8275..6506074fd1d 100644 --- a/pkg/result/result.go +++ b/pkg/result/result.go @@ -36,8 +36,11 @@ const ( // StepResultType default step result value StepResultType ResultType = 4 - // StepArtifactsResultType default artifacts result value + // StepArtifactsResultType default step artifacts result value StepArtifactsResultType ResultType = 5 + + // TaskRunArtifactsResultType default taskRun artifacts result value + TaskRunArtifactsResultType ResultType = 6 ) // RunResult is used to write key/value pairs to TaskRun pod termination messages. @@ -93,6 +96,8 @@ func (r *ResultType) UnmarshalJSON(data []byte) error { *r = InternalTektonResultType case "StepArtifactsResult": *r = StepArtifactsResultType + case "TaskRunArtifactsResult": + *r = TaskRunArtifactsResultType default: *r = UnknownResultType } diff --git a/pkg/result/result_test.go b/pkg/result/result_test.go index 4ab14598f35..34654ec3f0c 100644 --- a/pkg/result/result_test.go +++ b/pkg/result/result_test.go @@ -43,9 +43,13 @@ func TestRunResult_UnmarshalJSON(t *testing.T) { data: "{\"key\":\"resultName\",\"value\":\"\", \"type\": \"InternalTektonResult\"}", pr: RunResult{Key: "resultName", Value: "", ResultType: InternalTektonResultType}, }, { - name: "type defined as string - ArtifactsResult", + name: "type defined as string - StepArtifactsResultType", data: "{\"key\":\"resultName\",\"value\":\"\", \"type\": \"StepArtifactsResult\"}", pr: RunResult{Key: "resultName", Value: "", ResultType: StepArtifactsResultType}, + }, { + name: "type defined as string - TaskRunArtifactResult", + data: "{\"key\":\"resultName\",\"value\":\"\", \"type\": \"TaskRunArtifactsResult\"}", + pr: RunResult{Key: "resultName", Value: "", ResultType: TaskRunArtifactsResultType}, }, { name: "type defined as int", data: "{\"key\":\"resultName\",\"value\":\"\", \"type\": 1}", diff --git a/test/artifacts_test.go b/test/artifacts_test.go index 5680594c34f..1ae89116803 100644 --- a/test/artifacts_test.go +++ b/test/artifacts_test.go @@ -252,10 +252,10 @@ func TestConsumeArtifacts(t *testing.T) { task := simpleArtifactProducerTask(t, namespace, fqImageName) task.Spec.Steps = append(task.Spec.Steps, v1.Step{Name: "consume-outputs", Image: fqImageName, - Command: []string{"sh", "-c", "echo -n $(steps.hello.outputs) >> $(step.results.result1.path)"}, + Command: []string{"sh", "-c", "echo -n $(steps.hello.outputs.image) >> $(step.results.result1.path)"}, Results: []v1.StepResult{{Name: "result1", Type: v1.ResultsTypeString}}}, v1.Step{Name: "consume-inputs", Image: fqImageName, - Command: []string{"sh", "-c", "echo -n $(steps.hello.inputs) >> $(step.results.result2.path)"}, + Command: []string{"sh", "-c", "echo -n $(steps.hello.inputs.source) >> $(step.results.result2.path)"}, Results: []v1.StepResult{{Name: "result2", Type: v1.ResultsTypeString}}}, ) if _, err := c.V1TaskClient.Create(ctx, task, metav1.CreateOptions{}); err != nil {