Skip to content

Commit d345e99

Browse files
ericzzzzzzztekton-robot
ericzzzzzzz
authored and
tekton-robot
committed
kind/feat: passing artifacts between tasks
1 parent 7f6c6ec commit d345e99

39 files changed

+1799
-246
lines changed

‎docs/artifacts.md

Lines changed: 314 additions & 51 deletions
Large diffs are not rendered by default.

‎docs/pipeline-api.md

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1428,6 +1428,9 @@ string
14281428
</table>
14291429
<h3 id="tekton.dev/v1.Artifacts">Artifacts
14301430
</h3>
1431+
<p>
1432+
(<em>Appears on:</em><a href="#tekton.dev/v1.TaskRunStatusFields">TaskRunStatusFields</a>)
1433+
</p>
14311434
<div>
14321435
<p>Artifacts represents the collection of input and output artifacts associated with
14331436
a task run or a similar process. Artifacts in this context are units of data or resources
@@ -5902,6 +5905,20 @@ All TaskRunStatus stored in RetriesStatus will have no date within the RetriesSt
59025905
</tr>
59035906
<tr>
59045907
<td>
5908+
<code>artifacts</code><br/>
5909+
<em>
5910+
<a href="#tekton.dev/v1.Artifacts">
5911+
Artifacts
5912+
</a>
5913+
</em>
5914+
</td>
5915+
<td>
5916+
<em>(Optional)</em>
5917+
<p>Artifacts are the list of artifacts written out by the task&rsquo;s containers</p>
5918+
</td>
5919+
</tr>
5920+
<tr>
5921+
<td>
59055922
<code>sidecars</code><br/>
59065923
<em>
59075924
<a href="#tekton.dev/v1.SidecarState">

‎docs/variables.md

Lines changed: 56 additions & 53 deletions
Large diffs are not rendered by default.

‎examples/v1/pipelineruns/alpha/consume-artifacts-from-task.yaml

Lines changed: 60 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,60 @@
1+
apiVersion: tekton.dev/v1
2+
kind: PipelineRun
3+
metadata:
4+
generateName: pipelinerun-consume-tasks-artifacts
5+
spec:
6+
pipelineSpec:
7+
tasks:
8+
- name: produce-artifacts-task
9+
taskSpec:
10+
description: |
11+
A simple task that produces artifacts
12+
steps:
13+
- name: produce-artifacts
14+
image: bash:latest
15+
script: |
16+
#!/usr/bin/env bash
17+
cat > $(artifacts.path) << EOF
18+
{
19+
"inputs":[
20+
{
21+
"name":"input-artifacts",
22+
"values":[
23+
{
24+
"uri":"pkg:example.github.com/inputs",
25+
"digest":{
26+
"sha256":"b35cacccfdb1e24dc497d15d553891345fd155713ffe647c281c583269eaaae0"
27+
}
28+
}
29+
]
30+
}
31+
],
32+
"outputs":[
33+
{
34+
"name":"image",
35+
"values":[
36+
{
37+
"uri":"pkg:github/package-url/purl-spec@244fd47e07d1004f0aed9c",
38+
"digest":{
39+
"sha256":"df85b9e3983fe2ce20ef76ad675ecf435cc99fc9350adc54fa230bae8c32ce48",
40+
"sha1":"95588b8f34c31eb7d62c92aaa4e6506639b06ef2"
41+
}
42+
}
43+
]
44+
}
45+
]
46+
}
47+
EOF
48+
- name: consume-artifacts
49+
runAfter:
50+
- produce-artifacts-task
51+
taskSpec:
52+
steps:
53+
- name: artifacts-consumer-python
54+
image: python:latest
55+
script: |
56+
#!/usr/bin/env python3
57+
import json
58+
data = json.loads('$(tasks.produce-artifacts-task.outputs.image)')
59+
if data[0]['uri'] != "pkg:github/package-url/purl-spec@244fd47e07d1004f0aed9c":
60+
exit(1)

‎examples/v1/taskruns/alpha/produce-consume-artifacts.yaml

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -44,12 +44,11 @@ spec:
4444
- name: artifacts-consumer
4545
image: docker.io/library/bash:latest
4646
script: |
47-
echo $(steps.artifacts-producer.outputs)
4847
echo $(steps.artifacts-producer.inputs.input-artifacts)
4948
- name: artifacts-consumer-python
5049
image: docker.io/library/python:latest
5150
script: |
5251
#!/usr/bin/env python3
5352
import json
54-
data = json.loads('$(steps.artifacts-producer.outputs)')
53+
data = json.loads('$(steps.artifacts-producer.outputs.image)')
5554
print(data[0]['uri'])

‎examples/v1/taskruns/alpha/task-artifacts.yaml

Lines changed: 44 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,44 @@
1+
apiVersion: tekton.dev/v1
2+
kind: TaskRun
3+
metadata:
4+
generateName: task-run-artifacts
5+
spec:
6+
taskSpec:
7+
description: |
8+
A simple task that produces artifacts
9+
steps:
10+
- name: produce-artifacts
11+
image: bash:latest
12+
script: |
13+
#!/usr/bin/env bash
14+
cat > $(artifacts.path) << EOF
15+
{
16+
"inputs":[
17+
{
18+
"name":"input-artifacts",
19+
"values":[
20+
{
21+
"uri":"pkg:example.github.com/inputs",
22+
"digest":{
23+
"sha256":"b35cacccfdb1e24dc497d15d553891345fd155713ffe647c281c583269eaaae0"
24+
}
25+
}
26+
]
27+
}
28+
],
29+
"outputs":[
30+
{
31+
"name":"image",
32+
"values":[
33+
{
34+
"uri":"pkg:github/package-url/purl-spec@244fd47e07d1004f0aed9c",
35+
"digest":{
36+
"sha256":"df85b9e3983fe2ce20ef76ad675ecf435cc99fc9350adc54fa230bae8c32ce48",
37+
"sha1":"95588b8f34c31eb7d62c92aaa4e6506639b06ef2"
38+
}
39+
}
40+
]
41+
}
42+
]
43+
}
44+
EOF

‎internal/artifactref/artifactref.go

Lines changed: 12 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -2,10 +2,17 @@ package artifactref
22

33
import "regexp"
44

5-
// case 1: steps.<step-name>.inputs
6-
// case 2: steps.<step-name>.outputs
7-
// case 3: steps.<step-name>.inputs.<artifact-category-name>
8-
// case 4: steps.<step-name>.outputs.<artifact-category-name>
9-
const stepArtifactUsagePattern = `\$\(steps\.([^.]+)\.(?:inputs|outputs)(?:\.([^.^\)]+))?\)`
5+
// case 1: steps.<step-name>.inputs.<artifact-category-name>
6+
// case 2: steps.<step-name>.outputs.<artifact-category-name>
7+
const stepArtifactUsagePattern = `\$\(steps\.([^.]+)\.(?:inputs|outputs)\.([^.)]+)\)`
8+
9+
// case 1: tasks.<task-name>.inputs.<artifact-category-name>
10+
// case 2: tasks.<task-name>.outputs.<artifact-category-name>
11+
const taskArtifactUsagePattern = `\$\(tasks\.([^.]+)\.(?:inputs|outputs)\.([^.)]+)\)`
12+
13+
const StepArtifactPathPattern = `step.artifacts.path`
14+
15+
const TaskArtifactPathPattern = `artifacts.path`
1016

1117
var StepArtifactRegex = regexp.MustCompile(stepArtifactUsagePattern)
18+
var TaskArtifactRegex = regexp.MustCompile(taskArtifactUsagePattern)

‎internal/sidecarlogresults/sidecarlogresults.go

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -49,6 +49,7 @@ const (
4949
stepResultType SidecarLogResultType = "step"
5050

5151
stepArtifactType SidecarLogResultType = "stepArtifact"
52+
taskArtifactType SidecarLogResultType = "taskArtifact"
5253
sidecarResultNameSeparator string = "."
5354
)
5455

@@ -285,6 +286,8 @@ func parseResults(resultBytes []byte, maxResultLimit int) (result.RunResult, err
285286
resultType = result.StepResultType
286287
case stepArtifactType:
287288
resultType = result.StepArtifactsResultType
289+
case taskArtifactType:
290+
resultType = result.TaskRunArtifactsResultType
288291
default:
289292
return result.RunResult{}, fmt.Errorf("invalid sidecar result type %v. Must be %v or %v or %v", res.Type, taskResultType, stepResultType, stepArtifactType)
290293
}

‎internal/sidecarlogresults/sidecarlogresults_test.go

Lines changed: 65 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -338,6 +338,39 @@ func TestParseResults(t *testing.T) {
338338
}`,
339339
Type: "stepArtifact",
340340
},
341+
{
342+
Name: "task-run-artifacts-result",
343+
Value: `{
344+
"inputs":[
345+
{
346+
"name":"input-artifacts",
347+
"values":[
348+
{
349+
"uri":"pkg:example.github.com/inputs",
350+
"digest":{
351+
"sha256":"b35cacccfdb1e24dc497d15d553891345fd155713ffe647c281c583269eaaae0"
352+
}
353+
}
354+
]
355+
}
356+
],
357+
"outputs":[
358+
{
359+
"name":"image",
360+
"values":[
361+
{
362+
"uri":"pkg:github/package-url/purl-spec@244fd47e07d1004f0aed9c",
363+
"digest":{
364+
"sha256":"df85b9e3983fe2ce20ef76ad675ecf435cc99fc9350adc54fa230bae8c32ce48",
365+
"sha1":"95588b8f34c31eb7d62c92aaa4e6506639b06ef2"
366+
}
367+
}
368+
]
369+
}
370+
]
371+
}`,
372+
Type: "taskArtifact",
373+
},
341374
}
342375
podLogs := []string{}
343376
for _, r := range results {
@@ -400,6 +433,38 @@ func TestParseResults(t *testing.T) {
400433
]
401434
}`,
402435
ResultType: result.StepArtifactsResultType,
436+
}, {
437+
Key: "task-run-artifacts-result",
438+
Value: `{
439+
"inputs":[
440+
{
441+
"name":"input-artifacts",
442+
"values":[
443+
{
444+
"uri":"pkg:example.github.com/inputs",
445+
"digest":{
446+
"sha256":"b35cacccfdb1e24dc497d15d553891345fd155713ffe647c281c583269eaaae0"
447+
}
448+
}
449+
]
450+
}
451+
],
452+
"outputs":[
453+
{
454+
"name":"image",
455+
"values":[
456+
{
457+
"uri":"pkg:github/package-url/purl-spec@244fd47e07d1004f0aed9c",
458+
"digest":{
459+
"sha256":"df85b9e3983fe2ce20ef76ad675ecf435cc99fc9350adc54fa230bae8c32ce48",
460+
"sha1":"95588b8f34c31eb7d62c92aaa4e6506639b06ef2"
461+
}
462+
}
463+
]
464+
}
465+
]
466+
}`,
467+
ResultType: result.TaskRunArtifactsResultType,
403468
}}
404469
stepResults := []result.RunResult{}
405470
for _, plog := range podLogs {

‎pkg/apis/pipeline/paths.go

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -30,4 +30,6 @@ const (
3030
StepsDir = "/tekton/steps"
3131

3232
ScriptDir = "/tekton/scripts"
33+
34+
ArtifactsDir = "/tekton/artifacts"
3335
)

‎pkg/apis/pipeline/v1/artifact_types.go

Lines changed: 89 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,25 @@
1+
/*
2+
Copyright 2024 The Tekton Authors
3+
4+
Licensed under the Apache License, Version 2.0 (the "License");
5+
you may not use this file except in compliance with the License.
6+
You may obtain a copy of the License at
7+
8+
http://www.apache.org/licenses/LICENSE-2.0
9+
10+
Unless required by applicable law or agreed to in writing, software
11+
distributed under the License is distributed on an "AS IS" BASIS,
12+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
See the License for the specific language governing permissions and
14+
limitations under the License.
15+
*/
16+
117
package v1
218

19+
import (
20+
"github.com/google/go-cmp/cmp"
21+
)
22+
323
// Algorithm Standard cryptographic hash algorithm
424
type Algorithm string
525

@@ -27,3 +47,72 @@ type Artifacts struct {
2747
Inputs []Artifact `json:"inputs,omitempty"`
2848
Outputs []Artifact `json:"outputs,omitempty"`
2949
}
50+
51+
func (a *Artifacts) Merge(another Artifacts) {
52+
inputMap := make(map[string][]ArtifactValue)
53+
var newInputs []Artifact
54+
55+
for _, v := range a.Inputs {
56+
inputMap[v.Name] = v.Values
57+
}
58+
59+
for _, v := range another.Inputs {
60+
_, ok := inputMap[v.Name]
61+
if !ok {
62+
inputMap[v.Name] = []ArtifactValue{}
63+
}
64+
for _, vv := range v.Values {
65+
exists := false
66+
for _, av := range inputMap[v.Name] {
67+
if cmp.Equal(vv, av) {
68+
exists = true
69+
break
70+
}
71+
}
72+
if !exists {
73+
inputMap[v.Name] = append(inputMap[v.Name], vv)
74+
}
75+
}
76+
}
77+
78+
for k, v := range inputMap {
79+
newInputs = append(newInputs, Artifact{
80+
Name: k,
81+
Values: v,
82+
})
83+
}
84+
85+
outputMap := make(map[string][]ArtifactValue)
86+
var newOutputs []Artifact
87+
for _, v := range a.Outputs {
88+
outputMap[v.Name] = v.Values
89+
}
90+
91+
for _, v := range another.Outputs {
92+
_, ok := outputMap[v.Name]
93+
if !ok {
94+
outputMap[v.Name] = []ArtifactValue{}
95+
}
96+
for _, vv := range v.Values {
97+
exists := false
98+
for _, av := range outputMap[v.Name] {
99+
if cmp.Equal(vv, av) {
100+
exists = true
101+
break
102+
}
103+
}
104+
if !exists {
105+
outputMap[v.Name] = append(outputMap[v.Name], vv)
106+
}
107+
}
108+
}
109+
110+
for k, v := range outputMap {
111+
newOutputs = append(newOutputs, Artifact{
112+
Name: k,
113+
Values: v,
114+
})
115+
}
116+
a.Inputs = newInputs
117+
a.Outputs = newOutputs
118+
}

0 commit comments

Comments
 (0)