Saved grouped by as artifact!

Author: saulshanabrook

k8/Makefile

@@ -72,7 +72,7 @@ $(info )
 # Pass in the WORKFLOW_NAME as a variable
 download-workflow:
 	argo get ${WORKFLOW_NAME} -o json \
-		| jq -r '.status.outputs.artifacts[0].s3 | "artifact/" + .bucket + "/" + .key' \
+		| jq -r '.status.outputs.artifacts[] | select(.name == "api") | .s3 | "artifact/" + .bucket + "/" + .key' \
 		| xargs mc cat \
 		| tar -xvzf - -C ../data/api/
 	mv -f ../data/api/api.json ../data/api/${WORKFLOW_LABEL}.json

k8/argo/workflow.yml

@@ -35,6 +35,10 @@ spec:
             template: groupby-lines
         - - name: infer-api
             template: infer-api
+            arguments:
+              artifacts:
+                - name: grouped
+                  from: "{{workflow.outputs.artifacts.grouped}}"
         - - name: trigger-github-action
             template: trigger-github-action
     - name: record-api
@@ -58,14 +62,19 @@ spec:
           - name: data
             mountPath: /tmp/vol
     - name: groupby-lines
+      outputs:
+        artifacts:
+          - name: grouped
+            path: /tmp/grouped.jsonl
+            globalName: "grouped"
       container:
         image: "{{workflow.parameters.base-image}}"
         command: [python, -m, record_api.line_counts]
         env:
           - name: PYTHON_RECORD_API_INPUT
             value: /tmp/vol/raw.jsonl
           - name: PYTHON_RECORD_API_OUTPUT
-            value: /tmp/vol/grouped.jsonl
+            value: /tmp/grouped.jsonl
         resources:
           requests:
             memory: 6Gi
@@ -77,11 +86,15 @@ spec:
           - name: data
             mountPath: /tmp/vol
     - name: infer-api
+      inputs:
+        artifacts:
+          - name: grouped
+            path: /tmp/grouped.jsonl
       outputs:
         artifacts:
           - name: api
             path: /tmp/api.json
-            globalName: 'api'
+            globalName: "api"
       container:
         image: "{{workflow.parameters.base-image}}"
         command: [python, -m, record_api.infer_apis]
@@ -91,19 +104,16 @@ spec:
           - name: PYTHON_RECORD_API_MODULES
             value: pandas,numpy
           - name: PYTHON_RECORD_API_INPUT
-            value: /tmp/vol/grouped.jsonl
+            value: /tmp/grouped.jsonl
           - name: PYTHON_RECORD_API_OUTPUT
             value: /tmp/api.json
         resources:
           requests:
-            memory: 1Gi
-            cpu: 500m
+            memory: 6Gi
+            cpu: 100m
           limits:
-            memory: 8Gi
+            memory: 16Gi
             cpu: 4000m
-        volumeMounts:
-          - name: data
-            mountPath: /tmp/vol
     - name: trigger-github-action
       container:
         image: "curlimages/curl"