Skip to content

Commit 62e2940

Browse files
rawmind0rawmind0
committed
Add new cloudflare-r2-upload workflow to allow upload from github artifacts or release
1 parent c921b7c commit 62e2940

File tree

1 file changed

+355
-0
lines changed

1 file changed

+355
-0
lines changed

‎.github/workflows/_cloudflare-r2-upload.yaml

Lines changed: 355 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,355 @@
1+
name: Upload to Cloudflare R2
2+
3+
on:
4+
workflow_call:
5+
inputs:
6+
# Source configuration
7+
source_type:
8+
description: 'Source of files to upload: "artifacts" or "release"'
9+
required: false
10+
type: string
11+
default: 'artifacts'
12+
13+
# For artifacts mode
14+
artifact_pattern:
15+
description: 'Pattern to match artifacts (for artifacts mode)'
16+
required: false
17+
type: string
18+
default: '*'
19+
20+
# For release mode
21+
release_tag:
22+
description: 'GitHub release tag to download from (for release mode, defaults to current tag)'
23+
required: false
24+
type: string
25+
default: ''
26+
27+
release_pattern:
28+
description: 'Pattern to match release assets (for release mode)'
29+
required: false
30+
type: string
31+
default: '*'
32+
33+
# R2 Configuration
34+
r2_account_id:
35+
description: 'Cloudflare account ID'
36+
required: true
37+
type: string
38+
39+
r2_bucket:
40+
description: 'R2 bucket name'
41+
required: true
42+
type: string
43+
44+
r2_destination_dir:
45+
description: 'Destination directory in R2 bucket'
46+
required: false
47+
type: string
48+
default: ''
49+
50+
upload_to_latest:
51+
description: 'Also upload to "latest" folder'
52+
required: false
53+
type: boolean
54+
default: true
55+
56+
upload_to_versioned:
57+
description: 'Upload to version/commit folder'
58+
required: false
59+
type: boolean
60+
default: true
61+
62+
# Multipart upload configuration
63+
multipart_size:
64+
description: 'Minimum file size (in MB) to use multipart upload'
65+
required: false
66+
type: number
67+
default: 100
68+
69+
max_retries:
70+
description: 'Maximum number of retries for uploads'
71+
required: false
72+
type: number
73+
default: 5
74+
75+
multipart_concurrent:
76+
description: 'Use concurrent multipart uploads'
77+
required: false
78+
type: boolean
79+
default: true
80+
81+
# Environment
82+
environment:
83+
description: 'GitHub environment for secrets'
84+
required: false
85+
type: string
86+
default: ''
87+
88+
# GitHub App auth (optional)
89+
github_app_auth:
90+
description: 'Use GitHub App for authentication'
91+
required: false
92+
type: boolean
93+
default: false
94+
95+
github_app_repos:
96+
description: 'Repositories for GitHub App auth'
97+
required: false
98+
type: string
99+
default: ''
100+
101+
secrets:
102+
r2_access_key_id:
103+
required: true
104+
r2_secret_access_key:
105+
required: true
106+
app_id:
107+
required: false
108+
app_pem:
109+
required: false
110+
111+
outputs:
112+
upload_urls:
113+
description: 'URLs of uploaded files'
114+
value: ${{ jobs.upload-to-r2.outputs.upload_urls }}
115+
latest_path:
116+
description: 'Path to latest folder in R2'
117+
value: ${{ jobs.upload-to-r2.outputs.latest_path }}
118+
versioned_path:
119+
description: 'Path to versioned folder in R2'
120+
value: ${{ jobs.upload-to-r2.outputs.versioned_path }}
121+
122+
jobs:
123+
upload-to-r2:
124+
name: Upload to Cloudflare R2
125+
runs-on: ubuntu-latest
126+
environment: ${{ inputs.environment }}
127+
128+
outputs:
129+
upload_urls: ${{ steps.generate-outputs.outputs.upload_urls }}
130+
latest_path: ${{ steps.paths.outputs.latest_path }}
131+
versioned_path: ${{ steps.paths.outputs.versioned_path }}
132+
133+
steps:
134+
- name: Checkout with GitHub App
135+
if: inputs.github_app_auth
136+
uses: zondax/actions/checkout-with-app@v1
137+
with:
138+
github_app_auth: ${{ inputs.github_app_auth }}
139+
github_app_repos: ${{ inputs.github_app_repos }}
140+
use_sudo: true
141+
app_id: ${{ secrets.app_id }}
142+
app_pem: ${{ secrets.app_pem }}
143+
144+
- name: Checkout repository
145+
if: ${{ !inputs.github_app_auth }}
146+
uses: actions/checkout@v4
147+
148+
# ARTIFACTS MODE: Download from GitHub Actions artifacts
149+
- name: Download artifacts
150+
if: inputs.source_type == 'artifacts'
151+
uses: actions/download-artifact@v4
152+
with:
153+
pattern: ${{ inputs.artifact_pattern }}
154+
path: download
155+
156+
# RELEASE MODE: Download from GitHub release
157+
- name: Download from GitHub release
158+
if: inputs.source_type == 'release'
159+
env:
160+
GH_TOKEN: ${{ github.token }}
161+
run: |
162+
mkdir -p download
163+
164+
# Determine release tag
165+
RELEASE_TAG="${{ inputs.release_tag }}"
166+
if [ -z "$RELEASE_TAG" ]; then
167+
# Use current tag if not specified
168+
if [[ "${{ github.ref_type }}" == "tag" ]]; then
169+
RELEASE_TAG="${GITHUB_REF#refs/tags/}"
170+
else
171+
echo "❌ Error: release_tag is required when not running on a tag"
172+
exit 1
173+
fi
174+
fi
175+
176+
echo "📦 Downloading from release: $RELEASE_TAG"
177+
178+
# Parse pattern into array for multiple patterns
179+
IFS=' ' read -ra PATTERNS <<< "${{ inputs.release_pattern }}"
180+
181+
# Download each pattern
182+
for pattern in "${PATTERNS[@]}"; do
183+
echo " Downloading pattern: $pattern"
184+
gh release download "$RELEASE_TAG" \
185+
--repo ${{ github.repository }} \
186+
--pattern "$pattern" \
187+
--dir download || true
188+
done
189+
190+
echo "📋 Downloaded files:"
191+
ls -la download/
192+
193+
# Prepare upload directory (same for both modes)
194+
- name: Prepare upload directory
195+
run: |
196+
mkdir -p upload
197+
198+
# Handle both flat and nested structures
199+
if [ -d "download" ]; then
200+
# For artifacts mode (might have subdirectories)
201+
find download -type f -exec cp {} upload/ \;
202+
fi
203+
204+
echo "📋 Files to upload:"
205+
ls -la upload/
206+
207+
# Count files
208+
FILE_COUNT=$(find upload -type f | wc -l)
209+
echo "Total files to upload: $FILE_COUNT"
210+
211+
if [ "$FILE_COUNT" -eq 0 ]; then
212+
echo "⚠️ No files found to upload!"
213+
exit 1
214+
fi
215+
216+
# Determine folder names
217+
- name: Determine upload paths
218+
id: paths
219+
run: |
220+
# Version/tag based path
221+
if [[ "${{ github.ref_type }}" == "tag" ]]; then
222+
FOLDER="${GITHUB_REF#refs/tags/}"
223+
echo "📦 Production build - using tag: $FOLDER"
224+
else
225+
SHORT_SHA=$(echo "${{ github.sha }}" | cut -c1-7)
226+
BRANCH="${GITHUB_REF#refs/heads/}"
227+
FOLDER="${BRANCH}-${SHORT_SHA}"
228+
echo "🔧 Development build - using branch-commit: $FOLDER"
229+
fi
230+
231+
echo "folder=$FOLDER" >> $GITHUB_OUTPUT
232+
233+
# Construct full paths
234+
BASE_DIR="${{ inputs.r2_destination_dir }}"
235+
if [[ -n "$BASE_DIR" ]]; then
236+
LATEST_PATH="${BASE_DIR}/latest"
237+
VERSIONED_PATH="${BASE_DIR}/${FOLDER}"
238+
echo "🗂️ Using base directory: $BASE_DIR"
239+
else
240+
LATEST_PATH="latest"
241+
VERSIONED_PATH="${FOLDER}"
242+
echo "🗂️ Uploading to bucket root"
243+
fi
244+
245+
echo "latest_path=$LATEST_PATH" >> $GITHUB_OUTPUT
246+
echo "versioned_path=$VERSIONED_PATH" >> $GITHUB_OUTPUT
247+
248+
# Install rclone for R2 upload
249+
- name: Install rclone
250+
run: |
251+
curl https://rclone.org/install.sh | sudo bash
252+
rclone version
253+
254+
# Configure rclone for R2
255+
- name: Configure rclone for R2
256+
run: |
257+
mkdir -p ~/.config/rclone
258+
cat > ~/.config/rclone/rclone.conf << EOF
259+
[r2]
260+
type = s3
261+
provider = Cloudflare
262+
access_key_id = ${{ secrets.r2_access_key_id }}
263+
secret_access_key = ${{ secrets.r2_secret_access_key }}
264+
endpoint = https://${{ inputs.r2_account_id }}.r2.cloudflarestorage.com
265+
acl = private
266+
no_check_bucket = true
267+
EOF
268+
269+
# Multipart upload for large files
270+
- name: Configure multipart settings
271+
id: multipart
272+
run: |
273+
# Calculate chunk size based on input
274+
CHUNK_SIZE="${{ inputs.multipart_size }}M"
275+
echo "chunk_size=$CHUNK_SIZE" >> $GITHUB_OUTPUT
276+
277+
# Set concurrent transfers
278+
if [[ "${{ inputs.multipart_concurrent }}" == "true" ]]; then
279+
echo "transfers=4" >> $GITHUB_OUTPUT
280+
else
281+
echo "transfers=1" >> $GITHUB_OUTPUT
282+
fi
283+
284+
# Upload to versioned folder
285+
- name: Upload to versioned folder
286+
if: inputs.upload_to_versioned
287+
run: |
288+
echo "📦 Uploading to versioned folder: ${{ steps.paths.outputs.versioned_path }}"
289+
290+
rclone sync upload/ r2:${{ inputs.r2_bucket }}/${{ steps.paths.outputs.versioned_path }}/ \
291+
--s3-chunk-size ${{ steps.multipart.outputs.chunk_size }} \
292+
--transfers ${{ steps.multipart.outputs.transfers }} \
293+
--retries ${{ inputs.max_retries }} \
294+
--progress
295+
296+
echo "✅ Upload to versioned folder complete"
297+
298+
# Upload to latest folder
299+
- name: Upload to latest folder
300+
if: inputs.upload_to_latest
301+
run: |
302+
echo "🔄 Uploading to latest folder: ${{ steps.paths.outputs.latest_path }}"
303+
304+
rclone sync upload/ r2:${{ inputs.r2_bucket }}/${{ steps.paths.outputs.latest_path }}/ \
305+
--s3-chunk-size ${{ steps.multipart.outputs.chunk_size }} \
306+
--transfers ${{ steps.multipart.outputs.transfers }} \
307+
--retries ${{ inputs.max_retries }} \
308+
--delete-during \
309+
--progress
310+
311+
echo "✅ Upload to latest folder complete"
312+
313+
# List uploaded files
314+
- name: List uploaded files
315+
id: list-files
316+
run: |
317+
echo "📋 Uploaded files:"
318+
319+
if [[ "${{ inputs.upload_to_versioned }}" == "true" ]]; then
320+
echo ""
321+
echo "Versioned folder (${{ steps.paths.outputs.versioned_path }}):"
322+
rclone ls r2:${{ inputs.r2_bucket }}/${{ steps.paths.outputs.versioned_path }}/ || true
323+
fi
324+
325+
if [[ "${{ inputs.upload_to_latest }}" == "true" ]]; then
326+
echo ""
327+
echo "Latest folder (${{ steps.paths.outputs.latest_path }}):"
328+
rclone ls r2:${{ inputs.r2_bucket }}/${{ steps.paths.outputs.latest_path }}/ || true
329+
fi
330+
331+
# Generate output URLs
332+
- name: Generate output URLs
333+
id: generate-outputs
334+
run: |
335+
echo "=== Upload Results ==="
336+
echo "📦 R2 upload completed successfully!"
337+
echo ""
338+
339+
URLS=""
340+
341+
if [[ "${{ inputs.upload_to_latest }}" == "true" ]]; then
342+
LATEST_URL="R2 Latest: ${{ inputs.r2_bucket }}/${{ steps.paths.outputs.latest_path }}"
343+
echo "🔄 $LATEST_URL"
344+
URLS="${URLS}${LATEST_URL}\n"
345+
fi
346+
347+
if [[ "${{ inputs.upload_to_versioned }}" == "true" ]]; then
348+
VERSIONED_URL="R2 Versioned: ${{ inputs.r2_bucket }}/${{ steps.paths.outputs.versioned_path }}"
349+
echo "📂 $VERSIONED_URL"
350+
URLS="${URLS}${VERSIONED_URL}"
351+
fi
352+
353+
echo "upload_urls<<EOF" >> $GITHUB_OUTPUT
354+
echo -e "$URLS" >> $GITHUB_OUTPUT
355+
echo "EOF" >> $GITHUB_OUTPUT

0 commit comments

Comments
 (0)