Skip to content

Commit c68e7fb

Browse files
abetlenabetlen
committed
fix: pull all gh releases for self-hosted python index
1 parent e251a0b commit c68e7fb

File tree

3 files changed

+133
-39
lines changed

3 files changed

+133
-39
lines changed

‎.github/workflows/generate-index-from-release.yaml

+3
Original file line numberDiff line numberDiff line change
@@ -35,7 +35,10 @@ jobs:
3535
- name: Setup Pages
3636
uses: actions/configure-pages@v5
3737
- name: Build
38+
env:
39+
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
3840
run: |
41+
./scripts/get-releases.sh
3942
./scripts/releases-to-pep-503.sh index/whl/cpu '^[v]?[0-9]+\.[0-9]+\.[0-9]+$'
4043
./scripts/releases-to-pep-503.sh index/whl/cu121 '^[v]?[0-9]+\.[0-9]+\.[0-9]+-cu121$'
4144
./scripts/releases-to-pep-503.sh index/whl/cu122 '^[v]?[0-9]+\.[0-9]+\.[0-9]+-cu122$'

‎scripts/get-releases.sh

+47
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,47 @@
1+
#!/bin/bash
2+
3+
# Function to get all releases
4+
get_all_releases() {
5+
local page=1
6+
local per_page=100
7+
local releases=""
8+
local new_releases
9+
10+
# Prepare headers
11+
local headers=(-H "Accept: application/vnd.github.v3+json")
12+
if [ -n "$GITHUB_TOKEN" ]; then
13+
headers+=(-H "Authorization: Bearer $GITHUB_TOKEN")
14+
fi
15+
16+
while true; do
17+
response=$(curl -s "${headers[@]}" \
18+
"https://api.github.com/repos/abetlen/llama-cpp-python/releases?page=$page&per_page=$per_page")
19+
20+
# Check if the response is valid JSON
21+
if ! echo "$response" | jq empty > /dev/null 2>&1; then
22+
echo "Error: Invalid response from GitHub API" >&2
23+
echo "Response: $response" >&2
24+
return 1
25+
fi
26+
27+
new_releases=$(echo "$response" | jq -r '.[].tag_name')
28+
if [ -z "$new_releases" ]; then
29+
break
30+
fi
31+
releases="$releases $new_releases"
32+
((page++))
33+
done
34+
35+
echo $releases
36+
}
37+
38+
# Get all releases and save to file
39+
releases=$(get_all_releases)
40+
if [ $? -ne 0 ]; then
41+
echo "Failed to fetch releases. Please check your internet connection and try again later." >&2
42+
exit 1
43+
fi
44+
45+
echo "$releases" | tr ' ' '\n' > all_releases.txt
46+
47+
echo "All releases have been saved to all_releases.txt"

‎scripts/releases-to-pep-503.sh

+83-39
Original file line numberDiff line numberDiff line change
@@ -1,60 +1,104 @@
11
#!/bin/bash
22

3+
# Enable exit on error
4+
set -e
5+
6+
# Function for logging
7+
log_error() {
8+
echo "ERROR: $1" >&2
9+
}
10+
11+
log_info() {
12+
echo "INFO: $1"
13+
}
14+
315
# Get output directory or default to index/whl/cpu
416
output_dir=${1:-"index/whl/cpu"}
517

6-
# Create output directory
7-
mkdir -p $output_dir
8-
9-
# Change to output directory
10-
pushd $output_dir
18+
# Get pattern from second arg or default to valid python package version pattern
19+
pattern=${2:-"^[v]?[0-9]+\.[0-9]+\.[0-9]+$"}
1120

12-
# Create an index html file
13-
echo "<!DOCTYPE html>" > index.html
14-
echo "<html>" >> index.html
15-
echo " <head></head>" >> index.html
16-
echo " <body>" >> index.html
17-
echo " <a href=\"llama-cpp-python/\">llama-cpp-python</a>" >> index.html
18-
echo " <br>" >> index.html
19-
echo " </body>" >> index.html
20-
echo "</html>" >> index.html
21-
echo "" >> index.html
21+
# Get the current directory (where the script is run from)
22+
current_dir="$(pwd)"
2223

23-
# Create llama-cpp-python directory
24-
mkdir -p llama-cpp-python
24+
# Check if all_releases.txt exists
25+
if [ ! -f "$current_dir/all_releases.txt" ]; then
26+
log_error "all_releases.txt not found in the current directory."
27+
exit 1
28+
fi
2529

26-
# Change to llama-cpp-python directory
27-
pushd llama-cpp-python
30+
# Create output directory
31+
mkdir -p "$output_dir"
2832

2933
# Create an index html file
30-
echo "<!DOCTYPE html>" > index.html
31-
echo "<html>" >> index.html
32-
echo " <body>" >> index.html
33-
echo " <h1>Links for llama-cpp-python</h1>" >> index.html
34+
cat << EOF > "$output_dir/index.html"
35+
<!DOCTYPE html>
36+
<html>
37+
<head></head>
38+
<body>
39+
<a href="llama-cpp-python/">llama-cpp-python</a>
40+
<br>
41+
</body>
42+
</html>
3443
35-
# Get all releases
36-
releases=$(curl -s https://api.github.com/repos/abetlen/llama-cpp-python/releases | jq -r .[].tag_name)
44+
EOF
3745

38-
# Get pattern from second arg or default to valid python package version pattern
39-
pattern=${2:-"^[v]?[0-9]+\.[0-9]+\.[0-9]+$"}
46+
# Create llama-cpp-python directory
47+
mkdir -p "$output_dir/llama-cpp-python"
48+
49+
# Create an index html file in llama-cpp-python directory
50+
cat << EOF > "$output_dir/llama-cpp-python/index.html"
51+
<!DOCTYPE html>
52+
<html>
53+
<body>
54+
<h1>Links for llama-cpp-python</h1>
55+
EOF
4056

4157
# Filter releases by pattern
42-
releases=$(echo $releases | tr ' ' '\n' | grep -E $pattern)
58+
releases=$(grep -E "$pattern" "$current_dir/all_releases.txt")
59+
60+
# Prepare curl headers
61+
headers=('--header' 'Accept: application/vnd.github.v3+json')
62+
if [ -n "$GITHUB_TOKEN" ]; then
63+
headers+=('--header' "authorization: Bearer $GITHUB_TOKEN")
64+
fi
65+
headers+=('--header' 'content-type: application/json')
4366

4467
# For each release, get all assets
4568
for release in $releases; do
46-
assets=$(curl -s https://api.github.com/repos/abetlen/llama-cpp-python/releases/tags/$release | jq -r .assets)
69+
log_info "Processing release: $release"
70+
response=$(curl -s "${headers[@]}" \
71+
"https://api.github.com/repos/abetlen/llama-cpp-python/releases/tags/$release")
72+
73+
if [ -z "$response" ]; then
74+
log_error "Empty response from GitHub API for release $release"
75+
continue
76+
fi
77+
78+
if ! echo "$response" | jq -e '.assets' > /dev/null 2>&1; then
79+
log_error "Invalid or unexpected response from GitHub API for release $release"
80+
log_error "Response: $response"
81+
continue
82+
fi
83+
4784
# Get release version from release ie v0.1.0-cu121 -> v0.1.0
48-
release_version=$(echo $release | grep -oE "^[v]?[0-9]+\.[0-9]+\.[0-9]+")
49-
echo " <h2>$release_version</h2>" >> index.html
50-
for asset in $(echo $assets | jq -r .[].browser_download_url); do
51-
if [[ $asset == *".whl" ]]; then
52-
echo " <a href=\"$asset\">$asset</a>" >> index.html
53-
echo " <br>" >> index.html
54-
fi
85+
release_version=$(echo "$release" | grep -oE "^[v]?[0-9]+\.[0-9]+\.[0-9]+")
86+
echo " <h2>$release_version</h2>" >> "$output_dir/llama-cpp-python/index.html"
87+
88+
wheel_urls=$(echo "$response" | jq -r '.assets[] | select(.name | endswith(".whl")) | .browser_download_url')
89+
if [ -z "$wheel_urls" ]; then
90+
log_error "No wheel files found for release $release"
91+
continue
92+
fi
93+
94+
echo "$wheel_urls" | while read -r asset; do
95+
echo " <a href=\"$asset\">$asset</a>" >> "$output_dir/llama-cpp-python/index.html"
96+
echo " <br>" >> "$output_dir/llama-cpp-python/index.html"
5597
done
5698
done
5799

58-
echo " </body>" >> index.html
59-
echo "</html>" >> index.html
60-
echo "" >> index.html
100+
echo " </body>" >> "$output_dir/llama-cpp-python/index.html"
101+
echo "</html>" >> "$output_dir/llama-cpp-python/index.html"
102+
echo "" >> "$output_dir/llama-cpp-python/index.html"
103+
104+
log_info "Index generation complete. Output directory: $output_dir"

0 commit comments

Comments
 (0)