Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Mirror more data per repo and allow arbitrary user mirroring #6

Open
wants to merge 3 commits into
base: master
Choose a base branch
from
Open

Mirror more data per repo and allow arbitrary user mirroring #6

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
3b7b9ea
Enable LFS and wiki mirroring, and use auth token for all repo mirrors
Brycey92 Mar 4, 2024
7d943f4
Improve clarity of curl error handling
Brycey92 Mar 5, 2024
b0b2c9e
Allow `--mode user` to fetch repos of arbitrary `--user` value
Brycey92 Mar 5, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
15 changes: 10 additions & 5 deletions README.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,18 +30,18 @@ There are four modes of operation, which the script can print out to the console
./github2gitea-mirror -m

Usage: ./github2gitea-mirror
-m, --mode {org,star,repo,user} Mode to use; either mirror an organization or mirror all starred repositories.
-m, --mode {org,star,repo,user} Mode to use; mirror an organization, mirror all starred repositories, mirror a single repo, or mirror a user.
-o, --org $organization GitHub organization to mirror and/or the target organization in Gitea.
-u, --user $github_user GitHub user to gather the starred repositories from.
-v, --visibility {public,private} Visibility for the created Gitea organization.
-u, --user $github_user GitHub user to gather repositories from.
-v, --visibility {public,private} Visibility for the created Gitea organization or user.
-r, --repo $repo_url GitHub URL of a single repo to create a mirror for.
```

- `--mode`
- `org`: Mirror a complete oranization with all its public/private repositories.
- `star`: Mirror all starred repositories by user `$github_user`.
- `repo`: Mirror a single (public or private) repository.
- `user`: Mirror a complete user with all its public/private repositories.
- `user`: Mirror a complete user `$github_user` with all its public/private repositories.

#### Examples

Expand All @@ -60,11 +60,16 @@ Mirror a single GitHub repository:
./github2gitea-mirror -m repo https://github.com/maxkratz/github2gitea-mirror -u $myGitHubUser
```

Mirror a complete GitHub user:
Mirror your own complete GitHub user:
```bash
./github2gitea-mirror -m user -u $myGitHubUser
```

Mirror someone else's complete GitHub user:
```bash
./github2gitea-mirror -m user -v public -u $someGitHubUser
```

#### TODOs
- [x] Mirror a complete GitHub organization
- [x] Mirror (single) private repos
Expand Down
9 changes: 7 additions & 2 deletions delete_gitea_org
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -66,7 +66,10 @@ fetch_orga_repos() {
log "Fetch organization repos."
if ! $CURL -X GET $GITEA_URL/api/v1/orgs/${gitea_organization}/repos "${header_options[@]}" >${jsonoutput}/1.json 2>${jsonoutput}/stderr.txt; then
local code=$(<${jsonoutput}/result.txt)
if (( code != 404 ));then # 404 == orga not found
if (( code == 404 ));then # 404 == orga not found
log "404: Gitea orga not found. Nothing to do. Exiting."
exit 1
else
cat ${jsonoutput}/stderr.txt >&2
fi
fi
Expand Down Expand Up @@ -95,7 +98,9 @@ delete_orga() {
log "Delete orga."
if ! $CURL -X DELETE $GITEA_URL/api/v1/orgs/${gitea_organization} "${header_options[@]}" > ${jsonoutput}/result.txt 2>${jsonoutput}/stderr.txt; then
local code=$(<${jsonoutput}/result.txt)
if (( code != 404 ));then # 404 == orga not found
if (( code == 404 ));then # 404 == orga not found
log "404: Gitea orga not found. Is it already deleted?"
else
cat ${jsonoutput}/stderr.txt >&2
fi
fi
Expand Down
75 changes: 67 additions & 8 deletions github2gitea-mirror
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,7 @@ fi

# Parse input arguments
if [[ -z "$1" ]]; then
log "No parameter(s) given. Exit."
echo -e "No parameter(s) given. Exit."
exit 1
fi
while [[ "$#" -gt 0 ]]; do
Expand All @@ -45,7 +45,7 @@ while [[ "$#" -gt 0 ]]; do
-u|--user) github_user="$2"; shift ;;
-v|--visibility) visibility="$2"; shift ;;
-r|--repo) repo="$2"; shift ;;
*) log "Unknown parameter passed: $1"; exit 1 ;;
*) echo -e "Unknown parameter passed: $1"; exit 1 ;;
esac
shift
done
Expand Down Expand Up @@ -114,7 +114,26 @@ set_uid() {

# Sets the uid to the specified Gitea user
set_uid_user() {
uid=$($CURL "${header_options[@]}" $GITEA_URL/api/v1/users/${github_user} | jq .id)
if $CURL "${header_options[@]}" $GITEA_URL/api/v1/users/${github_user} >${jsonoutput}/result.txt ; then
uid=$(jq .id <${jsonoutput}/result.txt)
else
if [[ $(jq <${jsonoutput}/result.txt '. | length') -eq 0 ]] ; then
log "Gitea user not found. Creating user."
if [[ -z "${visibility}" ]]; then
echo -e "Visibility not set."
exit 1
fi
create_migration_user ${visibility}
if $CURL "${header_options[@]}" $GITEA_URL/api/v1/users/${github_user} >${jsonoutput}/result.txt ; then
uid=$(jq .id <${jsonoutput}/result.txt)
else
echo -e "Failed to get Gitea user after attempted creation. Exiting."
exit 1
fi
else
exit 1
fi
fi
}

# Fetches all starred repos of the given user to JSON files
Expand Down Expand Up @@ -142,9 +161,21 @@ fetch_orga_repos() {
# Fetches all public/private repos of the given GitHub user to JSON files
fetch_user_repos() {
log "Fetch user repos."
$CURL "https://api.github.com/user" -u "${github_user}:${GITHUB_TOKEN}" >${jsonoutput}/user.txt
if (( $(jq <${jsonoutput}/user.txt '. | length') == 0 ))
then
echo -e "Empty response or 404 from GitHub. Exiting."
exit 1
else
if [[ $(jq <${jsonoutput}/user.txt '.login') == "\"${github_user}\"" ]] ; then
local url_segment="user"
else
local url_segment="users/${github_user}"
fi
fi
i=1
# GitHub API just returns empty arrays instead of 404
while $CURL "https://api.github.com/user/repos?affiliation=owner&page=${i}&per_page=100" -u "${github_user}:${GITHUB_TOKEN}" >${jsonoutput}/${i}.json \
while $CURL "https://api.github.com/${url_segment}/repos?affiliation=owner&page=${i}&per_page=100" -u "${github_user}:${GITHUB_TOKEN}" >${jsonoutput}/${i}.json \
&& (( $(jq <${jsonoutput}/${i}.json '. | length') > 0 )) ; do
(( i++ ))
done
Expand All @@ -163,8 +194,27 @@ create_migration_repo() {
log "Create migration repo."
if ! $CURL -w "%{http_code}\n" "${header_options[@]}" -d @- -X POST $GITEA_URL/api/v1/repos/migrate > ${jsonoutput}/result.txt 2>${jsonoutput}/stderr.txt; then
local code=$(<${jsonoutput}/result.txt)
if (( code != 409 ));then # 409 == repo already exits
if (( code == 409 ));then # 409 == repo already exits
log "409: Gitea repo already exists. Skipping migration."
else
cat ${jsonoutput}/stderr.txt >&2
exit 1
fi
fi
}

# Creates a specific public/private user on Gitea
create_migration_user() {
visibility="${1:-}"
log "Create migration user with name: ${github_user}"
local password=$(tr -dc 'A-Za-z0-9!@#$%^&*()_+\-=[]{}\|;:,.<>/`~' < /dev/urandom | head -c 64)
if ! $CURL -X POST $GITEA_URL/api/v1/admin/users "${header_options[@]}" --data '{"username": "'"${github_user}"'", "visibility": "'"${visibility}"'", "email": "'"${github_user}@example.com"'", "password": "'"${password}"'", "must_change_password": true}' > ${jsonoutput}/result.txt 2>${jsonoutput}/stderr.txt; then
Copy link
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Despite the Gitea API documentation saying the password field is not required, it is required, and Gitea will return a 400 Bad Request error without it.

local code=$(<${jsonoutput}/result.txt)
if (( code == 422 ));then # 422 == user already exits
log "422: Gitea user already exists. Skipping creation."
else
cat ${jsonoutput}/stderr.txt >&2
exit 1
fi
fi
}
Expand All @@ -175,8 +225,11 @@ create_migration_orga() {
log "Create migration orga with name: ${gitea_organization}"
if ! $CURL -X POST $GITEA_URL/api/v1/orgs "${header_options[@]}" --data '{"username": "'"${gitea_organization}"'", "visibility": "'"${visibility}"'"}' > ${jsonoutput}/result.txt 2>${jsonoutput}/stderr.txt; then
local code=$(<${jsonoutput}/result.txt)
if (( code != 422 ));then # 422 == orga already exits
if (( code == 422 ));then # 422 == orga already exits
log "422: Gitea orga already exists. Skipping creation."
else
cat ${jsonoutput}/stderr.txt >&2
exit 1
fi
fi
}
Expand All @@ -196,11 +249,14 @@ repos_to_migration() {
if(.visibility==\"private\") then .private=true else .private=false end |\
if(.visibility==\"private\") then .auth_username=\"${github_user}\" else . end | \
if(.visibility==\"private\") then .auth_password=\"${GITHUB_TOKEN}\" else . end | \
.auth_token=\"${GITHUB_TOKEN}\" | \
.lfs=true | \
.mirror=true | \
.wiki=true | \
.clone_addr=.clone_url | \
.description=.description[0:255] | \
.repo_name=.name | \
{uid,repo_name,clone_addr,description,mirror,private,auth_username,auth_password}" <$f)
{uid,wiki,repo_name,clone_addr,description,mirror,lfs,private,auth_token,auth_username,auth_password}" <$f)
echo "Migrating repo" $(jq ".[$i] | .uid=${uid} | .name" <$f)
echo $mig_data | create_migration_repo
done
Expand All @@ -216,11 +272,14 @@ one_repo_to_migration() {
if(.visibility==\"private\") then .private=true else .private=false end |\
if(.visibility==\"private\") then .auth_username=\"${github_user}\" else . end | \
if(.visibility==\"private\") then .auth_password=\"${GITHUB_TOKEN}\" else . end | \
.auth_token=\"${GITHUB_TOKEN}\" | \
.lfs=true | \
.mirror=true | \
.wiki=true | \
.clone_addr=.clone_url | \
.description=.description[0:255] | \
.repo_name=.name | \
{repo_owner,repo_name,clone_addr,description,mirror,private,auth_username,auth_password}" <$f)
{wiki,repo_owner,repo_name,clone_addr,description,mirror,lfs,private,auth_token,auth_username,auth_password}" <$f)
echo "Migrating repo" $(jq ".name" <$f)
echo $mig_data | create_migration_repo
done
Expand Down