Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

fix system tests #1265

Merged
merged 2 commits into from
Apr 11, 2024
Merged

fix system tests #1265

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
4590bfc
fix system tests
neolynx Apr 10, 2024
9d9dda7
openpgp: export full key for internal gpg
neolynx Apr 11, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 0 additions & 6 deletions .github/workflows/ci.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -46,12 +46,6 @@ jobs:
make version
go generate
- name: golangci-lint
uses: golangci/golangci-lint-action@v3
with:
version: v1.54.2
args: --timeout=5m

- name: Setup Python
uses: actions/setup-python@v4
with:
Expand Down
11 changes: 7 additions & 4 deletions Makefile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,6 @@ dev:

check: system/env
ifeq ($(RUN_LONG_TESTS), yes)
golangci-lint run
system/env/bin/flake8
endif

Expand All @@ -57,6 +56,10 @@ ifeq ($(RUN_LONG_TESTS), yes)
PATH=$(BINPATH)/:$(PATH) && . system/env/bin/activate && APTLY_VERSION=$(VERSION) $(PYTHON) system/run.py --long $(TESTS) --coverage-dir $(COVERAGE_DIR) $(CAPTURE)
endif

docker-test: install
go test -v -coverpkg="./..." -c -tags testruncli
PATH=$(BINPATH)/:$(PATH) APTLY_VERSION=$(VERSION) $(PYTHON) system/run.py --long $(TESTS) --coverage-dir $(COVERAGE_DIR) $(CAPTURE) $(TEST)

test:
go test -v ./... -gocheck.v=true -coverprofile=unit.out

Expand Down Expand Up @@ -90,10 +93,10 @@ version: ## Print aptly version
@echo $(VERSION)

docker-build-system-tests: ## Build system-test docker image
docker build -f system/Dockerfile . -t aptly-system-test
docker build -f system/Dockerfile --no-cache . -t aptly-system-test

docker-system-tests: ## Run system tests in docker container
docker run -t --rm -v ${PWD}:/app aptly-system-test
docker-system-tests: ## Run system tests in docker container (add TEST=t04_mirror to run only specific tests)
docker run -t --rm -v ${PWD}:/app aptly-system-test /app/system/run-system-tests $(TEST)

golangci-lint: ## Run golangci-line in docker container
docker run -t --rm -v ~/.cache/golangci-lint/v1.56.2:/root/.cache -v ${PWD}:/app -w /app golangci/golangci-lint:v1.56.2 golangci-lint run
Expand Down
11 changes: 4 additions & 7 deletions http/download.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -176,7 +176,7 @@
expected *utils.ChecksumInfo, ignoreMismatch bool) error {

if downloader.progress != nil {
downloader.progress.Printf("Downloading %s...\n", url)
downloader.progress.Printf("Downloading: %s\n", url)
defer downloader.progress.Flush()
}
req, err := downloader.newRequest(ctx, "GET", url)
Expand All @@ -192,7 +192,7 @@
if err != nil {
if retryableError(err) {
if downloader.progress != nil {
downloader.progress.Printf("Error downloading %s: %s retrying...\n", url, err)
downloader.progress.Printf("Error (retrying): %s\n", err)
}
maxTries--
time.Sleep(delay)
Expand All @@ -203,15 +203,12 @@
}
} else {
if downloader.progress != nil {
downloader.progress.Printf("Error downloading %s: %s cannot retry...\n", url, err)
downloader.progress.Printf("Error: %s \n", err)

Check warning on line 206 in http/download.go

View check run for this annotation

Codecov / codecov/patch

http/download.go#L206 

Added line #L206 was not covered by tests
}
break
}
} else {
// get out of the loop
if downloader.progress != nil {
downloader.progress.Printf("Success downloading %s\n", url)
}
break
}
if downloader.progress != nil {
Expand All @@ -222,7 +219,7 @@
// still an error after retrying, giving up
if err != nil {
if downloader.progress != nil {
downloader.progress.Printf("Giving up on %s...\n", url)
downloader.progress.Printf("Download Error: %s\n", url)
}
return err
}
Expand Down
2 changes: 1 addition & 1 deletion http/grab.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -59,7 +59,7 @@ func (d *GrabDownloader) DownloadWithChecksum(ctx context.Context, url string, d
// Success
break
}
d.log("Error downloading %s: %v\n", url, err)
d.log("Download Error: %v\n", err)
if retryableError(err) {
maxTries--
d.log("Retrying download %s: %d\n", url, maxTries)
Expand Down
7 changes: 4 additions & 3 deletions system/Dockerfile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,15 +4,16 @@ RUN apt-get update -y && apt-get install -y --no-install-recommends curl gnupg &

RUN echo deb http://deb.debian.org/debian bookworm-backports main > /etc/apt/sources.list.d/backports.list
RUN apt-get update && \
apt-get install -y --no-install-recommends apg bzip2 xz-utils ca-certificates golang/bookworm-backports golang-go/bookworm-backports golang-doc/bookworm-backports golang-src/bookworm-backports make git python3 python3-venv && \
apt-get install -y --no-install-recommends apg bzip2 xz-utils ca-certificates golang/bookworm-backports golang-go/bookworm-backports golang-doc/bookworm-backports golang-src/bookworm-backports make git python3 python3-requests-unixsocket && \
apt-get clean && rm -rf /var/lib/apt/lists/*

RUN useradd -m --shell /bin/sh --home-dir /var/lib/aptly aptly

RUN mkdir app
WORKDIR /app

RUN su - aptly -c "git clone https://github.com/aptly-dev/aptly-fixture-db.git"
RUN su - aptly -c "git clone https://github.com/aptly-dev/aptly-fixture-pool.git"
RUN mkdir /home/runner
RUN cd /home/runner; git clone https://github.com/aptly-dev/aptly-fixture-db.git
RUN cd /home/runner; git clone https://github.com/aptly-dev/aptly-fixture-pool.git

CMD /app/system/run-system-tests
5 changes: 4 additions & 1 deletion system/lib.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -245,7 +245,10 @@ def prepare_fixture(self):

if hasattr(self, "fixtureCmds"):
for cmd in self.fixtureCmds:
self.run_cmd(cmd)
output = self.run_cmd(cmd)
print("\n")
for line in output.decode("utf-8").split("\n"):
print(f" {line}")

def sort_lines(self, output):
return "\n".join(sorted(self.ensure_utf8(output).split("\n")))
Expand Down
12 changes: 4 additions & 8 deletions system/run-system-tests
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,16 +1,12 @@
#!/bin/sh

# cleanup
rm -rf /app/tmp
rm -rf /tmp/aptly*
rm -rf /home/test/.*
rm -rf /home/test/aptly-fixture-*

usermod -u `stat -c %u /app` aptly
chown -R `stat -c %u /app` /var/lib/aptly

mkdir -p /home/runner
# use same /home/runner dir as in github workflow
chown -R `stat -c %u /app` /home/runner

su - aptly -c "cd /app; python3 -m venv system/env"
su - aptly -c "cd /app; go mod tidy; HOME=/home/runner make system-test"

rm -rf system/env
su - aptly -c "cd /app; export HOME=/home/runner; go mod tidy; make docker-test TEST=$@"
9 changes: 3 additions & 6 deletions system/t04_mirror/CreateMirror10Test_gold
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,16 +1,13 @@
Downloading http://repo.aptly.info/system-tests/archive.debian.org/debian-archive/debian/dists/stretch-backports/InRelease...
Success downloading http://repo.aptly.info/system-tests/archive.debian.org/debian-archive/debian/dists/stretch-backports/InRelease
Downloading: http://repo.aptly.info/system-tests/archive.debian.org/debian-archive/debian/dists/stretch-backports/InRelease

gpgv: Signature made Thu Mar 30 14:21:34 2023 UTC
gpgv: using RSA key 0146DC6D4A0B2914BDED34DB648ACFD622F3D138

gpgv: Signature made Thu Mar 30 14:22:13 2023 UTC
gpgv: using RSA key A7236886F3CCCAAD148A27F80E98404D386FA1D9

Downloading http://repo.aptly.info/system-tests/archive.debian.org/debian-archive/debian/dists/stretch-backports/Release...
Success downloading http://repo.aptly.info/system-tests/archive.debian.org/debian-archive/debian/dists/stretch-backports/Release
Downloading http://repo.aptly.info/system-tests/archive.debian.org/debian-archive/debian/dists/stretch-backports/Release.gpg...
Success downloading http://repo.aptly.info/system-tests/archive.debian.org/debian-archive/debian/dists/stretch-backports/Release.gpg
Downloading: http://repo.aptly.info/system-tests/archive.debian.org/debian-archive/debian/dists/stretch-backports/Release
Downloading: http://repo.aptly.info/system-tests/archive.debian.org/debian-archive/debian/dists/stretch-backports/Release.gpg

gpgv: Signature made Thu Mar 30 14:20:54 2023 UTC
gpgv: using RSA key 0146DC6D4A0B2914BDED34DB648ACFD622F3D138
Expand Down
12 changes: 5 additions & 7 deletions system/t04_mirror/CreateMirror11Test_gold
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,11 +1,9 @@
Downloading http://repo.aptly.info/system-tests/archive.debian.org/debian-archive/debian/dists/stretch/InRelease...
Error downloading http://repo.aptly.info/system-tests/archive.debian.org/debian-archive/debian/dists/stretch/InRelease: HTTP code 404 while fetching http://repo.aptly.info/system-tests/archive.debian.org/debian-archive/debian/dists/stretch/InRelease retrying...
Downloading: http://repo.aptly.info/system-tests/archive.debian.org/debian-archive/debian/dists/stretch/InRelease
Error (retrying): HTTP code 404 while fetching http://repo.aptly.info/system-tests/archive.debian.org/debian-archive/debian/dists/stretch/InRelease
Retrying 0 http://repo.aptly.info/system-tests/archive.debian.org/debian-archive/debian/dists/stretch/InRelease...
Giving up on http://repo.aptly.info/system-tests/archive.debian.org/debian-archive/debian/dists/stretch/InRelease...
Downloading http://repo.aptly.info/system-tests/archive.debian.org/debian-archive/debian/dists/stretch/Release...
Success downloading http://repo.aptly.info/system-tests/archive.debian.org/debian-archive/debian/dists/stretch/Release
Downloading http://repo.aptly.info/system-tests/archive.debian.org/debian-archive/debian/dists/stretch/Release.gpg...
Success downloading http://repo.aptly.info/system-tests/archive.debian.org/debian-archive/debian/dists/stretch/Release.gpg
Download Error: http://repo.aptly.info/system-tests/archive.debian.org/debian-archive/debian/dists/stretch/InRelease
Downloading: http://repo.aptly.info/system-tests/archive.debian.org/debian-archive/debian/dists/stretch/Release
Downloading: http://repo.aptly.info/system-tests/archive.debian.org/debian-archive/debian/dists/stretch/Release.gpg
gpgv: can't allocate lock for '/home/runner/.gnupg/aptlytest.gpg'
gpgv: Signature made Sat Aug 14 07:43:24 2021 UTC
gpgv: using RSA key 16E90B3FDF65EDE3AA7F323C04EE7237B7D453EC
Expand Down
12 changes: 5 additions & 7 deletions system/t04_mirror/CreateMirror12Test_gold
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,11 +1,9 @@
Downloading http://repo.aptly.info/system-tests/archive.debian.org/debian-archive/debian/dists/stretch/InRelease...
Error downloading http://repo.aptly.info/system-tests/archive.debian.org/debian-archive/debian/dists/stretch/InRelease: HTTP code 404 while fetching http://repo.aptly.info/system-tests/archive.debian.org/debian-archive/debian/dists/stretch/InRelease retrying...
Downloading: http://repo.aptly.info/system-tests/archive.debian.org/debian-archive/debian/dists/stretch/InRelease
Error (retrying): HTTP code 404 while fetching http://repo.aptly.info/system-tests/archive.debian.org/debian-archive/debian/dists/stretch/InRelease
Retrying 0 http://repo.aptly.info/system-tests/archive.debian.org/debian-archive/debian/dists/stretch/InRelease...
Giving up on http://repo.aptly.info/system-tests/archive.debian.org/debian-archive/debian/dists/stretch/InRelease...
Downloading http://repo.aptly.info/system-tests/archive.debian.org/debian-archive/debian/dists/stretch/Release...
Success downloading http://repo.aptly.info/system-tests/archive.debian.org/debian-archive/debian/dists/stretch/Release
Downloading http://repo.aptly.info/system-tests/archive.debian.org/debian-archive/debian/dists/stretch/Release.gpg...
Success downloading http://repo.aptly.info/system-tests/archive.debian.org/debian-archive/debian/dists/stretch/Release.gpg
Download Error: http://repo.aptly.info/system-tests/archive.debian.org/debian-archive/debian/dists/stretch/InRelease
Downloading: http://repo.aptly.info/system-tests/archive.debian.org/debian-archive/debian/dists/stretch/Release
Downloading: http://repo.aptly.info/system-tests/archive.debian.org/debian-archive/debian/dists/stretch/Release.gpg

gpgv: Signature made Sat Aug 14 07:43:24 2021 UTC
gpgv: using RSA key 16E90B3FDF65EDE3AA7F323C04EE7237B7D453EC
Expand Down
3 changes: 1 addition & 2 deletions system/t04_mirror/CreateMirror13Test_gold
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,4 @@
Downloading http://repo.aptly.info/system-tests/archive.debian.org/debian-archive/debian/dists/stretch/Release...
Success downloading http://repo.aptly.info/system-tests/archive.debian.org/debian-archive/debian/dists/stretch/Release
Downloading: http://repo.aptly.info/system-tests/archive.debian.org/debian-archive/debian/dists/stretch/Release

Mirror [mirror13]: http://repo.aptly.info/system-tests/archive.debian.org/debian-archive/debian/ stretch successfully added.
You can run 'aptly mirror update mirror13' to download repository contents.
3 changes: 1 addition & 2 deletions system/t04_mirror/CreateMirror14Test_gold
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,4 @@
Downloading http://repo.aptly.info/system-tests/cloud.r-project.org/bin/linux/debian/bullseye-cran40/InRelease...
Success downloading http://repo.aptly.info/system-tests/cloud.r-project.org/bin/linux/debian/bullseye-cran40/InRelease
Downloading: http://repo.aptly.info/system-tests/cloud.r-project.org/bin/linux/debian/bullseye-cran40/InRelease
gpgv: can't allocate lock for '/home/runner/.gnupg/aptlytest.gpg'
gpgv: Signature made Thu Nov 2 07:43:52 2023 UTC
gpgv: using RSA key 7BA040A510E4E66ED3743EC1B8F25A8A73EACF41
Expand Down
3 changes: 1 addition & 2 deletions system/t04_mirror/CreateMirror16Test_gold
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,3 +1,2 @@
Downloading http://repo.aptly.info/system-tests/archive.debian.org/debian-archive/debian/dists/stretch/Release...
Success downloading http://repo.aptly.info/system-tests/archive.debian.org/debian-archive/debian/dists/stretch/Release
Downloading: http://repo.aptly.info/system-tests/archive.debian.org/debian-archive/debian/dists/stretch/Release
ERROR: unable to fetch mirror: architecture source not available in repo [mirror16]: http://repo.aptly.info/system-tests/archive.debian.org/debian-archive/debian/ stretch, use -force-architectures to override
3 changes: 1 addition & 2 deletions system/t04_mirror/CreateMirror17Test_gold
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,4 @@
Downloading http://repo.aptly.info/system-tests/archive.debian.org/debian-archive/debian/dists/stretch/Release...
Success downloading http://repo.aptly.info/system-tests/archive.debian.org/debian-archive/debian/dists/stretch/Release
Downloading: http://repo.aptly.info/system-tests/archive.debian.org/debian-archive/debian/dists/stretch/Release

Mirror [mirror17]: http://repo.aptly.info/system-tests/archive.debian.org/debian-archive/debian/ stretch [src] successfully added.
You can run 'aptly mirror update mirror17' to download repository contents.
12 changes: 5 additions & 7 deletions system/t04_mirror/CreateMirror18Test_gold
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,11 +1,9 @@
Downloading http://ppa.launchpad.net/gladky-anton/gnuplot/ubuntu/dists/maverick/InRelease...
Error downloading http://ppa.launchpad.net/gladky-anton/gnuplot/ubuntu/dists/maverick/InRelease: HTTP code 404 while fetching http://ppa.launchpad.net/gladky-anton/gnuplot/ubuntu/dists/maverick/InRelease retrying...
Downloading: http://ppa.launchpad.net/gladky-anton/gnuplot/ubuntu/dists/maverick/InRelease
Error (retrying): HTTP code 404 while fetching http://ppa.launchpad.net/gladky-anton/gnuplot/ubuntu/dists/maverick/InRelease
Retrying 0 http://ppa.launchpad.net/gladky-anton/gnuplot/ubuntu/dists/maverick/InRelease...
Giving up on http://ppa.launchpad.net/gladky-anton/gnuplot/ubuntu/dists/maverick/InRelease...
Downloading http://ppa.launchpad.net/gladky-anton/gnuplot/ubuntu/dists/maverick/Release...
Success downloading http://ppa.launchpad.net/gladky-anton/gnuplot/ubuntu/dists/maverick/Release
Downloading http://ppa.launchpad.net/gladky-anton/gnuplot/ubuntu/dists/maverick/Release.gpg...
Success downloading http://ppa.launchpad.net/gladky-anton/gnuplot/ubuntu/dists/maverick/Release.gpg
Download Error: http://ppa.launchpad.net/gladky-anton/gnuplot/ubuntu/dists/maverick/InRelease
Downloading: http://ppa.launchpad.net/gladky-anton/gnuplot/ubuntu/dists/maverick/Release
Downloading: http://ppa.launchpad.net/gladky-anton/gnuplot/ubuntu/dists/maverick/Release.gpg
gpgv: can't allocate lock for '/home/runner/.gnupg/aptlytest.gpg'
gpgv: Signature made Mon Oct 22 13:19:50 2012 UTC
gpgv: using RSA key A5279A973B1F56C0
Expand Down
3 changes: 1 addition & 2 deletions system/t04_mirror/CreateMirror19Test_gold
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,4 @@
Downloading http://repo.aptly.info/system-tests/archive.debian.org/debian-security/dists/stretch/updates/InRelease...
Success downloading http://repo.aptly.info/system-tests/archive.debian.org/debian-security/dists/stretch/updates/InRelease
Downloading: http://repo.aptly.info/system-tests/archive.debian.org/debian-security/dists/stretch/updates/InRelease
gpgv: can't allocate lock for '/home/runner/.gnupg/aptlytest.gpg'
gpgv: Signature made Sat Feb 18 04:22:45 2023 UTC
gpgv: using RSA key 379483D8B60160B155B372DDAA8E81B4331F7F50
Expand Down
3 changes: 1 addition & 2 deletions system/t04_mirror/CreateMirror1Test_gold
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,4 @@
Downloading http://repo.aptly.info/system-tests/archive.debian.org/debian-archive/debian/dists/stretch/Release...
Success downloading http://repo.aptly.info/system-tests/archive.debian.org/debian-archive/debian/dists/stretch/Release
Downloading: http://repo.aptly.info/system-tests/archive.debian.org/debian-archive/debian/dists/stretch/Release

Mirror [mirror1]: http://repo.aptly.info/system-tests/archive.debian.org/debian-archive/debian/ stretch successfully added.
You can run 'aptly mirror update mirror1' to download repository contents.
12 changes: 6 additions & 6 deletions system/t04_mirror/CreateMirror20Test_gold
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,9 +1,9 @@
Downloading http://repo.aptly.info/system-tests/archive.debian.org/debian-security/dists/stretch/updates/InRelease...
Error downloading http://repo.aptly.info/system-tests/archive.debian.org/debian-security/dists/stretch/updates/InRelease: http://repo.aptly.info/system-tests/archive.debian.org/debian-security/dists/stretch/updates/InRelease: Get "http://repo.aptly.info/system-tests/archive.debian.org/debian-security/dists/stretch/updates/InRelease": http: error connecting to proxy http://127.0.0.1:3137: dial tcp 127.0.0.1:3137: connection refused retrying...
Downloading: http://repo.aptly.info/system-tests/archive.debian.org/debian-security/dists/stretch/updates/InRelease
Error (retrying): http://repo.aptly.info/system-tests/archive.debian.org/debian-security/dists/stretch/updates/InRelease: Get "http://repo.aptly.info/system-tests/archive.debian.org/debian-security/dists/stretch/updates/InRelease": http: error connecting to proxy http://127.0.0.1:3137: dial tcp 127.0.0.1:3137: connection refused
Retrying 0 http://repo.aptly.info/system-tests/archive.debian.org/debian-security/dists/stretch/updates/InRelease...
Giving up on http://repo.aptly.info/system-tests/archive.debian.org/debian-security/dists/stretch/updates/InRelease...
Downloading http://repo.aptly.info/system-tests/archive.debian.org/debian-security/dists/stretch/updates/Release...
Error downloading http://repo.aptly.info/system-tests/archive.debian.org/debian-security/dists/stretch/updates/Release: http://repo.aptly.info/system-tests/archive.debian.org/debian-security/dists/stretch/updates/Release: Get "http://repo.aptly.info/system-tests/archive.debian.org/debian-security/dists/stretch/updates/Release": http: error connecting to proxy http://127.0.0.1:3137: dial tcp 127.0.0.1:3137: connection refused retrying...
Download Error: http://repo.aptly.info/system-tests/archive.debian.org/debian-security/dists/stretch/updates/InRelease
Downloading: http://repo.aptly.info/system-tests/archive.debian.org/debian-security/dists/stretch/updates/Release
Error (retrying): http://repo.aptly.info/system-tests/archive.debian.org/debian-security/dists/stretch/updates/Release: Get "http://repo.aptly.info/system-tests/archive.debian.org/debian-security/dists/stretch/updates/Release": http: error connecting to proxy http://127.0.0.1:3137: dial tcp 127.0.0.1:3137: connection refused
Retrying 0 http://repo.aptly.info/system-tests/archive.debian.org/debian-security/dists/stretch/updates/Release...
Giving up on http://repo.aptly.info/system-tests/archive.debian.org/debian-security/dists/stretch/updates/Release...
Download Error: http://repo.aptly.info/system-tests/archive.debian.org/debian-security/dists/stretch/updates/Release
ERROR: unable to fetch mirror: http://repo.aptly.info/system-tests/archive.debian.org/debian-security/dists/stretch/updates/Release: Get "http://repo.aptly.info/system-tests/archive.debian.org/debian-security/dists/stretch/updates/Release": http: error connecting to proxy http://127.0.0.1:3137: dial tcp 127.0.0.1:3137: connection refused
Loading
Loading