diff --git a/.github/dependabot.yaml b/.github/dependabot.yaml new file mode 100644 index 0000000..aafb8a2 --- /dev/null +++ b/.github/dependabot.yaml @@ -0,0 +1,11 @@ +version: 2 +updates: + - package-ecosystem: github-actions + directory: / + schedule: + interval: weekly + + - package-ecosystem: gomod + directory: / + schedule: + interval: weekly diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml new file mode 100644 index 0000000..eb05472 --- /dev/null +++ b/.github/workflows/ci.yaml @@ -0,0 +1,108 @@ +name: CI + +on: + pull_request: + push: + branches: [main] + tags: [v*] + +permissions: + contents: write + +jobs: + test: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-go@v5 + with: + go-version: ~1.22 + + - name: Lint + uses: secondlife-3p/golangci-lint-action@v5 + with: + version: latest + + - name: Test + run: | + go mod tidy + go test -v + build-image: + runs-on: ubuntu-latest + needs: test + steps: + - uses: actions/checkout@v4 + + - name: Docker meta + id: meta + uses: docker/metadata-action@v5 + with: + images: lindenlab/debserve + tags: | + type=ref,event=branch + type=ref,event=pr + type=semver,pattern={{version}} + type=semver,pattern={{major}}.{{minor}} + type=semver,pattern={{major}} + + - name: Set up QEMU + uses: docker/setup-qemu-action@v3 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Login to Docker Hub + uses: docker/login-action@v3 + if: startsWith(github.ref, 'refs/tags/v') + with: + username: ${{ secrets.SHARED_DOCKERHUB_USER }} + password: ${{ secrets.SHARED_DOCKERHUB_TOKEN }} + + - name: Build and push + uses: docker/build-push-action@v5 + with: + context: . + push: ${{ startsWith(github.ref, 'refs/tags/v') }} + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + platforms: linux/amd64,linux/arm64 + + - name: Docker Hub Description + uses: secondlife-3p/dockerhub-description@v4 + if: startsWith(github.ref, 'refs/tags/v') + with: + username: ${{ secrets.SHARED_DOCKERHUB_USER }} + password: ${{ secrets.SHARED_DOCKERHUB_TOKEN }} + repository: lindenlab/debserve + short-description: Self-contained debian package server + build-package: + runs-on: ubuntu-latest + needs: test + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Choose GoReleaser args + shell: bash + env: + IS_TAG: ${{ startsWith(github.ref, 'refs/tags/v') }} + id: goreleaser-args + run: | + if [[ "$IS_TAG" == "true" ]] + then + echo "Building for a tag: do a fully regular gorelease" >&2 + echo "value=" >> $GITHUB_OUTPUT + else + echo "Not building for a tag: do the gorelease in snapshot mode" >&2 + echo "value=--snapshot" >> $GITHUB_OUTPUT + fi + + - name: Run GoReleaser + uses: goreleaser/goreleaser-action@v5 + with: + version: v1.25.1 + args: release ${{ steps.goreleaser-args.outputs.value }} + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} \ No newline at end of file diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..6a20680 --- /dev/null +++ b/.gitignore @@ -0,0 +1,5 @@ +.vscode/ +debserve +Packages +Packages.* +dist/ diff --git a/.goreleaser.yaml b/.goreleaser.yaml new file mode 100644 index 0000000..f0c5c54 --- /dev/null +++ b/.goreleaser.yaml @@ -0,0 +1,28 @@ +version: 1 + +before: + hooks: + - go mod tidy + +builds: + - env: + - CGO_ENABLED=0 + goos: + - linux + - windows + - darwin + +archives: + - format: tar.gz + # this name template makes the OS and Arch compatible with the results of `uname`. + name_template: >- + {{ .ProjectName }}_ + {{- title .Os }}_ + {{- if eq .Arch "amd64" }}x86_64 + {{- else if eq .Arch "386" }}i386 + {{- else }}{{ .Arch }}{{ end }} + {{- if .Arm }}v{{ .Arm }}{{ end }} + # use zip for windows archives + format_overrides: + - goos: windows + format: zip \ No newline at end of file diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000..beacb3b --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,18 @@ +repos: + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: v4.5.0 + hooks: + - id: check-yaml + - id: trailing-whitespace + exclude: main_test.go + - repo: https://github.com/golangci/golangci-lint + rev: v1.58.0 + hooks: + - id: golangci-lint + - repo: local + hooks: + - id: test + name: go test + entry: go test + language: system + types: [go] \ No newline at end of file diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..c5cda5d --- /dev/null +++ b/Dockerfile @@ -0,0 +1,19 @@ +FROM golang:1.22-alpine AS builder + +WORKDIR /app + +COPY go.mod go.sum ./ +RUN go mod download +COPY *.go ./ +RUN CGO_ENABLED=0 GOOS=linux go build -o debserve + +FROM alpine:3 + +WORKDIR /packages + +COPY ./as-pwd /usr/local/bin/as-pwd +RUN apk add --no-cache su-exec +COPY --from=builder /app/debserve /usr/local/bin/debserve + +ENTRYPOINT ["as-pwd"] +CMD ["debserve", "--watch", "--listen", ":80"] \ No newline at end of file diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..9e7dddb --- /dev/null +++ b/LICENSE @@ -0,0 +1,20 @@ +Copyright (c) 2024 Linden Research, Inc. + +Permission is hereby granted, free of charge, to any person obtaining +a copy of this software and associated documentation files (the +"Software"), to deal in the Software without restriction, including +without limitation the rights to use, copy, modify, merge, publish, +distribute, sublicense, and/or sell copies of the Software, and to +permit persons to whom the Software is furnished to do so, subject to +the following conditions: + +The above copyright notice and this permission notice shall be +included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. \ No newline at end of file diff --git a/README.md b/README.md new file mode 100644 index 0000000..255b5af --- /dev/null +++ b/README.md @@ -0,0 +1,42 @@ +# Debserve + +**debserve** is a small, self-contained debian package indexer and server +designed to scratch a very specific itch: installing local debian files +into docker images. It may have other utility. + +## Instructions + +By default, **debserve** will scan for \*.deb files in the current directory +and host them at localhost:8080. Additional options: + +``` +A self-contained debian package indexer and server. + + Usage: ./debserve [options] [folder] + + -l string + HTTP server listen location (shorthand) (default "localhost:8080") + -listen string + HTTP server listen location (default "localhost:8080") + -r Search child directories (shorthand) + -recursive + Search child directories + -s Enable silent mode (shorthand) + -silent + Enable silent mode + -v Enable verbose mode (shorthand) + -verbose + Enable verbose mode + -w Enable watch mode (shorthand) + -watch + Enable watch mode +``` + +## Docker image + +A docker image, `lindenlab/debserve`, is also available, and can use a local volume mount +to index and serve local packages: + +```sh +docker run -it --rm -v $(pwd):/packages -p 12321:80 lindenlab/debserve +``` \ No newline at end of file diff --git a/as-pwd b/as-pwd new file mode 100755 index 0000000..6e5ddbc --- /dev/null +++ b/as-pwd @@ -0,0 +1,11 @@ +#!/bin/sh + +# Run the command as the user who owns the pwd + +set -e + +pwd=$(pwd) +user="$(stat -c "%u" "$pwd")" +group="$(stat -c "%g" "$pwd")" + +exec su-exec $user:$group "$@" \ No newline at end of file diff --git a/examples/test-pkg1_1.0.0_all.deb b/examples/test-pkg1_1.0.0_all.deb new file mode 100644 index 0000000..51cc8c5 Binary files /dev/null and b/examples/test-pkg1_1.0.0_all.deb differ diff --git a/examples/test-pkg2_1.0.0_amd64.deb b/examples/test-pkg2_1.0.0_amd64.deb new file mode 100644 index 0000000..f8f2847 Binary files /dev/null and b/examples/test-pkg2_1.0.0_amd64.deb differ diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..58aca02 --- /dev/null +++ b/go.mod @@ -0,0 +1,12 @@ +module github.com/secondlife/debserve + +go 1.22.2 + +require ( + github.com/blakesmith/ar v0.0.0-20190502131153-809d4375e1fb + github.com/dsnet/compress v0.0.1 + github.com/fsnotify/fsnotify v1.7.0 + github.com/ulikunitz/xz v0.5.12 +) + +require golang.org/x/sys v0.4.0 // indirect diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..90f9c4b --- /dev/null +++ b/go.sum @@ -0,0 +1,14 @@ +github.com/blakesmith/ar v0.0.0-20190502131153-809d4375e1fb h1:m935MPodAbYS46DG4pJSv7WO+VECIWUQ7OJYSoTrMh4= +github.com/blakesmith/ar v0.0.0-20190502131153-809d4375e1fb/go.mod h1:PkYb9DJNAwrSvRx5DYA+gUcOIgTGVMNkfSCbZM8cWpI= +github.com/dsnet/compress v0.0.1 h1:PlZu0n3Tuv04TzpfPbrnI0HW/YwodEXDS+oPKahKF0Q= +github.com/dsnet/compress v0.0.1/go.mod h1:Aw8dCMJ7RioblQeTqt88akK31OvO8Dhf5JflhBbQEHo= +github.com/dsnet/golib v0.0.0-20171103203638-1ea166775780/go.mod h1:Lj+Z9rebOhdfkVLjJ8T6VcRQv3SXugXy999NBtR9aFY= +github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nosvA= +github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyTJwOpGvigM= +github.com/klauspost/compress v1.4.1/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= +github.com/klauspost/cpuid v1.2.0/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= +github.com/ulikunitz/xz v0.5.6/go.mod h1:2bypXElzHzzJZwzH67Y6wb67pO62Rzfn7BSiF4ABRW8= +github.com/ulikunitz/xz v0.5.12 h1:37Nm15o69RwBkXM0J6A5OlE67RZTfzUxTj8fB3dfcsc= +github.com/ulikunitz/xz v0.5.12/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14= +golang.org/x/sys v0.4.0 h1:Zr2JFtRQNX3BCZ8YtxRE9hNJYC8J6I1MVbMg6owUp18= +golang.org/x/sys v0.4.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= diff --git a/main.go b/main.go new file mode 100644 index 0000000..80273b2 --- /dev/null +++ b/main.go @@ -0,0 +1,304 @@ +package main + +import ( + "archive/tar" + "compress/gzip" + "crypto/md5" + "crypto/sha1" + "crypto/sha256" + "encoding/hex" + "flag" + "fmt" + "io" + "io/fs" + "log" + "log/slog" + "net/http" + "os" + "path" + "path/filepath" + "strings" + "time" + + "github.com/blakesmith/ar" + "github.com/dsnet/compress/bzip2" + "github.com/fsnotify/fsnotify" + "github.com/ulikunitz/xz" + "github.com/ulikunitz/xz/lzma" +) + +func ExtractStanza(debfile string, filepath string, w io.Writer) error { + slog.Debug("Loading metadata", "deb", debfile) + + f, err := os.Open(debfile) + if err != nil { + return err + } + defer f.Close() + + fi, err := f.Stat() + if err != nil { + return err + } + + h1 := md5.New() + h2 := sha1.New() + h3 := sha256.New() + tee := io.TeeReader(f, io.MultiWriter(h1, h2, h3)) + + // Open the *.deb ar archive and search for control.{gz,xz,bz2,lzma} + a := ar.NewReader(tee) + var controlName string + for { + hdr, err := a.Next() + if err == io.EOF { + break + } + + if err != nil { + return err + } + + if strings.HasPrefix(hdr.Name, "control.tar") { + controlName = hdr.Name + break + } + } + if controlName == "" { + return fmt.Errorf("control file not found in %s", debfile) + } + + // Select the correct decompression method for the file + var cr io.Reader + switch controlName { + case "control.tar.gz": + if cr, err = gzip.NewReader(a); err != nil { + return err + } + case "control.tar.xz": + if cr, err = xz.NewReader(a); err != nil { + return err + } + case "control.tar.lzma": + if cr, err = lzma.NewReader(a); err != nil { + return err + } + case "control.tar.bz2": + if cr, err = bzip2.NewReader(a, nil); err != nil { + return err + } + default: + return fmt.Errorf("unsupported control archive format: %s", controlName) + } + + // Read the control file contents directly to the output writer + tr := tar.NewReader(cr) + for { + hdr, err := tr.Next() + if err == io.EOF { + break + } + + if err != nil { + return err + } + + if hdr.Name == "./control" { + if _, err := io.Copy(w, tr); err != nil { + return err + } + break + } + } + + // Advance the reader to the end of the file + if _, err = io.Copy(io.Discard, tee); err != nil { + return err + } + + filename := filepath + if filepath == "" { + filename = debfile + } + + if _, err = w.Write([]byte("Filename: " + filename + "\nSize: " + fmt.Sprint(fi.Size()) + "\n")); err != nil { + return err + } + + // Print the checksums + _, err = w.Write([]byte( + "MD5sum: " + hex.EncodeToString(h1.Sum(nil)) + "\n" + + "SHA1: " + hex.EncodeToString(h2.Sum(nil)) + "\n" + + "SHA256: " + hex.EncodeToString(h3.Sum(nil)) + "\n\n", + )) + + return err +} + +func main() { + flag.Usage = func() { + fmt.Fprintf(flag.CommandLine.Output(), "A self-contained debian package indexer and server.\n\n Usage: %s [options] [folder]\n\n", os.Args[0]) + flag.PrintDefaults() + } + + watch := flag.Bool("watch", false, "Enable watch mode") + flag.BoolVar(watch, "w", false, "Enable watch mode (shorthand)") + + listen := flag.String("listen", "localhost:8080", "HTTP server listen location") + flag.StringVar(listen, "l", "localhost:8080", "HTTP server listen location (shorthand)") + + silent := flag.Bool("silent", false, "Enable silent mode") + flag.BoolVar(silent, "s", false, "Enable silent mode (shorthand)") + + verbose := flag.Bool("verbose", false, "Enable verbose mode") + flag.BoolVar(verbose, "v", false, "Enable verbose mode (shorthand)") + + recursive := flag.Bool("recursive", false, "Search child directories") + flag.BoolVar(recursive, "r", false, "Search child directories (shorthand)") + + flag.Parse() + + slog.SetLogLoggerLevel(slog.LevelInfo) + if *verbose { + slog.SetLogLoggerLevel(slog.LevelDebug) + } else if *silent { + slog.SetLogLoggerLevel(slog.LevelError) + } + + maxdepth := 1 + if *recursive { + maxdepth = 5000 + } + + folder := "." + if flag.NArg() > 0 { + folder = flag.Arg(0) + } + + scan := func() error { + start := time.Now() + count, err := ScanAndWritePackages(folder, maxdepth) + slog.Info(fmt.Sprintf("Indexed %d package(s) in %v", count, time.Since(start))) + return err + } + + if *watch { + watcher, err := fsnotify.NewWatcher() + if err != nil { + log.Fatal(err) + } + + defer watcher.Close() + + go func() { + for { + select { + case ev, ok := <-watcher.Events: + if !ok { + return + } + + f := path.Base(ev.Name) + if f == "Packages" || f == "Packages.bz2" || f == "Packages.gz" { + continue + } + + if err = scan(); err != nil { + log.Fatal(err) + } + + case err, ok := <-watcher.Errors: + if !ok { + return + } + + log.Fatal(err) + } + } + }() + + err = watcher.Add(folder) + if err != nil { + log.Fatal(err) + } + } + + if err := scan(); err != nil { + log.Fatal(err) + } + + // Create a file server handler + fs := http.FileServer(http.Dir(folder)) + + // Create a custom handler function to log the requests + handler := func(w http.ResponseWriter, r *http.Request) { + slog.Debug("Request", "method", r.Method, "path", r.URL.Path) + fs.ServeHTTP(w, r) + } + + slog.Info(fmt.Sprintf("Starting debserver on %s", *listen)) + log.Fatal(http.ListenAndServe(*listen, http.HandlerFunc(handler))) +} + +func ScanAndWritePackages(folder string, maxdepth int) (int, error) { + f, err := os.Create(path.Join(folder, "Packages")) + if err != nil { + return 0, err + } + gzf, err := os.Create(path.Join(folder, "Packages.gz")) + if err != nil { + return 0, err + } + bzf, err := os.Create(path.Join(folder, "Packages.bz2")) + if err != nil { + return 0, err + } + + defer f.Close() + defer gzf.Close() + defer bzf.Close() + + gzw := gzip.NewWriter(gzf) + bzw, err := bzip2.NewWriter(bzf, nil) + if err != nil { + return 0, err + } + + defer gzw.Close() + defer bzw.Close() + + w := io.MultiWriter(f, gzw, bzw) + count, err := ScanPackages(folder, maxdepth, w) + + return count, err +} + +// Scan and produce Packages file +func ScanPackages(folder string, maxdepth int, w io.Writer) (int, error) { + count := 0 + + err := filepath.WalkDir(folder, func(path string, d fs.DirEntry, err error) error { + if err != nil { + return err + } + + if d.IsDir() { + if strings.Count(path, string(filepath.Separator)) > maxdepth { + return fs.SkipDir + } + } else if filepath.Ext(path) == ".deb" { + relpath, err := filepath.Rel(folder, path) + if err != nil { + return err + } + if err = ExtractStanza(path, "./"+relpath, w); err != nil { + return err + } + count++ + } + + return nil + }) + + return count, err +} diff --git a/main_test.go b/main_test.go new file mode 100644 index 0000000..8233d94 --- /dev/null +++ b/main_test.go @@ -0,0 +1,71 @@ +package main + +import ( + "bufio" + "bytes" + "testing" +) + +const stanza1 = `Package: test-pkg1 +Version: 1.0.0 +Section: +Priority: optional +Architecture: all +Maintainer: Unset Maintainer +Installed-Size: 0 +Description: no description given +Filename: ./examples/test-pkg1_1.0.0_all.deb +Size: 510 +MD5sum: 0df74607c7ce1414c7b5d4a1a9f01e80 +SHA1: 5a3f1d87ea4db2c204803ceb9400c828aae211d9 +SHA256: 9b6e51e7d30e5a8b9bebaeeb7383910c31f82d655e68d45c65d9dc647b920d07 + +` + +const stanza2 = `Package: test-pkg2 +Version: 1.0.0 +Section: +Priority: optional +Architecture: amd64 +Maintainer: Unset Maintainer +Installed-Size: 0 +Description: no description given +Filename: ./examples/test-pkg2_1.0.0_amd64.deb +Size: 512 +MD5sum: aeda61515dbdf90cd08922254e33abcc +SHA1: 5cf1ef88800084ac70faf69f27ec807782caf092 +SHA256: a0af1aa05a98536b21e1a60ae5199d0d749b0e9d81fca12aeb13d10c1e53b055 + +` + +func TestExtractStanza(t *testing.T) { + var buf bytes.Buffer + w := bufio.NewWriter(&buf) + err := ExtractStanza("./examples/test-pkg1_1.0.0_all.deb", "", w) + if err != nil { + t.Error(err) + } + w.Flush() + got := buf.String() + if got != stanza1 { + t.Errorf("Expected:\n%s\ngot:\n%s", stanza1, got) + } +} + +func TestScanPackages(t *testing.T) { + var buf bytes.Buffer + w := bufio.NewWriter(&buf) + i, err := ScanPackages(".", 2, w) + if err != nil { + t.Error(err) + } + if i != 2 { + t.Errorf("Expected 2 packages, got %d", i) + } + w.Flush() + got := buf.String() + expected := stanza1 + stanza2 + if got != expected { + t.Errorf("Expected:\n%s\ngot:\n%s", expected, got) + } +}