Arch packages implementation

go.mod

@@ -16,6 +16,7 @@ require (
 	github.com/42wim/sshsig v0.0.0-20211121163825-841cf5bbc121
 	github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358
 	github.com/NYTimes/gziphandler v1.1.1
+	github.com/ProtonMail/gopenpgp/v2 v2.7.1
 	github.com/PuerkitoBio/goquery v1.8.1
 	github.com/alecthomas/chroma/v2 v2.7.0
 	github.com/blakesmith/ar v0.0.0-20190502131153-809d4375e1fb
@@ -137,6 +138,7 @@ require (
 	github.com/Masterminds/sprig/v3 v3.2.3 // indirect
 	github.com/Microsoft/go-winio v0.6.1 // indirect
 	github.com/ProtonMail/go-crypto v0.0.0-20230528122434-6f98819771a1 // indirect
+	github.com/ProtonMail/go-mime v0.0.0-20230322103455-7d82a3887f2f // indirect
 	github.com/RoaringBitmap/roaring v1.2.3 // indirect
 	github.com/acomagu/bufpipe v1.0.4 // indirect
 	github.com/andybalholm/brotli v1.0.5 // indirect

go.sum

@@ -105,8 +105,13 @@ github.com/NYTimes/gziphandler v1.1.1 h1:ZUDjpQae29j0ryrS0u/B8HZfJBtBQHjqw2rQ2cq
 github.com/NYTimes/gziphandler v1.1.1/go.mod h1:n/CVRwUEOgIxrgPvAQhUUr9oeUtvrhMomdKFjzJNB0c=
 github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU=
 github.com/ProtonMail/go-crypto v0.0.0-20230217124315-7d5c6f04bbb8/go.mod h1:I0gYDMZ6Z5GRU7l58bNFSkPTFN6Yl12dsUlAZ8xy98g=
+github.com/ProtonMail/go-crypto v0.0.0-20230321155629-9a39f2531310/go.mod h1:8TI4H3IbrackdNgv+92dI+rhpCaLqM0IfpgCgenFvRE=
 github.com/ProtonMail/go-crypto v0.0.0-20230528122434-6f98819771a1 h1:JMDGhoQvXNTqH6Y3MC0IUw6tcZvaUdujNqzK2HYWZc8=
 github.com/ProtonMail/go-crypto v0.0.0-20230528122434-6f98819771a1/go.mod h1:EjAoLdwvbIOoOQr3ihjnSoLZRtE8azugULFRteWMNc0=
+github.com/ProtonMail/go-mime v0.0.0-20230322103455-7d82a3887f2f h1:tCbYj7/299ekTTXpdwKYF8eBlsYsDVoggDAuAjoK66k=
+github.com/ProtonMail/go-mime v0.0.0-20230322103455-7d82a3887f2f/go.mod h1:gcr0kNtGBqin9zDW9GOHcVntrwnjrK+qdJ06mWYBybw=
+github.com/ProtonMail/gopenpgp/v2 v2.7.1 h1:Awsg7MPc2gD3I7IFac2qE3Gdls0lZW8SzrFZ3k1oz0s=
+github.com/ProtonMail/gopenpgp/v2 v2.7.1/go.mod h1:/BU5gfAVwqyd8EfC3Eu7zmuhwYQpKs+cGD8M//iiaxs=
 github.com/PuerkitoBio/goquery v1.8.1 h1:uQxhNlArOIdbrH1tr0UXwdVFgDcZDrZVdcpygAcwmWM=
 github.com/PuerkitoBio/goquery v1.8.1/go.mod h1:Q8ICL1kNUJ2sXGoAhPGUdYDJvgQgHzJsnnd3H7Ho5jQ=
 github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0=

models/packages/descriptor.go

@@ -13,6 +13,7 @@ import (
 	user_model "code.gitea.io/gitea/models/user"
 	"code.gitea.io/gitea/modules/json"
 	"code.gitea.io/gitea/modules/packages/alpine"
+	"code.gitea.io/gitea/modules/packages/arch"
 	"code.gitea.io/gitea/modules/packages/cargo"
 	"code.gitea.io/gitea/modules/packages/chef"
 	"code.gitea.io/gitea/modules/packages/composer"
@@ -140,6 +141,8 @@ func GetPackageDescriptor(ctx context.Context, pv *PackageVersion) (*PackageDesc
 	switch p.Type {
 	case TypeAlpine:
 		metadata = &alpine.VersionMetadata{}
+	case TypeArch:
+		metadata = &arch.Metadata{}
 	case TypeCargo:
 		metadata = &cargo.Metadata{}
 	case TypeChef:

models/packages/package.go

@@ -31,6 +31,7 @@ type Type string
 // List of supported packages
 const (
 	TypeAlpine    Type = "alpine"
+	TypeArch      Type = "arch"
 	TypeCargo     Type = "cargo"
 	TypeChef      Type = "chef"
 	TypeComposer  Type = "composer"
@@ -55,6 +56,7 @@ const (
 
 var TypeList = []Type{
 	TypeAlpine,
+	TypeArch,
 	TypeCargo,
 	TypeChef,
 	TypeComposer,
@@ -82,6 +84,8 @@ func (pt Type) Name() string {
 	switch pt {
 	case TypeAlpine:
 		return "Alpine"
+	case TypeArch:
+		return "Arch"
 	case TypeCargo:
 		return "Cargo"
 	case TypeChef:
@@ -131,6 +135,8 @@ func (pt Type) SVGName() string {
 	switch pt {
 	case TypeAlpine:
 		return "gitea-alpine"
+	case TypeArch:
+		return "octicon-arch"
 	case TypeCargo:
 		return "gitea-cargo"
 	case TypeChef:

modules/packages/arch/metadata.go

@@ -0,0 +1,278 @@
+// Copyright 2023 The Gitea Authors. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+package arch
+
+import (
+	"bytes"
+	"crypto/md5"
+	"crypto/sha256"
+	"encoding/hex"
+	"errors"
+	"fmt"
+	"io"
+	"io/fs"
+	"os"
+	"path"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/mholt/archiver/v3"
+)
+
+// Metadata for arch package.
+type Metadata struct {
+	Filename          string
+	Name              string
+	Base              string
+	Version           string
+	Description       string
+	CompressedSize    int64
+	CompressedSizeMib string
+	InstalledSize     int64
+	InstalledSizeMib  string
+	MD5               string
+	SHA256            string
+	URL               string
+	BuildDate         int64
+	BuildDateStr      string
+	BaseDomain        string
+	Packager          string
+	Provides          []string
+	License           []string
+	Arch              []string
+	Depends           []string
+	OptDepends        []string
+	MakeDepends       []string
+	CheckDepends      []string
+	Backup            []string
+}
+
+// Function that recieves arch package archive data and returns it's metadata.
+func EjectMetadata(filename, domain string, pkg []byte) (*Metadata, error) {
+	pkgreader := io.LimitReader(bytes.NewReader(pkg), 250000)
+	var buf bytes.Buffer
+	err := archiver.DefaultZstd.Decompress(pkgreader, &buf)
+	if err != nil {
+		if !errors.Is(err, io.ErrUnexpectedEOF) {
+			return nil, err
+		}
+	}
+	splt := strings.Split(buf.String(), "PKGINFO")
+	if len(splt) < 2 {
+		return nil, errors.New("unable to eject .PKGINFO from archive")
+	}
+	raw := splt[1][0:10000]
+	inssize := int64(len(pkg))
+	compsize := ejectInt64(raw, "size")
+	unixbuilddate := ejectInt64(raw, "builddate")
+	return &Metadata{
+		Filename:          filename,
+		Name:              ejectString(raw, "pkgname"),
+		Base:              ejectString(raw, "pkgbase"),
+		Version:           ejectString(raw, "pkgver"),
+		Description:       ejectString(raw, "pkgdesc"),
+		CompressedSize:    inssize,
+		CompressedSizeMib: ByteCountSI(inssize),
+		InstalledSize:     compsize,
+		InstalledSizeMib:  ByteCountSI(compsize),
+		MD5:               md5sum(pkg),
+		SHA256:            sha256sum(pkg),
+		URL:               ejectString(raw, "url"),
+		BuildDate:         unixbuilddate,
+		BuildDateStr:      ReadableTime(unixbuilddate),
+		BaseDomain:        domain,
+		Packager:          ejectString(raw, "packager"),
+		Provides:          ejectStrings(raw, "provides"),
+		License:           ejectStrings(raw, "license"),
+		Arch:              ejectStrings(raw, "arch"),
+		Depends:           ejectStrings(raw, "depend"),
+		OptDepends:        ejectStrings(raw, "optdepend"),
+		MakeDepends:       ejectStrings(raw, "makedepend"),
+		CheckDepends:      ejectStrings(raw, "checkdepend"),
+		Backup:            ejectStrings(raw, "backup"),
+	}, nil
+}
+
+func ejectString(raw, field string) string {
+	splitted := strings.Split(raw, "\n"+field+" = ")
+	if len(splitted) < 2 {
+		return ``
+	}
+	return strings.Split(splitted[1], "\n")[0]
+}
+
+func ejectStrings(raw, field string) []string {
+	splitted := strings.Split(raw, "\n"+field+" = ")
+	if len(splitted) < 2 {
+		return nil
+	}
+	var rez []string
+	for i, v := range splitted {
+		if i == 0 {
+			continue
+		}
+		rez = append(rez, strings.Split(v, "\n")[0])
+	}
+	return rez
+}
+
+func ejectInt64(raw, field string) int64 {
+	splitted := strings.Split(raw, "\n"+field+" = ")
+	if len(splitted) < 2 {
+		return 0
+	}
+	i, err := strconv.ParseInt(strings.Split(splitted[1], "\n")[0], 10, 64)
+	if err != nil {
+		return 0
+	}
+	return i
+}
+
+func ByteCountSI(b int64) string {
+	const unit = 1000
+	if b < unit {
+		return fmt.Sprintf("%d B", b)
+	}
+	div, exp := int64(unit), 0
+	for n := b / unit; n >= unit; n /= unit {
+		div *= unit
+		exp++
+	}
+	return fmt.Sprintf("%.1f %cB", float64(b)/float64(div), "kMGTPE"[exp])
+}
+
+func ReadableTime(unix int64) string {
+	return time.Unix(unix, 0).Format(time.DateTime)
+}
+
+func md5sum(data []byte) string {
+	sum := md5.Sum(data)
+	return hex.EncodeToString(sum[:])
+}
+
+func sha256sum(data []byte) string {
+	sum := sha256.Sum256(data)
+	return hex.EncodeToString(sum[:])
+}
+
+// This function returns pacman package description in unarchived raw database
+// format.
+func (m *Metadata) GetDbDesc() string {
+	return strings.Join(rmEmptyStrings([]string{
+		formatField("FILENAME", m.Filename),
+		formatField("NAME", m.Name),
+		formatField("BASE", m.Base),
+		formatField("VERSION", m.Version),
+		formatField("DESC", m.Description),
+		formatField("CSIZE", m.CompressedSize),
+		formatField("ISIZE", m.InstalledSize),
+		formatField("MD5SUM", m.MD5),
+		formatField("SHA256SUM", m.SHA256),
+		formatField("URL", m.URL),
+		formatField("LICENSE", m.License),
+		formatField("ARCH", m.Arch),
+		formatField("BUILDDATE", m.BuildDate),
+		formatField("PACKAGER", m.Packager),
+		formatField("PROVIDES", m.Provides),
+		formatField("DEPENDS", m.Depends),
+		formatField("OPTDEPENDS", m.OptDepends),
+		formatField("MAKEDEPENDS", m.MakeDepends),
+		formatField("CHECKDEPENDS", m.CheckDepends),
+	}), "\n\n") + "\n\n"
+}
+
+func formatField(field string, value any) string {
+	switch value := value.(type) {
+	case []string:
+		if value == nil {
+			return ``
+		}
+		val := strings.Join(value, "\n")
+		return fmt.Sprintf("%%%s%%\n%s", field, val)
+	case string:
+		return fmt.Sprintf("%%%s%%\n%s", field, value)
+	case int64:
+		return fmt.Sprintf("%%%s%%\n%d", field, value)
+	}
+	return ``
+}
+
+func rmEmptyStrings(s []string) []string {
+	var r []string
+	for _, str := range s {
+		if str != "" {
+			r = append(r, str)
+		}
+	}
+	return r
+}
+
+// Function takes path to directory with pacman database and updates package
+// it with current metadata.
+func (m *Metadata) PutToDb(dir string, mode fs.FileMode) error {
+	descdir := path.Join(dir, m.Name+"-"+m.Version)
+	err := os.MkdirAll(descdir, mode)
+	if err != nil {
+		return err
+	}
+	return os.WriteFile(path.Join(descdir, "desc"), []byte(m.GetDbDesc()), mode)
+}
+
+// Function takes raw database archive bytes and destination directory as
+// arguements and unpacks database contents to destination directory.
+func UnpackDb(src, dst string) error {
+	return archiver.DefaultTarGz.Unarchive(src, dst)
+}
+
+// Function takes path to source directory with raw pacman description files
+// for pacman database, creates db.tar.gz archive and related symlink for
+// provided path.
+func PackDb(src, dst string) error {
+	if !strings.HasSuffix(dst, ".db.tar.gz") {
+		return fmt.Errorf("dst should end with '.db.tar.gz': %s", dst)
+	}
+	symlink := strings.TrimSuffix(dst, ".tar.gz")
+	if _, err := os.Stat(dst); err == nil {
+		err = os.RemoveAll(dst)
+		if err != nil {
+			return err
+		}
+		err = os.RemoveAll(symlink)
+		if err != nil {
+			return err
+		}
+	}
+	des, err := os.ReadDir(src)
+	if err != nil {
+		return err
+	}
+	var pkgdescs []string
+	for _, de := range des {
+		pkgdescs = append(pkgdescs, path.Join(src, de.Name()))
+	}
+	err = archiver.DefaultTarGz.Archive(pkgdescs, dst)
+	if err != nil {
+		return err
+	}
+	return os.Symlink(dst, symlink)
+}
+
+// Join database or package names to prevent collisions with same packages in
+// different user spaces. Skips empty strings and returns name joined with
+// dots.
+func Join(s ...string) string {
+	rez := ""
+	for i, v := range s {
+		if v == "" {
+			continue
+		}
+		if i+1 == len(s) {
+			rez += v
+			continue
+		}
+		rez += v + "."
+	}
+	return rez
+}

modules/packages/content_store.go

@@ -4,6 +4,7 @@
 package packages
 
 import (
+	"bytes"
 	"io"
 	"path"
 	"strings"
@@ -63,3 +64,17 @@ func RelativePathToKey(relativePath string) (BlobHash256Key, error) {
 
 	return BlobHash256Key(parts[2]), nil
 }
+
+// Save data with specified string key.
+func (s *ContentStore) SaveStrBytes(key string, data []byte) error {
+	return s.Save(BlobHash256Key(key), bytes.NewReader(data), int64(len(data)))
+}
+
+// Get data related to provided key.
+func (s *ContentStore) GetStrBytes(key string) ([]byte, error) {
+	obj, err := s.Get(BlobHash256Key(key))
+	if err != nil {
+		return nil, err
+	}
+	return io.ReadAll(obj)
+}

options/locale/locale_en-US.ini

@@ -3254,6 +3254,10 @@ alpine.repository = Repository Info
 alpine.repository.branches = Branches
 alpine.repository.repositories = Repositories
 alpine.repository.architectures = Architectures
+arch.pacmanconf = Add server to pacman.conf:
+arch.pacmansync = And sync package with pacman:
+arch.documentation = For more information on the Arch registry, see <a target="_blank" rel="noopener noreferrer" href="%s">the documentation</a>.
+arch.properties = Package properties
 cargo.registry = Setup this registry in the Cargo configuration file (for example <code>~/.cargo/config.toml</code>):
 cargo.install = To install the package using Cargo, run the following command:
 cargo.documentation = For more information on the Cargo registry, see <a target="_blank" rel="noopener noreferrer" href="%s">the documentation</a>.