feat: support grouping by any path for arch package (#4903)

Previous arch package grouping was not well-suited for complex or multi-architecture environments. It now supports the following content:

- Support grouping by any path.
- New support for packages in `xz` format.
- Fix clean up rules

<!--start release-notes-assistant-->

## Draft release notes
<!--URL:https://codeberg.org/forgejo/forgejo-->
- Features
  - [PR](https://codeberg.org/forgejo/forgejo/pulls/4903): <!--number 4903 --><!--line 0 --><!--description c3VwcG9ydCBncm91cGluZyBieSBhbnkgcGF0aCBmb3IgYXJjaCBwYWNrYWdl-->support grouping by any path for arch package<!--description-->
<!--end release-notes-assistant-->

Reviewed-on: https://codeberg.org/forgejo/forgejo/pulls/4903
Reviewed-by: Earl Warren <earl-warren@noreply.codeberg.org>
Co-authored-by: Exploding Dragon <explodingfkl@gmail.com>
Co-committed-by: Exploding Dragon <explodingfkl@gmail.com>
This commit is contained in:
Exploding Dragon 2024-08-11 10:35:11 +00:00 committed by Earl Warren
parent a4da672134
commit 87d50eca87
7 changed files with 309 additions and 218 deletions

View file

@ -41,11 +41,15 @@ var (
reVer = regexp.MustCompile(`^[a-zA-Z0-9:_.+]+-+[0-9]+$`)
reOptDep = regexp.MustCompile(`^[a-zA-Z0-9@._+-]+$|^[a-zA-Z0-9@._+-]+(:.*)`)
rePkgVer = regexp.MustCompile(`^[a-zA-Z0-9@._+-]+$|^[a-zA-Z0-9@._+-]+(>.*)|^[a-zA-Z0-9@._+-]+(<.*)|^[a-zA-Z0-9@._+-]+(=.*)`)
magicZSTD = []byte{0x28, 0xB5, 0x2F, 0xFD}
magicXZ = []byte{0xFD, 0x37, 0x7A, 0x58, 0x5A}
)
type Package struct {
Name string `json:"name"`
Version string `json:"version"` // Includes version, release and epoch
CompressType string `json:"compress_type"`
VersionMetadata VersionMetadata
FileMetadata FileMetadata
}
@ -89,18 +93,38 @@ func ParsePackage(r *packages.HashedBuffer) (*Package, error) {
if err != nil {
return nil, err
}
zstd := archiver.NewTarZstd()
err = zstd.Open(r, 0)
header := make([]byte, 5)
_, err = r.Read(header)
if err != nil {
return nil, err
}
defer zstd.Close()
_, err = r.Seek(0, io.SeekStart)
if err != nil {
return nil, err
}
var tarball archiver.Reader
var tarballType string
if bytes.Equal(header[:len(magicZSTD)], magicZSTD) {
tarballType = "zst"
tarball = archiver.NewTarZstd()
} else if bytes.Equal(header[:len(magicXZ)], magicXZ) {
tarballType = "xz"
tarball = archiver.NewTarXz()
} else {
return nil, errors.New("not supported compression")
}
err = tarball.Open(r, 0)
if err != nil {
return nil, err
}
defer tarball.Close()
var pkg *Package
var mtree bool
for {
f, err := zstd.Read()
f, err := tarball.Read()
if err == io.EOF {
break
}
@ -111,7 +135,7 @@ func ParsePackage(r *packages.HashedBuffer) (*Package, error) {
switch f.Name() {
case ".PKGINFO":
pkg, err = ParsePackageInfo(f)
pkg, err = ParsePackageInfo(tarballType, f)
if err != nil {
return nil, err
}
@ -137,8 +161,10 @@ func ParsePackage(r *packages.HashedBuffer) (*Package, error) {
// ParsePackageInfo Function that accepts reader for .PKGINFO file from package archive,
// validates all field according to PKGBUILD spec and returns package.
func ParsePackageInfo(r io.Reader) (*Package, error) {
p := &Package{}
func ParsePackageInfo(compressType string, r io.Reader) (*Package, error) {
p := &Package{
CompressType: compressType,
}
scanner := bufio.NewScanner(r)
for scanner.Scan() {
@ -281,7 +307,7 @@ func ValidatePackageSpec(p *Package) error {
// Desc Create pacman package description file.
func (p *Package) Desc() string {
entries := []string{
"FILENAME", fmt.Sprintf("%s-%s-%s.pkg.tar.zst", p.Name, p.Version, p.FileMetadata.Arch),
"FILENAME", fmt.Sprintf("%s-%s-%s.pkg.tar.%s", p.Name, p.Version, p.FileMetadata.Arch, p.CompressType),
"NAME", p.Name,
"BASE", p.VersionMetadata.Base,
"VERSION", p.Version,

View file

@ -158,11 +158,12 @@ checkdepend = ola
makedepend = cmake
backup = usr/bin/paket1
`
p, err := ParsePackageInfo(strings.NewReader(PKGINFO))
p, err := ParsePackageInfo("zst", strings.NewReader(PKGINFO))
require.NoError(t, err)
require.Equal(t, Package{
Name: "a",
Version: "1-2",
CompressType: "zst",
Name: "a",
Version: "1-2",
VersionMetadata: VersionMetadata{
Base: "b",
Description: "comment",
@ -417,8 +418,9 @@ dummy6
`
md := &Package{
Name: "zstd",
Version: "1.5.5-1",
CompressType: "zst",
Name: "zstd",
Version: "1.5.5-1",
VersionMetadata: VersionMetadata{
Base: "zstd",
Description: "Zstandard - Fast real-time compression algorithm",

View file

@ -143,10 +143,59 @@ func CommonRoutes() *web.Route {
r.Head("", arch.GetRepositoryKey)
r.Get("", arch.GetRepositoryKey)
})
r.Group("/{distro}", func() {
r.Put("", reqPackageAccess(perm.AccessModeWrite), arch.PushPackage)
r.Get("/{arch}/{file}", arch.GetPackageOrDB)
r.Delete("/{package}/{version}", reqPackageAccess(perm.AccessModeWrite), arch.RemovePackage)
r.Methods("HEAD,GET,PUT,DELETE", "*", func(ctx *context.Context) {
pathGroups := strings.Split(strings.Trim(ctx.Params("*"), "/"), "/")
groupLen := len(pathGroups)
isGetHead := ctx.Req.Method == "HEAD" || ctx.Req.Method == "GET"
isPut := ctx.Req.Method == "PUT"
isDelete := ctx.Req.Method == "DELETE"
if isGetHead {
if groupLen < 2 {
ctx.Status(http.StatusNotFound)
return
}
if groupLen == 2 {
ctx.SetParams("group", "")
ctx.SetParams("arch", pathGroups[0])
ctx.SetParams("file", pathGroups[1])
} else {
ctx.SetParams("group", strings.Join(pathGroups[:groupLen-2], "/"))
ctx.SetParams("arch", pathGroups[groupLen-2])
ctx.SetParams("file", pathGroups[groupLen-1])
}
arch.GetPackageOrDB(ctx)
return
} else if isPut {
ctx.SetParams("group", strings.Join(pathGroups, "/"))
reqPackageAccess(perm.AccessModeWrite)(ctx)
if ctx.Written() {
return
}
arch.PushPackage(ctx)
return
} else if isDelete {
if groupLen < 2 {
ctx.Status(http.StatusBadRequest)
return
}
if groupLen == 2 {
ctx.SetParams("group", "")
ctx.SetParams("package", pathGroups[0])
ctx.SetParams("version", pathGroups[1])
} else {
ctx.SetParams("group", strings.Join(pathGroups[:groupLen-2], "/"))
ctx.SetParams("package", pathGroups[groupLen-2])
ctx.SetParams("version", pathGroups[groupLen-1])
}
reqPackageAccess(perm.AccessModeWrite)(ctx)
if ctx.Written() {
return
}
arch.RemovePackage(ctx)
return
}
ctx.Status(http.StatusNotFound)
})
}, reqPackageAccess(perm.AccessModeRead))
r.Group("/cargo", func() {

View file

@ -9,6 +9,7 @@ import (
"fmt"
"io"
"net/http"
"regexp"
"strings"
packages_model "code.gitea.io/gitea/models/packages"
@ -21,6 +22,11 @@ import (
arch_service "code.gitea.io/gitea/services/packages/arch"
)
var (
archPkgOrSig = regexp.MustCompile(`^.*\.pkg\.tar\.\w+(\.sig)*$`)
archDBOrSig = regexp.MustCompile(`^.*.db(\.tar\.gz)*(\.sig)*$`)
)
func apiError(ctx *context.Context, status int, obj any) {
helper.LogAndProcessError(ctx, status, obj, func(message string) {
ctx.PlainText(status, message)
@ -41,7 +47,7 @@ func GetRepositoryKey(ctx *context.Context) {
}
func PushPackage(ctx *context.Context) {
distro := ctx.Params("distro")
group := ctx.Params("group")
upload, needToClose, err := ctx.UploadStream()
if err != nil {
@ -61,7 +67,7 @@ func PushPackage(ctx *context.Context) {
p, err := arch_module.ParsePackage(buf)
if err != nil {
apiError(ctx, http.StatusInternalServerError, err)
apiError(ctx, http.StatusBadRequest, err)
return
}
@ -97,7 +103,7 @@ func PushPackage(ctx *context.Context) {
properties := map[string]string{
arch_module.PropertyDescription: p.Desc(),
arch_module.PropertyArch: p.FileMetadata.Arch,
arch_module.PropertyDistribution: distro,
arch_module.PropertyDistribution: group,
}
version, _, err := packages_service.CreatePackageOrAddFileToExisting(
@ -114,8 +120,8 @@ func PushPackage(ctx *context.Context) {
},
&packages_service.PackageFileCreationInfo{
PackageFileInfo: packages_service.PackageFileInfo{
Filename: fmt.Sprintf("%s-%s-%s.pkg.tar.zst", p.Name, p.Version, p.FileMetadata.Arch),
CompositeKey: distro,
Filename: fmt.Sprintf("%s-%s-%s.pkg.tar.%s", p.Name, p.Version, p.FileMetadata.Arch, p.CompressType),
CompositeKey: group,
},
OverwriteExisting: false,
IsLead: true,
@ -138,8 +144,8 @@ func PushPackage(ctx *context.Context) {
// add sign file
_, err = packages_service.AddFileToPackageVersionInternal(ctx, version, &packages_service.PackageFileCreationInfo{
PackageFileInfo: packages_service.PackageFileInfo{
CompositeKey: distro,
Filename: fmt.Sprintf("%s-%s-%s.pkg.tar.zst.sig", p.Name, p.Version, p.FileMetadata.Arch),
CompositeKey: group,
Filename: fmt.Sprintf("%s-%s-%s.pkg.tar.%s.sig", p.Name, p.Version, p.FileMetadata.Arch, p.CompressType),
},
OverwriteExisting: true,
IsLead: false,
@ -149,7 +155,7 @@ func PushPackage(ctx *context.Context) {
if err != nil {
apiError(ctx, http.StatusInternalServerError, err)
}
if err = arch_service.BuildPacmanDB(ctx, ctx.Package.Owner.ID, distro, p.FileMetadata.Arch); err != nil {
if err = arch_service.BuildPacmanDB(ctx, ctx.Package.Owner.ID, group, p.FileMetadata.Arch); err != nil {
apiError(ctx, http.StatusInternalServerError, err)
return
}
@ -158,13 +164,12 @@ func PushPackage(ctx *context.Context) {
func GetPackageOrDB(ctx *context.Context) {
var (
file = ctx.Params("file")
distro = ctx.Params("distro")
arch = ctx.Params("arch")
file = ctx.Params("file")
group = ctx.Params("group")
arch = ctx.Params("arch")
)
if strings.HasSuffix(file, ".pkg.tar.zst") || strings.HasSuffix(file, ".pkg.tar.zst.sig") {
pkg, err := arch_service.GetPackageFile(ctx, distro, file, ctx.Package.Owner.ID)
if archPkgOrSig.MatchString(file) {
pkg, err := arch_service.GetPackageFile(ctx, group, file, ctx.Package.Owner.ID)
if err != nil {
if errors.Is(err, util.ErrNotExist) {
apiError(ctx, http.StatusNotFound, err)
@ -180,11 +185,8 @@ func GetPackageOrDB(ctx *context.Context) {
return
}
if strings.HasSuffix(file, ".db.tar.gz") ||
strings.HasSuffix(file, ".db") ||
strings.HasSuffix(file, ".db.tar.gz.sig") ||
strings.HasSuffix(file, ".db.sig") {
pkg, err := arch_service.GetPackageDBFile(ctx, distro, arch, ctx.Package.Owner.ID,
if archDBOrSig.MatchString(file) {
pkg, err := arch_service.GetPackageDBFile(ctx, group, arch, ctx.Package.Owner.ID,
strings.HasSuffix(file, ".sig"))
if err != nil {
if errors.Is(err, util.ErrNotExist) {
@ -205,9 +207,9 @@ func GetPackageOrDB(ctx *context.Context) {
func RemovePackage(ctx *context.Context) {
var (
distro = ctx.Params("distro")
pkg = ctx.Params("package")
ver = ctx.Params("version")
group = ctx.Params("group")
pkg = ctx.Params("package")
ver = ctx.Params("version")
)
pv, err := packages_model.GetVersionByNameAndVersion(
ctx, ctx.Package.Owner.ID, packages_model.TypeArch, pkg, ver,
@ -227,7 +229,7 @@ func RemovePackage(ctx *context.Context) {
}
deleted := false
for _, file := range files {
if file.CompositeKey == distro {
if file.CompositeKey == group {
deleted = true
err := packages_service.RemovePackageFileAndVersionIfUnreferenced(ctx, ctx.ContextUser, file)
if err != nil {
@ -237,7 +239,7 @@ func RemovePackage(ctx *context.Context) {
}
}
if deleted {
err = arch_service.BuildCustomRepositoryFiles(ctx, ctx.Package.Owner.ID, distro)
err = arch_service.BuildCustomRepositoryFiles(ctx, ctx.Package.Owner.ID, group)
if err != nil {
apiError(ctx, http.StatusInternalServerError, err)
}

View file

@ -15,7 +15,7 @@ import (
type PackageCleanupRuleForm struct {
ID int64
Enabled bool
Type string `binding:"Required;In(alpine,cargo,chef,composer,conan,conda,container,cran,debian,generic,go,helm,maven,npm,nuget,pub,pypi,rpm,rubygems,swift,vagrant)"`
Type string `binding:"Required;In(alpine,arch,cargo,chef,composer,conan,conda,container,cran,debian,generic,go,helm,maven,npm,nuget,pub,pypi,rpm,rubygems,swift,vagrant)"`
KeepCount int `binding:"In(0,1,5,10,25,50,100)"`
KeepPattern string `binding:"RegexPattern"`
RemoveDays int `binding:"In(0,7,14,30,60,90,180)"`

View file

@ -11,6 +11,7 @@ import (
"fmt"
"io"
"os"
"path/filepath"
"sort"
"strings"
@ -43,7 +44,7 @@ func BuildAllRepositoryFiles(ctx context.Context, ownerID int64) error {
}
for _, pf := range pfs {
if strings.HasSuffix(pf.Name, ".db") {
arch := strings.TrimSuffix(strings.TrimPrefix(pf.Name, fmt.Sprintf("%s-", pf.CompositeKey)), ".db")
arch := strings.TrimSuffix(pf.Name, ".db")
if err := BuildPacmanDB(ctx, ownerID, pf.CompositeKey, arch); err != nil {
return err
}
@ -99,7 +100,7 @@ func NewFileSign(ctx context.Context, ownerID int64, input io.Reader) (*packages
}
// BuildPacmanDB Create db signature cache
func BuildPacmanDB(ctx context.Context, ownerID int64, distro, arch string) error {
func BuildPacmanDB(ctx context.Context, ownerID int64, group, arch string) error {
pv, err := GetOrCreateRepositoryVersion(ctx, ownerID)
if err != nil {
return err
@ -110,15 +111,15 @@ func BuildPacmanDB(ctx context.Context, ownerID int64, distro, arch string) erro
return err
}
for _, pf := range pfs {
if pf.CompositeKey == distro && strings.HasPrefix(pf.Name, fmt.Sprintf("%s-%s", distro, arch)) {
// remove distro and arch
if pf.CompositeKey == group && pf.Name == fmt.Sprintf("%s.db", arch) {
// remove group and arch
if err := packages_service.DeletePackageFile(ctx, pf); err != nil {
return err
}
}
}
db, err := flushDB(ctx, ownerID, distro, arch)
db, err := createDB(ctx, ownerID, group, arch)
if errors.Is(err, io.EOF) {
return nil
} else if err != nil {
@ -140,13 +141,13 @@ func BuildPacmanDB(ctx context.Context, ownerID int64, distro, arch string) erro
return err
}
for name, data := range map[string]*packages_module.HashedBuffer{
fmt.Sprintf("%s-%s.db", distro, arch): db,
fmt.Sprintf("%s-%s.db.sig", distro, arch): sig,
fmt.Sprintf("%s.db", arch): db,
fmt.Sprintf("%s.db.sig", arch): sig,
} {
_, err = packages_service.AddFileToPackageVersionInternal(ctx, pv, &packages_service.PackageFileCreationInfo{
PackageFileInfo: packages_service.PackageFileInfo{
Filename: name,
CompositeKey: distro,
CompositeKey: group,
},
Creator: user_model.NewGhostUser(),
Data: data,
@ -160,7 +161,7 @@ func BuildPacmanDB(ctx context.Context, ownerID int64, distro, arch string) erro
return nil
}
func flushDB(ctx context.Context, ownerID int64, distro, arch string) (*packages_module.HashedBuffer, error) {
func createDB(ctx context.Context, ownerID int64, group, arch string) (*packages_module.HashedBuffer, error) {
pkgs, err := packages_model.GetPackagesByType(ctx, ownerID, packages_model.TypeArch)
if err != nil {
return nil, err
@ -185,17 +186,29 @@ func flushDB(ctx context.Context, ownerID int64, distro, arch string) (*packages
sort.Slice(versions, func(i, j int) bool {
return versions[i].CreatedUnix > versions[j].CreatedUnix
})
for _, ver := range versions {
file := fmt.Sprintf("%s-%s-%s.pkg.tar.zst", pkg.Name, ver.Version, arch)
pf, err := packages_model.GetFileForVersionByName(ctx, ver.ID, file, distro)
files, err := packages_model.GetFilesByVersionID(ctx, ver.ID)
if err != nil {
// add any arch package
file = fmt.Sprintf("%s-%s-any.pkg.tar.zst", pkg.Name, ver.Version)
pf, err = packages_model.GetFileForVersionByName(ctx, ver.ID, file, distro)
if err != nil {
continue
return nil, errors.Join(tw.Close(), gw.Close(), db.Close(), err)
}
var pf *packages_model.PackageFile
for _, file := range files {
ext := filepath.Ext(file.Name)
if file.CompositeKey == group && ext != "" && ext != ".db" && ext != ".sig" {
if pf == nil && strings.HasSuffix(file.Name, fmt.Sprintf("any.pkg.tar%s", ext)) {
pf = file
}
if strings.HasSuffix(file.Name, fmt.Sprintf("%s.pkg.tar%s", arch, ext)) {
pf = file
break
}
}
}
if pf == nil {
// file not exists
continue
}
pps, err := packages_model.GetPropertiesByName(
ctx, packages_model.PropertyTypeFile, pf.ID, arch_module.PropertyDescription,
)
@ -230,8 +243,8 @@ func flushDB(ctx context.Context, ownerID int64, distro, arch string) (*packages
// GetPackageFile Get data related to provided filename and distribution, for package files
// update download counter.
func GetPackageFile(ctx context.Context, distro, file string, ownerID int64) (io.ReadSeekCloser, error) {
pf, err := getPackageFile(ctx, distro, file, ownerID)
func GetPackageFile(ctx context.Context, group, file string, ownerID int64) (io.ReadSeekCloser, error) {
pf, err := getPackageFile(ctx, group, file, ownerID)
if err != nil {
return nil, err
}
@ -241,7 +254,7 @@ func GetPackageFile(ctx context.Context, distro, file string, ownerID int64) (io
}
// Ejects parameters required to get package file property from file name.
func getPackageFile(ctx context.Context, distro, file string, ownerID int64) (*packages_model.PackageFile, error) {
func getPackageFile(ctx context.Context, group, file string, ownerID int64) (*packages_model.PackageFile, error) {
var (
splt = strings.Split(file, "-")
pkgname = strings.Join(splt[0:len(splt)-3], "-")
@ -253,23 +266,23 @@ func getPackageFile(ctx context.Context, distro, file string, ownerID int64) (*p
return nil, err
}
pkgfile, err := packages_model.GetFileForVersionByName(ctx, version.ID, file, distro)
pkgfile, err := packages_model.GetFileForVersionByName(ctx, version.ID, file, group)
if err != nil {
return nil, err
}
return pkgfile, nil
}
func GetPackageDBFile(ctx context.Context, distro, arch string, ownerID int64, signFile bool) (io.ReadSeekCloser, error) {
func GetPackageDBFile(ctx context.Context, group, arch string, ownerID int64, signFile bool) (io.ReadSeekCloser, error) {
pv, err := GetOrCreateRepositoryVersion(ctx, ownerID)
if err != nil {
return nil, err
}
fileName := fmt.Sprintf("%s-%s.db", distro, arch)
fileName := fmt.Sprintf("%s.db", arch)
if signFile {
fileName = fmt.Sprintf("%s-%s.db.sig", distro, arch)
fileName = fmt.Sprintf("%s.db.sig", arch)
}
file, err := packages_model.GetFileForVersionByName(ctx, pv.ID, fileName, distro)
file, err := packages_model.GetFileForVersionByName(ctx, pv.ID, fileName, group)
if err != nil {
return nil, err
}

View file

@ -81,18 +81,18 @@ jMBmtEhxyCnCZdUAwYKxAxeRFVk4TCL0aYgWjt3kHTg9SjVStppI2YCSWshUEFGdmJmyCVGpnqIU
KNlA0hEjIOACGSLqYpXAD5SSNVT2MJRJwREAF4FRHPBlCJMSNwFguGAWDJBg+KIArkIJGNtCydUL
TuN1oBh/+zKkEblAsgjGqVgUwKLP+UOMOGCpAhICtg6ncFJH`),
"other": unPack(`
KLUv/QBYbRMABuOHS9BSNQdQ56F+xNFoV3CijY54JYt3VqV1iUU3xmj00y2pyBOCuokbhDYpvNsj
ZJeCxqH+nQFpMf4Wa92okaZoF4eH6HsXXCBo+qy3Fn4AigBgAEaYrLCQEuAom6YbHyuKZAFYksqi
sSOFiRs0WDmlACk0CnpnaAeKiCS3BlwVkViJEbDS43lFNbLkZEmGhc305Nn4AMLGiUkBDiMTG5Vz
q4ZISjCofEfR1NpXijvP2X95Hu1e+zLalc0+mjeT3Z/FPGvt62WymbX2dXMDIYKDLjjP8n03RrPf
A1vOApwGOh2MgE2LpgZrgXLDF2CUJ15idG2J8GCSgcc2ZVRgA8+RHD0k2VJjg6mRUgGGhBWEyEcz
5EePLhUeWlYhoFCKONxUiBiIUiQeDIqiQwkjLiyqnF5eGs6a2gGRapbU9JRyuXAlPemYajlJojJd
GBBJjo5GxFRkITOAvLhSCr2TDz4uzdU8Yh3i/SHP4qh3vTG2s9198NP8M+pdR73BvIP6qPeDjzsW
gTi+jXrXWOe5P/jZxOeod/287v6JljzNP99RNM0a+/x4ljz3LNV2t5v9qHfW2Pyg24u54zSfObWX
Y9bYrCTHtwdfPPPOYiU5fvB5FssfNN2V5EIPfg9LnM+JhtVEO8+FZw5LXA068YNPhimu9sHPQiWv
qc6fE9BTnxIe/LTKatab+WYu7T74uWNRxJW5W5Ux0bDLuG1ioCwjg4DvGgBcgB8cUDHJ1RQ89neE
wvjbNUMiIZdo5hbHgEpANwMkDnL0Jr7kVFg+0pZKjBkmklNgBH1YI8dQOAAKbr6EF5wYM80KWnAd
nYAR`),
/Td6WFoAAATm1rRGBMCyBIAYIQEWAAAAAAAAABaHRszgC/8CKl0AFxNGhTWwfXmuDQEJlHgNLrkq
VxpJY6d9iRTt6gB4uCj0481rnYfXaUADHzOFuF3490RPrM6juPXrknqtVyuWJ5efW19BgwctN6xk
UiXiZaXVAWVWJWy2XHJiyYCMWBfIjUfo1ccOgwolwgFHJ64ZJjbayA3k6lYPcImuAqYL5NEVHpwl
Z8CWIjiXXSMQGsB3gxMdq9nySZbHQLK/KCKQ+oseF6kXyIgSEyuG4HhjVBBYIwTvWzI06kjNUXEy
2sw0n50uocLSAwJ/3mdX3n3XF5nmmuQMPtFbdQgQtC2VhyVd3TdIF+pT6zAEzXFJJ3uLkNbKSS88
ZdBny6X/ftT5lQpNi/Wg0xLEQA4m4fu4fRAR0kOKzHM2svNLbTxa/wOPidqPzR6b/jfKmHkXxBNa
jFafty0a5K2S3F6JpwXZ2fqti/zG9NtMc+bbuXycC327EofXRXNtuOupELDD+ltTOIBF7CcTswyi
MZDP1PBie6GqDV2GuPz+0XXmul/ds+XysG19HIkKbJ+cQKp5o7Y0tI7EHM8GhwMl7MjgpQGj5nuv
0u2hqt4NXPNYqaMm9bFnnIUxEN82HgNWBcXf2baWKOdGzPzCuWg2fAM4zxHnBWcimxLXiJgaI8mU
J/QqTPWE0nJf1PW/J9yFQVR1Xo0TJyiX8/ObwmbqUPpxRGjKlYRBvn0jbTdUAENBSn+QVcASRGFE
SB9OM2B8Bg4jR/oojs8Beoq7zbIblgAAAACfRtXvhmznOgABzgSAGAAAKklb4rHEZ/sCAAAAAARZ
Wg==`), // this is tar.xz file
}
t.Run("RepositoryKey", func(t *testing.T) {
@ -105,155 +105,154 @@ nYAR`),
require.Contains(t, resp.Body.String(), "-----BEGIN PGP PUBLIC KEY BLOCK-----")
})
t.Run("Upload", func(t *testing.T) {
defer tests.PrintCurrentTest(t)()
req := NewRequestWithBody(t, "PUT", rootURL+"/default", bytes.NewReader(pkgs["any"]))
MakeRequest(t, req, http.StatusUnauthorized)
req = NewRequestWithBody(t, "PUT", rootURL+"/default", bytes.NewReader(pkgs["any"])).
AddBasicAuth(user.Name)
MakeRequest(t, req, http.StatusCreated)
pvs, err := packages.GetVersionsByPackageType(db.DefaultContext, user.ID, packages.TypeArch)
require.NoError(t, err)
require.Len(t, pvs, 1)
pd, err := packages.GetPackageDescriptor(db.DefaultContext, pvs[0])
require.NoError(t, err)
require.Nil(t, pd.SemVer)
require.IsType(t, &arch_model.VersionMetadata{}, pd.Metadata)
require.Equal(t, "test", pd.Package.Name)
require.Equal(t, "1.0.0-1", pd.Version.Version)
pfs, err := packages.GetFilesByVersionID(db.DefaultContext, pvs[0].ID)
require.NoError(t, err)
require.Len(t, pfs, 2) // zst and zst.sig
require.True(t, pfs[0].IsLead)
pb, err := packages.GetBlobByID(db.DefaultContext, pfs[0].BlobID)
require.NoError(t, err)
require.Equal(t, int64(len(pkgs["any"])), pb.Size)
req = NewRequestWithBody(t, "PUT", rootURL+"/default", bytes.NewReader(pkgs["any"])).
AddBasicAuth(user.Name)
MakeRequest(t, req, http.StatusConflict)
req = NewRequestWithBody(t, "PUT", rootURL+"/default", bytes.NewReader(pkgs["x86_64"])).
AddBasicAuth(user.Name)
MakeRequest(t, req, http.StatusCreated)
req = NewRequestWithBody(t, "PUT", rootURL+"/other", bytes.NewReader(pkgs["any"])).
AddBasicAuth(user.Name)
MakeRequest(t, req, http.StatusCreated)
req = NewRequestWithBody(t, "PUT", rootURL+"/other", bytes.NewReader(pkgs["aarch64"])).
AddBasicAuth(user.Name)
MakeRequest(t, req, http.StatusCreated)
req = NewRequestWithBody(t, "PUT", rootURL+"/base", bytes.NewReader(pkgs["other"])).
AddBasicAuth(user.Name)
MakeRequest(t, req, http.StatusCreated)
req = NewRequestWithBody(t, "PUT", rootURL+"/base", bytes.NewReader(pkgs["x86_64"])).
AddBasicAuth(user.Name)
MakeRequest(t, req, http.StatusCreated)
req = NewRequestWithBody(t, "PUT", rootURL+"/base", bytes.NewReader(pkgs["aarch64"])).
AddBasicAuth(user.Name)
MakeRequest(t, req, http.StatusCreated)
})
t.Run("Download", func(t *testing.T) {
defer tests.PrintCurrentTest(t)()
req := NewRequest(t, "GET", rootURL+"/default/x86_64/test-1.0.0-1-x86_64.pkg.tar.zst")
resp := MakeRequest(t, req, http.StatusOK)
require.Equal(t, pkgs["x86_64"], resp.Body.Bytes())
req = NewRequest(t, "GET", rootURL+"/default/x86_64/test-1.0.0-1-any.pkg.tar.zst")
resp = MakeRequest(t, req, http.StatusOK)
require.Equal(t, pkgs["any"], resp.Body.Bytes())
req = NewRequest(t, "GET", rootURL+"/default/x86_64/test-1.0.0-1-aarch64.pkg.tar.zst")
MakeRequest(t, req, http.StatusNotFound)
req = NewRequest(t, "GET", rootURL+"/other/x86_64/test-1.0.0-1-x86_64.pkg.tar.zst")
MakeRequest(t, req, http.StatusNotFound)
req = NewRequest(t, "GET", rootURL+"/other/x86_64/test-1.0.0-1-any.pkg.tar.zst")
resp = MakeRequest(t, req, http.StatusOK)
require.Equal(t, pkgs["any"], resp.Body.Bytes())
})
t.Run("SignVerify", func(t *testing.T) {
defer tests.PrintCurrentTest(t)()
req := NewRequest(t, "GET", rootURL+"/repository.key")
respPub := MakeRequest(t, req, http.StatusOK)
req = NewRequest(t, "GET", rootURL+"/other/x86_64/test-1.0.0-1-any.pkg.tar.zst")
respPkg := MakeRequest(t, req, http.StatusOK)
req = NewRequest(t, "GET", rootURL+"/other/x86_64/test-1.0.0-1-any.pkg.tar.zst.sig")
respSig := MakeRequest(t, req, http.StatusOK)
if err := gpgVerify(respPub.Body.Bytes(), respSig.Body.Bytes(), respPkg.Body.Bytes()); err != nil {
t.Fatal(err)
for _, group := range []string{"", "arch", "arch/os", "x86_64"} {
groupURL := rootURL
if group != "" {
groupURL = groupURL + "/" + group
}
})
t.Run(fmt.Sprintf("Upload[%s]", group), func(t *testing.T) {
defer tests.PrintCurrentTest(t)()
t.Run("Repository", func(t *testing.T) {
defer tests.PrintCurrentTest(t)()
req := NewRequest(t, "GET", rootURL+"/repository.key")
respPub := MakeRequest(t, req, http.StatusOK)
req := NewRequestWithBody(t, "PUT", groupURL, bytes.NewReader(pkgs["any"]))
MakeRequest(t, req, http.StatusUnauthorized)
req = NewRequest(t, "GET", rootURL+"/base/x86_64/base.db")
respPkg := MakeRequest(t, req, http.StatusOK)
req = NewRequestWithBody(t, "PUT", groupURL, bytes.NewReader(pkgs["any"])).
AddBasicAuth(user.Name)
MakeRequest(t, req, http.StatusCreated)
req = NewRequest(t, "GET", rootURL+"/base/x86_64/base.db.sig")
respSig := MakeRequest(t, req, http.StatusOK)
req = NewRequestWithBody(t, "PUT", groupURL, bytes.NewBuffer([]byte("any string"))).
AddBasicAuth(user.Name)
MakeRequest(t, req, http.StatusBadRequest)
if err := gpgVerify(respPub.Body.Bytes(), respSig.Body.Bytes(), respPkg.Body.Bytes()); err != nil {
t.Fatal(err)
}
files, err := listGzipFiles(respPkg.Body.Bytes())
require.NoError(t, err)
require.Len(t, files, 2)
for s, d := range files {
name := getProperty(string(d.Data), "NAME")
ver := getProperty(string(d.Data), "VERSION")
require.Equal(t, name+"-"+ver+"/desc", s)
fn := getProperty(string(d.Data), "FILENAME")
pgp := getProperty(string(d.Data), "PGPSIG")
req = NewRequest(t, "GET", rootURL+"/base/x86_64/"+fn+".sig")
respSig := MakeRequest(t, req, http.StatusOK)
decodeString, err := base64.StdEncoding.DecodeString(pgp)
pvs, err := packages.GetVersionsByPackageType(db.DefaultContext, user.ID, packages.TypeArch)
require.NoError(t, err)
require.Equal(t, respSig.Body.Bytes(), decodeString)
}
})
t.Run("Delete", func(t *testing.T) {
defer tests.PrintCurrentTest(t)()
req := NewRequestWithBody(t, "DELETE", rootURL+"/base/notfound/1.0.0-1", nil).
AddBasicAuth(user.Name)
MakeRequest(t, req, http.StatusNotFound)
require.Len(t, pvs, 1)
req = NewRequestWithBody(t, "DELETE", rootURL+"/base/test/1.0.0-1", nil).
AddBasicAuth(user.Name)
MakeRequest(t, req, http.StatusNoContent)
pd, err := packages.GetPackageDescriptor(db.DefaultContext, pvs[0])
require.NoError(t, err)
require.Nil(t, pd.SemVer)
require.IsType(t, &arch_model.VersionMetadata{}, pd.Metadata)
require.Equal(t, "test", pd.Package.Name)
require.Equal(t, "1.0.0-1", pd.Version.Version)
req = NewRequest(t, "GET", rootURL+"/base/x86_64/base.db")
respPkg := MakeRequest(t, req, http.StatusOK)
files, err := listGzipFiles(respPkg.Body.Bytes())
require.NoError(t, err)
require.Len(t, files, 1)
pfs, err := packages.GetFilesByVersionID(db.DefaultContext, pvs[0].ID)
require.NoError(t, err)
size := 0
for _, pf := range pfs {
if pf.CompositeKey == group {
size++
}
}
require.Equal(t, 2, size) // zst and zst.sig
req = NewRequestWithBody(t, "DELETE", rootURL+"/base/test2/1.0.0-1", nil).
AddBasicAuth(user.Name)
MakeRequest(t, req, http.StatusNoContent)
req = NewRequest(t, "GET", rootURL+"/base/x86_64/base.db")
MakeRequest(t, req, http.StatusNotFound)
pb, err := packages.GetBlobByID(db.DefaultContext, pfs[0].BlobID)
require.NoError(t, err)
require.Equal(t, int64(len(pkgs["any"])), pb.Size)
req = NewRequest(t, "GET", rootURL+"/default/x86_64/base.db")
respPkg = MakeRequest(t, req, http.StatusOK)
files, err = listGzipFiles(respPkg.Body.Bytes())
require.NoError(t, err)
require.Len(t, files, 1)
})
req = NewRequestWithBody(t, "PUT", groupURL, bytes.NewReader(pkgs["any"])).
AddBasicAuth(user.Name) // exists
MakeRequest(t, req, http.StatusConflict)
req = NewRequestWithBody(t, "PUT", groupURL, bytes.NewReader(pkgs["x86_64"])).
AddBasicAuth(user.Name)
MakeRequest(t, req, http.StatusCreated)
req = NewRequestWithBody(t, "PUT", groupURL, bytes.NewReader(pkgs["aarch64"])).
AddBasicAuth(user.Name)
MakeRequest(t, req, http.StatusCreated)
req = NewRequestWithBody(t, "PUT", groupURL, bytes.NewReader(pkgs["aarch64"])).
AddBasicAuth(user.Name) // exists again
MakeRequest(t, req, http.StatusConflict)
})
t.Run(fmt.Sprintf("Download[%s]", group), func(t *testing.T) {
defer tests.PrintCurrentTest(t)()
req := NewRequest(t, "GET", groupURL+"/x86_64/test-1.0.0-1-x86_64.pkg.tar.zst")
resp := MakeRequest(t, req, http.StatusOK)
require.Equal(t, pkgs["x86_64"], resp.Body.Bytes())
req = NewRequest(t, "GET", groupURL+"/x86_64/test-1.0.0-1-any.pkg.tar.zst")
resp = MakeRequest(t, req, http.StatusOK)
require.Equal(t, pkgs["any"], resp.Body.Bytes())
// get other group
req = NewRequest(t, "GET", rootURL+"/unknown/x86_64/test-1.0.0-1-aarch64.pkg.tar.zst")
MakeRequest(t, req, http.StatusNotFound)
})
t.Run(fmt.Sprintf("SignVerify[%s]", group), func(t *testing.T) {
defer tests.PrintCurrentTest(t)()
req := NewRequest(t, "GET", rootURL+"/repository.key")
respPub := MakeRequest(t, req, http.StatusOK)
req = NewRequest(t, "GET", groupURL+"/x86_64/test-1.0.0-1-any.pkg.tar.zst")
respPkg := MakeRequest(t, req, http.StatusOK)
req = NewRequest(t, "GET", groupURL+"/x86_64/test-1.0.0-1-any.pkg.tar.zst.sig")
respSig := MakeRequest(t, req, http.StatusOK)
if err := gpgVerify(respPub.Body.Bytes(), respSig.Body.Bytes(), respPkg.Body.Bytes()); err != nil {
t.Fatal(err)
}
})
t.Run(fmt.Sprintf("RepositoryDB[%s]", group), func(t *testing.T) {
defer tests.PrintCurrentTest(t)()
req := NewRequest(t, "GET", rootURL+"/repository.key")
respPub := MakeRequest(t, req, http.StatusOK)
req = NewRequest(t, "GET", groupURL+"/x86_64/base.db")
respPkg := MakeRequest(t, req, http.StatusOK)
req = NewRequest(t, "GET", groupURL+"/x86_64/base.db.sig")
respSig := MakeRequest(t, req, http.StatusOK)
if err := gpgVerify(respPub.Body.Bytes(), respSig.Body.Bytes(), respPkg.Body.Bytes()); err != nil {
t.Fatal(err)
}
files, err := listTarGzFiles(respPkg.Body.Bytes())
require.NoError(t, err)
require.Len(t, files, 1)
for s, d := range files {
name := getProperty(string(d.Data), "NAME")
ver := getProperty(string(d.Data), "VERSION")
require.Equal(t, name+"-"+ver+"/desc", s)
fn := getProperty(string(d.Data), "FILENAME")
pgp := getProperty(string(d.Data), "PGPSIG")
req = NewRequest(t, "GET", groupURL+"/x86_64/"+fn+".sig")
respSig := MakeRequest(t, req, http.StatusOK)
decodeString, err := base64.StdEncoding.DecodeString(pgp)
require.NoError(t, err)
require.Equal(t, respSig.Body.Bytes(), decodeString)
}
})
t.Run(fmt.Sprintf("Delete[%s]", group), func(t *testing.T) {
defer tests.PrintCurrentTest(t)()
// test data
req := NewRequestWithBody(t, "PUT", groupURL, bytes.NewReader(pkgs["other"])).
AddBasicAuth(user.Name)
MakeRequest(t, req, http.StatusCreated)
req = NewRequestWithBody(t, "DELETE", rootURL+"/base/notfound/1.0.0-1", nil).
AddBasicAuth(user.Name)
MakeRequest(t, req, http.StatusNotFound)
req = NewRequestWithBody(t, "DELETE", groupURL+"/test/1.0.0-1", nil).
AddBasicAuth(user.Name)
MakeRequest(t, req, http.StatusNoContent)
req = NewRequest(t, "GET", groupURL+"/x86_64/base.db")
respPkg := MakeRequest(t, req, http.StatusOK)
files, err := listTarGzFiles(respPkg.Body.Bytes())
require.NoError(t, err)
require.Len(t, files, 1) // other pkg in L225
req = NewRequestWithBody(t, "DELETE", groupURL+"/test2/1.0.0-1", nil).
AddBasicAuth(user.Name)
MakeRequest(t, req, http.StatusNoContent)
req = NewRequest(t, "GET", groupURL+"/x86_64/base.db")
MakeRequest(t, req, http.StatusNotFound)
})
}
}
func getProperty(data, key string) string {
@ -270,7 +269,7 @@ func getProperty(data, key string) string {
}
}
func listGzipFiles(data []byte) (fstest.MapFS, error) {
func listTarGzFiles(data []byte) (fstest.MapFS, error) {
reader, err := gzip.NewReader(bytes.NewBuffer(data))
defer reader.Close()
if err != nil {