added support for zstandard compressed debian data archives

fixed an issue causing debian repositories to not fail when a package could not be found
added more test fixtures

go.mod

@@ -11,6 +11,7 @@ require (
 	github.com/go-logr/logr v1.2.4
 	github.com/google/go-containerregistry v0.16.1
 	github.com/hashicorp/go-getter v1.7.2
+	github.com/klauspost/compress v1.17.0
 	github.com/knqyf263/go-deb-version v0.0.0-20230223133812-3ed183d23422
 	github.com/spf13/cobra v1.7.0
 	github.com/stretchr/testify v1.8.4
@@ -85,7 +86,6 @@ require (
 	github.com/inconshreveable/mousetrap v1.1.0 // indirect
 	github.com/jmespath/go-jmespath v0.4.0 // indirect
 	github.com/json-iterator/go v1.1.12 // indirect
-	github.com/klauspost/compress v1.17.0 // indirect
 	github.com/levigross/grequests v0.0.0-20190908174114-253788527a1a // indirect
 	github.com/mitchellh/go-homedir v1.1.0 // indirect
 	github.com/mitchellh/go-testing-interface v1.14.1 // indirect

pkg/archiveutil/tar.go

@@ -7,6 +7,7 @@ import (
 	"errors"
 	"github.com/chainguard-dev/go-apk/pkg/fs"
 	"github.com/go-logr/logr"
+	"github.com/klauspost/compress/zstd"
 	"github.com/ulikunitz/xz"
 	"io"
 	"os"
@@ -24,6 +25,7 @@ func Guntar(ctx context.Context, r io.Reader, rootfs fs.FullFS) error {
 	return Untar(ctx, gzp, rootfs)
 }
 
+// XZuntar is the same as Untar, but it first decodes the XZ archive
 func XZuntar(ctx context.Context, r io.Reader, rootfs fs.FullFS) error {
 	xzp, err := xz.NewReader(r)
 	if err != nil {
@@ -32,6 +34,16 @@ func XZuntar(ctx context.Context, r io.Reader, rootfs fs.FullFS) error {
 	return Untar(ctx, xzp, rootfs)
 }
 
+// Zuntar is the same as Untar, but it first decodes the Zstandard archive
+func Zuntar(ctx context.Context, r io.Reader, rootfs fs.FullFS) error {
+	zp, err := zstd.NewReader(r)
+	if err != nil {
+		return err
+	}
+	defer zp.Close()
+	return Untar(ctx, zp, rootfs)
+}
+
 // Untar expands a tar archive into the given path.
 func Untar(ctx context.Context, r io.Reader, rootfs fs.FullFS) error {
 	log := logr.FromContextOrDiscard(ctx)

pkg/debian/index.go

@@ -28,6 +28,7 @@ func NewIndex(ctx context.Context, repository, release, component, arch string)
 	}
 	defer resp.Body.Close()
 	if resp.StatusCode < 200 || resp.StatusCode > 299 {
+		log.Info("failed to locate Packages.gz file", "url", target)
 		return nil, fmt.Errorf("http response failed with code: %d", resp.StatusCode)
 	}
 	log.V(1).Info("successfully downloaded index", "code", resp.StatusCode)

pkg/packages/debian/debian.go

@@ -2,6 +2,7 @@ package debian
 
 import (
 	"context"
+	"errors"
 	"fmt"
 	"github.com/chainguard-dev/go-apk/pkg/fs"
 	v1 "github.com/djcass44/all-your-base/pkg/api/v1"
@@ -38,7 +39,7 @@ func NewPackageKeeper(ctx context.Context, repositories []string) (*PackageKeepe
 	}, nil
 }
 
-func (*PackageKeeper) Unpack(ctx context.Context, pkg string, rootfs fs.FullFS) error {
+func (p *PackageKeeper) Unpack(ctx context.Context, pkg string, rootfs fs.FullFS) error {
 	log := logr.FromContextOrDiscard(ctx).WithValues("pkg", pkg)
 	log.Info("unpacking deb")
 
@@ -57,15 +58,44 @@ func (*PackageKeeper) Unpack(ctx context.Context, pkg string, rootfs fs.FullFS)
 		return err
 	}
 
-	// then we need to untar the 'data.tar.gz' file
+	// then we need to unpack the 'data.tar.X' file
 	// that contains the filesystem
 
-	dataF, err := tmpFs.Open("/data.tar.xz")
+	if _, err := tmpFs.Stat(dataXZ); err == nil {
+		return p.unpackXZ(ctx, tmpFs, rootfs)
+	}
+	if _, err := tmpFs.Stat(dataZstd); err == nil {
+		return p.unpackZstd(ctx, tmpFs, rootfs)
+	}
+
+	return errors.New("unknown or unsupported data archive")
+}
+
+const (
+	dataXZ   = "/data.tar.xz"
+	dataZstd = "/data.tar.zst"
+)
+
+func (*PackageKeeper) unpackZstd(ctx context.Context, src fs.FullFS, dst fs.FullFS) error {
+	log := logr.FromContextOrDiscard(ctx)
+	log.V(1).Info("unpacking zstandard data archive")
+	f, err := src.Open(dataZstd)
 	if err != nil {
-		log.Error(err, "failed to open data.tar.gz file")
+		log.Error(err, "failed to open data.tar.xz file")
 		return err
 	}
-	return archiveutil.XZuntar(ctx, dataF, rootfs)
+	return archiveutil.Zuntar(ctx, f, dst)
+}
+
+func (*PackageKeeper) unpackXZ(ctx context.Context, src fs.FullFS, dst fs.FullFS) error {
+	log := logr.FromContextOrDiscard(ctx)
+	log.V(1).Info("unpacking xz data archive")
+	f, err := src.Open(dataXZ)
+	if err != nil {
+		log.Error(err, "failed to open data.tar.xz file")
+		return err
+	}
+	return archiveutil.XZuntar(ctx, f, dst)
 }
 
 func (p *PackageKeeper) Resolve(ctx context.Context, pkg string) ([]lockfile.Package, error) {
@@ -76,6 +106,9 @@ func (p *PackageKeeper) Resolve(ctx context.Context, pkg string) ([]lockfile.Pac
 		if err != nil {
 			return nil, err
 		}
+		if len(out) == 0 {
+			continue
+		}
 		names := make([]lockfile.Package, len(out))
 		for i := range out {
 			names[i] = lockfile.Package{