From 7cac1d5f3b311f09a278c410bc8688899b76bff0 Mon Sep 17 00:00:00 2001 From: unclegedd Date: Tue, 27 Feb 2024 12:18:49 -0600 Subject: [PATCH] chore: refactor pull --- src/cmd/uds.go | 3 +- src/config/config.go | 3 ++ src/pkg/bundle/provider.go | 4 +- src/pkg/bundle/pull.go | 25 +++--------- src/pkg/bundle/remote.go | 77 ++++++++++++++++++------------------- src/pkg/bundle/tarball.go | 65 +++++++++++++++---------------- src/pkg/cache/cache.go | 8 ++-- src/test/e2e/bundle_test.go | 2 +- 8 files changed, 85 insertions(+), 102 deletions(-) diff --git a/src/cmd/uds.go b/src/cmd/uds.go index e6630f08f..ce266a7ec 100644 --- a/src/cmd/uds.go +++ b/src/cmd/uds.go @@ -255,8 +255,7 @@ func configureZarf() { TempDirectory: config.CommonOptions.TempDirectory, OCIConcurrency: config.CommonOptions.OCIConcurrency, Confirm: config.CommonOptions.Confirm, - // todo: decouple Zarf cache? - CachePath: config.CommonOptions.CachePath, + CachePath: config.CommonOptions.CachePath, // use uds-cache instead of zarf-cache } } diff --git a/src/config/config.go b/src/config/config.go index ae9c6101e..c122fb200 100644 --- a/src/config/config.go +++ b/src/config/config.go @@ -47,6 +47,9 @@ const ( // UDSCache is the directory containing cached bundle layers UDSCache = ".uds-cache" + // UDSCacheLayers is the directory in the cache containing cached bundle layers + UDSCacheLayers = "layers" + // TasksYAML is the default name of the uds run cmd file TasksYAML = "tasks.yaml" diff --git a/src/pkg/bundle/provider.go b/src/pkg/bundle/provider.go index 214de09c8..7d05c06be 100644 --- a/src/pkg/bundle/provider.go +++ b/src/pkg/bundle/provider.go @@ -34,7 +34,7 @@ type Provider interface { // LoadBundle loads a bundle into the temporary directory and returns a map of the bundle's files // // (currently only the remote provider utilizes the concurrency parameter) - LoadBundle(concurrency int) (types.PathMap, error) + LoadBundle(options types.BundlePullOptions, concurrency int) (*types.UDSBundle, types.PathMap, error) // CreateBundleSBOM creates a bundle-level SBOM from the underlying Zarf packages, if the Zarf package contains an SBOM CreateBundleSBOM(extractSBOM bool) error @@ -43,7 +43,7 @@ type Provider interface { PublishBundle(bundle types.UDSBundle, remote *oci.OrasRemote) error // getBundleManifest gets the bundle's root manifest - getBundleManifest() error + getBundleManifest() (*oci.ZarfOCIManifest, error) // ZarfPackageNameMap returns a map of the zarf package name specified in the uds-bundle.yaml to the actual zarf package name ZarfPackageNameMap() (map[string]string, error) diff --git a/src/pkg/bundle/pull.go b/src/pkg/bundle/pull.go index ed6bf690a..059a8f70a 100644 --- a/src/pkg/bundle/pull.go +++ b/src/pkg/bundle/pull.go @@ -21,10 +21,11 @@ import ( ocispec "github.com/opencontainers/image-spec/specs-go/v1" ) -// Pull pulls a bundle and saves it locally + caches it +// Pull pulls a bundle and saves it locally func (b *Bundle) Pull() error { + // use uds-cache/packages as the dst dir for the pull to get auto caching + // we use an ORAS ocistore to make that dir look like an OCI artifact cacheDir := filepath.Join(zarfConfig.GetAbsCachePath(), "packages") - // create the cache directory if it doesn't exist if err := utils.CreateDirectory(cacheDir, 0755); err != nil { return err } @@ -41,28 +42,12 @@ func (b *Bundle) Pull() error { return err } - // pull the bundle's metadata + sig - loadedMetadata, err := provider.LoadBundleMetadata() - if err != nil { - return err - } - if err := utils.ReadYaml(loadedMetadata[config.BundleYAML], &b.bundle); err != nil { - return err - } - - // validate the sig (if present) - if err := ValidateBundleSignature(loadedMetadata[config.BundleYAML], loadedMetadata[config.BundleYAMLSignature], b.cfg.PullOpts.PublicKeyPath); err != nil { - return err - } - // pull the bundle's uds-bundle.yaml and it's Zarf pkgs - // todo: refactor this fn, think about pulling the rootDesc first and getting the hashes from there - // today, we are getting the Zarf image manifest hashes from the uds-bundle.yaml - // in that logic we end up pulling the root manifest twice, once in LoadBundle and the other below in remote.ResolveRoot() - loaded, err := provider.LoadBundle(zarfConfig.CommonOptions.OCIConcurrency) + bundle, loaded, err := provider.LoadBundle(b.cfg.PullOpts, zarfConfig.CommonOptions.OCIConcurrency) if err != nil { return err } + b.bundle = *bundle // create a remote client just to resolve the root descriptor platform := ocispec.Platform{ diff --git a/src/pkg/bundle/remote.go b/src/pkg/bundle/remote.go index 74ad523f6..c96bb5e57 100644 --- a/src/pkg/bundle/remote.go +++ b/src/pkg/bundle/remote.go @@ -43,19 +43,14 @@ type ociProvider struct { src string dst string *oci.OrasRemote - manifest *oci.ZarfOCIManifest } -func (op *ociProvider) getBundleManifest() error { - if op.manifest != nil { - return nil - } +func (op *ociProvider) getBundleManifest() (*oci.ZarfOCIManifest, error) { root, err := op.FetchRoot() if err != nil { - return err + return nil, err } - op.manifest = root - return nil + return root, nil } // LoadBundleMetadata loads a remote bundle's metadata @@ -79,10 +74,6 @@ func (op *ociProvider) LoadBundleMetadata() (types.PathMap, error) { } loaded[rel] = absSha } - err = op.getBundleManifest() - if err != nil { - return nil, err - } return loaded, nil } @@ -149,55 +140,60 @@ func (op *ociProvider) CreateBundleSBOM(extractSBOM bool) error { return nil } -// LoadBundle loads a bundle's uds-bundle.yaml and Zarf packages from a remote source -func (op *ociProvider) LoadBundle(_ int) (types.PathMap, error) { - var layersToPull []ocispec.Descriptor - estimatedBytes := int64(0) - - if err := op.getBundleManifest(); err != nil { - return nil, err - } - - loaded, err := op.LoadBundleMetadata() // todo: remove? this seems redundant, can we pass the "loaded" var in +// LoadBundle loads a bundle from a remote source +func (op *ociProvider) LoadBundle(opts types.BundlePullOptions, _ int) (*types.UDSBundle, types.PathMap, error) { + var bundle types.UDSBundle + // pull the bundle's metadata + sig + loaded, err := op.LoadBundleMetadata() if err != nil { - return nil, err + return nil, nil, err + } + if err := zarfUtils.ReadYaml(loaded[config.BundleYAML], &bundle); err != nil { + return nil, nil, err } - b, err := os.ReadFile(loaded[config.BundleYAML]) - if err != nil { - return nil, err + // validate the sig (if present) before pulling the whole bundle + if err := ValidateBundleSignature(loaded[config.BundleYAML], loaded[config.BundleYAMLSignature], opts.PublicKeyPath); err != nil { + return nil, nil, err } - var bundle types.UDSBundle - if err := goyaml.Unmarshal(b, &bundle); err != nil { - return nil, err + var layersToPull []ocispec.Descriptor + estimatedBytes := int64(0) + + // get the bundle's root manifest + rootManifest, err := op.getBundleManifest() + if err != nil { + return nil, nil, err } for _, pkg := range bundle.Packages { + // grab sha of zarf image manifest and pull it down sha := strings.Split(pkg.Ref, "@sha256:")[1] // this is where we use the SHA appended to the Zarf pkg inside the bundle - manifestDesc := op.manifest.Locate(sha) + manifestDesc := rootManifest.Locate(sha) if err != nil { - return nil, err + return nil, nil, err } manifestBytes, err := op.FetchLayer(manifestDesc) if err != nil { - return nil, err + return nil, nil, err } + // unmarshal the zarf image manifest and add it to the layers to pull var manifest oci.ZarfOCIManifest if err := json.Unmarshal(manifestBytes, &manifest); err != nil { - return nil, err + return nil, nil, err } layersToPull = append(layersToPull, manifestDesc) progressBar := message.NewProgressBar(int64(len(manifest.Layers)), fmt.Sprintf("Verifying layers in Zarf package: %s", pkg.Name)) + // go through the layers in the zarf image manifest and check if they exist in the remote for _, layer := range manifest.Layers { ok, err := op.Repo().Blobs().Exists(op.ctx, layer) progressBar.Add(1) estimatedBytes += layer.Size if err != nil { - return nil, err + return nil, nil, err } // if the layer exists in the remote, add it to the layers to pull if ok { @@ -209,13 +205,13 @@ func (op *ociProvider) LoadBundle(_ int) (types.PathMap, error) { store, err := ocistore.NewWithContext(op.ctx, op.dst) if err != nil { - return nil, err + return nil, nil, err } // grab the bundle root manifest and add it to the layers to pull rootDesc, err := op.ResolveRoot() if err != nil { - return nil, err + return nil, nil, err } layersToPull = append(layersToPull, rootDesc) @@ -232,7 +228,7 @@ func (op *ociProvider) LoadBundle(_ int) (types.PathMap, error) { _, err = oras.Copy(op.ctx, op.Repo(), op.Repo().Reference.String(), store, op.Repo().Reference.String(), copyOpts) if err != nil { doneSaving <- 1 - return nil, err + return nil, nil, err } doneSaving <- 1 @@ -243,7 +239,7 @@ func (op *ociProvider) LoadBundle(_ int) (types.PathMap, error) { loaded[sha] = filepath.Join(op.dst, config.BlobsDir, sha) } - return loaded, nil + return &bundle, loaded, nil } func (op *ociProvider) PublishBundle(_ types.UDSBundle, _ *oci.OrasRemote) error { @@ -337,7 +333,8 @@ func CheckOCISourcePath(source string) (string, error) { // ZarfPackageNameMap returns the uds bundle zarf package name to actual zarf package name mappings from the oci provider func (op *ociProvider) ZarfPackageNameMap() (map[string]string, error) { - if err := op.getBundleManifest(); err != nil { + rootManifest, err := op.getBundleManifest() + if err != nil { return nil, err } @@ -359,7 +356,7 @@ func (op *ociProvider) ZarfPackageNameMap() (map[string]string, error) { nameMap := make(map[string]string) for _, pkg := range bundle.Packages { sha := strings.Split(pkg.Ref, "@sha256:")[1] // this is where we use the SHA appended to the Zarf pkg inside the bundle - manifestDesc := op.manifest.Locate(sha) + manifestDesc := rootManifest.Locate(sha) nameMap[manifestDesc.Annotations[config.UDSPackageNameAnnotation]] = manifestDesc.Annotations[config.ZarfPackageNameAnnotation] } return nameMap, nil diff --git a/src/pkg/bundle/tarball.go b/src/pkg/bundle/tarball.go index 360b4f41b..3812988e3 100644 --- a/src/pkg/bundle/tarball.go +++ b/src/pkg/bundle/tarball.go @@ -26,16 +26,15 @@ import ( ) type tarballBundleProvider struct { - ctx context.Context - src string - dst string - bundleRootManifest *oci.ZarfOCIManifest - bundleRootDesc ocispec.Descriptor + ctx context.Context + src string + dst string + bundleRootDesc ocispec.Descriptor // populated in getBundleManifest as a side effect } // CreateBundleSBOM creates a bundle-level SBOM from the underlying Zarf packages, if the Zarf package contains an SBOM func (tp *tarballBundleProvider) CreateBundleSBOM(extractSBOM bool) error { - err := tp.getBundleManifest() + rootManifest, err := tp.getBundleManifest() if err != nil { return err } @@ -47,7 +46,7 @@ func (tp *tarballBundleProvider) CreateBundleSBOM(extractSBOM bool) error { SBOMArtifactPathMap := make(types.PathMap) containsSBOMs := false - for _, layer := range tp.bundleRootManifest.Layers { + for _, layer := range rootManifest.Layers { // get Zarf image manifests from bundle manifest if layer.Annotations[ocispec.AnnotationTitle] == config.BundleYAML { continue @@ -113,19 +112,18 @@ func (tp *tarballBundleProvider) CreateBundleSBOM(extractSBOM bool) error { return nil } -func (tp *tarballBundleProvider) getBundleManifest() error { - if tp.bundleRootManifest != nil { - return nil - } +func (tp *tarballBundleProvider) getBundleManifest() (*oci.ZarfOCIManifest, error) { // Create a secure temporary directory for handling files + + // todo: make fn for grabbing the bundleRootDesc, we would also use it in publish secureTempDir, err := zarfUtils.MakeTempDir(config.CommonOptions.TempDirectory) if err != nil { - return fmt.Errorf("failed to create a secure temporary directory: %w", err) + return nil, fmt.Errorf("failed to create a secure temporary directory: %w", err) } defer os.RemoveAll(secureTempDir) // Ensure cleanup of the temp directory if err := av3.Extract(tp.src, "index.json", secureTempDir); err != nil { - return fmt.Errorf("failed to extract index.json from %s: %w", tp.src, err) + return nil, fmt.Errorf("failed to extract index.json from %s: %w", tp.src, err) } indexPath := filepath.Join(secureTempDir, "index.json") @@ -133,26 +131,25 @@ func (tp *tarballBundleProvider) getBundleManifest() error { b, err := os.ReadFile(indexPath) if err != nil { - return fmt.Errorf("failed to read index.json: %w", err) + return nil, fmt.Errorf("failed to read index.json: %w", err) } var index ocispec.Index - if err := json.Unmarshal(b, &index); err != nil { - return fmt.Errorf("failed to unmarshal index.json: %w", err) + return nil, fmt.Errorf("failed to unmarshal index.json: %w", err) } // local bundles only have one manifest entry in their index.json bundleManifestDesc := index.Manifests[0] - tp.bundleRootDesc = bundleManifestDesc + tp.bundleRootDesc = bundleManifestDesc // side effect! need to save this desc bc it's needed in publish if len(index.Manifests) > 1 { - return fmt.Errorf("expected only one manifest in index.json, found %d", len(index.Manifests)) + return nil, fmt.Errorf("expected only one manifest in index.json, found %d", len(index.Manifests)) } manifestRelativePath := filepath.Join(config.BlobsDir, bundleManifestDesc.Digest.Encoded()) if err := av3.Extract(tp.src, manifestRelativePath, secureTempDir); err != nil { - return fmt.Errorf("failed to extract %s from %s: %w", bundleManifestDesc.Digest.Encoded(), tp.src, err) + return nil, fmt.Errorf("failed to extract %s from %s: %w", bundleManifestDesc.Digest.Encoded(), tp.src, err) } manifestPath := filepath.Join(secureTempDir, manifestRelativePath) @@ -160,32 +157,32 @@ func (tp *tarballBundleProvider) getBundleManifest() error { defer os.Remove(manifestPath) if err := zarfUtils.SHAsMatch(manifestPath, bundleManifestDesc.Digest.Encoded()); err != nil { - return err + return nil, err } b, err = os.ReadFile(manifestPath) if err != nil { - return err + return nil, err } var manifest *oci.ZarfOCIManifest if err := json.Unmarshal(b, &manifest); err != nil { - return err + return nil, err } - tp.bundleRootManifest = manifest - return nil + return manifest, nil } // LoadBundle loads a bundle from a tarball -func (tp *tarballBundleProvider) LoadBundle(_ int) (types.PathMap, error) { - return nil, fmt.Errorf("uds pull does not support pulling local bundles") +func (tp *tarballBundleProvider) LoadBundle(_ types.BundlePullOptions, _ int) (*types.UDSBundle, types.PathMap, error) { + return nil, nil, fmt.Errorf("uds pull does not support pulling local bundles") } // LoadBundleMetadata loads a bundle's metadata from a tarball func (tp *tarballBundleProvider) LoadBundleMetadata() (types.PathMap, error) { - if err := tp.getBundleManifest(); err != nil { + bundleRootManifest, err := tp.getBundleManifest() + if err != nil { return nil, err } pathsToExtract := config.BundleAlwaysPull @@ -193,7 +190,7 @@ func (tp *tarballBundleProvider) LoadBundleMetadata() (types.PathMap, error) { loaded := make(types.PathMap) for _, path := range pathsToExtract { - layer := tp.bundleRootManifest.Locate(path) + layer := bundleRootManifest.Locate(path) if !oci.IsEmptyDescriptor(layer) { pathInTarball := filepath.Join(config.BlobsDir, layer.Digest.Encoded()) abs := filepath.Join(tp.dst, pathInTarball) @@ -241,7 +238,8 @@ func (tp *tarballBundleProvider) getZarfLayers(store *ocistore.Store, pkgManifes // PublishBundle publishes a local bundle to a remote OCI registry func (tp *tarballBundleProvider) PublishBundle(bundle types.UDSBundle, remote *oci.OrasRemote) error { var layersToPush []ocispec.Descriptor - if err := tp.getBundleManifest(); err != nil { + bundleRootManifest, err := tp.getBundleManifest() + if err != nil { return err } estimatedBytes := int64(0) @@ -252,7 +250,7 @@ func (tp *tarballBundleProvider) PublishBundle(bundle types.UDSBundle, remote *o return err } // push bundle layers to remote - for _, manifestDesc := range tp.bundleRootManifest.Layers { + for _, manifestDesc := range bundleRootManifest.Layers { layersToPush = append(layersToPush, manifestDesc) if manifestDesc.Annotations[ocispec.AnnotationTitle] == config.BundleYAML { continue // uds-bundle.yaml doesn't have layers @@ -266,7 +264,7 @@ func (tp *tarballBundleProvider) PublishBundle(bundle types.UDSBundle, remote *o } // grab image config - layersToPush = append(layersToPush, tp.bundleRootManifest.Config) + layersToPush = append(layersToPush, bundleRootManifest.Config) // copy bundle copyOpts := utils.CreateCopyOpts(layersToPush, config.CommonOptions.OCIConcurrency) @@ -301,12 +299,13 @@ func (tp *tarballBundleProvider) PublishBundle(bundle types.UDSBundle, remote *o // ZarfPackageNameMap gets zarf package name mappings from tarball provider func (tp *tarballBundleProvider) ZarfPackageNameMap() (map[string]string, error) { - if err := tp.getBundleManifest(); err != nil { + bundleRootManifest, err := tp.getBundleManifest() + if err != nil { return nil, err } nameMap := make(map[string]string) - for _, layer := range tp.bundleRootManifest.Layers { + for _, layer := range bundleRootManifest.Layers { if layer.MediaType == oci.ZarfLayerMediaTypeBlob { // only the uds bundle layer will have AnnotationTitle set if layer.Annotations[ocispec.AnnotationTitle] != config.BundleYAML { diff --git a/src/pkg/cache/cache.go b/src/pkg/cache/cache.go index 8e0520ecd..bfc9bb0ff 100644 --- a/src/pkg/cache/cache.go +++ b/src/pkg/cache/cache.go @@ -30,7 +30,7 @@ func expandTilde(cachePath string) string { func Add(filePathToAdd string) error { // ensure cache dir exists cacheDir := config.CommonOptions.CachePath - if err := os.MkdirAll(filepath.Join(cacheDir, "images"), 0755); err != nil { + if err := os.MkdirAll(filepath.Join(cacheDir, config.UDSCacheLayers), 0755); err != nil { return err } @@ -46,7 +46,7 @@ func Add(filePathToAdd string) error { } defer srcFile.Close() - dstFile, err := os.Create(filepath.Join(cacheDir, "images", filename)) + dstFile, err := os.Create(filepath.Join(cacheDir, config.UDSCacheLayers, filename)) if err != nil { return err } @@ -58,7 +58,7 @@ func Add(filePathToAdd string) error { // Exists checks if a layer exists in the cache func Exists(layerDigest string) bool { cacheDir := config.CommonOptions.CachePath - layerCachePath := filepath.Join(expandTilde(cacheDir), "images", layerDigest) + layerCachePath := filepath.Join(expandTilde(cacheDir), config.UDSCacheLayers, layerDigest) _, err := os.Stat(layerCachePath) return !os.IsNotExist(err) } @@ -66,7 +66,7 @@ func Exists(layerDigest string) bool { // Use copies a layer from the cache to the dst dir func Use(layerDigest, dstDir string) error { cacheDir := config.CommonOptions.CachePath - layerCachePath := filepath.Join(expandTilde(cacheDir), "images", layerDigest) + layerCachePath := filepath.Join(expandTilde(cacheDir), config.UDSCacheLayers, layerDigest) srcFile, err := os.Open(layerCachePath) if err != nil { return err diff --git a/src/test/e2e/bundle_test.go b/src/test/e2e/bundle_test.go index 766484ca2..bfa65f99c 100644 --- a/src/test/e2e/bundle_test.go +++ b/src/test/e2e/bundle_test.go @@ -21,7 +21,7 @@ import ( ) func zarfPublish(t *testing.T, path string, reg string) { - args := strings.Split(fmt.Sprintf("zarf package publish %s oci://%s --insecure --oci-concurrency=10", path, reg), " ") + args := strings.Split(fmt.Sprintf("zarf package publish %s oci://%s --insecure --oci-concurrency=10 -l debug", path, reg), " ") _, _, err := e2e.UDS(args...) require.NoError(t, err) }