diff --git a/go.mod b/go.mod index e9228eb9a0d..ea480cc6c0b 100644 --- a/go.mod +++ b/go.mod @@ -46,7 +46,7 @@ require ( github.com/kastenhq/goversion v0.0.0-20230811215019-93b2f8823953 github.com/knqyf263/go-rpmdb v0.0.0-20230301153543-ba94b245509b github.com/mholt/archiver/v3 v3.5.1 - github.com/microsoft/go-rustaudit v0.0.0-20220730194248-4b17361d90a5 + github.com/microsoft/go-rustaudit v0.0.0-20220808201409-204dfee52032 github.com/mitchellh/go-homedir v1.1.0 github.com/mitchellh/hashstructure/v2 v2.0.2 github.com/mitchellh/mapstructure v1.5.0 @@ -76,6 +76,11 @@ require ( modernc.org/sqlite v1.27.0 ) +require ( + github.com/samber/lo v1.38.1 + golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 +) + require ( dario.cat/mergo v1.0.0 // indirect github.com/AdaLogics/go-fuzz-headers v0.0.0-20230811130428-ced1acdcaa24 // indirect @@ -215,7 +220,6 @@ require ( golang.org/x/term v0.14.0 // indirect golang.org/x/text v0.14.0 // indirect golang.org/x/tools v0.13.0 // indirect - golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect google.golang.org/genproto v0.0.0-20230913181813-007df8e322eb // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20230920204549-e6e6cdab5c13 // indirect google.golang.org/grpc v1.58.3 // indirect diff --git a/go.sum b/go.sum index 4a6c6575a69..d3443abb7cd 100644 --- a/go.sum +++ b/go.sum @@ -549,8 +549,8 @@ github.com/mgutz/ansi v0.0.0-20200706080929-d51e80ef957d h1:5PJl274Y63IEHC+7izoQ github.com/mgutz/ansi v0.0.0-20200706080929-d51e80ef957d/go.mod h1:01TrycV0kFyexm33Z7vhZRXopbI8J3TDReVlkTgMUxE= github.com/mholt/archiver/v3 v3.5.1 h1:rDjOBX9JSF5BvoJGvjqK479aL70qh9DIpZCl+k7Clwo= github.com/mholt/archiver/v3 v3.5.1/go.mod h1:e3dqJ7H78uzsRSEACH1joayhuSyhnonssnDhppzS1L4= -github.com/microsoft/go-rustaudit v0.0.0-20220730194248-4b17361d90a5 h1:tQRHcLQwnwrPq2j2Qra/NnyjyESBGwdeBeVdAE9kXYg= -github.com/microsoft/go-rustaudit v0.0.0-20220730194248-4b17361d90a5/go.mod h1:vYT9HE7WCvL64iVeZylKmCsWKfE+JZ8105iuh2Trk8g= +github.com/microsoft/go-rustaudit v0.0.0-20220808201409-204dfee52032 h1:TLygBUBxikNJJfLwgm+Qwdgq1FtfV8Uh7bcxRyTzK8s= +github.com/microsoft/go-rustaudit v0.0.0-20220808201409-204dfee52032/go.mod h1:vYT9HE7WCvL64iVeZylKmCsWKfE+JZ8105iuh2Trk8g= github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= github.com/miekg/dns v1.1.26/go.mod h1:bPDLeHnStXmXAq1m/Ch/hvfNHr14JKNPMBo3VZKjuso= github.com/miekg/dns v1.1.41/go.mod h1:p6aan82bvRIyn+zDIv9xYNUpwa73JcSh9BKwknJysuI= @@ -683,6 +683,8 @@ github.com/sagikazarmark/slog-shim v0.1.0 h1:diDBnUNK9N/354PgrxMywXnAwEr1QZcOr6g github.com/sagikazarmark/slog-shim v0.1.0/go.mod h1:SrcSrq8aKtyuqEI1uvTDTK1arOWRIczQRv+GVI1AkeQ= github.com/saintfish/chardet v0.0.0-20230101081208-5e3ef4b5456d h1:hrujxIzL1woJ7AwssoOcM/tq5JjjG2yYOc8odClEiXA= github.com/saintfish/chardet v0.0.0-20230101081208-5e3ef4b5456d/go.mod h1:uugorj2VCxiV1x+LzaIdVa9b4S4qGAcH6cbhh4qVxOU= +github.com/samber/lo v1.38.1 h1:j2XEAqXKb09Am4ebOg31SpvzUTTs6EN3VfgeLUhPdXM= +github.com/samber/lo v1.38.1/go.mod h1:+m/ZKRl6ClXCE2Lgf3MsQlWfh4bn1bz6CXEOxnEXnEA= github.com/sanity-io/litter v1.5.5 h1:iE+sBxPBzoK6uaEP5Lt3fHNgpKcHXc/A2HGETy0uJQo= github.com/sanity-io/litter v1.5.5/go.mod h1:9gzJgR2i4ZpjZHsKvUXIRQVk7P+yM3e+jAF7bU2UI5U= github.com/sassoftware/go-rpmutils v0.2.0 h1:pKW0HDYMFWQ5b4JQPiI3WI12hGsVoW0V8+GMoZiI/JE= diff --git a/syft/pkg/cataloger/cataloger.go b/syft/pkg/cataloger/cataloger.go index a66a0569eb6..bac606d1c07 100644 --- a/syft/pkg/cataloger/cataloger.go +++ b/syft/pkg/cataloger/cataloger.go @@ -86,6 +86,7 @@ func DirectoryCatalogers(cfg Config) []pkg.Cataloger { java.NewPomCataloger(), java.NewNativeImageCataloger(), javascript.NewLockCataloger(), + javascript.NewJavaScriptCataloger(), nix.NewStoreCataloger(), php.NewComposerLockCataloger(), gentoo.NewPortageCataloger(), @@ -125,6 +126,7 @@ func AllCatalogers(cfg Config) []pkg.Cataloger { java.NewPomCataloger(), java.NewNativeImageCataloger(), javascript.NewLockCataloger(), + javascript.NewJavaScriptCataloger(), javascript.NewPackageCataloger(), kernel.NewLinuxKernelCataloger(cfg.LinuxKernel), nix.NewStoreCataloger(), diff --git a/syft/pkg/cataloger/common/cpe/generate.go b/syft/pkg/cataloger/common/cpe/generate.go index 2077d7e3b88..1215ea3f98b 100644 --- a/syft/pkg/cataloger/common/cpe/generate.go +++ b/syft/pkg/cataloger/common/cpe/generate.go @@ -180,7 +180,7 @@ func candidateVendors(p pkg.Package) []string { case pkg.ApkDBEntry: vendors.union(candidateVendorsForAPK(p)) case pkg.NpmPackage: - vendors.union(candidateVendorsForJavascript(p)) + vendors.union(candidateVendorsForJavaScript(p)) } // We should no longer be generating vendor candidates with these values ["" and "*"] diff --git a/syft/pkg/cataloger/common/cpe/javascript.go b/syft/pkg/cataloger/common/cpe/javascript.go index 881bc658577..5465cf50fb9 100644 --- a/syft/pkg/cataloger/common/cpe/javascript.go +++ b/syft/pkg/cataloger/common/cpe/javascript.go @@ -2,7 +2,7 @@ package cpe import "github.com/anchore/syft/syft/pkg" -func candidateVendorsForJavascript(p pkg.Package) fieldCandidateSet { +func candidateVendorsForJavaScript(p pkg.Package) fieldCandidateSet { if _, ok := p.Metadata.(pkg.NpmPackage); !ok { return nil } diff --git a/syft/pkg/cataloger/generic/cataloger.go b/syft/pkg/cataloger/generic/cataloger.go index 41faaf2856a..e1051645de3 100644 --- a/syft/pkg/cataloger/generic/cataloger.go +++ b/syft/pkg/cataloger/generic/cataloger.go @@ -1,6 +1,10 @@ package generic import ( + "path/filepath" + + "github.com/bmatcuk/doublestar/v4" + "github.com/anchore/syft/internal" "github.com/anchore/syft/internal/log" "github.com/anchore/syft/syft/artifact" @@ -10,12 +14,19 @@ import ( ) type processor func(resolver file.Resolver, env Environment) []request +type groupedProcessor func(resolver file.Resolver, env Environment) []groupedRequest type request struct { file.Location Parser } +type groupedRequest struct { + Locations []file.Location + PrimaryFileLocation file.Location + GroupedParser +} + // Cataloger implements the Catalog interface and is responsible for dispatching the proper parser function for // a given path or glob pattern. This is intended to be reusable across many package cataloger types. type Cataloger struct { @@ -23,6 +34,92 @@ type Cataloger struct { upstreamCataloger string } +// GroupedCataloger is a special case of Cataloger that will process files together +// this is needed for the case of package.json and package-lock.json files for example +type GroupedCataloger struct { + groupedProcessor []groupedProcessor + upstreamCataloger string +} + +func (c *GroupedCataloger) Name() string { + return c.upstreamCataloger +} + +func isPrimaryFileGlobPresent(primaryFileGlob string, globs []string) bool { + for _, g := range globs { + if g == primaryFileGlob { + return true + } + } + return false +} + +func generateGroupedProcessor(parser GroupedParser, primaryFileGlob string, globs []string) func(resolver file.Resolver, env Environment) []groupedRequest { + return func(resolver file.Resolver, env Environment) []groupedRequest { + var requests []groupedRequest + colocatedFiles := collectColocatedFiles(resolver, globs) + + // Filter to only directories that contain all specified files + for _, files := range colocatedFiles { + allMatched, primaryFileLocation := isAllGlobsMatched(files, globs, primaryFileGlob) + if allMatched { + requests = append(requests, makeGroupedRequests(parser, files, primaryFileLocation)) + } + } + + return requests + } +} + +func collectColocatedFiles(resolver file.Resolver, globs []string) map[string][]file.Location { + colocatedFiles := make(map[string][]file.Location) + for _, g := range globs { + log.WithFields("glob", g).Trace("searching for paths matching glob") + matches, err := resolver.FilesByGlob(g) + if err != nil { + log.Warnf("unable to process glob=%q: %+v", g, err) + continue + } + for _, match := range matches { + dir := filepath.Dir(match.RealPath) + colocatedFiles[dir] = append(colocatedFiles[dir], match) + } + } + return colocatedFiles +} + +func isAllGlobsMatched(files []file.Location, globs []string, primaryFileGlob string) (bool, file.Location) { + globMatches := make(map[string]bool) + var primaryFileLocation file.Location + + for _, g := range globs { + for _, file := range files { + if matched, _ := doublestar.PathMatch(g, file.RealPath); matched { + if g == primaryFileGlob { + primaryFileLocation = file + } + globMatches[g] = true + break + } + } + } + + return len(globMatches) == len(globs), primaryFileLocation +} + +// WithParserByGlobColocation is a special case of WithParserByGlob that will only match files that are colocated +// with all of the provided globs. This is useful for cases where a package is defined by multiple files (e.g. package.json + package-lock.json). +// This function will only match files that are colocated with all of the provided globs. +func (c *GroupedCataloger) WithParserByGlobColocation(parser GroupedParser, primaryFileGlob string, globs []string) *GroupedCataloger { + if !isPrimaryFileGlobPresent(primaryFileGlob, globs) { + log.Warnf("primary file glob=%q not present in globs=%+v", primaryFileGlob, globs) + return c + } + + c.groupedProcessor = append(c.groupedProcessor, generateGroupedProcessor(parser, primaryFileGlob, globs)) + return c +} + func (c *Cataloger) WithParserByGlobs(parser Parser, globs ...string) *Cataloger { c.processor = append(c.processor, func(resolver file.Resolver, env Environment) []request { @@ -43,6 +140,69 @@ func (c *Cataloger) WithParserByGlobs(parser Parser, globs ...string) *Cataloger return c } +// selectFiles takes a set of file trees and resolves and file references of interest for future cataloging +func (c *GroupedCataloger) selectFiles(resolver file.Resolver) []groupedRequest { + var requests []groupedRequest + for _, proc := range c.groupedProcessor { + requests = append(requests, proc(resolver, Environment{})...) + } + return requests +} + +// Catalog is given an object to resolve file references and content, this function returns any discovered Packages after analyzing the catalog source. +func (c *GroupedCataloger) Catalog(resolver file.Resolver) ([]pkg.Package, []artifact.Relationship, error) { + var packages []pkg.Package + var relationships []artifact.Relationship + + logger := log.Nested("cataloger", c.upstreamCataloger) + + env := Environment{ + // TODO: consider passing into the cataloger, this would affect the cataloger interface (and all implementations). This can be deferred until later. + LinuxRelease: linux.IdentifyRelease(resolver), + } + + for _, req := range c.selectFiles(resolver) { + parser := req.GroupedParser + var readClosers []file.LocationReadCloser + + for _, location := range req.Locations { + log.WithFields("path", location.RealPath).Trace("parsing file contents") + contentReader, err := resolver.FileContentsByLocation(location) + if err != nil { + logger.WithFields("location", location.RealPath, "error", err).Warn("unable to fetch contents") + continue + } + readClosers = append(readClosers, file.NewLocationReadCloser(location, contentReader)) + } + + // If your parser is expecting multiple file contents, ensure its signature reflects this change + discoveredPackages, discoveredRelationships, err := parser(resolver, &env, readClosers) + for _, rc := range readClosers { + internal.CloseAndLogError(rc, rc.Path()) + } + if err != nil { + logger.WithFields("error", err).Warnf("cataloger failed") + continue + } + + for _, p := range discoveredPackages { + p.FoundBy = c.upstreamCataloger + packages = append(packages, p) + } + + relationships = append(relationships, discoveredRelationships...) + } + return packages, relationships, nil +} + +func makeGroupedRequests(parser GroupedParser, locations []file.Location, primaryFileLocation file.Location) groupedRequest { + return groupedRequest{ + Locations: locations, + PrimaryFileLocation: primaryFileLocation, + GroupedParser: parser, + } +} + func (c *Cataloger) WithParserByMimeTypes(parser Parser, types ...string) *Cataloger { c.processor = append(c.processor, func(resolver file.Resolver, env Environment) []request { @@ -98,6 +258,12 @@ func NewCataloger(upstreamCataloger string) *Cataloger { } } +func NewGroupedCataloger(upstreamCataloger string) *GroupedCataloger { + return &GroupedCataloger{ + upstreamCataloger: upstreamCataloger, + } +} + // Name returns a string that uniquely describes the upstream cataloger that this Generic Cataloger represents. func (c *Cataloger) Name() string { return c.upstreamCataloger @@ -125,7 +291,6 @@ func (c *Cataloger) Catalog(resolver file.Resolver) ([]pkg.Package, []artifact.R logger.WithFields("location", location.RealPath, "error", err).Warn("unable to fetch contents") continue } - discoveredPackages, discoveredRelationships, err := parser(resolver, &env, file.NewLocationReadCloser(location, contentReader)) internal.CloseAndLogError(contentReader, location.AccessPath) if err != nil { diff --git a/syft/pkg/cataloger/generic/cataloger_test.go b/syft/pkg/cataloger/generic/cataloger_test.go index c99a91ff029..dd1d86834f7 100644 --- a/syft/pkg/cataloger/generic/cataloger_test.go +++ b/syft/pkg/cataloger/generic/cataloger_test.go @@ -13,6 +13,41 @@ import ( "github.com/anchore/syft/syft/pkg" ) +func Test_WithParserByGlobColocation(t *testing.T) { + matchedFilesPaths := make(map[string]bool) + parser := func(resolver file.Resolver, env *Environment, readers []file.LocationReadCloser) ([]pkg.Package, []artifact.Relationship, error) { + var packages []pkg.Package + var relationships []artifact.Relationship + + for _, reader := range readers { + matchedFilesPaths[reader.Path()] = true + } + return packages, relationships, nil + } + + upstream := "colocation-cataloger" + expectedCollocatedPaths := []string{ + "test-fixtures/pkg-json/package.json", + "test-fixtures/pkg-json/package-lock.json", + } + + resolver := file.NewMockResolverForPaths(expectedCollocatedPaths...) + + cataloger := NewGroupedCataloger(upstream). + WithParserByGlobColocation(parser, "**/package-lock.json", []string{"**/package.json", "**/package-lock.json"}) + + _, _, err := cataloger.Catalog(resolver) + assert.NoError(t, err) + + for path := range matchedFilesPaths { + t.Logf("Matched file path: %s", path) // Log each matched file + } + + // Assert that the expected files were matched + require.True(t, matchedFilesPaths["test-fixtures/pkg-json/package.json"]) + require.True(t, matchedFilesPaths["test-fixtures/pkg-json/package-lock.json"]) +} + func Test_Cataloger(t *testing.T) { allParsedPaths := make(map[string]bool) parser := func(resolver file.Resolver, env *Environment, reader file.LocationReadCloser) ([]pkg.Package, []artifact.Relationship, error) { diff --git a/syft/pkg/cataloger/generic/parser.go b/syft/pkg/cataloger/generic/parser.go index c95808fc175..47051dc19ec 100644 --- a/syft/pkg/cataloger/generic/parser.go +++ b/syft/pkg/cataloger/generic/parser.go @@ -12,3 +12,4 @@ type Environment struct { } type Parser func(file.Resolver, *Environment, file.LocationReadCloser) ([]pkg.Package, []artifact.Relationship, error) +type GroupedParser func(file.Resolver, *Environment, []file.LocationReadCloser) ([]pkg.Package, []artifact.Relationship, error) diff --git a/syft/pkg/cataloger/generic/test-fixtures/pkg-json/package-lock.json b/syft/pkg/cataloger/generic/test-fixtures/pkg-json/package-lock.json new file mode 100644 index 00000000000..e69de29bb2d diff --git a/syft/pkg/cataloger/generic/test-fixtures/pkg-json/package.json b/syft/pkg/cataloger/generic/test-fixtures/pkg-json/package.json new file mode 100644 index 00000000000..e69de29bb2d diff --git a/syft/pkg/cataloger/internal/pkgtest/test_generic_parser.go b/syft/pkg/cataloger/internal/pkgtest/test_generic_parser.go index f39523706f3..5729953c957 100644 --- a/syft/pkg/cataloger/internal/pkgtest/test_generic_parser.go +++ b/syft/pkg/cataloger/internal/pkgtest/test_generic_parser.go @@ -225,6 +225,34 @@ func (p *CatalogTester) TestParser(t *testing.T, parser generic.Parser) { p.assertPkgs(t, pkgs, relationships) } +func (p *CatalogTester) TestGroupedCataloger(t *testing.T, cataloger pkg.Cataloger) { + t.Helper() + + resolver := NewObservingResolver(p.resolver) + + pkgs, relationships, err := cataloger.Catalog(resolver) + + // this is a minimum set, the resolver may return more than just this list + for _, path := range p.expectedPathResponses { + assert.Truef(t, resolver.ObservedPathResponses(path), "expected path query for %q was not observed", path) + } + + // this is a full set, any other queries are unexpected (and will fail the test) + if len(p.expectedContentQueries) > 0 { + assert.ElementsMatchf(t, p.expectedContentQueries, resolver.AllContentQueries(), "unexpected content queries observed: diff %s", cmp.Diff(p.expectedContentQueries, resolver.AllContentQueries())) + } + + if p.assertResultExpectations { + p.wantErr(t, err) + p.assertPkgs(t, pkgs, relationships) + } else { + resolver.PruneUnfulfilledPathResponses(p.ignoreUnfulfilledPathResponses, p.ignoreAnyUnfulfilledPaths...) + + // if we aren't testing the results, we should focus on what was searched for (for glob-centric tests) + assert.Falsef(t, resolver.HasUnfulfilledPathRequests(), "unfulfilled path requests: \n%v", resolver.PrettyUnfulfilledPathRequests()) + } +} + func (p *CatalogTester) TestCataloger(t *testing.T, cataloger pkg.Cataloger) { t.Helper() @@ -276,6 +304,7 @@ func (p *CatalogTester) assertPkgs(t *testing.T, pkgs []pkg.Package, relationshi p.compareOptions = append(p.compareOptions, cmpopts.IgnoreFields(pkg.Package{}, "id"), // note: ID is not deterministic for test purposes + cmpopts.IgnoreFields(pkg.Package{}, "FoundBy"), cmpopts.SortSlices(pkg.Less), cmpopts.SortSlices(relationshipLess), cmp.Comparer( diff --git a/syft/pkg/cataloger/javascript/cataloger.go b/syft/pkg/cataloger/javascript/cataloger.go index 56127e373cd..97ee921f36f 100644 --- a/syft/pkg/cataloger/javascript/cataloger.go +++ b/syft/pkg/cataloger/javascript/cataloger.go @@ -20,3 +20,13 @@ func NewLockCataloger() *generic.Cataloger { WithParserByGlobs(parseYarnLock, "**/yarn.lock"). WithParserByGlobs(parsePnpmLock, "**/pnpm-lock.yaml") } + +// NewJavaScriptCataloger returns a new JavaScript cataloger object based on detection +// of npm based packages and lock files to provide a complete dependency graph of the +// packages. +func NewJavaScriptCataloger() *generic.GroupedCataloger { + return generic.NewGroupedCataloger("javascript-lock-cataloger"). + WithParserByGlobColocation(parseJavaScript, "**/yarn.lock", []string{"**/package.json", "**/yarn.lock"}). + WithParserByGlobColocation(parseJavaScript, "**/package-lock.json", []string{"**/package.json", "**/package-lock.json"}). + WithParserByGlobColocation(parseJavaScript, "**/pnpm-lock.yaml", []string{"**/package.json", "**/pnpm-lock.yaml"}) +} diff --git a/syft/pkg/cataloger/javascript/cataloger_test.go b/syft/pkg/cataloger/javascript/cataloger_test.go index c4db1754596..25739f155ed 100644 --- a/syft/pkg/cataloger/javascript/cataloger_test.go +++ b/syft/pkg/cataloger/javascript/cataloger_test.go @@ -3,137 +3,709 @@ package javascript import ( "testing" + "github.com/anchore/syft/syft/artifact" "github.com/anchore/syft/syft/file" "github.com/anchore/syft/syft/pkg" "github.com/anchore/syft/syft/pkg/cataloger/internal/pkgtest" ) -func Test_JavascriptCataloger(t *testing.T) { - locationSet := file.NewLocationSet(file.NewLocation("package-lock.json")) - expectedPkgs := []pkg.Package{ - { - Name: "@actions/core", - Version: "1.6.0", - FoundBy: "javascript-lock-cataloger", - PURL: "pkg:npm/%40actions/core@1.6.0", - Locations: locationSet, - Language: pkg.JavaScript, - Type: pkg.NpmPkg, - Licenses: pkg.NewLicenseSet( - pkg.NewLicenseFromLocations("MIT", file.NewLocation("package-lock.json")), - ), - Metadata: pkg.NpmPackageLockEntry{Resolved: "https://registry.npmjs.org/@actions/core/-/core-1.6.0.tgz", Integrity: "sha512-NB1UAZomZlCV/LmJqkLhNTqtKfFXJZAUPcfl/zqG7EfsQdeUJtaWO98SGbuQ3pydJ3fHl2CvI/51OKYlCYYcaw=="}, - }, - { - Name: "ansi-regex", - Version: "3.0.0", - FoundBy: "javascript-lock-cataloger", - PURL: "pkg:npm/ansi-regex@3.0.0", - Locations: locationSet, - Language: pkg.JavaScript, - Type: pkg.NpmPkg, - Metadata: pkg.NpmPackageLockEntry{Resolved: "https://registry.npmjs.org/ansi-regex/-/ansi-regex-3.0.0.tgz", Integrity: "sha1-7QMXwyIGT3lGbAKWa922Bas32Zg="}, - }, - { - Name: "cowsay", - Version: "1.4.0", - FoundBy: "javascript-lock-cataloger", - PURL: "pkg:npm/cowsay@1.4.0", - Locations: locationSet, - Language: pkg.JavaScript, - Type: pkg.NpmPkg, - Licenses: pkg.NewLicenseSet( - pkg.NewLicenseFromLocations("MIT", file.NewLocation("package-lock.json")), - ), - Metadata: pkg.NpmPackageLockEntry{Resolved: "https://registry.npmjs.org/cowsay/-/cowsay-1.4.0.tgz", Integrity: "sha512-rdg5k5PsHFVJheO/pmE3aDg2rUDDTfPJau6yYkZYlHFktUz+UxbE+IgnUAEyyCyv4noL5ltxXD0gZzmHPCy/9g=="}, - }, - { - Name: "get-stdin", - Version: "5.0.1", - FoundBy: "javascript-lock-cataloger", - PURL: "pkg:npm/get-stdin@5.0.1", - Locations: locationSet, - Language: pkg.JavaScript, - Type: pkg.NpmPkg, - Metadata: pkg.NpmPackageLockEntry{Resolved: "https://registry.npmjs.org/get-stdin/-/get-stdin-5.0.1.tgz", Integrity: "sha1-Ei4WFZHiH/TFJTAwVpPyDmOTo5g="}, - }, - { - Name: "is-fullwidth-code-point", - Version: "2.0.0", - FoundBy: "javascript-lock-cataloger", - PURL: "pkg:npm/is-fullwidth-code-point@2.0.0", - Locations: locationSet, - Language: pkg.JavaScript, - Type: pkg.NpmPkg, - Metadata: pkg.NpmPackageLockEntry{Resolved: "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-2.0.0.tgz", Integrity: "sha1-o7MKXE8ZkYMWeqq5O+764937ZU8="}, - }, - { - Name: "minimist", - Version: "0.0.10", - FoundBy: "javascript-lock-cataloger", - PURL: "pkg:npm/minimist@0.0.10", - Locations: locationSet, - Language: pkg.JavaScript, - Type: pkg.NpmPkg, - Metadata: pkg.NpmPackageLockEntry{Resolved: "https://registry.npmjs.org/minimist/-/minimist-0.0.10.tgz", Integrity: "sha1-3j+YVD2/lggr5IrRoMfNqDYwHc8="}, - }, - { - Name: "optimist", - Version: "0.6.1", - FoundBy: "javascript-lock-cataloger", - PURL: "pkg:npm/optimist@0.6.1", - Locations: locationSet, - Language: pkg.JavaScript, - Type: pkg.NpmPkg, - Metadata: pkg.NpmPackageLockEntry{Resolved: "https://registry.npmjs.org/optimist/-/optimist-0.6.1.tgz", Integrity: "sha1-2j6nRob6IaGaERwybpDrFaAZZoY="}, - }, - { - Name: "string-width", - Version: "2.1.1", - FoundBy: "javascript-lock-cataloger", - PURL: "pkg:npm/string-width@2.1.1", - Locations: locationSet, - Language: pkg.JavaScript, - Type: pkg.NpmPkg, - Metadata: pkg.NpmPackageLockEntry{Resolved: "https://registry.npmjs.org/string-width/-/string-width-2.1.1.tgz", Integrity: "sha512-nOqH59deCq9SRHlxq1Aw85Jnt4w6KvLKqWVik6oA9ZklXLNIOlqg4F2yrT1MVaTjAqvVwdfeZ7w7aCvJD7ugkw=="}, - }, - { - Name: "strip-ansi", - Version: "4.0.0", - FoundBy: "javascript-lock-cataloger", - PURL: "pkg:npm/strip-ansi@4.0.0", - Locations: locationSet, - Language: pkg.JavaScript, - Type: pkg.NpmPkg, - Metadata: pkg.NpmPackageLockEntry{Resolved: "https://registry.npmjs.org/strip-ansi/-/strip-ansi-4.0.0.tgz", Integrity: "sha1-qEeQIusaw2iocTibY1JixQXuNo8="}, - }, - { - Name: "strip-eof", - Version: "1.0.0", - FoundBy: "javascript-lock-cataloger", - PURL: "pkg:npm/strip-eof@1.0.0", - Locations: locationSet, - Language: pkg.JavaScript, - Type: pkg.NpmPkg, - Metadata: pkg.NpmPackageLockEntry{Resolved: "https://registry.npmjs.org/strip-eof/-/strip-eof-1.0.0.tgz", Integrity: "sha1-u0P/VZim6wXYm1n80SnJgzE2Br8="}, - }, - { - Name: "wordwrap", - Version: "0.0.3", - FoundBy: "javascript-lock-cataloger", - PURL: "pkg:npm/wordwrap@0.0.3", - Locations: locationSet, - Language: pkg.JavaScript, - Type: pkg.NpmPkg, - Metadata: pkg.NpmPackageLockEntry{Resolved: "https://registry.npmjs.org/wordwrap/-/wordwrap-0.0.3.tgz", Integrity: "sha1-o9XabNXAvAAI03I0u68b7WMFkQc="}, +func expectedPackagesAndRelationshipsLockV1(locationSet file.LocationSet, metadata bool) ([]pkg.Package, []artifact.Relationship) { + metadataMap := map[string]pkg.NpmPackageLockEntry{ + "rxjs": { + Resolved: "https://registry.npmjs.org/rxjs/-/rxjs-7.5.7.tgz", + Integrity: "sha512-z9MzKh/UcOqB3i20H6rtrlaE/CgjLOvheWK/9ILrbhROGTweAi1BaFsTT9FbwZi5Trr1qNRs+MXkhmR06awzQA==", + }, + "test-app": { + Resolved: "", + Integrity: "", + }, + "typescript": { + Resolved: "https://registry.npmjs.org/typescript/-/typescript-4.7.4.tgz", + Integrity: "sha512-C0WQT0gezHuw6AdY1M2jxUO83Rjf0HP7Sk1DtXj6j1EwkQNZrHAg2XPWlq62oqEhYvONq5pkC2Y9oPljWToLmQ==", + }, + "tslib": { + Resolved: "https://registry.npmjs.org/tslib/-/tslib-2.6.2.tgz", + Integrity: "sha512-AEYxH93jGFPn/a2iVAwW87VuUIkR1FVUKB77NwMF7nBTDkDrrT/Hpt/IrCJ0QXhW27jTBDcf5ZY7w6RiqTMw2Q==", + }, + "zone.js": { + Resolved: "https://registry.npmjs.org/zone.js/-/zone.js-0.11.8.tgz", + Integrity: "sha512-82bctBg2hKcEJ21humWIkXRlLBBmrc3nN7DFh5LGGhcyycO2S7FN8NmdvlcKaGFDNVL4/9kFLmwmInTavdJERA==", + }, + } + rxjs := pkg.Package{ + Name: "rxjs", + Version: "7.5.7", + FoundBy: "javascript-cataloger", + PURL: "pkg:npm/rxjs@7.5.7", + Locations: locationSet, + Language: pkg.JavaScript, + Type: pkg.NpmPkg, + Metadata: pkg.NpmPackageLockEntry{}, + } + testApp := pkg.Package{ + Name: "test-app", + Version: "0.0.0", + FoundBy: "javascript-cataloger", + PURL: "pkg:npm/test-app@0.0.0", + Locations: locationSet, + Language: pkg.JavaScript, + Type: pkg.NpmPkg, + Metadata: pkg.NpmPackageLockEntry{}, + } + tslib := pkg.Package{ + Name: "tslib", + Version: "2.6.2", + FoundBy: "javascript-cataloger", + PURL: "pkg:npm/tslib@2.6.2", + Locations: locationSet, + Language: pkg.JavaScript, + Type: pkg.NpmPkg, + Metadata: pkg.NpmPackageLockEntry{}, + } + typescript := pkg.Package{ + Name: "typescript", + Version: "4.7.4", + FoundBy: "javascript-cataloger", + PURL: "pkg:npm/typescript@4.7.4", + Locations: locationSet, + Language: pkg.JavaScript, + Type: pkg.NpmPkg, + Metadata: pkg.NpmPackageLockEntry{}, + } + zonejs := pkg.Package{ + Name: "zone.js", + Version: "0.11.8", + FoundBy: "javascript-cataloger", + PURL: "pkg:npm/zone.js@0.11.8", + Locations: locationSet, + Language: pkg.JavaScript, + Type: pkg.NpmPkg, + Metadata: pkg.NpmPackageLockEntry{}, + } + + l := []*pkg.Package{ + &rxjs, + &testApp, + &tslib, + &typescript, + &zonejs, + } + + var expectedPkgs []pkg.Package + for i := range l { + if metadata { + l[i].Metadata = metadataMap[l[i].Name] + expectedPkgs = append(expectedPkgs, *l[i]) + } else { + expectedPkgs = append(expectedPkgs, *l[i]) + } + } + + expectedRelationships := []artifact.Relationship{ + { + From: rxjs, + To: testApp, + Type: artifact.DependencyOfRelationship, + Data: nil, + }, + { + From: tslib, + To: testApp, + Type: artifact.DependencyOfRelationship, + Data: nil, + }, + { + From: typescript, + To: testApp, + Type: artifact.DependencyOfRelationship, + Data: nil, + }, + { + From: zonejs, + To: testApp, + Type: artifact.DependencyOfRelationship, + Data: nil, + }, + } + + return expectedPkgs, expectedRelationships +} + +func expectedPackagesAndRelationshipsLockV2(locationSet file.LocationSet, metadata bool) ([]pkg.Package, []artifact.Relationship) { + metadataMap := map[string]pkg.NpmPackageLockEntry{ + "rxjs": { + Resolved: "https://registry.npmjs.org/rxjs/-/rxjs-7.5.7.tgz", + Integrity: "sha512-z9MzKh/UcOqB3i20H6rtrlaE/CgjLOvheWK/9ILrbhROGTweAi1BaFsTT9FbwZi5Trr1qNRs+MXkhmR06awzQA==", + }, + "test-app": { + Resolved: "", + Integrity: "", + }, + "tslib": { + Resolved: "https://registry.npmjs.org/tslib/-/tslib-2.4.1.tgz", + Integrity: "sha512-tGyy4dAjRIEwI7BzsB0lynWgOpfqjUdq91XXAlIWD2OwKBH7oCl/GZG/HT4BOHrTlPMOASlMQ7veyTqpmRcrNA==", + }, + "typescript": { + Resolved: "https://registry.npmjs.org/typescript/-/typescript-4.7.4.tgz", + Integrity: "sha512-C0WQT0gezHuw6AdY1M2jxUO83Rjf0HP7Sk1DtXj6j1EwkQNZrHAg2XPWlq62oqEhYvONq5pkC2Y9oPljWToLmQ==", + }, + "zone.js": { + Resolved: "https://registry.npmjs.org/zone.js/-/zone.js-0.11.8.tgz", + Integrity: "sha512-82bctBg2hKcEJ21humWIkXRlLBBmrc3nN7DFh5LGGhcyycO2S7FN8NmdvlcKaGFDNVL4/9kFLmwmInTavdJERA==", + }, + } + rxjs := pkg.Package{ + Name: "rxjs", + Version: "7.5.7", + PURL: "pkg:npm/rxjs@7.5.7", + Locations: locationSet, + Licenses: pkg.NewLicenseSet(), + Language: pkg.JavaScript, + Type: pkg.NpmPkg, + Metadata: pkg.NpmPackageLockEntry{}, + } + testApp := pkg.Package{ + Name: "test-app", + Version: "0.0.0", + PURL: "pkg:npm/test-app@0.0.0", + Locations: locationSet, + Licenses: pkg.NewLicenseSet(), + Language: pkg.JavaScript, + Type: pkg.NpmPkg, + Metadata: pkg.NpmPackageLockEntry{}, + } + tslib := pkg.Package{ + Name: "tslib", + Version: "2.4.1", + PURL: "pkg:npm/tslib@2.4.1", + Locations: locationSet, + Licenses: pkg.NewLicenseSet(), + Language: pkg.JavaScript, + Type: pkg.NpmPkg, + Metadata: pkg.NpmPackageLockEntry{}, + } + typescript := pkg.Package{ + Name: "typescript", + Version: "4.7.4", + FoundBy: "javascript-cataloger", + PURL: "pkg:npm/typescript@4.7.4", + Locations: locationSet, + Language: pkg.JavaScript, + Type: pkg.NpmPkg, + Metadata: pkg.NpmPackageLockEntry{}, + } + zonejs := pkg.Package{ + Name: "zone.js", + Version: "0.11.8", + PURL: "pkg:npm/zone.js@0.11.8", + Locations: locationSet, + Licenses: pkg.NewLicenseSet(), + Language: pkg.JavaScript, + Type: pkg.NpmPkg, + Metadata: pkg.NpmPackageLockEntry{}, + } + + l := []*pkg.Package{ + &rxjs, + &testApp, + &tslib, + &typescript, + &zonejs, + } + + var expectedPkgs []pkg.Package + for i := range l { + if metadata { + l[i].Metadata = metadataMap[l[i].Name] + expectedPkgs = append(expectedPkgs, *l[i]) + } else { + expectedPkgs = append(expectedPkgs, *l[i]) + } + } + + expectedRelationships := []artifact.Relationship{ + { + From: rxjs, + To: testApp, + Type: artifact.DependencyOfRelationship, + Data: nil, + }, + { + From: tslib, + To: rxjs, + Type: artifact.DependencyOfRelationship, + Data: nil, + }, + { + From: tslib, + To: testApp, + Type: artifact.DependencyOfRelationship, + Data: nil, + }, + { + From: tslib, + To: zonejs, + Type: artifact.DependencyOfRelationship, + Data: nil, + }, + { + From: typescript, + To: testApp, + Type: artifact.DependencyOfRelationship, + Data: nil, + }, + { + From: zonejs, + To: testApp, + Type: artifact.DependencyOfRelationship, + Data: nil, }, } + return expectedPkgs, expectedRelationships +} + +func expectedPackagesAndRelationshipsYarnLock(locationSet file.LocationSet, metadata bool) ([]pkg.Package, []artifact.Relationship) { + metadataMap := map[string]pkg.NpmPackageLockEntry{ + "rxjs": { + Resolved: "https://registry.npmjs.org/rxjs/-/rxjs-7.5.7.tgz", + Integrity: "sha512-z9MzKh/UcOqB3i20H6rtrlaE/CgjLOvheWK/9ILrbhROGTweAi1BaFsTT9FbwZi5Trr1qNRs+MXkhmR06awzQA==", + }, + "test-app": { + Resolved: "", + Integrity: "", + }, + "tslib": { + Resolved: "https://registry.npmjs.org/tslib/-/tslib-2.4.1.tgz", + Integrity: "sha512-tGyy4dAjRIEwI7BzsB0lynWgOpfqjUdq91XXAlIWD2OwKBH7oCl/GZG/HT4BOHrTlPMOASlMQ7veyTqpmRcrNA==", + }, + "typescript": { + Resolved: "https://registry.npmjs.org/typescript/-/typescript-4.7.4.tgz", + Integrity: "sha512-C0WQT0gezHuw6AdY1M2jxUO83Rjf0HP7Sk1DtXj6j1EwkQNZrHAg2XPWlq62oqEhYvONq5pkC2Y9oPljWToLmQ==", + }, + "zone.js": { + Resolved: "https://registry.npmjs.org/zone.js/-/zone.js-0.11.8.tgz", + Integrity: "sha512-82bctBg2hKcEJ21humWIkXRlLBBmrc3nN7DFh5LGGhcyycO2S7FN8NmdvlcKaGFDNVL4/9kFLmwmInTavdJERA==", + }, + } + rxjs := pkg.Package{ + Name: "rxjs", + Version: "7.5.7", + FoundBy: "javascript-cataloger", + PURL: "pkg:npm/rxjs@7.5.7", + Locations: locationSet, + Licenses: pkg.NewLicenseSet(), + Language: pkg.JavaScript, + Type: pkg.NpmPkg, + Metadata: pkg.NpmPackageLockEntry{}, + } + testApp := pkg.Package{ + Name: "test-app", + Version: "0.0.0", + FoundBy: "javascript-cataloger", + PURL: "pkg:npm/test-app@0.0.0", + Locations: locationSet, + Licenses: pkg.NewLicenseSet(), + Language: pkg.JavaScript, + Type: pkg.NpmPkg, + Metadata: pkg.NpmPackageLockEntry{}, + } + tslib := pkg.Package{ + Name: "tslib", + Version: "2.4.1", + FoundBy: "javascript-cataloger", + PURL: "pkg:npm/tslib@2.4.1", + Locations: locationSet, + Licenses: pkg.NewLicenseSet(), + Language: pkg.JavaScript, + Type: pkg.NpmPkg, + Metadata: pkg.NpmPackageLockEntry{}, + } + typescript := pkg.Package{ + Name: "typescript", + Version: "4.7.4", + FoundBy: "javascript-cataloger", + PURL: "pkg:npm/typescript@4.7.4", + Locations: locationSet, + Licenses: pkg.NewLicenseSet(), + Language: pkg.JavaScript, + Type: pkg.NpmPkg, + } + zonejs := pkg.Package{ + Name: "zone.js", + Version: "0.11.8", + FoundBy: "javascript-cataloger", + PURL: "pkg:npm/zone.js@0.11.8", + Locations: locationSet, + Licenses: pkg.NewLicenseSet(), + Language: pkg.JavaScript, + Type: pkg.NpmPkg, + + Metadata: pkg.NpmPackageLockEntry{}, + } + + l := []*pkg.Package{ + &rxjs, + &testApp, + &tslib, + &typescript, + &zonejs, + } + + var expectedPkgs []pkg.Package + for i := range l { + if metadata { + l[i].Metadata = metadataMap[l[i].Name] + expectedPkgs = append(expectedPkgs, *l[i]) + } else { + expectedPkgs = append(expectedPkgs, *l[i]) + } + } + + expectedRelationships := []artifact.Relationship{ + { + From: rxjs, + To: testApp, + Type: artifact.DependencyOfRelationship, + Data: nil, + }, + { + From: tslib, + To: rxjs, + Type: artifact.DependencyOfRelationship, + Data: nil, + }, + { + From: tslib, + To: testApp, + Type: artifact.DependencyOfRelationship, + Data: nil, + }, + { + From: tslib, + To: zonejs, + Type: artifact.DependencyOfRelationship, + Data: nil, + }, + { + From: typescript, + To: testApp, + Type: artifact.DependencyOfRelationship, + Data: nil, + }, + { + From: zonejs, + To: testApp, + Type: artifact.DependencyOfRelationship, + Data: nil, + }, + } + + return expectedPkgs, expectedRelationships +} + +func expectedPackagesAndRelationshipsLockV3(locationSet file.LocationSet, metadata bool) ([]pkg.Package, []artifact.Relationship) { + metadataMap := map[string]pkg.NpmPackageLockEntry{ + "rxjs": { + Resolved: "https://registry.npmjs.org/rxjs/-/rxjs-7.5.0.tgz", + Integrity: "sha512-fuCKAfFawVYX0pyFlETtYnXI+5iiY9Dftgk+VdgeOq+Qyi9ZDWckHZRDaXRt5WCNbbLkmAheoSGDiceyCIKNZA==", + }, + "test-app": { + Resolved: "", + Integrity: "", + }, + "tslib": { + Resolved: "https://registry.npmjs.org/tslib/-/tslib-2.6.2.tgz", + Integrity: "sha512-AEYxH93jGFPn/a2iVAwW87VuUIkR1FVUKB77NwMF7nBTDkDrrT/Hpt/IrCJ0QXhW27jTBDcf5ZY7w6RiqTMw2Q==", + }, + "typescript": { + Resolved: "https://registry.npmjs.org/typescript/-/typescript-4.7.4.tgz", + Integrity: "sha512-C0WQT0gezHuw6AdY1M2jxUO83Rjf0HP7Sk1DtXj6j1EwkQNZrHAg2XPWlq62oqEhYvONq5pkC2Y9oPljWToLmQ==", + }, + "zone.js": { + Resolved: "https://registry.npmjs.org/zone.js/-/zone.js-0.11.8.tgz", + Integrity: "sha512-82bctBg2hKcEJ21humWIkXRlLBBmrc3nN7DFh5LGGhcyycO2S7FN8NmdvlcKaGFDNVL4/9kFLmwmInTavdJERA==", + }, + } + rxjs := pkg.Package{ + Name: "rxjs", + Version: "7.5.0", + FoundBy: "javascript-cataloger", + PURL: "pkg:npm/rxjs@7.5.0", + Locations: locationSet, + Language: pkg.JavaScript, + Type: pkg.NpmPkg, + } + testApp := pkg.Package{ + Name: "test-app", + Version: "0.0.0", + FoundBy: "javascript-cataloger", + PURL: "pkg:npm/test-app@0.0.0", + Locations: locationSet, + Language: pkg.JavaScript, + Type: pkg.NpmPkg, + } + tslib := pkg.Package{ + Name: "tslib", + Version: "2.6.2", + FoundBy: "javascript-cataloger", + PURL: "pkg:npm/tslib@2.6.2", + Locations: locationSet, + Language: pkg.JavaScript, + Type: pkg.NpmPkg, + } + typescript := pkg.Package{ + Name: "typescript", + Version: "4.7.4", + FoundBy: "javascript-cataloger", + PURL: "pkg:npm/typescript@4.7.4", + Locations: locationSet, + Language: pkg.JavaScript, + Type: pkg.NpmPkg, + } + zonejs := pkg.Package{ + Name: "zone.js", + Version: "0.11.8", + FoundBy: "javascript-cataloger", + PURL: "pkg:npm/zone.js@0.11.8", + Locations: locationSet, + Language: pkg.JavaScript, + Type: pkg.NpmPkg, + } + + l := []*pkg.Package{ + &rxjs, + &testApp, + &tslib, + &typescript, + &zonejs, + } + + var expectedPkgs []pkg.Package + for i := range l { + if metadata { + l[i].Metadata = metadataMap[l[i].Name] + expectedPkgs = append(expectedPkgs, *l[i]) + } else { + expectedPkgs = append(expectedPkgs, *l[i]) + } + } + + expectedRelationships := []artifact.Relationship{ + { + From: rxjs, + To: testApp, + Type: artifact.DependencyOfRelationship, + Data: nil, + }, + { + From: tslib, + To: rxjs, + Type: artifact.DependencyOfRelationship, + Data: nil, + }, + { + From: tslib, + To: testApp, + Type: artifact.DependencyOfRelationship, + Data: nil, + }, + { + From: tslib, + To: zonejs, + Type: artifact.DependencyOfRelationship, + Data: nil, + }, + { + From: typescript, + To: testApp, + Type: artifact.DependencyOfRelationship, + Data: nil, + }, + { + From: zonejs, + To: testApp, + Type: artifact.DependencyOfRelationship, + Data: nil, + }, + } + + return expectedPkgs, expectedRelationships +} + +func expectedPackagesAndRelationshipsPnpmLock(locationSet file.LocationSet, metadata bool) ([]pkg.Package, []artifact.Relationship) { + metadataMap := map[string]pkg.NpmPackageLockEntry{ + "rxjs": { + Resolved: "https://registry.npmjs.org/rxjs/-/rxjs-7.5.7.tgz", + Integrity: "sha512-z9MzKh/UcOqB3i20H6rtrlaE/CgjLOvheWK/9ILrbhROGTweAi1BaFsTT9FbwZi5Trr1qNRs+MXkhmR06awzQA==", + }, + "test-app": { + Resolved: "", + Integrity: "", + }, + "tslib": { + Resolved: "https://registry.npmjs.org/tslib/-/tslib-2.6.2.tgz", + Integrity: "sha512-tGyy4dAjRIEwI7BzsB0lynWgOpfqjUdq91XXAlIWD2OwKBH7oCl/GZG/HT4BOHrTlPMOASlMQ7veyTqpmRcrNA==", + }, + "typescript": { + Resolved: "https://registry.npmjs.org/typescript/-/typescript-4.7.4.tgz", + Integrity: "sha512-C0WQT0gezHuw6AdY1M2jxUO83Rjf0HP7Sk1DtXj6j1EwkQNZrHAg2XPWlq62oqEhYvONq5pkC2Y9oPljWToLmQ==", + }, + "zone.js": { + Resolved: "https://registry.npmjs.org/zone.js/-/zone.js-0.11.8.tgz", + Integrity: "sha512-82bctBg2hKcEJ21humWIkXRlLBBmrc3nN7DFh5LGGhcyycO2S7FN8NmdvlcKaGFDNVL4/9kFLmwmInTavdJERA==", + }, + } + rxjs := pkg.Package{ + Name: "rxjs", + Version: "7.5.7", + FoundBy: "javascript-cataloger", + PURL: "pkg:npm/rxjs@7.5.7", + Locations: locationSet, + Language: pkg.JavaScript, + Type: pkg.NpmPkg, + + Metadata: pkg.NpmPackageLockEntry{}, + } + testApp := pkg.Package{ + Name: "test-app", + Version: "0.0.0", + FoundBy: "javascript-cataloger", + PURL: "pkg:npm/test-app@0.0.0", + Locations: locationSet, + Language: pkg.JavaScript, + Type: pkg.NpmPkg, + + Metadata: pkg.NpmPackageLockEntry{}, + } + tslib := pkg.Package{ + Name: "tslib", + Version: "2.6.2", + FoundBy: "javascript-cataloger", + PURL: "pkg:npm/tslib@2.6.2", + Locations: locationSet, + Language: pkg.JavaScript, + Type: pkg.NpmPkg, + Metadata: pkg.NpmPackageLockEntry{}, + } + typescript := pkg.Package{ + Name: "typescript", + Version: "4.7.4", + FoundBy: "javascript-cataloger", + PURL: "pkg:npm/typescript@4.7.4", + Locations: locationSet, + Language: pkg.JavaScript, + Type: pkg.NpmPkg, + Metadata: pkg.NpmPackageLockEntry{}, + } + zonejs := pkg.Package{ + Name: "zone.js", + Version: "0.11.8", + FoundBy: "javascript-cataloger", + PURL: "pkg:npm/zone.js@0.11.8", + Locations: locationSet, + Language: pkg.JavaScript, + Type: pkg.NpmPkg, + Metadata: pkg.NpmPackageLockEntry{}, + } + + l := []*pkg.Package{ + &rxjs, + &testApp, + &tslib, + &typescript, + &zonejs, + } + + var expectedPkgs []pkg.Package + for i := range l { + if metadata { + l[i].Metadata = metadataMap[l[i].Name] + expectedPkgs = append(expectedPkgs, *l[i]) + } else { + expectedPkgs = append(expectedPkgs, *l[i]) + } + } + + expectedRelationships := []artifact.Relationship{ + { + From: rxjs, + To: testApp, + Type: artifact.DependencyOfRelationship, + Data: nil, + }, + { + From: tslib, + To: rxjs, + Type: artifact.DependencyOfRelationship, + Data: nil, + }, + { + From: tslib, + To: testApp, + Type: artifact.DependencyOfRelationship, + Data: nil, + }, + { + From: tslib, + To: zonejs, + Type: artifact.DependencyOfRelationship, + Data: nil, + }, + { + From: zonejs, + To: testApp, + Type: artifact.DependencyOfRelationship, + Data: nil, + }, + } + + return expectedPkgs, expectedRelationships +} + +func Test_JavaScriptCataloger_PkgLock_v1(t *testing.T) { + locationSet := file.NewLocationSet(file.NewLocation("package-lock.json")) + expectedPkgs, expectedRelationships := expectedPackagesAndRelationshipsLockV1(locationSet, true) + pkgtest.NewCatalogTester(). + FromDirectory(t, "test-fixtures/pkg-json-and-lock/v1"). + Expects(expectedPkgs, expectedRelationships). + TestGroupedCataloger(t, NewJavaScriptCataloger()) +} + +func Test_JavaScriptCataloger_PkgLock_v2(t *testing.T) { + locationSet := file.NewLocationSet(file.NewLocation("package-lock.json")) + expectedPkgs, expectedRelationships := expectedPackagesAndRelationshipsLockV2(locationSet, true) pkgtest.NewCatalogTester(). - FromDirectory(t, "test-fixtures/pkg-lock"). - Expects(expectedPkgs, nil). - TestCataloger(t, NewLockCataloger()) + FromDirectory(t, "test-fixtures/pkg-json-and-lock/v2"). + Expects(expectedPkgs, expectedRelationships). + TestGroupedCataloger(t, NewJavaScriptCataloger()) +} + +func Test_JavaScriptCataloger_PkgLock_v3(t *testing.T) { + locationSet := file.NewLocationSet(file.NewLocation("package-lock.json")) + expectedPkgs, expectedRelationships := expectedPackagesAndRelationshipsLockV3(locationSet, true) + pkgtest.NewCatalogTester(). + FromDirectory(t, "test-fixtures/pkg-json-and-lock/v3"). + Expects(expectedPkgs, expectedRelationships). + TestGroupedCataloger(t, NewJavaScriptCataloger()) +} + +func Test_JavaScriptCataloger_YarnLock(t *testing.T) { + locationSet := file.NewLocationSet(file.NewLocation("yarn.lock")) + expectedPkgs, expectedRelationships := expectedPackagesAndRelationshipsYarnLock(locationSet, true) + pkgtest.NewCatalogTester(). + FromDirectory(t, "test-fixtures/pkg-json-and-yarn-lock"). + Expects(expectedPkgs, expectedRelationships). + TestGroupedCataloger(t, NewJavaScriptCataloger()) +} +func Test_JavaScriptCataloger_PnpmLock(t *testing.T) { + locationSet := file.NewLocationSet(file.NewLocation("pnpm-lock.yaml")) + expectedPkgs, expectedRelationships := expectedPackagesAndRelationshipsPnpmLock(locationSet, false) + pkgtest.NewCatalogTester(). + FromDirectory(t, "test-fixtures/pkg-json-and-pnpm-lock"). + Expects(expectedPkgs, expectedRelationships). + TestGroupedCataloger(t, NewJavaScriptCataloger()) } func Test_PackageCataloger_Globs(t *testing.T) { @@ -161,19 +733,34 @@ func Test_PackageCataloger_Globs(t *testing.T) { } } -func Test_LockCataloger_Globs(t *testing.T) { +func Test_JavaScriptCataloger_Globs(t *testing.T) { tests := []struct { name string fixture string expected []string }{ { - name: "obtain package files", - fixture: "test-fixtures/glob-paths", + name: "obtain package lock files", + fixture: "test-fixtures/pkg-json-and-lock/v1", + expected: []string{ + "package-lock.json", + "package.json", + }, + }, + { + name: "obtain yarn lock files", + fixture: "test-fixtures/pkg-json-and-yarn-lock", + expected: []string{ + "yarn.lock", + "package.json", + }, + }, + { + name: "obtain yarn lock files", + fixture: "test-fixtures/pkg-json-and-pnpm-lock", expected: []string{ - "src/package-lock.json", - "src/pnpm-lock.yaml", - "src/yarn.lock", + "pnpm-lock.yaml", + "package.json", }, }, } @@ -183,7 +770,7 @@ func Test_LockCataloger_Globs(t *testing.T) { pkgtest.NewCatalogTester(). FromDirectory(t, test.fixture). ExpectsResolverContentQueries(test.expected). - TestCataloger(t, NewLockCataloger()) + TestGroupedCataloger(t, NewJavaScriptCataloger()) }) } } diff --git a/syft/pkg/cataloger/javascript/filter/filter.go b/syft/pkg/cataloger/javascript/filter/filter.go new file mode 100644 index 00000000000..2961c8c9c04 --- /dev/null +++ b/syft/pkg/cataloger/javascript/filter/filter.go @@ -0,0 +1,23 @@ +package filter + +import "strings" + +func filterFunc(strFunc func(string, string) bool, args ...string) func(string) bool { + return func(filename string) bool { + for _, suffix := range args { + if strFunc(filename, suffix) { + return true + } + } + return false + } +} + +var ( + JavaScriptYarnLock = filterFunc(strings.HasSuffix, "yarn.lock") + JavaScriptPackageJSON = func(filename string) bool { + return strings.HasSuffix(filename, "package.json") + } + JavaScriptPackageLock = filterFunc(strings.HasSuffix, "package-lock.json") + JavaScriptPmpmLock = filterFunc(strings.HasSuffix, "pnpm-lock.yaml") +) diff --git a/syft/pkg/cataloger/javascript/key/key.go b/syft/pkg/cataloger/javascript/key/key.go new file mode 100644 index 00000000000..6e005571fd4 --- /dev/null +++ b/syft/pkg/cataloger/javascript/key/key.go @@ -0,0 +1,7 @@ +package key + +import "fmt" + +func NpmPackageKey(name, version string) string { + return fmt.Sprintf("%s:%s", name, version) +} diff --git a/syft/pkg/cataloger/javascript/key/key_test.go b/syft/pkg/cataloger/javascript/key/key_test.go new file mode 100644 index 00000000000..9e9cea99a0f --- /dev/null +++ b/syft/pkg/cataloger/javascript/key/key_test.go @@ -0,0 +1,26 @@ +package key + +import "testing" + +func TestNpmPackageKey(t *testing.T) { + tests := []struct { + name string + version string + expected string + }{ + {"lodash", "1.0.0", "lodash:1.0.0"}, + {"react", "16.8.0", "react:16.8.0"}, + {"", "1.0.0", ":1.0.0"}, + {"lodash", "", "lodash:"}, + {"", "", ":"}, + } + + for _, tt := range tests { + t.Run(tt.name+":"+tt.version, func(t *testing.T) { + got := NpmPackageKey(tt.name, tt.version) + if got != tt.expected { + t.Errorf("NpmPackageKey(%q, %q) = %q; want %q", tt.name, tt.version, got, tt.expected) + } + }) + } +} diff --git a/syft/pkg/cataloger/javascript/package.go b/syft/pkg/cataloger/javascript/package.go index 9a1331ac6e6..fa7459b7acf 100644 --- a/syft/pkg/cataloger/javascript/package.go +++ b/syft/pkg/cataloger/javascript/package.go @@ -12,7 +12,7 @@ import ( "github.com/anchore/syft/syft/pkg" ) -func newPackageJSONPackage(u packageJSON, indexLocation file.Location) pkg.Package { +func newPackageJSONRootPackage(u packageJSON, indexLocation file.Location) pkg.Package { licenseCandidates, err := u.licensesFromJSON() if err != nil { log.Warnf("unable to extract licenses from javascript package.json: %+v", err) @@ -23,7 +23,7 @@ func newPackageJSONPackage(u packageJSON, indexLocation file.Location) pkg.Packa Name: u.Name, Version: u.Version, PURL: packageURL(u.Name, u.Version), - Locations: file.NewLocationSet(indexLocation), + Locations: file.NewLocationSet(indexLocation.WithAnnotation(pkg.EvidenceAnnotationKey, pkg.PrimaryEvidenceAnnotation)), Language: pkg.JavaScript, Licenses: pkg.NewLicenseSet(license...), Type: pkg.NpmPkg, @@ -43,7 +43,7 @@ func newPackageJSONPackage(u packageJSON, indexLocation file.Location) pkg.Packa return p } -func newPackageLockV1Package(resolver file.Resolver, location file.Location, name string, u lockDependency) pkg.Package { +func newPackageLockV1Package(resolver file.Resolver, location file.Location, name string, u packageLockDependency) pkg.Package { version := u.Version const aliasPrefixPackageLockV1 = "npm:" @@ -74,7 +74,7 @@ func newPackageLockV1Package(resolver file.Resolver, location file.Location, nam ) } -func newPackageLockV2Package(resolver file.Resolver, location file.Location, name string, u lockPackage) pkg.Package { +func newPackageLockV2Package(resolver file.Resolver, location file.Location, name string, u packageLockPackage) pkg.Package { return finalizeLockPkg( resolver, location, @@ -91,36 +91,6 @@ func newPackageLockV2Package(resolver file.Resolver, location file.Location, nam ) } -func newPnpmPackage(resolver file.Resolver, location file.Location, name, version string) pkg.Package { - return finalizeLockPkg( - resolver, - location, - pkg.Package{ - Name: name, - Version: version, - Locations: file.NewLocationSet(location.WithAnnotation(pkg.EvidenceAnnotationKey, pkg.PrimaryEvidenceAnnotation)), - PURL: packageURL(name, version), - Language: pkg.JavaScript, - Type: pkg.NpmPkg, - }, - ) -} - -func newYarnLockPackage(resolver file.Resolver, location file.Location, name, version string) pkg.Package { - return finalizeLockPkg( - resolver, - location, - pkg.Package{ - Name: name, - Version: version, - Locations: file.NewLocationSet(location.WithAnnotation(pkg.EvidenceAnnotationKey, pkg.PrimaryEvidenceAnnotation)), - PURL: packageURL(name, version), - Language: pkg.JavaScript, - Type: pkg.NpmPkg, - }, - ) -} - func finalizeLockPkg(resolver file.Resolver, location file.Location, p pkg.Package) pkg.Package { licenseCandidate := addLicenses(p.Name, resolver, location) p.Licenses.Add(pkg.NewLicensesFromLocation(location, licenseCandidate...)...) diff --git a/syft/pkg/cataloger/javascript/parse_javascript.go b/syft/pkg/cataloger/javascript/parse_javascript.go new file mode 100644 index 00000000000..65e0b4c522f --- /dev/null +++ b/syft/pkg/cataloger/javascript/parse_javascript.go @@ -0,0 +1,89 @@ +package javascript + +import ( + "path" + "strings" + + "github.com/anchore/syft/syft/artifact" + "github.com/anchore/syft/syft/file" + "github.com/anchore/syft/syft/pkg" + "github.com/anchore/syft/syft/pkg/cataloger/generic" + "github.com/anchore/syft/syft/pkg/cataloger/javascript/filter" +) + +var _ generic.GroupedParser = parseJavaScript + +var path2dir = func(relpath string) string { return path.Dir(strings.ReplaceAll(relpath, `\`, `/`)) } + +func parseJavaScript(resolver file.Resolver, e *generic.Environment, readers []file.LocationReadCloser) (pkgs []pkg.Package, relationships []artifact.Relationship, err error) { + jsonMap := map[string]*packageJSON{} + lockMap := map[string]*packageLock{} + yarnMap := map[string]map[string]*yarnLockPackage{} + pnpmMap := map[string]map[string]*pnpmLockPackage{} + pnpmLock := map[string]pnpmLockYaml{} + pnpmLocation := file.Location{} + yarnLocation := file.Location{} + lockLocation := file.Location{} + + for _, reader := range readers { + // in the case we find matching files in the node_modules directories, skip those + // as the whole purpose of the lock file is for the specific dependencies of the root project + if pathContainsNodeModulesDirectory(reader.Path()) { + return nil, nil, nil + } + + thePath := reader.Location.Path() + if filter.JavaScriptYarnLock(thePath) { + yarnMap[path2dir(thePath)] = parseYarnLockFile(resolver, reader) + yarnLocation = reader.Location + } + if filter.JavaScriptPackageJSON(thePath) { + js, err := parsePackageJSONFile(resolver, e, reader) + if err != nil { + return nil, nil, err + } + js.File = thePath + jsonMap[js.Name] = js + } + if filter.JavaScriptPackageLock(thePath) { + lock, err := parsePackageLockFile(reader) + if err != nil { + return nil, nil, err + } + lockMap[lock.Name] = &lock + lockLocation = reader.Location + } + if filter.JavaScriptPmpmLock(thePath) { + pMap, pLock := parsePnpmLockFile(reader) + pnpmMap[path2dir(thePath)] = pMap + pnpmLock[path2dir(thePath)] = pLock + pnpmLocation = reader.Location + } + } + + for name, js := range jsonMap { + if lock, ok := lockMap[name]; ok { + p, rels := finalizePackageLockWithPackageJSON(resolver, js, lock, lockLocation) + pkgs = append(pkgs, p...) + relationships = append(relationships, rels...) + } + + if js.File != "" { + if yarn, ok := yarnMap[path2dir(js.File)]; ok { + p, rels := finalizeYarnLockWithPackageJSON(resolver, js, yarn, yarnLocation) + pkgs = append(pkgs, p...) + relationships = append(relationships, rels...) + } + if pnpm, ok := pnpmMap[path2dir(js.File)]; ok { + pLock := pnpmLock[path2dir(js.File)] + p, rels := finalizePnpmLockWithPackageJSON(resolver, js, &pLock, pnpm, pnpmLocation) + pkgs = append(pkgs, p...) + relationships = append(relationships, rels...) + } + } + } + + pkg.Sort(pkgs) + pkg.SortRelationships(relationships) + return pkgs, relationships, nil +} diff --git a/syft/pkg/cataloger/javascript/parse_package_json.go b/syft/pkg/cataloger/javascript/parse_package_json.go index d0e7edbc0be..09669e8c2bc 100644 --- a/syft/pkg/cataloger/javascript/parse_package_json.go +++ b/syft/pkg/cataloger/javascript/parse_package_json.go @@ -4,7 +4,6 @@ import ( "encoding/json" "errors" "fmt" - "io" "regexp" "github.com/mitchellh/mapstructure" @@ -20,19 +19,24 @@ import ( // integrity check var _ generic.Parser = parsePackageJSON -// packageJSON represents a JavaScript package.json file type packageJSON struct { - Version string `json:"version"` - Latest []string `json:"latest"` - Author author `json:"author"` - License json.RawMessage `json:"license"` - Licenses json.RawMessage `json:"licenses"` - Name string `json:"name"` - Homepage string `json:"homepage"` - Description string `json:"description"` - Dependencies map[string]string `json:"dependencies"` - Repository repository `json:"repository"` - Private bool `json:"private"` + Name string `json:"name"` + Version string `json:"version"` + Author author `json:"author"` + License json.RawMessage `json:"license"` + Licenses json.RawMessage `json:"licenses"` + Homepage string `json:"homepage"` + Private bool `json:"private"` + Description string `json:"description"` + Develop bool `json:"dev"` // lock v3 + Repository repository `json:"repository"` + Dependencies map[string]string `json:"dependencies"` + DevDependencies map[string]string `json:"devDependencies"` + PeerDependencies map[string]string `json:"peerDependencies"` + PeerDependenciesMeta map[string]struct { + Optional bool `json:"optional"` + } `json:"peerDependenciesMeta"` + File string `json:"-"` } type author struct { @@ -50,33 +54,31 @@ type repository struct { // ---> name: "Isaac Z. Schlueter" email: "i@izs.me" url: "http://blog.izs.me" var authorPattern = regexp.MustCompile(`^\s*(?P[^<(]*)(\s+<(?P.*)>)?(\s\((?P.*)\))?\s*$`) -// parsePackageJSON parses a package.json and returns the discovered JavaScript packages. -func parsePackageJSON(_ file.Resolver, _ *generic.Environment, reader file.LocationReadCloser) ([]pkg.Package, []artifact.Relationship, error) { - var pkgs []pkg.Package - dec := json.NewDecoder(reader) - - for { - var p packageJSON - if err := dec.Decode(&p); errors.Is(err, io.EOF) { - break - } else if err != nil { - return nil, nil, fmt.Errorf("failed to parse package.json file: %w", err) - } - - if !p.hasNameAndVersionValues() { - log.Debugf("encountered package.json file without a name and/or version field, ignoring (path=%q)", reader.Path()) - return nil, nil, nil - } +func parsePackageJSON(resolver file.Resolver, e *generic.Environment, reader file.LocationReadCloser) ([]pkg.Package, []artifact.Relationship, error) { + pkgjson, err := parsePackageJSONFile(resolver, e, reader) + if err != nil { + return nil, nil, err + } - pkgs = append( - pkgs, - newPackageJSONPackage(p, reader.Location.WithAnnotation(pkg.EvidenceAnnotationKey, pkg.PrimaryEvidenceAnnotation)), - ) + if !pkgjson.hasNameAndVersionValues() { + log.Debugf("encountered package.json file without a name and/or version field, ignoring (path=%q)", reader.Path()) + return nil, nil, nil } - pkg.Sort(pkgs) + rootPkg := newPackageJSONRootPackage(*pkgjson, reader.Location) + return []pkg.Package{rootPkg}, nil, nil +} + +// parsePackageJSON parses a package.json and returns the discovered JavaScript packages. +func parsePackageJSONFile(_ file.Resolver, _ *generic.Environment, reader file.LocationReadCloser) (*packageJSON, error) { + var js *packageJSON + decoder := json.NewDecoder(reader) + err := decoder.Decode(&js) + if err != nil { + return nil, err + } - return pkgs, nil, nil + return js, nil } func (a *author) UnmarshalJSON(b []byte) error { diff --git a/syft/pkg/cataloger/javascript/parse_package_json_test.go b/syft/pkg/cataloger/javascript/parse_package_json_test.go index 5c544580a89..f3ab113d7d3 100644 --- a/syft/pkg/cataloger/javascript/parse_package_json_test.go +++ b/syft/pkg/cataloger/javascript/parse_package_json_test.go @@ -16,7 +16,7 @@ func TestParsePackageJSON(t *testing.T) { ExpectedPkg pkg.Package }{ { - Fixture: "test-fixtures/pkg-json/package.json", + Fixture: "test-fixtures/pkg-json/pkg-json/package.json", ExpectedPkg: pkg.Package{ Name: "npm", Version: "6.14.6", @@ -24,7 +24,7 @@ func TestParsePackageJSON(t *testing.T) { Type: pkg.NpmPkg, Language: pkg.JavaScript, Licenses: pkg.NewLicenseSet( - pkg.NewLicenseFromLocations("Artistic-2.0", file.NewLocation("test-fixtures/pkg-json/package.json")), + pkg.NewLicenseFromLocations("Artistic-2.0", file.NewLocation("test-fixtures/pkg-json/pkg-json/package.json")), ), Metadata: pkg.NpmPackage{ Name: "npm", @@ -37,7 +37,7 @@ func TestParsePackageJSON(t *testing.T) { }, }, { - Fixture: "test-fixtures/pkg-json/package-license-object.json", + Fixture: "test-fixtures/pkg-json/license-object/package.json", ExpectedPkg: pkg.Package{ Name: "npm", Version: "6.14.6", @@ -45,7 +45,7 @@ func TestParsePackageJSON(t *testing.T) { Type: pkg.NpmPkg, Language: pkg.JavaScript, Licenses: pkg.NewLicenseSet( - pkg.NewLicenseFromLocations("ISC", file.NewLocation("test-fixtures/pkg-json/package-license-object.json")), + pkg.NewLicenseFromLocations("ISC", file.NewLocation("test-fixtures/pkg-json/license-object/package.json")), ), Metadata: pkg.NpmPackage{ Name: "npm", @@ -58,15 +58,15 @@ func TestParsePackageJSON(t *testing.T) { }, }, { - Fixture: "test-fixtures/pkg-json/package-license-objects.json", + Fixture: "test-fixtures/pkg-json/license-objects/package.json", ExpectedPkg: pkg.Package{ Name: "npm", Version: "6.14.6", PURL: "pkg:npm/npm@6.14.6", Type: pkg.NpmPkg, Licenses: pkg.NewLicenseSet( - pkg.NewLicenseFromLocations("MIT", file.NewLocation("test-fixtures/pkg-json/package-license-objects.json")), - pkg.NewLicenseFromLocations("Apache-2.0", file.NewLocation("test-fixtures/pkg-json/package-license-objects.json")), + pkg.NewLicenseFromLocations("MIT", file.NewLocation("test-fixtures/pkg-json/license-objects/package.json")), + pkg.NewLicenseFromLocations("Apache-2.0", file.NewLocation("test-fixtures/pkg-json/license-objects/package.json")), ), Language: pkg.JavaScript, Metadata: pkg.NpmPackage{ @@ -80,7 +80,7 @@ func TestParsePackageJSON(t *testing.T) { }, }, { - Fixture: "test-fixtures/pkg-json/package-malformed-license.json", + Fixture: "test-fixtures/pkg-json/malformed-license/package.json", ExpectedPkg: pkg.Package{ Name: "npm", Version: "6.14.6", @@ -98,7 +98,7 @@ func TestParsePackageJSON(t *testing.T) { }, }, { - Fixture: "test-fixtures/pkg-json/package-no-license.json", + Fixture: "test-fixtures/pkg-json/no-license/package.json", ExpectedPkg: pkg.Package{ Name: "npm", Version: "6.14.6", @@ -116,14 +116,14 @@ func TestParsePackageJSON(t *testing.T) { }, }, { - Fixture: "test-fixtures/pkg-json/package-nested-author.json", + Fixture: "test-fixtures/pkg-json/nested-author/package.json", ExpectedPkg: pkg.Package{ Name: "npm", Version: "6.14.6", PURL: "pkg:npm/npm@6.14.6", Type: pkg.NpmPkg, Licenses: pkg.NewLicenseSet( - pkg.NewLicenseFromLocations("Artistic-2.0", file.NewLocation("test-fixtures/pkg-json/package-nested-author.json")), + pkg.NewLicenseFromLocations("Artistic-2.0", file.NewLocation("test-fixtures/pkg-json/nested-author/package.json")), ), Language: pkg.JavaScript, Metadata: pkg.NpmPackage{ @@ -137,14 +137,14 @@ func TestParsePackageJSON(t *testing.T) { }, }, { - Fixture: "test-fixtures/pkg-json/package-repo-string.json", + Fixture: "test-fixtures/pkg-json/repo-string/package.json", ExpectedPkg: pkg.Package{ Name: "function-bind", Version: "1.1.1", PURL: "pkg:npm/function-bind@1.1.1", Type: pkg.NpmPkg, Licenses: pkg.NewLicenseSet( - pkg.NewLicenseFromLocations("MIT", file.NewLocation("test-fixtures/pkg-json/package-repo-string.json")), + pkg.NewLicenseFromLocations("MIT", file.NewLocation("test-fixtures/pkg-json/repo-string/package.json")), ), Language: pkg.JavaScript, Metadata: pkg.NpmPackage{ @@ -158,14 +158,14 @@ func TestParsePackageJSON(t *testing.T) { }, }, { - Fixture: "test-fixtures/pkg-json/package-private.json", + Fixture: "test-fixtures/pkg-json/private/package.json", ExpectedPkg: pkg.Package{ Name: "npm", Version: "6.14.6", PURL: "pkg:npm/npm@6.14.6", Type: pkg.NpmPkg, Licenses: pkg.NewLicenseSet( - pkg.NewLicenseFromLocations("Artistic-2.0", file.NewLocation("test-fixtures/pkg-json/package-private.json")), + pkg.NewLicenseFromLocations("Artistic-2.0", file.NewLocation("test-fixtures/pkg-json/private/package.json")), ), Language: pkg.JavaScript, Metadata: pkg.NpmPackage{ @@ -190,7 +190,7 @@ func TestParsePackageJSON(t *testing.T) { } func TestParsePackageJSON_Partial(t *testing.T) { // see https://github.com/anchore/syft/issues/311 - const fixtureFile = "test-fixtures/pkg-json/package-partial.json" + const fixtureFile = "test-fixtures/pkg-json/partial/package.json" pkgtest.TestFileParser(t, fixtureFile, parsePackageJSON, nil, nil) } diff --git a/syft/pkg/cataloger/javascript/parse_package_lock.go b/syft/pkg/cataloger/javascript/parse_package_lock.go index 45c5a0c1f53..23d57fb7329 100644 --- a/syft/pkg/cataloger/javascript/parse_package_lock.go +++ b/syft/pkg/cataloger/javascript/parse_package_lock.go @@ -5,12 +5,16 @@ import ( "errors" "fmt" "io" + "path" "strings" + "github.com/samber/lo" + "golang.org/x/xerrors" + "github.com/anchore/syft/internal/log" "github.com/anchore/syft/syft/artifact" "github.com/anchore/syft/syft/file" - "github.com/anchore/syft/syft/pkg" + syftPkg "github.com/anchore/syft/syft/pkg" "github.com/anchore/syft/syft/pkg/cataloger/generic" ) @@ -19,39 +23,49 @@ var _ generic.Parser = parsePackageLock // packageLock represents a JavaScript package.lock json file type packageLock struct { - Requires bool `json:"requires"` - LockfileVersion int `json:"lockfileVersion"` - Dependencies map[string]lockDependency - Packages map[string]lockPackage + Name string `json:"name"` + Version string `json:"version"` + LockfileVersion int `json:"lockfileVersion"` + Dependencies map[string]*packageLockDependency `json:"dependencies"` + Packages map[string]*packageLockPackage `json:"packages"` + Requires bool `json:"requires"` } -// lockDependency represents a single package dependency listed in the package.lock json file -type lockDependency struct { - Version string `json:"version"` - Resolved string `json:"resolved"` - Integrity string `json:"integrity"` +type packageLockPackage struct { + Name string `json:"name"` + Version string `json:"version"` + Integrity string `json:"integrity"` + Resolved string `json:"resolved"` + Dependencies map[string]string `json:"dependencies"` + Workspaces []string `json:"workspaces"` + OptionalDependencies map[string]string `json:"optionalDependencies"` + DevDependencies map[string]string `json:"devDependencies"` + Link bool `json:"link"` + License packageLockLicense `json:"license"` + Dev bool `json:"dev"` + Peer bool `json:"peer"` + Requires map[string]string `json:"requires"` } -type lockPackage struct { - Name string `json:"name"` // only present in the root package entry (named "") - Version string `json:"version"` - Resolved string `json:"resolved"` - Integrity string `json:"integrity"` - License packageLockLicense `json:"license"` +type packageLockDependency struct { + name string + Version string `json:"version"` + Requires map[string]string `json:"requires"` + Integrity string `json:"integrity"` + Resolved string `json:"resolved"` + Dev bool `json:"dev"` + Dependencies map[string]*packageLockDependency `json:"dependencies"` } // packageLockLicense type packageLockLicense []string -// parsePackageLock parses a package-lock.json and returns the discovered JavaScript packages. -func parsePackageLock(resolver file.Resolver, _ *generic.Environment, reader file.LocationReadCloser) ([]pkg.Package, []artifact.Relationship, error) { +func parsePackageLockFile(reader file.LocationReadCloser) (packageLock, error) { // in the case we find package-lock.json files in the node_modules directories, skip those // as the whole purpose of the lock file is for the specific dependencies of the root project if pathContainsNodeModulesDirectory(reader.Path()) { - return nil, nil, nil + return packageLock{}, nil } - - var pkgs []pkg.Package dec := json.NewDecoder(reader) var lock packageLock @@ -59,40 +73,288 @@ func parsePackageLock(resolver file.Resolver, _ *generic.Environment, reader fil if err := dec.Decode(&lock); errors.Is(err, io.EOF) { break } else if err != nil { - return nil, nil, fmt.Errorf("failed to parse package-lock.json file: %w", err) + return packageLock{}, fmt.Errorf("failed to parse package-lock.json file: %w", err) } } + return lock, nil +} - if lock.LockfileVersion == 1 { - for name, pkgMeta := range lock.Dependencies { - pkgs = append(pkgs, newPackageLockV1Package(resolver, reader.Location, name, pkgMeta)) - } +// parsePackageLock parses a package-lock.json and returns the discovered JavaScript packages. +func parsePackageLock(resolver file.Resolver, _ *generic.Environment, reader file.LocationReadCloser) ([]syftPkg.Package, []artifact.Relationship, error) { + lock, err := parsePackageLockFile(reader) + if err != nil { + return nil, nil, err + } + pkgs, rels := finalizePackageLockWithoutPackageJSON(resolver, &lock, reader.Location) + return pkgs, rels, nil +} + +func finalizePackageLockWithoutPackageJSON(resolver file.Resolver, pkglock *packageLock, indexLocation file.Location) ([]syftPkg.Package, []artifact.Relationship) { + var root syftPkg.Package + if pkglock.Name == "" { + name := rootNameFromPath(indexLocation) + p := packageLockPackage{Name: name, Version: "0.0.0"} + root = newPackageLockV2Package( + resolver, + indexLocation, + name, + p, + ) + } else { + p := packageLockPackage{Name: pkglock.Name, Version: pkglock.Version} + root = newPackageLockV2Package( + resolver, + indexLocation, + pkglock.Name, + p, + ) + } + + if pkglock.LockfileVersion == 1 { + return finalizePackageLockWithoutPackageJSONV1(resolver, pkglock, indexLocation, root) + } + + if pkglock.LockfileVersion == 3 || pkglock.LockfileVersion == 2 { + root = newPackageLockV2Package( + resolver, + indexLocation, + pkglock.Name, + *pkglock.Packages[""], + ) + return finalizePackageLockV2(resolver, pkglock, indexLocation, root) + } + + return nil, nil +} + +func finalizePackageLockWithPackageJSON(resolver file.Resolver, pkgjson *packageJSON, pkglock *packageLock, indexLocation file.Location) ([]syftPkg.Package, []artifact.Relationship) { + if pkgjson == nil { + return finalizePackageLockWithoutPackageJSON(resolver, pkglock, indexLocation) + } + + if !pkgjson.hasNameAndVersionValues() { + log.Debugf("encountered package.json file without a name and/or version field, ignoring (path=%q)", indexLocation.Path()) + return nil, nil + } + + p := packageLockPackage{Name: pkgjson.Name, Version: pkgjson.Version} + root := newPackageLockV2Package( + resolver, + indexLocation, + pkglock.Name, + p, + ) + + if pkglock.LockfileVersion == 1 { + return finalizePackageLockWithPackageJSONV1(resolver, pkgjson, pkglock, indexLocation, root) + } + + if pkglock.LockfileVersion == 3 || pkglock.LockfileVersion == 2 { + return finalizePackageLockV2(resolver, pkglock, indexLocation, root) + } + + return nil, nil +} + +func finalizePackageLockWithoutPackageJSONV1(resolver file.Resolver, pkglock *packageLock, indexLocation file.Location, root syftPkg.Package) ([]syftPkg.Package, []artifact.Relationship) { + if pkglock.LockfileVersion != 1 { + return nil, nil } + pkgs := []syftPkg.Package{} + pkgs = append(pkgs, root) - if lock.LockfileVersion == 2 || lock.LockfileVersion == 3 { - for name, pkgMeta := range lock.Packages { - if name == "" { - if pkgMeta.Name == "" { - continue + // create packages + for name, lockDep := range pkglock.Dependencies { + lockDep.name = name + pkg := newPackageLockV1Package( + resolver, + indexLocation, + name, + *lockDep, + ) + pkgs = append(pkgs, pkg) + } + syftPkg.Sort(pkgs) + return pkgs, nil +} + +func finalizePackageLockWithPackageJSONV1(resolver file.Resolver, pkgjson *packageJSON, pkglock *packageLock, indexLocation file.Location, root syftPkg.Package) ([]syftPkg.Package, []artifact.Relationship) { + if pkglock.LockfileVersion != 1 { + return nil, nil + } + pkgs := []syftPkg.Package{} + relationships := []artifact.Relationship{} + + pkgs = append(pkgs, root) + depnameMap := map[string]syftPkg.Package{} + + // create packages + for name, lockDep := range pkglock.Dependencies { + lockDep.name = name + pkg := newPackageLockV1Package( + resolver, + indexLocation, + name, + *lockDep, + ) + pkgs = append(pkgs, pkg) + depnameMap[name] = pkg + } + + // create relationships + for name, lockDep := range pkglock.Dependencies { + lockDep.name = name + for name, sub := range lockDep.Dependencies { + sub.name = name + if subPkg, ok := depnameMap[name]; ok { + rel := artifact.Relationship{ + From: subPkg, + To: depnameMap[name], + Type: artifact.DependencyOfRelationship, } - name = pkgMeta.Name + relationships = append(relationships, rel) } + } + for name := range lockDep.Requires { + if subPkg, ok := depnameMap[name]; ok { + rel := artifact.Relationship{ + From: subPkg, + To: depnameMap[name], + Type: artifact.DependencyOfRelationship, + } + relationships = append(relationships, rel) + } + } + } + + for name := range pkgjson.Dependencies { + rel := artifact.Relationship{ + From: depnameMap[name], + To: root, + Type: artifact.DependencyOfRelationship, + } + relationships = append(relationships, rel) + } + + for name := range pkgjson.DevDependencies { + rel := artifact.Relationship{ + From: depnameMap[name], + To: root, + Type: artifact.DependencyOfRelationship, + } + relationships = append(relationships, rel) + } + + syftPkg.Sort(pkgs) + syftPkg.SortRelationships(relationships) + return pkgs, relationships +} + +//nolint:funlen +func finalizePackageLockV2(resolver file.Resolver, pkglock *packageLock, indexLocation file.Location, root syftPkg.Package) ([]syftPkg.Package, []artifact.Relationship) { + if pkglock.LockfileVersion != 3 && pkglock.LockfileVersion != 2 { + return nil, nil + } + if _, ok := pkglock.Packages[""]; !ok { + log.Debugf("encountered a package-lock.json file without a root pakcage, ignoring (path=%q)", indexLocation.Path()) + return nil, nil + } + + pkgs := []syftPkg.Package{} + pkgs = append(pkgs, root) + relationships := []artifact.Relationship{} + packages := pkglock.Packages + pkgMap := map[artifact.ID]syftPkg.Package{} + + // Resolve links first + // https://docs.npmjs.com/cli/v9/configuring-npm/package-lock-json#packages + resolveLinks(packages) + + // directDeps := map[string]struct{}{} + for name, version := range lo.Assign(packages[""].Dependencies, packages[""].OptionalDependencies, packages[""].DevDependencies) { + pkgPath := joinPaths(nodeModulesDir, name) + if _, ok := packages[pkgPath]; !ok { + log.Debugf("Unable to find the direct dependency: '%s@%s'", name, version) + continue + } + // Store the package paths of direct dependencies + // e.g. node_modules/body-parser + // directDeps[pkgPath] = struct{}{} + + p := newPackageLockV2Package( + resolver, + indexLocation, + name, + *packages[pkgPath], + ) + pkgMap[p.ID()] = p + pkgs = append(pkgs, p) + + relationships = append(relationships, artifact.Relationship{ + From: p, + To: root, + Type: artifact.DependencyOfRelationship, + }) + } + + for pkgPath, pkg := range packages { + if !strings.HasPrefix(pkgPath, "node_modules") { + continue + } + + // skip peer dependencies + if pkg.Peer { + continue + } - // handles alias names - if pkgMeta.Name != "" { - name = pkgMeta.Name + // pkg.Name exists when package name != folder name + pkgName := pkg.Name + if pkgName == "" { + pkgName = pkgNameFromPath(pkgPath) + } + + p := newPackageLockV2Package( + resolver, + indexLocation, + pkgName, + *pkg, + ) + // add only if not already added + if _, ok := pkgMap[p.ID()]; !ok { + pkgs = append(pkgs, p) + } + + // npm builds graph using optional deps. e.g.: + // └─┬ watchpack@1.7.5 + // ├─┬ chokidar@3.5.3 - optional dependency + // │ └── glob-parent@5.1. + dependencies := lo.Assign(pkg.Dependencies, pkg.DevDependencies, pkg.OptionalDependencies) + // dependsOn := make([]string, 0, len(dependencies)) + for depName, depVersion := range dependencies { + dep, err := findDependsOn(pkgPath, depName, packages) + if err != nil { + log.Warnf("Cannot resolve the version: '%s@%s'", depName, depVersion) + continue } - pkgs = append( - pkgs, - newPackageLockV2Package(resolver, reader.Location, getNameFromPath(name), pkgMeta), + depPkg := newPackageLockV2Package( + resolver, + indexLocation, + depName, + *dep, ) + + relationships = append(relationships, artifact.Relationship{ + From: depPkg, + To: p, + Type: artifact.DependencyOfRelationship, + }) } } - pkg.Sort(pkgs) - - return pkgs, nil, nil + syftPkg.Sort(pkgs) + syftPkg.SortRelationships(relationships) + return pkgs, relationships } func (licenses *packageLockLicense) UnmarshalJSON(data []byte) (err error) { @@ -125,7 +387,103 @@ func (licenses *packageLockLicense) UnmarshalJSON(data []byte) (err error) { return nil } -func getNameFromPath(path string) string { - parts := strings.Split(path, "node_modules/") - return parts[len(parts)-1] +// for local package npm uses links. e.g.: +// function/func1 -> target of package +// node_modules/func1 -> link to target +// see `package-lock_v3_with_workspace.json` to better understanding +func resolveLinks(packages map[string]*packageLockPackage) { + links := lo.PickBy(packages, func(_ string, pkg *packageLockPackage) bool { + return pkg.Link + }) + + // Early return + if len(links) == 0 { + return + } + + rootPkg := packages[""] + if rootPkg.Dependencies == nil { + rootPkg.Dependencies = make(map[string]string) + } + + workspaces := rootPkg.Workspaces + for pkgPath, pkg := range packages { + for linkPath, link := range links { + if !strings.HasPrefix(pkgPath, link.Resolved) { + continue + } + // The target doesn't have the "resolved" field, so we need to copy it from the link. + if pkg.Resolved == "" { + pkg.Resolved = link.Resolved + } + + // Resolve the link package so all packages are located under "node_modules". + resolvedPath := strings.ReplaceAll(pkgPath, link.Resolved, linkPath) + packages[resolvedPath] = pkg + + // Delete the target package + delete(packages, pkgPath) + + if isWorkspace(pkgPath, workspaces) { + rootPkg.Dependencies[pkgNameFromPath(linkPath)] = pkg.Version + } + break + } + } + packages[""] = rootPkg +} + +const nodeModulesDir = "node_modules" + +func pkgNameFromPath(path string) string { + // lock file contains path to dependency in `node_modules`. e.g.: + // node_modules/string-width + // node_modules/string-width/node_modules/strip-ansi + // we renamed to `node_modules` directory prefixes `workspace` when resolving Links + // node_modules/function1 + // node_modules/nested_func/node_modules/debug + if index := strings.LastIndex(path, nodeModulesDir); index != -1 { + return path[index+len(nodeModulesDir)+1:] + } + log.Warnf("npm %q package path doesn't have `node_modules` prefix", path) + return path +} + +func isWorkspace(pkgPath string, workspaces []string) bool { + for _, workspace := range workspaces { + if match, err := path.Match(workspace, pkgPath); err != nil { + log.Debugf("unable to parse workspace %q for %s", workspace, pkgPath) + } else if match { + return true + } + } + return false +} + +func joinPaths(paths ...string) string { + return strings.Join(paths, "/") +} + +func findDependsOn(pkgPath, depName string, packages map[string]*packageLockPackage) (*packageLockPackage, error) { + depPath := joinPaths(pkgPath, nodeModulesDir) + paths := strings.Split(depPath, "/") + // Try to resolve the version with the nearest directory + // e.g. for pkgPath == `node_modules/body-parser/node_modules/debug`, depName == `ms`: + // - "node_modules/body-parser/node_modules/debug/node_modules/ms" + // - "node_modules/body-parser/node_modules/ms" + // - "node_modules/ms" + for i := len(paths) - 1; i >= 0; i-- { + if paths[i] != nodeModulesDir { + continue + } + path := joinPaths(paths[:i+1]...) + path = joinPaths(path, depName) + + if dep, ok := packages[path]; ok { + return dep, nil + } + } + + // It should not reach here. + return nil, xerrors.Errorf("can't find dependsOn for %s", depName) } diff --git a/syft/pkg/cataloger/javascript/parse_package_lock_test.go b/syft/pkg/cataloger/javascript/parse_package_lock_test.go index 34a1017c22c..44b90b39489 100644 --- a/syft/pkg/cataloger/javascript/parse_package_lock_test.go +++ b/syft/pkg/cataloger/javascript/parse_package_lock_test.go @@ -12,6 +12,14 @@ import ( func TestParsePackageLock(t *testing.T) { var expectedRelationships []artifact.Relationship expectedPkgs := []pkg.Package{ + { + Name: "pkg-lock", + Version: "0.0.0", + PURL: "pkg:npm/pkg-lock@0.0.0", + Language: pkg.JavaScript, + Type: pkg.NpmPkg, + Metadata: pkg.NpmPackageLockEntry{}, + }, { Name: "@actions/core", Version: "1.6.0", @@ -110,62 +118,96 @@ func TestParsePackageLock(t *testing.T) { } func TestParsePackageLockV2(t *testing.T) { - fixture := "test-fixtures/pkg-lock/package-lock-2.json" - var expectedRelationships []artifact.Relationship + fixture := "test-fixtures/pkg-lock/lock-2/package-lock.json" + locationSet := file.NewLocationSet(file.NewLocation(fixture)) + npm := pkg.Package{ + Name: "npm", + Version: "6.14.6", + Language: pkg.JavaScript, + Type: pkg.NpmPkg, + PURL: "pkg:npm/npm@6.14.6", + Locations: locationSet, + Metadata: pkg.NpmPackageLockEntry{}, + } + propTypes := pkg.Package{ + Name: "@types/prop-types", + Version: "15.7.5", + PURL: "pkg:npm/%40types/prop-types@15.7.5", + Language: pkg.JavaScript, + Type: pkg.NpmPkg, + Licenses: pkg.NewLicenseSet( + pkg.NewLicenseFromLocations("MIT", file.NewLocation(fixture)), + ), + Locations: locationSet, + Metadata: pkg.NpmPackageLockEntry{Resolved: "https://registry.npmjs.org/@types/prop-types/-/prop-types-15.7.5.tgz", Integrity: "sha1-XxnSuFqY6VWANvajysyIGUIPBc8="}, + } + typesReact := pkg.Package{ + Name: "@types/react", + Version: "18.0.17", + PURL: "pkg:npm/%40types/react@18.0.17", + Language: pkg.JavaScript, + Type: pkg.NpmPkg, + Licenses: pkg.NewLicenseSet( + pkg.NewLicenseFromLocations("MIT", file.NewLocation(fixture)), + ), + Locations: locationSet, + Metadata: pkg.NpmPackageLockEntry{Resolved: "https://registry.npmjs.org/@types/react/-/react-18.0.17.tgz", Integrity: "sha1-RYPZwyLWfv5LOak10iPtzHBQzPQ="}, + } + scheduler := pkg.Package{ + Name: "@types/scheduler", + Version: "0.16.2", + PURL: "pkg:npm/%40types/scheduler@0.16.2", + Language: pkg.JavaScript, + Type: pkg.NpmPkg, + Licenses: pkg.NewLicenseSet( + pkg.NewLicenseFromLocations("MIT", file.NewLocation(fixture)), + ), + Locations: locationSet, + Metadata: pkg.NpmPackageLockEntry{Resolved: "https://registry.npmjs.org/@types/scheduler/-/scheduler-0.16.2.tgz", Integrity: "sha1-GmL4lSVyPd4kuhsBsJK/XfitTTk="}, + } + csstype := pkg.Package{ + Name: "csstype", + Version: "3.1.0", + PURL: "pkg:npm/csstype@3.1.0", + Language: pkg.JavaScript, + Type: pkg.NpmPkg, + Licenses: pkg.NewLicenseSet( + pkg.NewLicenseFromLocations("MIT", file.NewLocation(fixture)), + ), + Locations: locationSet, + Metadata: pkg.NpmPackageLockEntry{Resolved: "https://registry.npmjs.org/csstype/-/csstype-3.1.0.tgz", Integrity: "sha1-TdysNxjXh8+d8NG30VAzklyPKfI="}, + } + expectedPkgs := []pkg.Package{ + npm, + propTypes, + typesReact, + scheduler, + csstype, + } + expectedRelationships := []artifact.Relationship{ { - Name: "npm", - Version: "6.14.6", - Language: pkg.JavaScript, - Type: pkg.NpmPkg, - PURL: "pkg:npm/npm@6.14.6", - Metadata: pkg.NpmPackageLockEntry{}, - }, - { - Name: "@types/prop-types", - Version: "15.7.5", - PURL: "pkg:npm/%40types/prop-types@15.7.5", - Language: pkg.JavaScript, - Type: pkg.NpmPkg, - Licenses: pkg.NewLicenseSet( - pkg.NewLicenseFromLocations("MIT", file.NewLocation(fixture)), - ), - Metadata: pkg.NpmPackageLockEntry{Resolved: "https://registry.npmjs.org/@types/prop-types/-/prop-types-15.7.5.tgz", Integrity: "sha1-XxnSuFqY6VWANvajysyIGUIPBc8="}, + From: propTypes, + To: typesReact, + Type: artifact.DependencyOfRelationship, }, { - Name: "@types/react", - Version: "18.0.17", - PURL: "pkg:npm/%40types/react@18.0.17", - Language: pkg.JavaScript, - Type: pkg.NpmPkg, - Licenses: pkg.NewLicenseSet( - pkg.NewLicenseFromLocations("MIT", file.NewLocation(fixture)), - ), - Metadata: pkg.NpmPackageLockEntry{Resolved: "https://registry.npmjs.org/@types/react/-/react-18.0.17.tgz", Integrity: "sha1-RYPZwyLWfv5LOak10iPtzHBQzPQ="}, + From: typesReact, + To: npm, + Type: artifact.DependencyOfRelationship, }, { - Name: "@types/scheduler", - Version: "0.16.2", - PURL: "pkg:npm/%40types/scheduler@0.16.2", - Language: pkg.JavaScript, - Type: pkg.NpmPkg, - Licenses: pkg.NewLicenseSet( - pkg.NewLicenseFromLocations("MIT", file.NewLocation(fixture)), - ), - Metadata: pkg.NpmPackageLockEntry{Resolved: "https://registry.npmjs.org/@types/scheduler/-/scheduler-0.16.2.tgz", Integrity: "sha1-GmL4lSVyPd4kuhsBsJK/XfitTTk="}, + From: scheduler, + To: typesReact, + Type: artifact.DependencyOfRelationship, }, { - Name: "csstype", - Version: "3.1.0", - PURL: "pkg:npm/csstype@3.1.0", - Language: pkg.JavaScript, - Type: pkg.NpmPkg, - Licenses: pkg.NewLicenseSet( - pkg.NewLicenseFromLocations("MIT", file.NewLocation(fixture)), - ), - Metadata: pkg.NpmPackageLockEntry{Resolved: "https://registry.npmjs.org/csstype/-/csstype-3.1.0.tgz", Integrity: "sha1-TdysNxjXh8+d8NG30VAzklyPKfI="}, + From: csstype, + To: typesReact, + Type: artifact.DependencyOfRelationship, }, } + for i := range expectedPkgs { expectedPkgs[i].Locations.Add(file.NewLocation(fixture)) } @@ -173,50 +215,85 @@ func TestParsePackageLockV2(t *testing.T) { } func TestParsePackageLockV3(t *testing.T) { - fixture := "test-fixtures/pkg-lock/package-lock-3.json" - var expectedRelationships []artifact.Relationship + fixture := "test-fixtures/pkg-lock/lock-3/package-lock.json" + locationSet := file.NewLocationSet(file.NewLocation(fixture)) + lockV3Fixture := pkg.Package{ + Name: "lock-v3-fixture", + Version: "1.0.0", + Language: pkg.JavaScript, + Type: pkg.NpmPkg, + PURL: "pkg:npm/lock-v3-fixture@1.0.0", + Locations: locationSet, + Metadata: pkg.NpmPackageLockEntry{}, + } + propTypes := pkg.Package{ + Name: "@types/prop-types", + Version: "15.7.5", + Language: pkg.JavaScript, + Type: pkg.NpmPkg, + PURL: "pkg:npm/%40types/prop-types@15.7.5", + Locations: locationSet, + Metadata: pkg.NpmPackageLockEntry{Resolved: "https://registry.npmjs.org/@types/prop-types/-/prop-types-15.7.5.tgz", Integrity: "sha512-JCB8C6SnDoQf0cNycqd/35A7MjcnK+ZTqE7judS6o7utxUCg6imJg3QK2qzHKszlTjcj2cn+NwMB2i96ubpj7w=="}, + } + typesReact := pkg.Package{ + Name: "@types/react", + Version: "18.0.20", + Language: pkg.JavaScript, + Type: pkg.NpmPkg, + PURL: "pkg:npm/%40types/react@18.0.20", + Locations: locationSet, + Metadata: pkg.NpmPackageLockEntry{Resolved: "https://registry.npmjs.org/@types/react/-/react-18.0.20.tgz", Integrity: "sha512-MWul1teSPxujEHVwZl4a5HxQ9vVNsjTchVA+xRqv/VYGCuKGAU6UhfrTdF5aBefwD1BHUD8i/zq+O/vyCm/FrA=="}, + } + scheduler := pkg.Package{ + Name: "@types/scheduler", + Version: "0.16.2", + Language: pkg.JavaScript, + Type: pkg.NpmPkg, + PURL: "pkg:npm/%40types/scheduler@0.16.2", + Locations: locationSet, + Metadata: pkg.NpmPackageLockEntry{Resolved: "https://registry.npmjs.org/@types/scheduler/-/scheduler-0.16.2.tgz", Integrity: "sha512-hppQEBDmlwhFAXKJX2KnWLYu5yMfi91yazPb2l+lbJiwW+wdo1gNeRA+3RgNSO39WYX2euey41KEwnqesU2Jew=="}, + } + csstype := pkg.Package{ + Name: "csstype", + Version: "3.1.1", + Language: pkg.JavaScript, + Type: pkg.NpmPkg, + PURL: "pkg:npm/csstype@3.1.1", + Locations: locationSet, + Metadata: pkg.NpmPackageLockEntry{Resolved: "https://registry.npmjs.org/csstype/-/csstype-3.1.1.tgz", Integrity: "sha512-DJR/VvkAvSZW9bTouZue2sSxDwdTN92uHjqeKVm+0dAqdfNykRzQ95tay8aXMBAAPpUiq4Qcug2L7neoRh2Egw=="}, + } + expectedPkgs := []pkg.Package{ + lockV3Fixture, + propTypes, + typesReact, + scheduler, + csstype, + } + + expectedRelationships := []artifact.Relationship{ { - Name: "lock-v3-fixture", - Version: "1.0.0", - Language: pkg.JavaScript, - Type: pkg.NpmPkg, - PURL: "pkg:npm/lock-v3-fixture@1.0.0", - Metadata: pkg.NpmPackageLockEntry{}, - }, - { - Name: "@types/prop-types", - Version: "15.7.5", - Language: pkg.JavaScript, - Type: pkg.NpmPkg, - PURL: "pkg:npm/%40types/prop-types@15.7.5", - Metadata: pkg.NpmPackageLockEntry{Resolved: "https://registry.npmjs.org/@types/prop-types/-/prop-types-15.7.5.tgz", Integrity: "sha512-JCB8C6SnDoQf0cNycqd/35A7MjcnK+ZTqE7judS6o7utxUCg6imJg3QK2qzHKszlTjcj2cn+NwMB2i96ubpj7w=="}, + From: propTypes, + To: typesReact, + Type: artifact.DependencyOfRelationship, }, { - Name: "@types/react", - Version: "18.0.20", - Language: pkg.JavaScript, - Type: pkg.NpmPkg, - PURL: "pkg:npm/%40types/react@18.0.20", - Metadata: pkg.NpmPackageLockEntry{Resolved: "https://registry.npmjs.org/@types/react/-/react-18.0.20.tgz", Integrity: "sha512-MWul1teSPxujEHVwZl4a5HxQ9vVNsjTchVA+xRqv/VYGCuKGAU6UhfrTdF5aBefwD1BHUD8i/zq+O/vyCm/FrA=="}, + From: typesReact, + To: lockV3Fixture, + Type: artifact.DependencyOfRelationship, }, { - Name: "@types/scheduler", - Version: "0.16.2", - Language: pkg.JavaScript, - Type: pkg.NpmPkg, - PURL: "pkg:npm/%40types/scheduler@0.16.2", - Metadata: pkg.NpmPackageLockEntry{Resolved: "https://registry.npmjs.org/@types/scheduler/-/scheduler-0.16.2.tgz", Integrity: "sha512-hppQEBDmlwhFAXKJX2KnWLYu5yMfi91yazPb2l+lbJiwW+wdo1gNeRA+3RgNSO39WYX2euey41KEwnqesU2Jew=="}, + From: scheduler, + To: typesReact, + Type: artifact.DependencyOfRelationship, }, { - Name: "csstype", - Version: "3.1.1", - Language: pkg.JavaScript, - Type: pkg.NpmPkg, - PURL: "pkg:npm/csstype@3.1.1", - Metadata: pkg.NpmPackageLockEntry{Resolved: "https://registry.npmjs.org/csstype/-/csstype-3.1.1.tgz", Integrity: "sha512-DJR/VvkAvSZW9bTouZue2sSxDwdTN92uHjqeKVm+0dAqdfNykRzQ95tay8aXMBAAPpUiq4Qcug2L7neoRh2Egw=="}, + From: csstype, + To: typesReact, + Type: artifact.DependencyOfRelationship, }, } + for i := range expectedPkgs { expectedPkgs[i].Locations.Add(file.NewLocation(fixture)) } @@ -226,6 +303,14 @@ func TestParsePackageLockV3(t *testing.T) { func TestParsePackageLockAlias(t *testing.T) { var expectedRelationships []artifact.Relationship commonPkgs := []pkg.Package{ + { + Name: "alias-check", + Version: "1.0.0", + PURL: "pkg:npm/alias-check@1.0.0", + Language: pkg.JavaScript, + Type: pkg.NpmPkg, + Metadata: pkg.NpmPackageLockEntry{}, + }, { Name: "case", Version: "1.6.2", @@ -252,9 +337,9 @@ func TestParsePackageLockAlias(t *testing.T) { }, } - packageLockV1 := "test-fixtures/pkg-lock/alias-package-lock-1.json" - packageLockV2 := "test-fixtures/pkg-lock/alias-package-lock-2.json" - packageLocks := []string{packageLockV1, packageLockV2} + packageLockV1 := "test-fixtures/pkg-lock/alias-1/package-lock.json" + packageLockV2 := "test-fixtures/pkg-lock/alias-2/package-lock.json" + packageLocks := []string{packageLockV1} v2Pkg := pkg.Package{ Name: "alias-check", @@ -284,43 +369,66 @@ func TestParsePackageLockAlias(t *testing.T) { } func TestParsePackageLockLicenseWithArray(t *testing.T) { - fixture := "test-fixtures/pkg-lock/array-license-package-lock.json" - var expectedRelationships []artifact.Relationship + fixture := "test-fixtures/pkg-lock/array-license/package-lock.json" + locationSet := file.NewLocationSet(file.NewLocation(fixture)) + pauseStream := pkg.Package{ + Name: "pause-stream", + Version: "0.0.11", + Language: pkg.JavaScript, + Type: pkg.NpmPkg, + Licenses: pkg.NewLicenseSet( + pkg.NewLicenseFromLocations("MIT", file.NewLocation(fixture)), + pkg.NewLicenseFromLocations("Apache2", file.NewLocation(fixture)), + ), + Locations: locationSet, + PURL: "pkg:npm/pause-stream@0.0.11", + + Metadata: pkg.NpmPackageLockEntry{}, + } + through := pkg.Package{ + Name: "through", + Version: "2.3.8", + Language: pkg.JavaScript, + Type: pkg.NpmPkg, + Licenses: pkg.NewLicenseSet( + pkg.NewLicenseFromLocations("MIT", file.NewLocation(fixture)), + ), + Locations: locationSet, + PURL: "pkg:npm/through@2.3.8", + + Metadata: pkg.NpmPackageLockEntry{}, + } + tmp := pkg.Package{ + Name: "tmp", + Version: "1.0.0", + Language: pkg.JavaScript, + Type: pkg.NpmPkg, + Licenses: pkg.NewLicenseSet( + pkg.NewLicenseFromLocations("ISC", file.NewLocation(fixture)), + ), + Locations: locationSet, + PURL: "pkg:npm/tmp@1.0.0", + + Metadata: pkg.NpmPackageLockEntry{}, + } + expectedPkgs := []pkg.Package{ + pauseStream, + through, + tmp, + } + expectedRelationships := []artifact.Relationship{ { - Name: "tmp", - Version: "1.0.0", - Language: pkg.JavaScript, - Type: pkg.NpmPkg, - Licenses: pkg.NewLicenseSet( - pkg.NewLicenseFromLocations("ISC", file.NewLocation(fixture)), - ), - PURL: "pkg:npm/tmp@1.0.0", - Metadata: pkg.NpmPackageLockEntry{}, - }, - { - Name: "pause-stream", - Version: "0.0.11", - Language: pkg.JavaScript, - Type: pkg.NpmPkg, - - Licenses: pkg.NewLicenseSet( - pkg.NewLicenseFromLocations("MIT", file.NewLocation(fixture)), - pkg.NewLicenseFromLocations("Apache2", file.NewLocation(fixture)), - ), - PURL: "pkg:npm/pause-stream@0.0.11", - Metadata: pkg.NpmPackageLockEntry{}, + From: pauseStream, + To: tmp, + Type: artifact.DependencyOfRelationship, + Data: nil, }, { - Name: "through", - Version: "2.3.8", - Language: pkg.JavaScript, - Type: pkg.NpmPkg, - Licenses: pkg.NewLicenseSet( - pkg.NewLicenseFromLocations("MIT", file.NewLocation(fixture)), - ), - PURL: "pkg:npm/through@2.3.8", - Metadata: pkg.NpmPackageLockEntry{}, + From: through, + To: pauseStream, + Type: artifact.DependencyOfRelationship, + Data: nil, }, } for i := range expectedPkgs { diff --git a/syft/pkg/cataloger/javascript/parse_pnpm_lock.go b/syft/pkg/cataloger/javascript/parse_pnpm_lock.go index 1b786752e67..5f719d7045c 100644 --- a/syft/pkg/cataloger/javascript/parse_pnpm_lock.go +++ b/syft/pkg/cataloger/javascript/parse_pnpm_lock.go @@ -1,7 +1,6 @@ package javascript import ( - "fmt" "io" "regexp" "strconv" @@ -14,67 +13,158 @@ import ( "github.com/anchore/syft/syft/file" "github.com/anchore/syft/syft/pkg" "github.com/anchore/syft/syft/pkg/cataloger/generic" + "github.com/anchore/syft/syft/pkg/cataloger/javascript/key" ) // integrity check var _ generic.Parser = parsePnpmLock +type pnpmLockPackage struct { + Name string + Version string + Integrity string + Resolved string + Dependencies map[string]string +} + type pnpmLockYaml struct { - Version string `json:"lockfileVersion" yaml:"lockfileVersion"` - Dependencies map[string]interface{} `json:"dependencies" yaml:"dependencies"` - Packages map[string]interface{} `json:"packages" yaml:"packages"` + Version string `yaml:"lockfileVersion"` + Specifiers map[string]interface{} `yaml:"specifiers"` + Dependencies map[string]interface{} `yaml:"dependencies"` + DevDependencies map[string]interface{} `yaml:"devDependencies"` + Packages map[string]*pnpmLockPackage `yaml:"packages"` } -func parsePnpmLock(resolver file.Resolver, _ *generic.Environment, reader file.LocationReadCloser) ([]pkg.Package, []artifact.Relationship, error) { - bytes, err := io.ReadAll(reader) - if err != nil { - return nil, nil, fmt.Errorf("failed to load pnpm-lock.yaml file: %w", err) +func newPnpmLockPackage(resolver file.Resolver, location file.Location, p *pnpmLockPackage) pkg.Package { + if p == nil { + return pkg.Package{} } + return finalizeLockPkg( + resolver, + location, + pkg.Package{ + Name: p.Name, + Version: p.Version, + Locations: file.NewLocationSet(location.WithAnnotation(pkg.EvidenceAnnotationKey, pkg.PrimaryEvidenceAnnotation)), + PURL: packageURL(p.Name, p.Version), + Language: pkg.JavaScript, + Type: pkg.NpmPkg, + Metadata: pkg.NpmPackageLockEntry{ + Resolved: p.Resolved, + Integrity: p.Integrity, + }, + }, + ) +} + +func parsePnpmLock(resolver file.Resolver, _ *generic.Environment, reader file.LocationReadCloser) ([]pkg.Package, []artifact.Relationship, error) { + pnpmMap, pnpmLock := parsePnpmLockFile(reader) + pkgs, _ := finalizePnpmLockWithoutPackageJSON(resolver, &pnpmLock, pnpmMap, reader.Location) + return pkgs, nil, nil +} + +func finalizePnpmLockWithoutPackageJSON(resolver file.Resolver, _ *pnpmLockYaml, pnpmMap map[string]*pnpmLockPackage, indexLocation file.Location) ([]pkg.Package, []artifact.Relationship) { + seenPkgMap := make(map[string]bool) var pkgs []pkg.Package - var lockFile pnpmLockYaml + var relationships []artifact.Relationship - if err := yaml.Unmarshal(bytes, &lockFile); err != nil { - return nil, nil, fmt.Errorf("failed to parse pnpm-lock.yaml file: %w", err) + name := rootNameFromPath(indexLocation) + if name == "" { + return nil, nil } + root := newPnpmLockPackage(resolver, indexLocation, &pnpmLockPackage{Name: name, Version: "0.0.0"}) + pkgs = append(pkgs, root) - lockVersion, _ := strconv.ParseFloat(lockFile.Version, 64) + for _, lockPkg := range pnpmMap { + if seenPkgMap[key.NpmPackageKey(lockPkg.Name, lockPkg.Version)] { + continue + } - for name, info := range lockFile.Dependencies { - version := "" - - switch info := info.(type) { - case string: - version = info - case map[string]interface{}: - v, ok := info["version"] - if !ok { - break - } - ver, ok := v.(string) - if ok { - version = parseVersion(ver) - } - default: - log.Tracef("unsupported pnpm dependency type: %+v", info) + pkg := newPnpmLockPackage(resolver, indexLocation, lockPkg) + pkgs = append(pkgs, pkg) + seenPkgMap[key.NpmPackageKey(pkg.Name, pkg.Version)] = true + } + + pkg.Sort(pkgs) + pkg.SortRelationships(relationships) + return pkgs, relationships +} + +func finalizePnpmLockWithPackageJSON(resolver file.Resolver, pkgjson *packageJSON, pnpmLock *pnpmLockYaml, pnpmMap map[string]*pnpmLockPackage, indexLocation file.Location) ([]pkg.Package, []artifact.Relationship) { + seenPkgMap := make(map[string]bool) + var pkgs []pkg.Package + var relationships []artifact.Relationship + + root := newPnpmLockPackage(resolver, indexLocation, &pnpmLockPackage{Name: pkgjson.Name, Version: pkgjson.Version}) + pkgs = append(pkgs, root) + + // create root relationships + for name, info := range pnpmLock.Dependencies { + version := parsePnpmDependencyInfo(info) + if version == "" { continue } - if hasPkg(pkgs, name, version) { + p := pnpmMap[key.NpmPackageKey(name, version)] + pkg := newPnpmLockPackage(resolver, indexLocation, p) + rel := artifact.Relationship{ + From: pkg, + To: root, + Type: artifact.DependencyOfRelationship, + } + relationships = append(relationships, rel) + } + + // create packages + for _, lockPkg := range pnpmMap { + if seenPkgMap[key.NpmPackageKey(lockPkg.Name, lockPkg.Version)] { continue } - pkgs = append(pkgs, newPnpmPackage(resolver, reader.Location, name, version)) + pkg := newPnpmLockPackage(resolver, indexLocation, lockPkg) + pkgs = append(pkgs, pkg) + seenPkgMap[key.NpmPackageKey(pkg.Name, pkg.Version)] = true + } + + // create pkg relationships + for _, lockPkg := range pnpmMap { + p := pnpmMap[key.NpmPackageKey(lockPkg.Name, lockPkg.Version)] + pkg := newPnpmLockPackage( + resolver, + indexLocation, + p, + ) + + for name, version := range lockPkg.Dependencies { + dep := pnpmMap[key.NpmPackageKey(name, version)] + depPkg := newPnpmLockPackage( + resolver, + indexLocation, + dep, + ) + rel := artifact.Relationship{ + From: depPkg, + To: pkg, + Type: artifact.DependencyOfRelationship, + } + relationships = append(relationships, rel) + } } + pkg.Sort(pkgs) + pkg.SortRelationships(relationships) + return pkgs, relationships +} + +func parsePnpmPackages(lockFile pnpmLockYaml, lockVersion float64, pnpmLock map[string]*pnpmLockPackage) { packageNameRegex := regexp.MustCompile(`^/?([^(]*)(?:\(.*\))*$`) splitChar := "/" if lockVersion >= 6.0 { splitChar = "@" } - // parse packages from packages section of pnpm-lock.yaml - for nameVersion := range lockFile.Packages { + for nameVersion, packageDetails := range lockFile.Packages { nameVersion = packageNameRegex.ReplaceAllString(nameVersion, "$1") nameVersionSplit := strings.Split(strings.TrimPrefix(nameVersion, "/"), splitChar) @@ -84,25 +174,77 @@ func parsePnpmLock(resolver file.Resolver, _ *generic.Environment, reader file.L // construct name from all array items other than last item (version) name := strings.Join(nameVersionSplit[:len(nameVersionSplit)-1], splitChar) - if hasPkg(pkgs, name, version) { - continue + if pnpmLock[key.NpmPackageKey(name, version)] != nil { + if pnpmLock[key.NpmPackageKey(name, version)].Version == version { + continue + } } - pkgs = append(pkgs, newPnpmPackage(resolver, reader.Location, name, version)) - } + packageDetails.Name = name + packageDetails.Version = version - pkg.Sort(pkgs) + pnpmLock[key.NpmPackageKey(name, version)] = packageDetails + } +} - return pkgs, nil, nil +func parsePnpmDependencyInfo(info interface{}) (version string) { + switch info := info.(type) { + case string: + version = info + case map[string]interface{}: + v, ok := info["version"] + if !ok { + break + } + ver, ok := v.(string) + if ok { + version = parseVersion(ver) + } + } + log.Tracef("unsupported pnpm dependency type: %+v", info) + return } -func hasPkg(pkgs []pkg.Package, name, version string) bool { - for _, p := range pkgs { - if p.Name == name && p.Version == version { - return true +func parsePnpmDependencies(lockFile pnpmLockYaml, pnpmLock map[string]*pnpmLockPackage) { + for name, info := range lockFile.Dependencies { + version := parsePnpmDependencyInfo(info) + if version == "" { + continue } + + if pnpmLock[key.NpmPackageKey(name, version)] != nil { + if pnpmLock[key.NpmPackageKey(name, version)].Version == version { + continue + } + } + + pnpmLock[key.NpmPackageKey(name, version)] = &pnpmLockPackage{ + Name: name, + Version: version, + } + } +} + +// parsePnpmLock parses a pnpm-lock.yaml file to get a list of packages +func parsePnpmLockFile(file file.LocationReadCloser) (map[string]*pnpmLockPackage, pnpmLockYaml) { + pnpmLock := map[string]*pnpmLockPackage{} + bytes, err := io.ReadAll(file) + if err != nil { + return pnpmLock, pnpmLockYaml{} + } + + var lockFile pnpmLockYaml + if err := yaml.Unmarshal(bytes, &lockFile); err != nil { + return pnpmLock, pnpmLockYaml{} } - return false + + lockVersion, _ := strconv.ParseFloat(lockFile.Version, 64) + + // parse packages from packages section of pnpm-lock.yaml + parsePnpmPackages(lockFile, lockVersion, pnpmLock) + parsePnpmDependencies(lockFile, pnpmLock) + + return pnpmLock, lockFile } func parseVersion(version string) string { diff --git a/syft/pkg/cataloger/javascript/parse_pnpm_lock_test.go b/syft/pkg/cataloger/javascript/parse_pnpm_lock_test.go index 7c0ed1c4db8..510c2f4307c 100644 --- a/syft/pkg/cataloger/javascript/parse_pnpm_lock_test.go +++ b/syft/pkg/cataloger/javascript/parse_pnpm_lock_test.go @@ -16,12 +16,22 @@ func TestParsePnpmLock(t *testing.T) { locationSet := file.NewLocationSet(file.NewLocation(fixture)) expectedPkgs := []pkg.Package{ + { + Name: "pnpm", + Version: "0.0.0", + PURL: "pkg:npm/pnpm@0.0.0", + Locations: locationSet, + Language: pkg.JavaScript, + Metadata: pkg.NpmPackageLockEntry{}, + Type: pkg.NpmPkg, + }, { Name: "nanoid", Version: "3.3.4", PURL: "pkg:npm/nanoid@3.3.4", Locations: locationSet, Language: pkg.JavaScript, + Metadata: pkg.NpmPackageLockEntry{}, Type: pkg.NpmPkg, }, { @@ -30,6 +40,7 @@ func TestParsePnpmLock(t *testing.T) { PURL: "pkg:npm/picocolors@1.0.0", Locations: locationSet, Language: pkg.JavaScript, + Metadata: pkg.NpmPackageLockEntry{}, Type: pkg.NpmPkg, }, { @@ -38,6 +49,7 @@ func TestParsePnpmLock(t *testing.T) { PURL: "pkg:npm/source-map-js@1.0.2", Locations: locationSet, Language: pkg.JavaScript, + Metadata: pkg.NpmPackageLockEntry{}, Type: pkg.NpmPkg, }, { @@ -46,6 +58,7 @@ func TestParsePnpmLock(t *testing.T) { PURL: "pkg:npm/%40bcoe/v8-coverage@0.2.3", Locations: locationSet, Language: pkg.JavaScript, + Metadata: pkg.NpmPackageLockEntry{}, Type: pkg.NpmPkg, }, } @@ -60,12 +73,22 @@ func TestParsePnpmV6Lock(t *testing.T) { locationSet := file.NewLocationSet(file.NewLocation(fixture)) expectedPkgs := []pkg.Package{ + { + Name: "pnpm-v6", + Version: "0.0.0", + PURL: "pkg:npm/pnpm-v6@0.0.0", + Locations: locationSet, + Language: pkg.JavaScript, + Metadata: pkg.NpmPackageLockEntry{}, + Type: pkg.NpmPkg, + }, { Name: "@testing-library/jest-dom", Version: "5.16.5", PURL: "pkg:npm/%40testing-library/jest-dom@5.16.5", Locations: locationSet, Language: pkg.JavaScript, + Metadata: pkg.NpmPackageLockEntry{}, Type: pkg.NpmPkg, }, { @@ -74,6 +97,7 @@ func TestParsePnpmV6Lock(t *testing.T) { PURL: "pkg:npm/%40testing-library/react@13.4.0", Locations: locationSet, Language: pkg.JavaScript, + Metadata: pkg.NpmPackageLockEntry{}, Type: pkg.NpmPkg, }, { @@ -82,6 +106,7 @@ func TestParsePnpmV6Lock(t *testing.T) { PURL: "pkg:npm/%40testing-library/user-event@13.5.0", Locations: locationSet, Language: pkg.JavaScript, + Metadata: pkg.NpmPackageLockEntry{}, Type: pkg.NpmPkg, }, { @@ -90,6 +115,7 @@ func TestParsePnpmV6Lock(t *testing.T) { PURL: "pkg:npm/react@18.2.0", Locations: locationSet, Language: pkg.JavaScript, + Metadata: pkg.NpmPackageLockEntry{}, Type: pkg.NpmPkg, }, { @@ -98,6 +124,7 @@ func TestParsePnpmV6Lock(t *testing.T) { PURL: "pkg:npm/react-dom@18.2.0", Locations: locationSet, Language: pkg.JavaScript, + Metadata: pkg.NpmPackageLockEntry{}, Type: pkg.NpmPkg, }, { @@ -106,6 +133,7 @@ func TestParsePnpmV6Lock(t *testing.T) { PURL: "pkg:npm/web-vitals@2.1.4", Locations: locationSet, Language: pkg.JavaScript, + Metadata: pkg.NpmPackageLockEntry{}, Type: pkg.NpmPkg, }, { @@ -114,6 +142,7 @@ func TestParsePnpmV6Lock(t *testing.T) { PURL: "pkg:npm/%40babel/core@7.21.4", Locations: locationSet, Language: pkg.JavaScript, + Metadata: pkg.NpmPackageLockEntry{}, Type: pkg.NpmPkg, }, { @@ -122,6 +151,7 @@ func TestParsePnpmV6Lock(t *testing.T) { PURL: "pkg:npm/%40types/eslint@8.37.0", Locations: locationSet, Language: pkg.JavaScript, + Metadata: pkg.NpmPackageLockEntry{}, Type: pkg.NpmPkg, }, { @@ -130,6 +160,7 @@ func TestParsePnpmV6Lock(t *testing.T) { PURL: "pkg:npm/read-cache@1.0.0", Locations: locationSet, Language: pkg.JavaScript, + Metadata: pkg.NpmPackageLockEntry{}, Type: pkg.NpmPkg, }, { @@ -138,6 +169,7 @@ func TestParsePnpmV6Lock(t *testing.T) { PURL: "pkg:npm/schema-utils@3.1.2", Locations: locationSet, Language: pkg.JavaScript, + Metadata: pkg.NpmPackageLockEntry{}, Type: pkg.NpmPkg, }, } diff --git a/syft/pkg/cataloger/javascript/parse_yarn_lock.go b/syft/pkg/cataloger/javascript/parse_yarn_lock.go index d42490ed30d..28b92ac38db 100644 --- a/syft/pkg/cataloger/javascript/parse_yarn_lock.go +++ b/syft/pkg/cataloger/javascript/parse_yarn_lock.go @@ -2,46 +2,26 @@ package javascript import ( "bufio" - "fmt" - "regexp" - - "github.com/scylladb/go-set/strset" + "strings" "github.com/anchore/syft/syft/artifact" "github.com/anchore/syft/syft/file" "github.com/anchore/syft/syft/pkg" "github.com/anchore/syft/syft/pkg/cataloger/generic" + "github.com/anchore/syft/syft/pkg/cataloger/javascript/key" + yarnparse "github.com/anchore/syft/syft/pkg/cataloger/javascript/parser/yarn" ) // integrity check var _ generic.Parser = parseYarnLock -var ( - // packageNameExp matches the name of the dependency in yarn.lock - // including scope/namespace prefix if found. - // For example: "aws-sdk@2.706.0" returns "aws-sdk" - // "@babel/code-frame@^7.0.0" returns "@babel/code-frame" - packageNameExp = regexp.MustCompile(`^"?((?:@\w[\w-_.]*\/)?\w[\w-_.]*)@`) - - // versionExp matches the "version" line of a yarn.lock entry and captures the version value. - // For example: version "4.10.1" (...and the value "4.10.1" is captured) - versionExp = regexp.MustCompile(`^\W+version(?:\W+"|:\W+)([\w-_.]+)"?`) - - // packageURLExp matches the name and version of the dependency in yarn.lock - // from the resolved URL, including scope/namespace prefix if any. - // For example: - // `resolved "https://registry.yarnpkg.com/async/-/async-3.2.3.tgz#ac53dafd3f4720ee9e8a160628f18ea91df196c9"` - // would return "async" and "3.2.3" - // - // `resolved "https://registry.yarnpkg.com/@4lolo/resize-observer-polyfill/-/resize-observer-polyfill-1.5.2.tgz#58868fc7224506236b5550d0c68357f0a874b84b"` - // would return "@4lolo/resize-observer-polyfill" and "1.5.2" - packageURLExp = regexp.MustCompile(`^\s+resolved\s+"https://registry\.(?:yarnpkg\.com|npmjs\.org)/(.+?)/-/(?:.+?)-(\d+\..+?)\.tgz`) -) - -const ( - noPackage = "" - noVersion = "" -) +type yarnLockPackage struct { + Name string + Version string + Integrity string + Resolved string + Dependencies map[string]string +} func parseYarnLock(resolver file.Resolver, _ *generic.Environment, reader file.LocationReadCloser) ([]pkg.Package, []artifact.Relationship, error) { // in the case we find yarn.lock files in the node_modules directories, skip those @@ -50,70 +30,238 @@ func parseYarnLock(resolver file.Resolver, _ *generic.Environment, reader file.L return nil, nil, nil } - var pkgs []pkg.Package - scanner := bufio.NewScanner(reader) - parsedPackages := strset.New() - currentPackage := noPackage - currentVersion := noVersion + yarnMap := parseYarnLockFile(resolver, reader) + pkgs, _ := finalizeYarnLockWithoutPackageJSON(resolver, yarnMap, reader.Location) + return pkgs, nil, nil +} - for scanner.Scan() { - line := scanner.Text() +func newYarnLockPackage(resolver file.Resolver, location file.Location, p *yarnLockPackage) pkg.Package { + if p == nil { + return pkg.Package{} + } - if packageName := findPackageName(line); packageName != noPackage { - // When we find a new package, check if we have unsaved identifiers - if currentPackage != noPackage && currentVersion != noVersion && !parsedPackages.Has(currentPackage+"@"+currentVersion) { - pkgs = append(pkgs, newYarnLockPackage(resolver, reader.Location, currentPackage, currentVersion)) - parsedPackages.Add(currentPackage + "@" + currentVersion) - } + return finalizeLockPkg( + resolver, + location, + pkg.Package{ + Name: p.Name, + Version: p.Version, + Locations: file.NewLocationSet(location.WithAnnotation(pkg.EvidenceAnnotationKey, pkg.PrimaryEvidenceAnnotation)), + PURL: packageURL(p.Name, p.Version), + Language: pkg.JavaScript, + Type: pkg.NpmPkg, + Metadata: pkg.NpmPackageLockEntry{ + Resolved: p.Resolved, + Integrity: p.Integrity, + }, + }, + ) +} - currentPackage = packageName - } else if version := findPackageVersion(line); version != noVersion { - currentVersion = version - } else if packageName, version := findPackageAndVersion(line); packageName != noPackage && version != noVersion && !parsedPackages.Has(packageName+"@"+version) { - pkgs = append(pkgs, newYarnLockPackage(resolver, reader.Location, packageName, version)) - parsedPackages.Add(packageName + "@" + version) +// parseYarnLockFile takes a yarn.lock file and returns a map of packages +func parseYarnLockFile(_ file.Resolver, file file.LocationReadCloser) map[string]*yarnLockPackage { + /* + name@version[, name@version]: + version "xxx" + resolved "xxx" + integrity "xxx" + dependencies: + name "xxx" + name "xxx" + */ + lineNumber := 1 + lockMap := map[string]*yarnLockPackage{} - // Cleanup to indicate no unsaved identifiers - currentPackage = noPackage - currentVersion = noVersion + scanner := bufio.NewScanner(file.ReadCloser) + scanner.Split(yarnparse.ScanBlocks) + for scanner.Scan() { + block := scanner.Bytes() + pkg, refVersions, newLine, err := parseYarnPkgBlock(block, lineNumber) + lineNumber = newLine + 2 + if err != nil { + return nil + } else if pkg.Name == "" { + continue } - } - // check if we have valid unsaved data after end-of-file has reached - if currentPackage != noPackage && currentVersion != noVersion && !parsedPackages.Has(currentPackage+"@"+currentVersion) { - pkgs = append(pkgs, newYarnLockPackage(resolver, reader.Location, currentPackage, currentVersion)) - parsedPackages.Add(currentPackage + "@" + currentVersion) + for _, refVersion := range refVersions { + lockMap[refVersion] = &pkg + } } if err := scanner.Err(); err != nil { - return nil, nil, fmt.Errorf("failed to parse yarn.lock file: %w", err) + return nil } - pkg.Sort(pkgs) + return lockMap +} - return pkgs, nil, nil +// rootNameFromPath returns a "fake" root name of a package based on it +// directory name. This is used when there is no package.json file +// to create a root package. +func rootNameFromPath(location file.Location) string { + splits := strings.Split(location.RealPath, "/") + if len(splits) < 2 { + return "" + } + return splits[len(splits)-2] } -func findPackageName(line string) string { - if matches := packageNameExp.FindStringSubmatch(line); len(matches) >= 2 { - return matches[1] +// finalizeYarnLockWithPackageJSON takes a yarn.lock file and a package.json file and returns a map of packages +// nolint:funlen +func finalizeYarnLockWithPackageJSON(resolver file.Resolver, pkgjson *packageJSON, yarnlock map[string]*yarnLockPackage, indexLocation file.Location) ([]pkg.Package, []artifact.Relationship) { + if pkgjson == nil { + return nil, nil + } + + var pkgs []pkg.Package + var relationships []artifact.Relationship + var root pkg.Package + seenPkgMap := make(map[string]bool) + + p := yarnLockPackage{ + Name: pkgjson.Name, + Version: pkgjson.Version, + } + root = newYarnLockPackage(resolver, indexLocation, &p) + + for name, version := range pkgjson.Dependencies { + depPkg := yarnlock[key.NpmPackageKey(name, version)] + dep := newYarnLockPackage(resolver, indexLocation, depPkg) + rel := artifact.Relationship{ + From: dep, + To: root, + Type: artifact.DependencyOfRelationship, + } + relationships = append(relationships, rel) + } + for name, version := range pkgjson.DevDependencies { + depPkg := yarnlock[key.NpmPackageKey(name, version)] + dep := newYarnLockPackage(resolver, indexLocation, depPkg) + rel := artifact.Relationship{ + From: dep, + To: root, + Type: artifact.DependencyOfRelationship, + } + relationships = append(relationships, rel) + } + pkgs = append(pkgs, root) + + // create packages + for _, lockPkg := range yarnlock { + if seenPkgMap[key.NpmPackageKey(lockPkg.Name, lockPkg.Version)] { + continue + } + + pkg := newYarnLockPackage(resolver, indexLocation, lockPkg) + pkgs = append(pkgs, pkg) + seenPkgMap[key.NpmPackageKey(lockPkg.Name, lockPkg.Version)] = true + } + + // create relationships + for _, lockPkg := range yarnlock { + pkg := newYarnLockPackage(resolver, indexLocation, lockPkg) + + for name, version := range lockPkg.Dependencies { + dep := yarnlock[key.NpmPackageKey(name, version)] + depPkg := newYarnLockPackage( + resolver, + indexLocation, + dep, + ) + + rel := artifact.Relationship{ + From: depPkg, + To: pkg, + Type: artifact.DependencyOfRelationship, + } + relationships = append(relationships, rel) + } } - return noPackage + pkg.Sort(pkgs) + pkg.SortRelationships(relationships) + return pkgs, relationships } -func findPackageVersion(line string) string { - if matches := versionExp.FindStringSubmatch(line); len(matches) >= 2 { - return matches[1] +// finalizeYarnLockWithoutPackageJSON takes a yarn.lock file and returns a map of packages +func finalizeYarnLockWithoutPackageJSON(resolver file.Resolver, yarnlock map[string]*yarnLockPackage, indexLocation file.Location) ([]pkg.Package, []artifact.Relationship) { + var pkgs []pkg.Package + var relationships []artifact.Relationship + seenPkgMap := make(map[string]bool) + + name := rootNameFromPath(indexLocation) + if name != "" { + p := yarnLockPackage{ + Name: name, + Version: "0.0.0", + } + root := newYarnLockPackage(resolver, indexLocation, &p) + pkgs = append(pkgs, root) + } + + // create packages + for _, lockPkg := range yarnlock { + if seenPkgMap[key.NpmPackageKey(lockPkg.Name, lockPkg.Version)] { + continue + } + + pkg := newYarnLockPackage(resolver, indexLocation, lockPkg) + pkgs = append(pkgs, pkg) + seenPkgMap[key.NpmPackageKey(lockPkg.Name, lockPkg.Version)] = true + } + + // create relationships + for _, lockPkg := range yarnlock { + pkg := newYarnLockPackage(resolver, indexLocation, lockPkg) + + for name, version := range lockPkg.Dependencies { + dep := yarnlock[key.NpmPackageKey(name, version)] + depPkg := newYarnLockPackage( + resolver, + indexLocation, + dep, + ) + + rel := artifact.Relationship{ + From: depPkg, + To: pkg, + Type: artifact.DependencyOfRelationship, + } + relationships = append(relationships, rel) + } } - return noVersion + pkg.Sort(pkgs) + pkg.SortRelationships(relationships) + return pkgs, relationships } -func findPackageAndVersion(line string) (string, string) { - if matches := packageURLExp.FindStringSubmatch(line); len(matches) >= 2 { - return matches[1], matches[2] +/* + parseYarnPkgBlock parses a yarn package block like this and return a yarnLockPackage struct + and refVersions which are "tslib@^2.1.0" and "tslib@^2.3.0" in this example + +"tslib@^2.1.0", "tslib@^2.3.0": + + "integrity" "sha512-tGyy4dAjRIEwI7BzsB0lynWgOpfqjUdq91XXAlIWD2OwKBH7oCl/GZG/HT4BOHrTlPMOASlMQ7veyTqpmRcrNA==" + "resolved" "https://registry.npmjs.org/tslib/-/tslib-2.4.1.tgz" + "version" "2.4.1" +*/ +func parseYarnPkgBlock(block []byte, lineNum int) (pkg yarnLockPackage, refVersions []string, newLine int, err error) { + pkgRef, lineNumber, err := yarnparse.ParseBlock(block, lineNum) + for _, pattern := range pkgRef.Patterns { + nv := strings.Split(pattern, ":") + if len(nv) != 2 { + continue + } + refVersions = append(refVersions, key.NpmPackageKey(nv[0], nv[1])) } - return noPackage, noVersion + return yarnLockPackage{ + Name: pkgRef.Name, + Version: pkgRef.Version, + Integrity: pkgRef.Integrity, + Resolved: pkgRef.Resolved, + Dependencies: pkgRef.Dependencies, + }, refVersions, lineNumber, err } diff --git a/syft/pkg/cataloger/javascript/parse_yarn_lock_test.go b/syft/pkg/cataloger/javascript/parse_yarn_lock_test.go index cb2dacc407c..584cc17cdd8 100644 --- a/syft/pkg/cataloger/javascript/parse_yarn_lock_test.go +++ b/syft/pkg/cataloger/javascript/parse_yarn_lock_test.go @@ -3,8 +3,6 @@ package javascript import ( "testing" - "github.com/stretchr/testify/assert" - "github.com/anchore/syft/syft/artifact" "github.com/anchore/syft/syft/file" "github.com/anchore/syft/syft/pkg" @@ -17,12 +15,22 @@ func TestParseYarnBerry(t *testing.T) { locations := file.NewLocationSet(file.NewLocation(fixture)) expectedPkgs := []pkg.Package{ + { + Name: "yarn-berry", + Version: "0.0.0", + Locations: locations, + PURL: "pkg:npm/yarn-berry@0.0.0", + Language: pkg.JavaScript, + Metadata: pkg.NpmPackageLockEntry{}, + Type: pkg.NpmPkg, + }, { Name: "@babel/code-frame", Version: "7.10.4", Locations: locations, PURL: "pkg:npm/%40babel/code-frame@7.10.4", Language: pkg.JavaScript, + Metadata: pkg.NpmPackageLockEntry{}, Type: pkg.NpmPkg, }, { @@ -31,6 +39,7 @@ func TestParseYarnBerry(t *testing.T) { Locations: locations, PURL: "pkg:npm/%40types/minimatch@3.0.3", Language: pkg.JavaScript, + Metadata: pkg.NpmPackageLockEntry{}, Type: pkg.NpmPkg, }, { @@ -39,6 +48,7 @@ func TestParseYarnBerry(t *testing.T) { Locations: locations, PURL: "pkg:npm/%40types/qs@6.9.4", Language: pkg.JavaScript, + Metadata: pkg.NpmPackageLockEntry{}, Type: pkg.NpmPkg, }, { @@ -47,6 +57,7 @@ func TestParseYarnBerry(t *testing.T) { Locations: locations, PURL: "pkg:npm/ajv@6.12.3", Language: pkg.JavaScript, + Metadata: pkg.NpmPackageLockEntry{}, Type: pkg.NpmPkg, }, { @@ -55,6 +66,7 @@ func TestParseYarnBerry(t *testing.T) { Locations: locations, PURL: "pkg:npm/asn1.js@4.10.1", Language: pkg.JavaScript, + Metadata: pkg.NpmPackageLockEntry{}, Type: pkg.NpmPkg, }, { @@ -63,6 +75,7 @@ func TestParseYarnBerry(t *testing.T) { Locations: locations, PURL: "pkg:npm/atob@2.1.2", Language: pkg.JavaScript, + Metadata: pkg.NpmPackageLockEntry{}, Type: pkg.NpmPkg, }, { @@ -71,6 +84,7 @@ func TestParseYarnBerry(t *testing.T) { PURL: "pkg:npm/aws-sdk@2.706.0", Locations: locations, Language: pkg.JavaScript, + Metadata: pkg.NpmPackageLockEntry{}, Type: pkg.NpmPkg, }, { @@ -79,6 +93,7 @@ func TestParseYarnBerry(t *testing.T) { Locations: locations, PURL: "pkg:npm/c0n-fab_u.laTION@7.7.7", Language: pkg.JavaScript, + Metadata: pkg.NpmPackageLockEntry{}, Type: pkg.NpmPkg, }, { @@ -87,12 +102,12 @@ func TestParseYarnBerry(t *testing.T) { Locations: locations, PURL: "pkg:npm/jhipster-core@7.3.4", Language: pkg.JavaScript, + Metadata: pkg.NpmPackageLockEntry{}, Type: pkg.NpmPkg, }, } pkgtest.TestFileParser(t, fixture, parseYarnLock, expectedPkgs, expectedRelationships) - } func TestParseYarnLock(t *testing.T) { @@ -101,13 +116,26 @@ func TestParseYarnLock(t *testing.T) { locations := file.NewLocationSet(file.NewLocation(fixture)) expectedPkgs := []pkg.Package{ + { + Name: "yarn", + Version: "0.0.0", + Locations: locations, + PURL: "pkg:npm/yarn@0.0.0", + Language: pkg.JavaScript, + Metadata: pkg.NpmPackageLockEntry{}, + Type: pkg.NpmPkg, + }, { Name: "@babel/code-frame", Version: "7.10.4", Locations: locations, PURL: "pkg:npm/%40babel/code-frame@7.10.4", Language: pkg.JavaScript, - Type: pkg.NpmPkg, + Metadata: pkg.NpmPackageLockEntry{ + Resolved: "https://registry.yarnpkg.com/@babel/code-frame/-/code-frame-7.10.4.tgz#168da1a36e90da68ae8d49c0f1b48c7c6249213a", + Integrity: "sha512-vG6SvB6oYEhvgisZNFRmRCUkLz11c7rp+tbNTynGqc6mS1d5ATd/sGyV6W0KZZnXRKMTzZDRgQT3Ou9jhpAfUg==", + }, + Type: pkg.NpmPkg, }, { Name: "@types/minimatch", @@ -115,7 +143,11 @@ func TestParseYarnLock(t *testing.T) { Locations: locations, PURL: "pkg:npm/%40types/minimatch@3.0.3", Language: pkg.JavaScript, - Type: pkg.NpmPkg, + Metadata: pkg.NpmPackageLockEntry{ + Resolved: "https://registry.yarnpkg.com/@types/minimatch/-/minimatch-3.0.3.tgz#3dca0e3f33b200fc7d1139c0cd96c1268cadfd9d", + Integrity: "sha512-tHq6qdbT9U1IRSGf14CL0pUlULksvY9OZ+5eEgl1N7t+OA3tGvNpxJCzuKQlsNgCVwbAs670L1vcVQi8j9HjnA==", + }, + Type: pkg.NpmPkg, }, { Name: "@types/qs", @@ -123,7 +155,11 @@ func TestParseYarnLock(t *testing.T) { Locations: locations, PURL: "pkg:npm/%40types/qs@6.9.4", Language: pkg.JavaScript, - Type: pkg.NpmPkg, + Metadata: pkg.NpmPackageLockEntry{ + Resolved: "https://registry.yarnpkg.com/@types/qs/-/qs-6.9.4.tgz#a59e851c1ba16c0513ea123830dd639a0a15cb6a", + Integrity: "sha512-+wYo+L6ZF6BMoEjtf8zB2esQsqdV6WsjRK/GP9WOgLPrq87PbNWgIxS76dS5uvl/QXtHGakZmwTznIfcPXcKlQ==", + }, + Type: pkg.NpmPkg, }, { Name: "ajv", @@ -131,7 +167,11 @@ func TestParseYarnLock(t *testing.T) { Locations: locations, PURL: "pkg:npm/ajv@6.12.3", Language: pkg.JavaScript, - Type: pkg.NpmPkg, + Metadata: pkg.NpmPackageLockEntry{ + Resolved: "https://registry.yarnpkg.com/ajv/-/ajv-6.12.3.tgz#18c5af38a111ddeb4f2697bd78d68abc1cabd706", + Integrity: "sha512-4K0cK3L1hsqk9xIb2z9vs/XU+PGJZ9PNpJRDS9YLzmNdX6jmVPfamLvTJr0aDAusnHyCHO6MjzlkAsgtqp9teA==", + }, + Type: pkg.NpmPkg, }, { Name: "asn1.js", @@ -139,16 +179,23 @@ func TestParseYarnLock(t *testing.T) { Locations: locations, PURL: "pkg:npm/asn1.js@4.10.1", Language: pkg.JavaScript, - Type: pkg.NpmPkg, + Metadata: pkg.NpmPackageLockEntry{ + Resolved: "https://registry.yarnpkg.com/asn1.js/-/asn1.js-4.10.1.tgz#b9c2bf5805f1e64aadeed6df3a2bfafb5a73f5a0", + Integrity: "sha512-p32cOF5q0Zqs9uBiONKYLm6BClCoBCM5O9JfeUSlnQLBTxYdTK+pW+nXflm8UkKd2UYlEbYz5qEi0JuZR9ckSw==", + }, + Type: pkg.NpmPkg, }, { Name: "atob", Version: "2.1.2", Locations: locations, - - PURL: "pkg:npm/atob@2.1.2", - Language: pkg.JavaScript, - Type: pkg.NpmPkg, + PURL: "pkg:npm/atob@2.1.2", + Language: pkg.JavaScript, + Metadata: pkg.NpmPackageLockEntry{ + Resolved: "https://registry.yarnpkg.com/atob/-/atob-2.1.2.tgz#6d9517eb9e030d2436666651e86bd9f6f13533c9", + Integrity: "sha512-Wm6ukoaOGJi/73p/cl2GvLjTI5JM1k/O14isD73YML8StrH/7/lRFgmg8nICZgD3bZZvjwCGxtMOD3wWNAu8cg==", + }, + Type: pkg.NpmPkg, }, { Name: "aws-sdk", @@ -156,7 +203,11 @@ func TestParseYarnLock(t *testing.T) { Locations: locations, PURL: "pkg:npm/aws-sdk@2.706.0", Language: pkg.JavaScript, - Type: pkg.NpmPkg, + Metadata: pkg.NpmPackageLockEntry{ + Resolved: "https://registry.yarnpkg.com/aws-sdk/-/aws-sdk-2.706.0.tgz#09f65e9a91ecac5a635daf934082abae30eca953", + Integrity: "sha512-7GT+yrB5Wb/zOReRdv/Pzkb2Qt+hz6B/8FGMVaoysX3NryHvQUdz7EQWi5yhg9CxOjKxdw5lFwYSs69YlSp1KA==", + }, + Type: pkg.NpmPkg, }, { Name: "jhipster-core", @@ -164,175 +215,25 @@ func TestParseYarnLock(t *testing.T) { Locations: locations, PURL: "pkg:npm/jhipster-core@7.3.4", Language: pkg.JavaScript, - Type: pkg.NpmPkg, + Metadata: pkg.NpmPackageLockEntry{ + Resolved: "https://registry.yarnpkg.com/jhipster-core/-/jhipster-core-7.3.4.tgz#c34b8c97c7f4e8b7518dae015517e2112c73cc80", + Integrity: "sha512-AUhT69kNkqppaJZVfan/xnKG4Gs9Ggj7YLtTZFVe+xg+THrbMb5Ng7PL07PDlDw4KAEA33GMCwuAf65E8EpC4g==", + }, + Type: pkg.NpmPkg, }, - { Name: "something-i-made-up", Version: "7.7.7", Locations: locations, PURL: "pkg:npm/something-i-made-up@7.7.7", Language: pkg.JavaScript, - Type: pkg.NpmPkg, + Metadata: pkg.NpmPackageLockEntry{ + Resolved: "https://registry.yarnpkg.com/something-i-made-up/-/c0n-fab_u.laTION-7.7.7.tgz#b9c2bf5805f1e64aadeed6df3a2bfafb5a73f5a0", + Integrity: "sha512-p32cOF5q0Zqs9uBiONKYLm6BClCoBCM5O9JfeUSlnQLBTxYdTK+pW+nXflm8UkKd2UYlEbYz5qEi0JuZR9ckSw==", + }, + Type: pkg.NpmPkg, }, } pkgtest.TestFileParser(t, fixture, parseYarnLock, expectedPkgs, expectedRelationships) - -} - -func TestParseYarnFindPackageNames(t *testing.T) { - tests := []struct { - line string - expected string - }{ - { - line: `"@babel/code-frame@npm:7.10.4":`, - expected: "@babel/code-frame", - }, - { - line: `"@babel/code-frame@^7.0.0", "@babel/code-frame@^7.10.4":`, - expected: "@babel/code-frame", - }, - { - line: "ajv@^6.10.2, ajv@^6.5.5:", - expected: "ajv", - }, - { - line: "aws-sdk@2.706.0:", - expected: "aws-sdk", - }, - { - line: "asn1.js@^4.0.0:", - expected: "asn1.js", - }, - { - line: "c0n-fab_u.laTION@^7.0.0", - expected: "c0n-fab_u.laTION", - }, - { - line: `"newtest@workspace:.":`, - expected: "newtest", - }, - { - line: `"color-convert@npm:^1.9.0":`, - expected: "color-convert", - }, - { - line: `"@npmcorp/code-frame@^7.1.0", "@npmcorp/code-frame@^7.10.4":`, - expected: "@npmcorp/code-frame", - }, - { - line: `"@npmcorp/code-frame@^7.2.3":`, - expected: "@npmcorp/code-frame", - }, - { - line: `"@s/odd-name@^7.1.2":`, - expected: "@s/odd-name", - }, - { - line: `"@/code-frame@^7.3.4":`, - expected: "", - }, - { - line: `"code-frame":`, - expected: "", - }, - } - - for _, test := range tests { - t.Run(test.expected, func(t *testing.T) { - t.Parallel() - actual := findPackageName(test.line) - assert.Equal(t, test.expected, actual) - }) - } -} - -func TestParseYarnFindPackageVersions(t *testing.T) { - tests := []struct { - line string - expected string - }{ - { - line: ` version "7.10.4"`, - expected: "7.10.4", - }, - { - line: ` version "7.11.5"`, - expected: "7.11.5", - }, - { - line: `version "7.12.6"`, - expected: "", - }, - { - line: ` version "0.0.0"`, - expected: "0.0.0", - }, - { - line: ` version "2" `, - expected: "2", - }, - { - line: ` version "9.3"`, - expected: "9.3", - }, - { - line: "ajv@^6.10.2, ajv@^6.5.5", - expected: "", - }, - { - line: "atob@^2.1.2:", - expected: "", - }, - { - line: `"color-convert@npm:^1.9.0":`, - expected: "", - }, - { - line: " version: 1.9.3", - expected: "1.9.3", - }, - { - line: " version: 2", - expected: "2", - }, - { - line: " version: 9.3", - expected: "9.3", - }, - { - line: "ajv@^6.10.2, ajv@^6.5.5", - expected: "", - }, - { - line: "atob@^2.1.2:", - expected: "", - }, - { - line: " version: 1.0.0-alpha+001", - expected: "1.0.0-alpha", - }, - { - line: " version: 1.0.0-beta_test+exp.sha.5114f85", - expected: "1.0.0-beta_test", - }, - { - line: " version: 1.0.0+21AF26D3-117B344092BD", - expected: "1.0.0", - }, - { - line: " version: 0.0.0-use.local", - expected: "0.0.0-use.local", - }, - } - - for _, test := range tests { - t.Run(test.expected, func(t *testing.T) { - t.Parallel() - actual := findPackageVersion(test.line) - assert.Equal(t, test.expected, actual) - }) - } } diff --git a/syft/pkg/cataloger/javascript/parser/yarn/parse.go b/syft/pkg/cataloger/javascript/parser/yarn/parse.go new file mode 100644 index 00000000000..1b670037e1a --- /dev/null +++ b/syft/pkg/cataloger/javascript/parser/yarn/parse.go @@ -0,0 +1,304 @@ +package yarn + +import ( + "bufio" + "bytes" + "errors" + "fmt" + "io" + "regexp" + "strings" + + "github.com/anchore/syft/internal/log" + "github.com/anchore/syft/syft/pkg/cataloger/javascript/key" +) + +var ( + yarnPatternRegexp = regexp.MustCompile(`^\s?\\?"?(?P\S+?)@(?:(?P\S+?):)?(?P.+?)\\?"?:?$`) + yarnPatternHTTPRegexp = regexp.MustCompile(`^\s?\\?"?(?P\S+?)@https:\/\/[^#]+#(?P.+?)\\?"?:?$`) + + yarnVersionRegexp = regexp.MustCompile(`^"?version:?"?\s+"?(?P[^"]+)"?`) + yarnDependencyRegexp = regexp.MustCompile(`\s{4,}"?(?P.+?)"?:?\s"?(?P[^"]+)"?`) + yarnIntegrityRegexp = regexp.MustCompile(`^"?integrity:?"?\s+"?(?P[^"]+)"?`) + yarnResolvedRegexp = regexp.MustCompile(`^"?resolved:?"?\s+"?(?P[^"]+)"?`) + // yarnPackageURLExp matches the name and version of the dependency in yarn.lock + // from the resolved URL, including scope/namespace prefix if any. + // For example: + // `"https://registry.yarnpkg.com/async/-/async-3.2.3.tgz#ac53dafd3f4720ee9e8a160628f18ea91df196c9"` + // would return "async" and "3.2.3" + // + // `"https://registry.yarnpkg.com/@4lolo/resize-observer-polyfill/-/resize-observer-polyfill-1.5.2.tgz#58868fc7224506236b5550d0c68357f0a874b84b"` + // would return "@4lolo/resize-observer-polyfill" and "1.5.2" + yarnPackageURLExp = regexp.MustCompile(`^https://registry\.(?:yarnpkg\.com|npmjs\.org)/(.+?)/-/(?:.+?)-(\d+\..+?)\.tgz`) +) + +type PkgRef struct { + Name string + Version string + Integrity string + Resolved string + Patterns []string + Dependencies map[string]string +} + +type LineScanner struct { + *bufio.Scanner + lineCount int +} + +func newLineScanner(r io.Reader) *LineScanner { + return &LineScanner{ + Scanner: bufio.NewScanner(r), + } +} + +func (s *LineScanner) Scan() bool { + scan := s.Scanner.Scan() + if scan { + s.lineCount++ + } + return scan +} + +func (s *LineScanner) LineNum(prevNum int) int { + return prevNum + s.lineCount - 1 +} + +func parseDependencies(scanner *LineScanner) map[string]string { + deps := map[string]string{} + for scanner.Scan() { + line := scanner.Text() + name, version, err := parseDependency(line) + if err != nil { + // finished dependencies block + return deps + } + deps[name] = version + } + + return deps +} + +func getDependency(target string) (name, version string, err error) { + capture := yarnDependencyRegexp.FindStringSubmatch(target) + if len(capture) < 3 { + return "", "", errors.New("not dependency") + } + return capture[1], capture[2], nil +} + +func getIntegrity(target string) (integrity string, err error) { + capture := yarnIntegrityRegexp.FindStringSubmatch(target) + if len(capture) < 2 { + return "", errors.New("not integrity") + } + return capture[1], nil +} + +func getResolved(target string) (resolved string, err error) { + capture := yarnResolvedRegexp.FindStringSubmatch(target) + if len(capture) < 2 { + return "", errors.New("not resolved") + } + return capture[1], nil +} + +func parseDependency(line string) (string, string, error) { + name, version, err := getDependency(line) + if err != nil { + return "", "", err + } + return name, version, nil +} + +func getVersion(target string) (version string, err error) { + capture := yarnVersionRegexp.FindStringSubmatch(target) + if len(capture) < 2 { + return "", fmt.Errorf("failed to parse version: '%s", target) + } + return capture[len(capture)-1], nil +} + +func getPackageNameFromResolved(resolution string) (pkgName string) { + if matches := yarnPackageURLExp.FindStringSubmatch(resolution); len(matches) >= 2 { + return matches[1] + } + return "" +} + +func parsePattern(target string) (packagename, protocol, version string, err error) { + var capture []string + var names []string + + if strings.Contains(target, "https://") { + capture = yarnPatternHTTPRegexp.FindStringSubmatch(target) + protocol = "https" + names = yarnPatternHTTPRegexp.SubexpNames() + } else { + capture = yarnPatternRegexp.FindStringSubmatch(target) + names = yarnPatternRegexp.SubexpNames() + } + + if len(capture) < 3 { + return "", "", "", errors.New("not package format") + } + for i, group := range names { + switch group { + case "package": + packagename = capture[i] + case "protocol": + protocol = capture[i] + case "version": + version = capture[i] + } + } + return +} + +func parsePackagePatterns(target string) (packagename, protocol string, patterns []string, err error) { + patternsSplit := strings.Split(target, ", ") + packagename, protocol, _, err = parsePattern(patternsSplit[0]) + if err != nil { + return "", "", nil, err + } + + var resultPatterns []string + for _, pattern := range patternsSplit { + _, _, version, _ := parsePattern(pattern) + resultPatterns = append(resultPatterns, key.NpmPackageKey(packagename, version)) + } + patterns = resultPatterns + return +} + +func validProtocol(protocol string) bool { + switch protocol { + // example: "jhipster-core@npm:7.3.4": + case "npm", "": + return true + // example: "my-pkg@workspace:." + case "workspace": + return true + // example: "should-type@https://github.com/shouldjs/type.git#1.3.0" + case "https": + return true + } + return false +} + +func ignoreProtocol(protocol string) bool { + switch protocol { + case "patch", "file", "link", "portal", "github", "git", "git+ssh", "git+http", "git+https", "git+file": + return true + } + return false +} + +func handleEmptyLinesAndComments(line string, skipBlock bool) (int, bool) { + if len(line) == 0 { + return 1, skipBlock + } + + if line[0] == '#' || skipBlock { + return 0, skipBlock + } + + if strings.HasPrefix(line, "__metadata") { + return 0, true + } + + return 0, skipBlock +} + +func handleLinePrefixes(line string, pkg *PkgRef, scanner *LineScanner) (err error) { + switch { + case strings.HasPrefix(line, "version"): + pkg.Version, err = getVersion(line) + case strings.HasPrefix(line, "integrity"): + pkg.Integrity, err = getIntegrity(line) + case strings.HasPrefix(line, "resolved"): + pkg.Resolved, err = getResolved(line) + case strings.HasPrefix(line, "dependencies:"): + pkg.Dependencies = parseDependencies(scanner) + } + return +} + +func ParseBlock(block []byte, lineNum int) (pkg PkgRef, lineNumber int, err error) { + var ( + emptyLines int // lib can start with empty lines first + skipBlock bool + ) + + scanner := newLineScanner(bytes.NewReader(block)) + for scanner.Scan() { + line := scanner.Text() + + var increment int + increment, skipBlock = handleEmptyLinesAndComments(line, skipBlock) + emptyLines += increment + + line = strings.TrimPrefix(strings.TrimSpace(line), "\"") + + if err := handleLinePrefixes(line, &pkg, scanner); err != nil { + skipBlock = true + } + + // try parse package patterns + if name, protocol, patterns, patternErr := parsePackagePatterns(line); patternErr == nil { + if patterns == nil || !validProtocol(protocol) { + skipBlock = true + if !ignoreProtocol(protocol) { + // we need to calculate the last line of the block in order to correctly determine the line numbers of the next blocks + // store the error. we will handle it later + err = fmt.Errorf("unknown protocol: '%s', line: %s", protocol, line) + continue + } + continue + } + pkg.Name = name + pkg.Patterns = patterns + continue + } + } + + // handles the case of namespaces packages like @4lolo/resize-observer-polyfill + // where the name might not be present in the name field, but only in the + // resolved field + resolvedPkgName := getPackageNameFromResolved(pkg.Resolved) + if resolvedPkgName != "" { + pkg.Name = resolvedPkgName + } + + // in case an unsupported protocol is detected + // show warning and continue parsing + if err != nil { + log.Debugf("failed to parse block: %s", err) + return pkg, scanner.LineNum(lineNum), nil + } + + if scanErr := scanner.Err(); scanErr != nil { + err = scanErr + } + + return pkg, scanner.LineNum(lineNum), err +} + +func ScanBlocks(data []byte, atEOF bool) (advance int, token []byte, err error) { + if atEOF && len(data) == 0 { + return 0, nil, nil + } + if i := bytes.Index(data, []byte("\n\n")); i >= 0 { + // We have a full newline-terminated line. + return i + 2, data[0:i], nil + } else if i := bytes.Index(data, []byte("\r\n\r\n")); i >= 0 { + return i + 4, data[0:i], nil + } + + // If we're at EOF, we have a final, non-terminated line. Return it. + if atEOF { + return len(data), data, nil + } + // Request more data. + return 0, nil, nil +} diff --git a/syft/pkg/cataloger/javascript/test-fixtures/pkg-json-and-lock/v1/package-lock.json b/syft/pkg/cataloger/javascript/test-fixtures/pkg-json-and-lock/v1/package-lock.json new file mode 100644 index 00000000000..3956cb31ac7 --- /dev/null +++ b/syft/pkg/cataloger/javascript/test-fixtures/pkg-json-and-lock/v1/package-lock.json @@ -0,0 +1,28 @@ +{ + "name": "test-app", + "version": "0.0.0", + "lockfileVersion": 1, + "dependencies": { + "rxjs": { + "version": "7.5.7", + "resolved": "https://registry.npmjs.org/rxjs/-/rxjs-7.5.7.tgz", + "integrity": "sha512-z9MzKh/UcOqB3i20H6rtrlaE/CgjLOvheWK/9ILrbhROGTweAi1BaFsTT9FbwZi5Trr1qNRs+MXkhmR06awzQA==" + }, + "tslib": { + "version": "2.6.2", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.6.2.tgz", + "integrity": "sha512-AEYxH93jGFPn/a2iVAwW87VuUIkR1FVUKB77NwMF7nBTDkDrrT/Hpt/IrCJ0QXhW27jTBDcf5ZY7w6RiqTMw2Q==" + }, + "typescript": { + "version": "4.7.4", + "resolved": "https://registry.npmjs.org/typescript/-/typescript-4.7.4.tgz", + "integrity": "sha512-C0WQT0gezHuw6AdY1M2jxUO83Rjf0HP7Sk1DtXj6j1EwkQNZrHAg2XPWlq62oqEhYvONq5pkC2Y9oPljWToLmQ==", + "dev": true + }, + "zone.js": { + "version": "0.11.8", + "resolved": "https://registry.npmjs.org/zone.js/-/zone.js-0.11.8.tgz", + "integrity": "sha512-82bctBg2hKcEJ21humWIkXRlLBBmrc3nN7DFh5LGGhcyycO2S7FN8NmdvlcKaGFDNVL4/9kFLmwmInTavdJERA==" + } + } +} diff --git a/syft/pkg/cataloger/javascript/test-fixtures/pkg-json-and-lock/v1/package.json b/syft/pkg/cataloger/javascript/test-fixtures/pkg-json-and-lock/v1/package.json new file mode 100644 index 00000000000..d273d6f6875 --- /dev/null +++ b/syft/pkg/cataloger/javascript/test-fixtures/pkg-json-and-lock/v1/package.json @@ -0,0 +1,20 @@ +{ + "name": "test-app", + "version": "0.0.0", + "scripts": { + "ng": "ng", + "start": "ng serve", + "build": "ng build", + "watch": "ng build --watch --configuration development", + "test": "ng test" + }, + "private": true, + "dependencies": { + "rxjs": "~7.5.0", + "tslib": "^2.3.0", + "zone.js": "~0.11.4" + }, + "devDependencies": { + "typescript": "~4.7.2" + } +} diff --git a/syft/pkg/cataloger/javascript/test-fixtures/pkg-json-and-lock/v2/package-lock.json b/syft/pkg/cataloger/javascript/test-fixtures/pkg-json-and-lock/v2/package-lock.json new file mode 100644 index 00000000000..e7715a4e9b4 --- /dev/null +++ b/syft/pkg/cataloger/javascript/test-fixtures/pkg-json-and-lock/v2/package-lock.json @@ -0,0 +1,83 @@ +{ + "name": "test-app", + "version": "0.0.0", + "lockfileVersion": 2, + "requires": true, + "packages": { + "": { + "name": "test-app", + "version": "0.0.0", + "dependencies": { + "rxjs": "~7.5.0", + "tslib": "^2.3.0", + "zone.js": "~0.11.4" + }, + "devDependencies": { + "typescript": "~4.7.2" + } + }, + "node_modules/rxjs": { + "version": "7.5.7", + "resolved": "https://registry.npmjs.org/rxjs/-/rxjs-7.5.7.tgz", + "integrity": "sha512-z9MzKh/UcOqB3i20H6rtrlaE/CgjLOvheWK/9ILrbhROGTweAi1BaFsTT9FbwZi5Trr1qNRs+MXkhmR06awzQA==", + "dependencies": { + "tslib": "^2.1.0" + } + }, + "node_modules/tslib": { + "version": "2.4.1", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.4.1.tgz", + "integrity": "sha512-tGyy4dAjRIEwI7BzsB0lynWgOpfqjUdq91XXAlIWD2OwKBH7oCl/GZG/HT4BOHrTlPMOASlMQ7veyTqpmRcrNA==" + }, + "node_modules/typescript": { + "version": "4.7.4", + "resolved": "https://registry.npmjs.org/typescript/-/typescript-4.7.4.tgz", + "integrity": "sha512-C0WQT0gezHuw6AdY1M2jxUO83Rjf0HP7Sk1DtXj6j1EwkQNZrHAg2XPWlq62oqEhYvONq5pkC2Y9oPljWToLmQ==", + "dev": true, + "bin": { + "tsc": "bin/tsc", + "tsserver": "bin/tsserver" + }, + "engines": { + "node": ">=4.2.0" + } + }, + "node_modules/zone.js": { + "version": "0.11.8", + "resolved": "https://registry.npmjs.org/zone.js/-/zone.js-0.11.8.tgz", + "integrity": "sha512-82bctBg2hKcEJ21humWIkXRlLBBmrc3nN7DFh5LGGhcyycO2S7FN8NmdvlcKaGFDNVL4/9kFLmwmInTavdJERA==", + "dependencies": { + "tslib": "^2.3.0" + } + } + }, + "dependencies": { + "rxjs": { + "version": "7.5.7", + "resolved": "https://registry.npmjs.org/rxjs/-/rxjs-7.5.7.tgz", + "integrity": "sha512-z9MzKh/UcOqB3i20H6rtrlaE/CgjLOvheWK/9ILrbhROGTweAi1BaFsTT9FbwZi5Trr1qNRs+MXkhmR06awzQA==", + "requires": { + "tslib": "^2.1.0" + } + }, + "tslib": { + "version": "2.4.1", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.4.1.tgz", + "integrity": "sha512-tGyy4dAjRIEwI7BzsB0lynWgOpfqjUdq91XXAlIWD2OwKBH7oCl/GZG/HT4BOHrTlPMOASlMQ7veyTqpmRcrNA==" + }, + "typescript": { + "version": "4.7.4", + "resolved": "https://registry.npmjs.org/typescript/-/typescript-4.7.4.tgz", + "integrity": "sha512-C0WQT0gezHuw6AdY1M2jxUO83Rjf0HP7Sk1DtXj6j1EwkQNZrHAg2XPWlq62oqEhYvONq5pkC2Y9oPljWToLmQ==", + "dev": true + }, + "zone.js": { + "version": "0.11.8", + "resolved": "https://registry.npmjs.org/zone.js/-/zone.js-0.11.8.tgz", + "integrity": "sha512-82bctBg2hKcEJ21humWIkXRlLBBmrc3nN7DFh5LGGhcyycO2S7FN8NmdvlcKaGFDNVL4/9kFLmwmInTavdJERA==", + "requires": { + "tslib": "^2.3.0" + } + } + } +} diff --git a/syft/pkg/cataloger/javascript/test-fixtures/pkg-json-and-lock/v2/package.json b/syft/pkg/cataloger/javascript/test-fixtures/pkg-json-and-lock/v2/package.json new file mode 100644 index 00000000000..d273d6f6875 --- /dev/null +++ b/syft/pkg/cataloger/javascript/test-fixtures/pkg-json-and-lock/v2/package.json @@ -0,0 +1,20 @@ +{ + "name": "test-app", + "version": "0.0.0", + "scripts": { + "ng": "ng", + "start": "ng serve", + "build": "ng build", + "watch": "ng build --watch --configuration development", + "test": "ng test" + }, + "private": true, + "dependencies": { + "rxjs": "~7.5.0", + "tslib": "^2.3.0", + "zone.js": "~0.11.4" + }, + "devDependencies": { + "typescript": "~4.7.2" + } +} diff --git a/syft/pkg/cataloger/javascript/test-fixtures/pkg-json-and-lock/v3/package-lock.json b/syft/pkg/cataloger/javascript/test-fixtures/pkg-json-and-lock/v3/package-lock.json new file mode 100644 index 00000000000..2dbd7c06de9 --- /dev/null +++ b/syft/pkg/cataloger/javascript/test-fixtures/pkg-json-and-lock/v3/package-lock.json @@ -0,0 +1,54 @@ +{ + "name": "test-app", + "version": "0.0.0", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "test-app", + "version": "0.0.0", + "dependencies": { + "rxjs": "~7.5.0", + "tslib": "^2.3.0", + "zone.js": "~0.11.4" + }, + "devDependencies": { + "typescript": "~4.7.2" + } + }, + "node_modules/rxjs": { + "version": "7.5.0", + "resolved": "https://registry.npmjs.org/rxjs/-/rxjs-7.5.0.tgz", + "integrity": "sha512-fuCKAfFawVYX0pyFlETtYnXI+5iiY9Dftgk+VdgeOq+Qyi9ZDWckHZRDaXRt5WCNbbLkmAheoSGDiceyCIKNZA==", + "dependencies": { + "tslib": "^2.1.0" + } + }, + "node_modules/tslib": { + "version": "2.6.2", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.6.2.tgz", + "integrity": "sha512-AEYxH93jGFPn/a2iVAwW87VuUIkR1FVUKB77NwMF7nBTDkDrrT/Hpt/IrCJ0QXhW27jTBDcf5ZY7w6RiqTMw2Q==" + }, + "node_modules/typescript": { + "version": "4.7.4", + "resolved": "https://registry.npmjs.org/typescript/-/typescript-4.7.4.tgz", + "integrity": "sha512-C0WQT0gezHuw6AdY1M2jxUO83Rjf0HP7Sk1DtXj6j1EwkQNZrHAg2XPWlq62oqEhYvONq5pkC2Y9oPljWToLmQ==", + "dev": true, + "bin": { + "tsc": "bin/tsc", + "tsserver": "bin/tsserver" + }, + "engines": { + "node": ">=4.2.0" + } + }, + "node_modules/zone.js": { + "version": "0.11.8", + "resolved": "https://registry.npmjs.org/zone.js/-/zone.js-0.11.8.tgz", + "integrity": "sha512-82bctBg2hKcEJ21humWIkXRlLBBmrc3nN7DFh5LGGhcyycO2S7FN8NmdvlcKaGFDNVL4/9kFLmwmInTavdJERA==", + "dependencies": { + "tslib": "^2.3.0" + } + } + } +} diff --git a/syft/pkg/cataloger/javascript/test-fixtures/pkg-json-and-lock/v3/package.json b/syft/pkg/cataloger/javascript/test-fixtures/pkg-json-and-lock/v3/package.json new file mode 100644 index 00000000000..d273d6f6875 --- /dev/null +++ b/syft/pkg/cataloger/javascript/test-fixtures/pkg-json-and-lock/v3/package.json @@ -0,0 +1,20 @@ +{ + "name": "test-app", + "version": "0.0.0", + "scripts": { + "ng": "ng", + "start": "ng serve", + "build": "ng build", + "watch": "ng build --watch --configuration development", + "test": "ng test" + }, + "private": true, + "dependencies": { + "rxjs": "~7.5.0", + "tslib": "^2.3.0", + "zone.js": "~0.11.4" + }, + "devDependencies": { + "typescript": "~4.7.2" + } +} diff --git a/syft/pkg/cataloger/javascript/test-fixtures/pkg-json-and-pnpm-lock/package.json b/syft/pkg/cataloger/javascript/test-fixtures/pkg-json-and-pnpm-lock/package.json new file mode 100644 index 00000000000..d273d6f6875 --- /dev/null +++ b/syft/pkg/cataloger/javascript/test-fixtures/pkg-json-and-pnpm-lock/package.json @@ -0,0 +1,20 @@ +{ + "name": "test-app", + "version": "0.0.0", + "scripts": { + "ng": "ng", + "start": "ng serve", + "build": "ng build", + "watch": "ng build --watch --configuration development", + "test": "ng test" + }, + "private": true, + "dependencies": { + "rxjs": "~7.5.0", + "tslib": "^2.3.0", + "zone.js": "~0.11.4" + }, + "devDependencies": { + "typescript": "~4.7.2" + } +} diff --git a/syft/pkg/cataloger/javascript/test-fixtures/pkg-json-and-pnpm-lock/pnpm-lock.yaml b/syft/pkg/cataloger/javascript/test-fixtures/pkg-json-and-pnpm-lock/pnpm-lock.yaml new file mode 100644 index 00000000000..930fbe73bd9 --- /dev/null +++ b/syft/pkg/cataloger/javascript/test-fixtures/pkg-json-and-pnpm-lock/pnpm-lock.yaml @@ -0,0 +1,39 @@ +lockfileVersion: 5.4 + +specifiers: + rxjs: ~7.5.0 + tslib: ^2.3.0 + typescript: ~4.7.2 + zone.js: ~0.11.4 + +dependencies: + rxjs: 7.5.7 + tslib: 2.6.2 + zone.js: 0.11.8 + +devDependencies: + typescript: 4.7.4 + +packages: + + /rxjs/7.5.7: + resolution: {integrity: sha512-z9MzKh/UcOqB3i20H6rtrlaE/CgjLOvheWK/9ILrbhROGTweAi1BaFsTT9FbwZi5Trr1qNRs+MXkhmR06awzQA==} + dependencies: + tslib: 2.6.2 + dev: false + + /tslib/2.6.2: + resolution: {integrity: sha512-AEYxH93jGFPn/a2iVAwW87VuUIkR1FVUKB77NwMF7nBTDkDrrT/Hpt/IrCJ0QXhW27jTBDcf5ZY7w6RiqTMw2Q==} + dev: false + + /typescript/4.7.4: + resolution: {integrity: sha512-C0WQT0gezHuw6AdY1M2jxUO83Rjf0HP7Sk1DtXj6j1EwkQNZrHAg2XPWlq62oqEhYvONq5pkC2Y9oPljWToLmQ==} + engines: {node: '>=4.2.0'} + hasBin: true + dev: true + + /zone.js/0.11.8: + resolution: {integrity: sha512-82bctBg2hKcEJ21humWIkXRlLBBmrc3nN7DFh5LGGhcyycO2S7FN8NmdvlcKaGFDNVL4/9kFLmwmInTavdJERA==} + dependencies: + tslib: 2.6.2 + dev: false diff --git a/syft/pkg/cataloger/javascript/test-fixtures/pkg-json-and-yarn-lock/package.json b/syft/pkg/cataloger/javascript/test-fixtures/pkg-json-and-yarn-lock/package.json new file mode 100644 index 00000000000..d273d6f6875 --- /dev/null +++ b/syft/pkg/cataloger/javascript/test-fixtures/pkg-json-and-yarn-lock/package.json @@ -0,0 +1,20 @@ +{ + "name": "test-app", + "version": "0.0.0", + "scripts": { + "ng": "ng", + "start": "ng serve", + "build": "ng build", + "watch": "ng build --watch --configuration development", + "test": "ng test" + }, + "private": true, + "dependencies": { + "rxjs": "~7.5.0", + "tslib": "^2.3.0", + "zone.js": "~0.11.4" + }, + "devDependencies": { + "typescript": "~4.7.2" + } +} diff --git a/syft/pkg/cataloger/javascript/test-fixtures/pkg-json-and-yarn-lock/yarn.lock b/syft/pkg/cataloger/javascript/test-fixtures/pkg-json-and-yarn-lock/yarn.lock new file mode 100644 index 00000000000..c9f4beb7011 --- /dev/null +++ b/syft/pkg/cataloger/javascript/test-fixtures/pkg-json-and-yarn-lock/yarn.lock @@ -0,0 +1,27 @@ +# THIS IS AN AUTOGENERATED FILE. DO NOT EDIT THIS FILE DIRECTLY. +# yarn lockfile v1 + + +"rxjs@~7.5.0": + "integrity" "sha512-z9MzKh/UcOqB3i20H6rtrlaE/CgjLOvheWK/9ILrbhROGTweAi1BaFsTT9FbwZi5Trr1qNRs+MXkhmR06awzQA==" + "resolved" "https://registry.npmjs.org/rxjs/-/rxjs-7.5.7.tgz" + "version" "7.5.7" + dependencies: + "tslib" "^2.1.0" + +"tslib@^2.1.0", "tslib@^2.3.0": + "integrity" "sha512-tGyy4dAjRIEwI7BzsB0lynWgOpfqjUdq91XXAlIWD2OwKBH7oCl/GZG/HT4BOHrTlPMOASlMQ7veyTqpmRcrNA==" + "resolved" "https://registry.npmjs.org/tslib/-/tslib-2.4.1.tgz" + "version" "2.4.1" + +"typescript@~4.7.2": + "integrity" "sha512-C0WQT0gezHuw6AdY1M2jxUO83Rjf0HP7Sk1DtXj6j1EwkQNZrHAg2XPWlq62oqEhYvONq5pkC2Y9oPljWToLmQ==" + "resolved" "https://registry.npmjs.org/typescript/-/typescript-4.7.4.tgz" + "version" "4.7.4" + +"zone.js@~0.11.4": + "integrity" "sha512-82bctBg2hKcEJ21humWIkXRlLBBmrc3nN7DFh5LGGhcyycO2S7FN8NmdvlcKaGFDNVL4/9kFLmwmInTavdJERA==" + "resolved" "https://registry.npmjs.org/zone.js/-/zone.js-0.11.8.tgz" + "version" "0.11.8" + dependencies: + "tslib" "^2.3.0" diff --git a/syft/pkg/cataloger/javascript/test-fixtures/pkg-json/package-license-object.json b/syft/pkg/cataloger/javascript/test-fixtures/pkg-json/license-object/package.json similarity index 100% rename from syft/pkg/cataloger/javascript/test-fixtures/pkg-json/package-license-object.json rename to syft/pkg/cataloger/javascript/test-fixtures/pkg-json/license-object/package.json diff --git a/syft/pkg/cataloger/javascript/test-fixtures/pkg-json/package-license-objects.json b/syft/pkg/cataloger/javascript/test-fixtures/pkg-json/license-objects/package.json similarity index 100% rename from syft/pkg/cataloger/javascript/test-fixtures/pkg-json/package-license-objects.json rename to syft/pkg/cataloger/javascript/test-fixtures/pkg-json/license-objects/package.json diff --git a/syft/pkg/cataloger/javascript/test-fixtures/pkg-json/package-malformed-license.json b/syft/pkg/cataloger/javascript/test-fixtures/pkg-json/malformed-license/package.json similarity index 100% rename from syft/pkg/cataloger/javascript/test-fixtures/pkg-json/package-malformed-license.json rename to syft/pkg/cataloger/javascript/test-fixtures/pkg-json/malformed-license/package.json diff --git a/syft/pkg/cataloger/javascript/test-fixtures/pkg-json/package-nested-author.json b/syft/pkg/cataloger/javascript/test-fixtures/pkg-json/nested-author/package.json similarity index 100% rename from syft/pkg/cataloger/javascript/test-fixtures/pkg-json/package-nested-author.json rename to syft/pkg/cataloger/javascript/test-fixtures/pkg-json/nested-author/package.json diff --git a/syft/pkg/cataloger/javascript/test-fixtures/pkg-json/package-no-license.json b/syft/pkg/cataloger/javascript/test-fixtures/pkg-json/no-license/package.json similarity index 100% rename from syft/pkg/cataloger/javascript/test-fixtures/pkg-json/package-no-license.json rename to syft/pkg/cataloger/javascript/test-fixtures/pkg-json/no-license/package.json diff --git a/syft/pkg/cataloger/javascript/test-fixtures/pkg-json/package-partial.json b/syft/pkg/cataloger/javascript/test-fixtures/pkg-json/partial/package.json similarity index 100% rename from syft/pkg/cataloger/javascript/test-fixtures/pkg-json/package-partial.json rename to syft/pkg/cataloger/javascript/test-fixtures/pkg-json/partial/package.json diff --git a/syft/pkg/cataloger/javascript/test-fixtures/pkg-json/package.json b/syft/pkg/cataloger/javascript/test-fixtures/pkg-json/pkg-json/package.json similarity index 100% rename from syft/pkg/cataloger/javascript/test-fixtures/pkg-json/package.json rename to syft/pkg/cataloger/javascript/test-fixtures/pkg-json/pkg-json/package.json diff --git a/syft/pkg/cataloger/javascript/test-fixtures/pkg-json/package-private.json b/syft/pkg/cataloger/javascript/test-fixtures/pkg-json/private/package.json similarity index 100% rename from syft/pkg/cataloger/javascript/test-fixtures/pkg-json/package-private.json rename to syft/pkg/cataloger/javascript/test-fixtures/pkg-json/private/package.json diff --git a/syft/pkg/cataloger/javascript/test-fixtures/pkg-json/package-repo-string.json b/syft/pkg/cataloger/javascript/test-fixtures/pkg-json/repo-string/package.json similarity index 100% rename from syft/pkg/cataloger/javascript/test-fixtures/pkg-json/package-repo-string.json rename to syft/pkg/cataloger/javascript/test-fixtures/pkg-json/repo-string/package.json diff --git a/syft/pkg/cataloger/javascript/test-fixtures/pkg-lock/alias-package-lock-1.json b/syft/pkg/cataloger/javascript/test-fixtures/pkg-lock/alias-1/package-lock.json similarity index 100% rename from syft/pkg/cataloger/javascript/test-fixtures/pkg-lock/alias-package-lock-1.json rename to syft/pkg/cataloger/javascript/test-fixtures/pkg-lock/alias-1/package-lock.json diff --git a/syft/pkg/cataloger/javascript/test-fixtures/pkg-lock/alias-package-lock-2.json b/syft/pkg/cataloger/javascript/test-fixtures/pkg-lock/alias-2/package-lock.json similarity index 100% rename from syft/pkg/cataloger/javascript/test-fixtures/pkg-lock/alias-package-lock-2.json rename to syft/pkg/cataloger/javascript/test-fixtures/pkg-lock/alias-2/package-lock.json diff --git a/syft/pkg/cataloger/javascript/test-fixtures/pkg-lock/array-license-package-lock.json b/syft/pkg/cataloger/javascript/test-fixtures/pkg-lock/array-license/package-lock.json similarity index 100% rename from syft/pkg/cataloger/javascript/test-fixtures/pkg-lock/array-license-package-lock.json rename to syft/pkg/cataloger/javascript/test-fixtures/pkg-lock/array-license/package-lock.json diff --git a/syft/pkg/cataloger/javascript/test-fixtures/pkg-lock/package-lock-2.json b/syft/pkg/cataloger/javascript/test-fixtures/pkg-lock/lock-2/package-lock.json similarity index 100% rename from syft/pkg/cataloger/javascript/test-fixtures/pkg-lock/package-lock-2.json rename to syft/pkg/cataloger/javascript/test-fixtures/pkg-lock/lock-2/package-lock.json diff --git a/syft/pkg/cataloger/javascript/test-fixtures/pkg-lock/package-lock-3.json b/syft/pkg/cataloger/javascript/test-fixtures/pkg-lock/lock-3/package-lock.json similarity index 100% rename from syft/pkg/cataloger/javascript/test-fixtures/pkg-lock/package-lock-3.json rename to syft/pkg/cataloger/javascript/test-fixtures/pkg-lock/lock-3/package-lock.json diff --git a/syft/pkg/relationships_test.go b/syft/pkg/relationships_test.go new file mode 100644 index 00000000000..503f2f0dc41 --- /dev/null +++ b/syft/pkg/relationships_test.go @@ -0,0 +1,164 @@ +package pkg + +import ( + "testing" + + "github.com/anchore/syft/syft/artifact" +) + +func TestSortRelationships(t *testing.T) { + rxjs := Package{ + Name: "rxjs", + Version: "7.5.0", + FoundBy: "javascript-cataloger", + PURL: "pkg:npm/rxjs@7.5.0", + Language: JavaScript, + Type: NpmPkg, + } + testApp := Package{ + Name: "test-app", + Version: "0.0.0", + FoundBy: "javascript-cataloger", + PURL: "pkg:npm/test-app@0.0.0", + Language: JavaScript, + Type: NpmPkg, + } + tslib := Package{ + Name: "tslib", + Version: "2.6.2", + FoundBy: "javascript-cataloger", + PURL: "pkg:npm/tslib@2.6.2", + Language: JavaScript, + Type: NpmPkg, + } + typescript := Package{ + Name: "typescript", + Version: "4.7.4", + FoundBy: "javascript-cataloger", + PURL: "pkg:npm/typescript@4.7.4", + Language: JavaScript, + Type: NpmPkg, + } + zonejs := Package{ + Name: "zone.js", + Version: "0.11.8", + FoundBy: "javascript-cataloger", + PURL: "pkg:npm/zone.js@0.11.8", + Language: JavaScript, + Type: NpmPkg, + } + + tests := []struct { + name string + input []artifact.Relationship + expected []artifact.Relationship + }{ + { + name: "basic sort", + input: []artifact.Relationship{ + { + From: testApp, + To: zonejs, + Type: artifact.DependencyOfRelationship, + Data: nil, + }, + { + From: testApp, + To: rxjs, + Type: artifact.DependencyOfRelationship, + Data: nil, + }, + { + From: testApp, + To: tslib, + Type: artifact.DependencyOfRelationship, + Data: nil, + }, + { + From: testApp, + To: typescript, + Type: artifact.DependencyOfRelationship, + Data: nil, + }, + { + From: zonejs, + To: tslib, + Type: artifact.DependencyOfRelationship, + Data: nil, + }, + { + From: rxjs, + To: tslib, + Type: artifact.DependencyOfRelationship, + Data: nil, + }, + }, + expected: []artifact.Relationship{ + { + From: rxjs, + To: tslib, + Type: artifact.DependencyOfRelationship, + Data: nil, + }, + { + From: testApp, + To: rxjs, + Type: artifact.DependencyOfRelationship, + Data: nil, + }, + { + From: testApp, + To: tslib, + Type: artifact.DependencyOfRelationship, + Data: nil, + }, + { + From: testApp, + To: typescript, + Type: artifact.DependencyOfRelationship, + Data: nil, + }, + { + From: testApp, + To: zonejs, + Type: artifact.DependencyOfRelationship, + Data: nil, + }, + { + From: zonejs, + To: tslib, + Type: artifact.DependencyOfRelationship, + Data: nil, + }, + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + SortRelationships(tt.input) + for i, got := range tt.input { + if !compareRelationships(got, tt.expected[i]) { + t.Errorf("Expected %v, got %v", tt.expected[i], got) + } + } + }) + } +} + +func compareRelationships(a, b artifact.Relationship) bool { + aFrom, ok1 := a.From.(Package) + bFrom, ok2 := b.From.(Package) + aTo, ok3 := a.To.(Package) + bTo, ok4 := b.To.(Package) + + if !(ok1 && ok2 && ok3 && ok4) { + return false + } + + return aFrom.Name == bFrom.Name && + aFrom.Version == bFrom.Version && + aTo.Name == bTo.Name && + aTo.Version == bTo.Version && + a.Type == b.Type +} diff --git a/test/integration/catalog_packages_cases_test.go b/test/integration/catalog_packages_cases_test.go index 3df13864a00..37a51553520 100644 --- a/test/integration/catalog_packages_cases_test.go +++ b/test/integration/catalog_packages_cases_test.go @@ -154,6 +154,8 @@ var dirOnlyTestCases = []testCase{ pkgType: pkg.NpmPkg, pkgLanguage: pkg.JavaScript, pkgInfo: map[string]string{ + "yarn": "0.0.0", + "package-lock": "0.0.0", "@babel/code-frame": "7.10.4", "get-stdin": "8.0.0", }, diff --git a/test/integration/node_packages_test.go b/test/integration/node_packages_test.go index dcf1c4a7381..b133d17bcaf 100644 --- a/test/integration/node_packages_test.go +++ b/test/integration/node_packages_test.go @@ -14,6 +14,8 @@ func TestNpmPackageLockDirectory(t *testing.T) { sbom, _ := catalogDirectory(t, "test-fixtures/npm-lock") foundPackages := strset.New() + // root pkg + foundPackages.Add("npm-lock") for actualPkg := range sbom.Artifacts.Packages.Enumerate(pkg.NpmPkg) { for _, actualLocation := range actualPkg.Locations.ToSlice() { @@ -25,7 +27,7 @@ func TestNpmPackageLockDirectory(t *testing.T) { } // ensure that integration test commonTestCases stay in sync with the available catalogers - const expectedPackageCount = 6 + const expectedPackageCount = 7 if foundPackages.Size() != expectedPackageCount { t.Errorf("found the wrong set of npm package-lock.json packages (expected: %d, actual: %d)", expectedPackageCount, foundPackages.Size()) } @@ -35,7 +37,7 @@ func TestYarnPackageLockDirectory(t *testing.T) { sbom, _ := catalogDirectory(t, "test-fixtures/yarn-lock") foundPackages := strset.New() - expectedPackages := strset.New("async@0.9.2", "async@3.2.3", "merge-objects@1.0.5", "should-type@1.3.0", "@4lolo/resize-observer-polyfill@1.5.2") + expectedPackages := strset.New("async@0.9.2", "async@3.2.3", "merge-objects@1.0.5", "should-type@1.3.0", "@4lolo/resize-observer-polyfill@1.5.2", "yarn-lock@1.0.0") for actualPkg := range sbom.Artifacts.Packages.Enumerate(pkg.NpmPkg) { for _, actualLocation := range actualPkg.Locations.ToSlice() {