Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Added (basic) support for build.gradle files #707

Closed
wants to merge 3 commits into from
Closed
Show file tree
Hide file tree
Changes from 2 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion syft/pkg/cataloger/apkdb/parse_apk_db.go
Original file line number Diff line number Diff line change
Expand Up @@ -71,7 +71,7 @@ func parseApkDB(_ string, reader io.Reader) ([]pkg.Package, []artifact.Relations
}

// nolint:funlen
// parseApkDBEntry reads and parses a single pkg.ApkMetadata element from the stream, returning nil if their are no more entries.
// parseApkDBEntry reads and parses a single pkg.ApkMetadata element from the stream, returning nil if there are no more entries.
func parseApkDBEntry(reader io.Reader) (*pkg.ApkMetadata, error) {
var entry pkg.ApkMetadata
pkgFields := make(map[string]interface{})
Expand Down
3 changes: 3 additions & 0 deletions syft/pkg/cataloger/cataloger.go
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,7 @@ func ImageCatalogers() []Cataloger {
deb.NewDpkgdbCataloger(),
rpmdb.NewRpmdbCataloger(),
java.NewJavaCataloger(),
java.NewJavaGradleCataloger(),
apkdb.NewApkdbCataloger(),
golang.NewGoModuleBinaryCataloger(),
}
Expand All @@ -57,6 +58,7 @@ func DirectoryCatalogers() []Cataloger {
deb.NewDpkgdbCataloger(),
rpmdb.NewRpmdbCataloger(),
java.NewJavaCataloger(),
java.NewJavaGradleCataloger(),
apkdb.NewApkdbCataloger(),
golang.NewGoModuleBinaryCataloger(),
golang.NewGoModFileCataloger(),
Expand All @@ -76,6 +78,7 @@ func AllCatalogers() []Cataloger {
deb.NewDpkgdbCataloger(),
rpmdb.NewRpmdbCataloger(),
java.NewJavaCataloger(),
java.NewJavaGradleCataloger(),
apkdb.NewApkdbCataloger(),
golang.NewGoModuleBinaryCataloger(),
golang.NewGoModFileCataloger(),
Expand Down
2 changes: 1 addition & 1 deletion syft/pkg/cataloger/common/cpe/generate_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -234,7 +234,7 @@ func TestGeneratePackageCPEs(t *testing.T) {
p: pkg.Package{
Name: "cxf-rt-bindings-xml",
Version: "3.3.10",
FoundBy: "java-cataloger",
FoundBy: "java-archive-cataloger",
Language: pkg.Java,
Type: pkg.JavaPkg,
MetadataType: pkg.JavaMetadataType,
Expand Down
11 changes: 10 additions & 1 deletion syft/pkg/cataloger/java/cataloger.go
Original file line number Diff line number Diff line change
Expand Up @@ -14,5 +14,14 @@ func NewJavaCataloger() *common.GenericCataloger {
globParsers[pattern] = parseJavaArchive
}

return common.NewGenericCataloger(nil, globParsers, "java-cataloger")
return common.NewGenericCataloger(nil, globParsers, "java-archive-cataloger")
}

// NewJavaGradleCataloger returns a new Java gradle cataloger object.
func NewJavaGradleCataloger() *common.GenericCataloger {
globParsers := map[string]common.ParserFn{
"**/build.gradle": parseJavaGradle,
}

return common.NewGenericCataloger(nil, globParsers, "java-gradle-cataloger")
}
181 changes: 181 additions & 0 deletions syft/pkg/cataloger/java/gradle_parser.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,181 @@
package java

import (
"bufio"
"fmt"
"io"
"strings"
"unicode"

"github.com/anchore/syft/internal/log"

"github.com/anchore/syft/internal/file"
"github.com/anchore/syft/syft/artifact"
"github.com/anchore/syft/syft/pkg"
"github.com/anchore/syft/syft/pkg/cataloger/common"
)

// integrity check
var _ common.ParserFn = parseJaveGradle

func parseJaveGradle(_ string, reader io.Reader) ([]pkg.Package, []artifact.Relationship, error) {
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

can you add tests around this parser function?

var packages []pkg.Package

dependencyScanner := bufio.NewScanner(reader)

variables := make(map[string][]byte)

// get each 'dependency {}' section
onDependency := func(data []byte, atEOF bool) (advance int, token []byte, err error) {
Copy link
Contributor

@wagoodman wagoodman Jan 3, 2022

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I see why this particular parser function is complex, mostly since the build.gradle file is groovy syntax and short of a groovy AST parser there isn't a straightforward way to extract the information in question. The split functions are pretty dense from a reading and understanding point of view, and I'm hesitant to change them without a body of tests around the split functions themselves.

Assuming there isn't any shared state in the stack that is leveraged, can you extract out all of the split functions and put them individually under test?

MAINLOOP:
for advance = -1; advance < len(data) - 16; {
// loop through each byte, stops looping once it gets near the end
for _, char := range []byte{'d','e','p','e','n','d','e','n','c','i','e','s'} {
// look for the string "dependencies"
advance++
if data[advance] != char {
if char == 'p' && data[advance] == 'f' {
// stupid way to parse variable definitions
advance++
for ; unicode.IsSpace(rune(data[advance])) && advance < len(data); advance++ {}
// loop until the start of the variable name
nameStart := advance
// remember the starting index of the variable name
for ; unicode.IsLetter(rune(data[advance])) && advance < len(data); advance++ {}
// loop until the end and record variable name
varName := string(data[nameStart:advance])
var terminator byte = '\n'
// character to read until. end of the quotes if there are any, else newline
// has to be explicitly set to byte so it can be compared with data[advance]
start := 0
for ; start == 0 && advance < len(data); advance++ {
// loop until the start of the variable('s value)
if unicode.IsLetter(rune(data[advance])) {
start = advance
}
for _, quote := range []byte{'"', '\''} {
if data[advance] == quote {
terminator = quote
start = advance + 1
}
}
}
for ; data[advance] != terminator && advance < len(data); advance++ {}
// loop until end of the variable and store it in the variables map
variables[varName] = data[start:advance]
}
// go doesn't have for ... else so we use this
continue MAINLOOP
}
}
advance++
for ; unicode.IsSpace(rune(data[advance])) && advance < len(data) - 2; advance++ {}
// if it found "dependencies", loop through whitespace
if data[advance] == '{' {
advance++
for start, nest := advance, 0; advance < len(data); advance++ {
// we've found a group of dependencies, loop to find the end
switch data[advance] {
case '$':
// if there's a `$`, try to insert a variable
advance++
varStart := advance
cutStart := advance - 1
cutEnd := 0
if data[advance] == '{' {
// if the next character is '', read until next ''
varStart++
cutEnd = 1
for ; data[advance] != '}' && advance < len(data); advance++ {}
} else {
// else, read until it's not a letter
for ; unicode.IsLetter(rune(data[advance]))&& advance < len(data); advance++ {}
}
variable := variables[string(data[varStart:advance])]
// get the variable's value from the variables map
// if it's not found, it just returns null
tmp := make([]byte, cutStart + len(variable) + len(data) - advance - cutEnd)
copy(tmp, data[:cutStart])
copy(tmp[cutStart:], variable)
copy(tmp[cutStart + len(variable):], data[advance + cutEnd:])
data = tmp
// stitch together the arrays to add the variable's value
case '{':
nest++
case '}':
if nest == 0 {
token = data[start:advance]
return
}
nest--
}
}
}
break
}
return 0, nil, nil
}

onEntry := func(data []byte, atEOF bool) (advance int, token []byte, err error) {
for ; advance < len(data) - 8; advance++ {
// split into individual strings
for _, quote := range []byte{'"','\''} {
// find beginning of the string by looping until there are quotes
if data[advance] == quote {
advance++
for start := advance; advance < len(data); advance++ {
if data[advance] == quote {
token = data[start:advance]
advance++
return
}
}
}
}
}
return 0, nil, nil
}

dependencyScanner.Split(onDependency)
for dependencyScanner.Scan() {
entryScanner := bufio.NewScanner(strings.NewReader(dependencyScanner.Text()))

entryScanner.Split(onEntry)
for entryScanner.Scan() {
name, version, err := parseGradleEntry(entryScanner.Text())
if err != nil {
return nil, nil, err
}
if metadata != nil {
packages = append(packages, pkg.Package{
Name: name,
Version: version,
FoundBy: "java-gradle-cataloger",
Language: pkg.Java,
Type: pkg.JavaPkg,
})
}
}
if err := entryScanner.Err(); err != nil {
return nil, nil, fmt.Errorf("failed to parse build.gradle file: %w", err)
}
}

if err := dependencyScanner.Err(); err != nil {
return nil, nil, fmt.Errorf("failed to parse build.gradle file: %w", err)
}

return packages, nil, nil
}

// nolint:funlen
// parseGradleEntry reads and parses a single pkg.GradleMetadata element from the stream, returning nil if the string is .
func parseGradleEntry(dependency string) (string, string, error) {
substrings := strings.Split(dependency, ":")

if len(substrings) != 3 {
return "", "", nil
}

return substrings[1], substrings[2], nil
}
2 changes: 1 addition & 1 deletion syft/pkg/cataloger/rpmdb/parse_rpmdb.go
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@ import (
"github.com/anchore/syft/syft/source"
)

// parseApkDb parses an "Packages" RPM DB and returns the Packages listed within it.
// parseRpmDb parses an "Packages" RPM DB and returns the Packages listed within it.
func parseRpmDB(resolver source.FilePathResolver, dbLocation source.Location, reader io.Reader) ([]pkg.Package, error) {
f, err := ioutil.TempFile("", internal.ApplicationName+"-rpmdb")
if err != nil {
Expand Down