Skip to content

Commit

Permalink
feat(serving): add build configs to FileTree serving data (#3527)
Browse files Browse the repository at this point in the history 
Also, fix API field name (i.e. build_config) to be consistent with xref.proto.
  • Loading branch information
@schroederc
schroederc committed Feb 19, 2019
1 parent b5eddd5 commit 448d46e
Show file tree
Hide file tree
Showing 8 changed files with 503 additions and 225 deletions.
20 changes: 19 additions & 1 deletion kythe/go/serving/filetree/filetree.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@ import (
"context"
"errors"
"fmt"
"log"
"path/filepath"
"strings"

Expand Down Expand Up @@ -74,7 +75,24 @@ func (t *Table) Directory(ctx context.Context, req *ftpb.DirectoryRequest) (*ftp
} else if err != nil {
return nil, fmt.Errorf("lookup error: %v", err)
}
entries, err := parseLegacyEntries(nil, ftpb.DirectoryReply_FILE, d.FileTicket)
entries := make([]*ftpb.DirectoryReply_Entry, 0, len(d.Entry))
for _, e := range d.Entry {
re := &ftpb.DirectoryReply_Entry{
Name: e.Name,
BuildConfig: e.BuildConfig,
}
switch e.Kind {
case srvpb.FileDirectory_FILE:
re.Kind = ftpb.DirectoryReply_FILE
case srvpb.FileDirectory_DIRECTORY:
re.Kind = ftpb.DirectoryReply_DIRECTORY
default:
log.Printf("WARNING: unknown directory entry type: %T", e)
continue
}
entries = append(entries, re)
}
entries, err := parseLegacyEntries(entries, ftpb.DirectoryReply_FILE, d.FileTicket)
if err != nil {
return nil, err
}
Expand Down
1 change: 1 addition & 0 deletions kythe/go/serving/pipeline/BUILD
View file
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,7 @@ go_library(
"@com_github_apache_beam//sdks/go/pkg/beam:go_default_library",
"@com_github_apache_beam//sdks/go/pkg/beam/transforms/filter:go_default_library",
"@com_github_golang_protobuf//proto:go_default_library",
"@org_bitbucket_creachadair_stringset//:go_default_library",
],
)

Expand Down
131 changes: 110 additions & 21 deletions kythe/go/serving/pipeline/filetree.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,21 +18,27 @@ package pipeline

import (
"fmt"
"log"
"path/filepath"
"reflect"
"sort"

"bitbucket.org/creachadair/stringset"
"kythe.io/kythe/go/serving/pipeline/nodes"
"kythe.io/kythe/go/util/kytheuri"
"kythe.io/kythe/go/util/compare"
"kythe.io/kythe/go/util/schema/facts"
kinds "kythe.io/kythe/go/util/schema/nodes"

"github.com/apache/beam/sdks/go/pkg/beam"

scpb "kythe.io/kythe/proto/schema_go_proto"
srvpb "kythe.io/kythe/proto/serving_go_proto"
spb "kythe.io/kythe/proto/storage_go_proto"
)

func init() {
beam.RegisterFunction(addCorpusRootsKey)
beam.RegisterFunction(anchorToFileBuildConfig)
beam.RegisterFunction(fileToCorpusRoot)
beam.RegisterFunction(fileToDirectories)

Expand All @@ -45,7 +51,7 @@ func (k *KytheBeam) getFileVNames() beam.PCollection {
return k.fileVNames
}
k.fileVNames = beam.DropValue(k.s, beam.Seq(k.s, k.nodes, &nodes.Filter{
FilterByKind: []string{"file"},
FilterByKind: []string{kinds.File},
IncludeFacts: []string{},
IncludeEdges: []string{},
}, moveSourceToKey))
Expand All @@ -68,31 +74,78 @@ func (k *KytheBeam) CorpusRoots() beam.PCollection {
func (k *KytheBeam) Directories() beam.PCollection {
s := k.s.Scope("Directories")
files := k.getFileVNames()
return beam.CombinePerKey(s, &combineDirectories{}, beam.ParDo(s, fileToDirectories, files))
anchors := beam.ParDo(s, &nodes.Filter{
FilterByKind: []string{kinds.Anchor},
IncludeFacts: []string{facts.BuildConfig},
IncludeEdges: []string{},
}, k.nodes)
return beam.CombinePerKey(s, &combineDirectories{}, beam.Flatten(s,
beam.ParDo(s, fileToDirectories, files),
beam.ParDo(s, anchorToFileBuildConfig, anchors),
))
}

// addCorpusRootsKey returns the given value with the Kythe corpus roots key constant.
func addCorpusRootsKey(val beam.T) (string, beam.T) { return "dirs:corpusRoots", val }

func dirTicket(corpus, root, dir string) string {
return fmt.Sprintf("dirs:%s\n%s\n%s", corpus, root, dir)
}

// anchorToFileBuildConfig emits a FileDirectory for each path component in the
// given anchor VName with its specified build config.
func anchorToFileBuildConfig(anchor *scpb.Node, emit func(string, *srvpb.FileDirectory)) {
// Clean the file path and remove any leading slash.
path := filepath.Clean(filepath.Join("/", anchor.Source.GetPath()))[1:]
dir := currentAsEmpty(filepath.Dir(path))
buildConfig := []string{""}
for _, f := range anchor.Fact {
if f.GetKytheName() == scpb.FactName_BUILD_CONFIG {
buildConfig[0] = string(f.Value)
}
}
corpus, root := anchor.Source.GetCorpus(), anchor.Source.GetRoot()
emit(dirTicket(corpus, root, dir), &srvpb.FileDirectory{
Entry: []*srvpb.FileDirectory_Entry{{
Name: filepath.Base(path),
Kind: srvpb.FileDirectory_FILE,
BuildConfig: buildConfig,
}},
})

for dir != "" {
name := filepath.Base(dir)
dir = currentAsEmpty(filepath.Dir(dir))
emit(dirTicket(corpus, root, dir), &srvpb.FileDirectory{
Entry: []*srvpb.FileDirectory_Entry{{
Name: name,
Kind: srvpb.FileDirectory_DIRECTORY,
BuildConfig: buildConfig,
}},
})
}
}

// fileToDirectories emits a FileDirectory for each path component in the given file VName.
func fileToDirectories(file *spb.VName, emit func(string, *srvpb.FileDirectory)) {
// Clean the file path and remove any leading slash.
path := filepath.Clean(filepath.Join("/", file.GetPath()))[1:]

dir := &spb.VName{
Corpus: file.Corpus,
Root: file.Root,
Path: currentAsEmpty(filepath.Dir(path)),
}
dirTicket := func() string { return fmt.Sprintf("dirs:%s\n%s\n%s", dir.Corpus, dir.Root, dir.Path) }
emit(dirTicket(), &srvpb.FileDirectory{
FileTicket: []string{kytheuri.ToString(file)},
dir := currentAsEmpty(filepath.Dir(path))
emit(dirTicket(file.Corpus, file.Root, dir), &srvpb.FileDirectory{
Entry: []*srvpb.FileDirectory_Entry{{
Name: filepath.Base(path),
Kind: srvpb.FileDirectory_FILE,
}},
})
for dir.Path != "" {
ticket := kytheuri.ToString(dir)
dir.Path = currentAsEmpty(filepath.Dir(dir.Path))
emit(dirTicket(), &srvpb.FileDirectory{
Subdirectory: []string{ticket},
for dir != "" {
name := filepath.Base(dir)
dir = currentAsEmpty(filepath.Dir(dir))
emit(dirTicket(file.Corpus, file.Root, dir), &srvpb.FileDirectory{
Entry: []*srvpb.FileDirectory_Entry{{
Name: name,
Kind: srvpb.FileDirectory_DIRECTORY,
}},
})
}
}
Expand Down Expand Up @@ -145,15 +198,51 @@ func (combineCorpusRoots) ExtractOutput(cr *srvpb.CorpusRoots) *srvpb.CorpusRoot
type combineDirectories struct{}

func (combineDirectories) MergeAccumulators(accum, dir *srvpb.FileDirectory) *srvpb.FileDirectory {
accum.Subdirectory = append(accum.Subdirectory, dir.Subdirectory...)
accum.FileTicket = append(accum.FileTicket, dir.FileTicket...)
accum.Entry = append(accum.Entry, dir.Entry...)
return accum
}

func (combineDirectories) ExtractOutput(dir *srvpb.FileDirectory) *srvpb.FileDirectory {
dir.FileTicket = removeDuplicates(dir.FileTicket)
dir.Subdirectory = removeDuplicates(dir.Subdirectory)
return dir
files := make(map[string]stringset.Set)
subdirs := make(map[string]stringset.Set)
for _, e := range dir.Entry {
switch e.Kind {
case srvpb.FileDirectory_FILE:
if configs, ok := files[e.Name]; ok {
configs.Add(e.BuildConfig...)
} else {
files[e.Name] = stringset.New(e.BuildConfig...)
}
case srvpb.FileDirectory_DIRECTORY:
if configs, ok := subdirs[e.Name]; ok {
configs.Add(e.BuildConfig...)
} else {
subdirs[e.Name] = stringset.New(e.BuildConfig...)
}
default:
log.Printf("WARNING: unknown FileDirectory kind: %v", e.Kind)
}
}
entries := make([]*srvpb.FileDirectory_Entry, 0, len(files)+len(subdirs))
for file, configs := range files {
entries = append(entries, &srvpb.FileDirectory_Entry{
Kind: srvpb.FileDirectory_FILE,
Name: file,
BuildConfig: configs.Elements(),
})
}
for subdir, configs := range subdirs {
entries = append(entries, &srvpb.FileDirectory_Entry{
Kind: srvpb.FileDirectory_DIRECTORY,
Name: subdir,
BuildConfig: configs.Elements(),
})
}
sort.Slice(entries, func(i, j int) bool {
return compare.Ints(int(entries[i].Kind), int(entries[j].Kind)).
AndThen(entries[i].Name, entries[j].Name) == compare.LT
})
return &srvpb.FileDirectory{Entry: entries}
}

func removeDuplicates(strs []string) []string {
Expand Down
75 changes: 64 additions & 11 deletions kythe/go/serving/pipeline/filetree_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -71,38 +71,91 @@ func TestDirectories(t *testing.T) {
testNodes := []*scpb.Node{{
Source: &spb.VName{Corpus: "corpus", Root: "root", Path: "path"},
Kind: &scpb.Node_KytheKind{scpb.NodeKind_FILE},
}, {
Source: &spb.VName{Corpus: "corpus", Root: "root", Path: "path", Signature: "a0"},
Kind: &scpb.Node_KytheKind{scpb.NodeKind_ANCHOR},
Fact: []*scpb.Fact{{
Name: &scpb.Fact_KytheName{scpb.FactName_BUILD_CONFIG},
Value: []byte("test-build-config"),
}},
}, {
Source: &spb.VName{Corpus: "corpus", Root: "root", Path: "path", Signature: "a1"},
Kind: &scpb.Node_KytheKind{scpb.NodeKind_ANCHOR},
Fact: []*scpb.Fact{{
Name: &scpb.Fact_KytheName{scpb.FactName_BUILD_CONFIG},
Value: []byte("test-build-config2"),
}},
}, {
Source: &spb.VName{Corpus: "corpus", Root: "root", Path: "path2"},
Kind: &scpb.Node_KytheKind{scpb.NodeKind_FILE},
}, {
Source: &spb.VName{Corpus: "corpus", Root: "root", Path: "path2", Signature: "a0"},
Kind: &scpb.Node_KytheKind{scpb.NodeKind_ANCHOR},
// no build-config
}, {
Source: &spb.VName{Corpus: "corpus", Root: "root", Path: "path3"},
Kind: &scpb.Node_KytheKind{scpb.NodeKind_FILE},
}, {
Source: &spb.VName{Corpus: "corpus", Path: "p/to/file.go"},
Kind: &scpb.Node_KytheKind{scpb.NodeKind_FILE},
}, {
Source: &spb.VName{Corpus: "corpus", Path: "p/to/file.go", Signature: "a0"},
Kind: &scpb.Node_KytheKind{scpb.NodeKind_ANCHOR},
Fact: []*scpb.Fact{{
Name: &scpb.Fact_KytheName{scpb.FactName_BUILD_CONFIG},
Value: []byte("test-build-config"),
}},
}, {
Source: &spb.VName{Corpus: "corpus2", Path: "p/to/file.go"},
Kind: &scpb.Node_KytheKind{scpb.NodeKind_FILE},
}}

expected := []*srvpb.FileDirectory{{
Subdirectory: []string{"kythe://corpus?path=p"},
Entry: []*srvpb.FileDirectory_Entry{{
Kind: srvpb.FileDirectory_DIRECTORY,
Name: "p",
}},
}, {
Subdirectory: []string{"kythe://corpus?path=p/to"},
Entry: []*srvpb.FileDirectory_Entry{{
Kind: srvpb.FileDirectory_DIRECTORY,
Name: "to",
}},
}, {
FileTicket: []string{"kythe://corpus?path=p/to/file.go"},
Entry: []*srvpb.FileDirectory_Entry{{
Kind: srvpb.FileDirectory_FILE,
Name: "file.go",
}},
}, {
Subdirectory: []string{"kythe://corpus2?path=p"},
Entry: []*srvpb.FileDirectory_Entry{{
Kind: srvpb.FileDirectory_DIRECTORY,
Name: "p",
BuildConfig: []string{"test-build-config"},
}},
}, {
Subdirectory: []string{"kythe://corpus2?path=p/to"},
Entry: []*srvpb.FileDirectory_Entry{{
Kind: srvpb.FileDirectory_DIRECTORY,
Name: "to",
BuildConfig: []string{"test-build-config"},
}},
}, {
FileTicket: []string{"kythe://corpus2?path=p/to/file.go"},
Entry: []*srvpb.FileDirectory_Entry{{
Kind: srvpb.FileDirectory_FILE,
Name: "file.go",
BuildConfig: []string{"test-build-config"},
}},
}, {
FileTicket: []string{
"kythe://corpus?path=path2?root=root",
"kythe://corpus?path=path3?root=root",
"kythe://corpus?path=path?root=root",
},
Entry: []*srvpb.FileDirectory_Entry{{
Kind: srvpb.FileDirectory_FILE,
Name: "path",
BuildConfig: []string{"test-build-config", "test-build-config2"},
}, {
Kind: srvpb.FileDirectory_FILE,
Name: "path2",
BuildConfig: []string{""},
}, {
Kind: srvpb.FileDirectory_FILE,
Name: "path3",
}},
}}

p, s, nodes := ptest.CreateList(testNodes)
Expand Down
2 changes: 1 addition & 1 deletion kythe/proto/filetree.proto
View file
Original file line number Diff line number Diff line change
Expand Up @@ -66,7 +66,7 @@ message DirectoryReply {

// Set of known build configurations of this FILE or all files recursively
// contained in/below this DIRECTORY.
repeated string build_configuration = 3;
repeated string build_config = 3;
}
enum Kind {
UNKNOWN = 0;
Expand Down

0 comments on commit 448d46e

Please sign in to comment.