/
getter.go
233 lines (208 loc) · 7.97 KB
/
getter.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
/*
Copyright (C) 2018 OpenControl Contributors. See LICENSE.md for license.
*/
package resources
import (
"fmt"
"github.com/opencontrol/compliance-masonry/pkg/lib/common"
"github.com/opencontrol/compliance-masonry/pkg/lib/opencontrol"
"github.com/opencontrol/compliance-masonry/tools/constants"
"github.com/opencontrol/compliance-masonry/tools/fs"
"github.com/opencontrol/compliance-masonry/tools/mapset"
"log"
"os"
"path/filepath"
)
//go:generate mockery -name Getter
// getAllLocalResources will get try to get the resources that are in the current "source" directory and place them
// in the final "destination" workspace directory.
func getAllLocalResources(source string, destination string, opencontrol common.OpenControl, getter Getter) error {
// Get Certifications
log.Println("Retrieving certifications")
err := getter.GetLocalResources(source, opencontrol.GetCertifications(), destination,
constants.DefaultCertificationsFolder, false, constants.Certifications)
if err != nil {
return err
}
// Get Standards
log.Println("Retrieving standards")
err = getter.GetLocalResources(source, opencontrol.GetStandards(), destination,
constants.DefaultStandardsFolder, false, constants.Standards)
if err != nil {
return err
}
// Get Components
log.Println("Retrieving components")
err = getter.GetLocalResources(source, opencontrol.GetComponents(), destination,
constants.DefaultComponentsFolder, true, constants.Components)
if err != nil {
return err
}
return nil
}
// getAllRemoteResources will get try to get the dependencies from their respective repositories and put them
// in the final "destination" workspace directory.
func getAllRemoteResources(destination string, opencontrol common.OpenControl, getter Getter) error {
// Get Certifications
log.Println("Retrieving dependent certifications")
err := getter.GetRemoteResources(destination, constants.DefaultCertificationsFolder,
opencontrol.GetCertificationsDependencies())
if err != nil {
return err
}
// Get Standards
log.Println("Retrieving dependent standards")
err = getter.GetRemoteResources(destination, constants.DefaultStandardsFolder,
opencontrol.GetStandardsDependencies())
if err != nil {
return err
}
// Get Components
log.Println("Retrieving dependent components")
err = getter.GetRemoteResources(destination, constants.DefaultComponentsFolder,
opencontrol.GetComponentsDependencies())
if err != nil {
return err
}
return nil
}
// GetResources will download all the resources that are specified by the schema first by copying the
// local resources then downloading the remote ones and letting their respective schema version handle
// how to get their resources.
func GetResources(source string, destination string, opencontrol common.OpenControl, getter Getter) error {
// Local
err := getAllLocalResources(source, destination, opencontrol, getter)
if err != nil {
return err
}
// Remote
err = getAllRemoteResources(destination, opencontrol, getter)
if err != nil {
return err
}
return nil
}
// Getter is an interface for how to get and place local and remote resources.
type Getter interface {
GetLocalResources(source string, resources []string, destination string, subfolder string, recursively bool,
resourceType constants.ResourceType) error
GetRemoteResources(destination string, subfolder string, entries []common.RemoteSource) error
}
// NewVCSAndLocalGetter constructs a new resource getter with the type of parser to use for the files.
func NewVCSAndLocalGetter(parser opencontrol.SchemaParser) Getter {
return &vcsAndLocalFSGetter{Downloader: NewVCSDownloader(), FSUtil: fs.OSUtil{}, Parser: parser,
ResourceMap: mapset.Init()}
}
// vcsAndLocalFSGetter is the resource getter that uses VCS for remote resource getting and local file system
// for local resources.
type vcsAndLocalFSGetter struct {
Downloader Downloader
FSUtil fs.Util
ResourceMap mapset.MapSet
Parser opencontrol.SchemaParser
}
// reserveLocalResourceDestination will attempt to make a unique reservation for a particular type of resource and make
// any necessary filesystem changes so that the resource can moved there.
func (g *vcsAndLocalFSGetter) reserveLocalResourceDestination(resourceType constants.ResourceType, subfolder string,
destination string, resource string) (string, error) {
// Attempt to make a unique reservation for the resource.
if result := g.ResourceMap.Reserve(string(resourceType), resource); !result.Success {
return "", result.Error
}
// Construct the folder of where the resource should be placed.
resourceDestinationFolder := filepath.Join(destination, subfolder)
// Construct the final path for the resource itself once placed in the destination path.
resourceDestination := filepath.Join(resourceDestinationFolder, filepath.Base(resource))
log.Printf("Ensuring directory %s exists\n", resourceDestinationFolder)
err := g.FSUtil.Mkdirs(resourceDestinationFolder)
if err != nil {
return "", err
}
return resourceDestination, nil
}
func (g *vcsAndLocalFSGetter) copyLocalResource(resourceSource string,
resourceDestination string, recursively bool) error {
var err error
log.Printf("Attempting to copy local resource %s into %s\n", resourceSource, resourceDestination)
if fi, err := os.Stat(resourceSource); err == nil {
if !fi.Mode().IsDir() {
recursively = false
}
}
if recursively {
log.Printf("Copying local resource %s recursively into %s\n",
resourceSource, resourceDestination)
err = g.FSUtil.CopyAll(resourceSource, resourceDestination)
} else {
log.Printf("Copying local resource %s into %s\n", resourceSource, resourceDestination)
err = g.FSUtil.Copy(resourceSource, resourceDestination)
}
if err != nil {
log.Printf("Copying local resources %s failed\n", resourceSource)
return err
}
return nil
}
// GetLocalResources is the implementation that uses the local file system to get local resources.
func (g *vcsAndLocalFSGetter) GetLocalResources(source string, resources []string, destination string,
subfolder string, recursively bool, resourceType constants.ResourceType) error {
for _, resource := range resources {
// Attempt to reserve a space for the local resource.
resourceDestination, err := g.reserveLocalResourceDestination(resourceType,
subfolder, destination, resource)
if err != nil {
return err
}
if resourceDestination == "" {
fmt.Printf("ERROR: Duplicate key '%s' in opencontrol.yaml\n", resource)
os.Exit(1)
}
// Find the final path of where the resource is originally located.
resourceSource := filepath.Join(source, resource)
// Attempt to copy the resource.
err = g.copyLocalResource(resourceSource, resourceDestination, recursively)
if err != nil {
return err
}
}
return nil
}
// GetRemoteResources is the implementation that uses VCS to get remote resources.
func (g *vcsAndLocalFSGetter) GetRemoteResources(destination string, subfolder string,
entries []common.RemoteSource) error {
// Create the temporary directory for where to clone all the remote resources.
tempResourcesDir, err := g.FSUtil.TempDir("", "opencontrol-resources")
if err != nil {
return err
}
defer os.RemoveAll(tempResourcesDir)
for _, entry := range entries {
// Create the final path for where to clone.
tempPath := filepath.Join(tempResourcesDir, subfolder, filepath.Base(entry.GetURL()))
// Clone repo
log.Printf("Attempting to clone %v into %s\n", entry, tempPath)
err := g.Downloader.DownloadRepo(entry, tempPath)
if err != nil {
return err
}
// If contextdir is defined, switch to that dir for content
if entry.GetContextDir() != "" {
tempPath = filepath.Join(tempPath, entry.GetContextDir())
}
// Parse the opencontrol.yaml.
configBytes, err := g.FSUtil.OpenAndReadFile(filepath.Join(tempPath, entry.GetConfigFile()))
if err != nil {
return err
}
opencontrol, err := g.Parser.Parse(configBytes)
if err != nil {
return err
}
// Get the resources specified in the OpenControl YAML
err = GetResources(tempPath, destination, opencontrol, g)
if err != nil {
return err
}
}
return nil
}