-
Notifications
You must be signed in to change notification settings - Fork 582
/
manager.go
333 lines (284 loc) · 11.2 KB
/
manager.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
// Copyright (c) 2021 Multus Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package config
import (
"context"
"encoding/json"
"fmt"
"os"
"path/filepath"
"sync"
"github.com/fsnotify/fsnotify"
"gopkg.in/k8snetworkplumbingwg/multus-cni.v4/pkg/logging"
)
// MultusDefaultNetworkName holds the default name of the multus network
const (
multusConfigFileName = "00-multus.conf"
MultusDefaultNetworkName = "multus-cni-network"
UserRWPermission = 0600
)
// Manager monitors the configuration of the primary CNI plugin, and
// regenerates multus configuration whenever it gets updated.
type Manager struct {
cniConfigData map[string]interface{}
configWatcher *fsnotify.Watcher
multusConfig *MultusConf
multusConfigDir string
multusConfigFilePath string
readinessIndicatorFilePath string
primaryCNIConfigPath string
}
// NewManager returns a config manager object, configured to read the
// primary CNI configuration in `config.MultusAutoconfigDir`. If
// `config.MultusMasterCni` is empty, this constructor will auto-discover the
// primary CNI for which it will delegate.
func NewManager(config MultusConf) (*Manager, error) {
var err error
defaultPluginName := config.MultusMasterCni
if defaultPluginName == "" {
defaultPluginName, err = getPrimaryCNIPluginName(config.MultusAutoconfigDir)
if err != nil {
_ = logging.Errorf("failed to find the primary CNI plugin: %v", err)
return nil, err
}
}
return newManager(config, defaultPluginName)
}
// overrideCNIVersion overrides cniVersion in cniConfigFile, it should be used only in kind case
func overrideCNIVersion(cniConfigFile string, multusCNIVersion string) error {
path, err := filepath.Abs(cniConfigFile)
if err != nil {
return fmt.Errorf("illegal path %s in cni config path %s: %w", path, cniConfigFile, err)
}
masterCNIConfigData, err := os.ReadFile(path)
if err != nil {
return fmt.Errorf("failed to read cni config %s: %v", path, err)
}
var primaryCNIConfigData map[string]interface{}
if err := json.Unmarshal(masterCNIConfigData, &primaryCNIConfigData); err != nil {
return fmt.Errorf("failed to unmarshall cni config %s: %w", cniConfigFile, err)
}
primaryCNIConfigData["cniVersion"] = multusCNIVersion
configBytes, err := json.Marshal(primaryCNIConfigData)
if err != nil {
return fmt.Errorf("couldn't update cluster network config: %v", err)
}
err = os.WriteFile(path, configBytes, 0644)
if err != nil {
return fmt.Errorf("couldn't update cluster network config: %v", err)
}
return nil
}
func newManager(config MultusConf, defaultCNIPluginName string) (*Manager, error) {
if config.ForceCNIVersion {
err := overrideCNIVersion(filepath.Join(config.MultusAutoconfigDir, defaultCNIPluginName), config.CNIVersion)
if err != nil {
return nil, err
}
}
readinessIndicatorPath := ""
if config.ReadinessIndicatorFile != "" {
readinessIndicatorPath = filepath.Dir(config.ReadinessIndicatorFile)
}
watcher, err := newWatcher(config.MultusAutoconfigDir, readinessIndicatorPath)
if err != nil {
return nil, err
}
if defaultCNIPluginName == fmt.Sprintf("%s/%s", config.MultusAutoconfigDir, multusConfigFileName) {
return nil, logging.Errorf("cannot specify %s/%s to prevent recursive config load", config.MultusAutoconfigDir, multusConfigFileName)
}
configManager := &Manager{
configWatcher: watcher,
multusConfig: &config,
multusConfigDir: config.MultusAutoconfigDir,
multusConfigFilePath: filepath.Join(config.CniConfigDir, multusConfigFileName),
primaryCNIConfigPath: filepath.Join(config.MultusAutoconfigDir, defaultCNIPluginName),
readinessIndicatorFilePath: config.ReadinessIndicatorFile,
}
if err := configManager.loadPrimaryCNIConfigFromFile(); err != nil {
return nil, fmt.Errorf("failed to load the primary CNI configuration as a multus delegate with error '%v'", err)
}
if config.OverrideNetworkName {
if err := configManager.overrideNetworkName(); err != nil {
return nil, logging.Errorf("could not override the network name: %v", err)
}
}
return configManager, nil
}
// Start generates an updated Multus config, writes it, and begins watching
// the config directory and readiness indicator files for changes
func (m *Manager) Start(ctx context.Context, wg *sync.WaitGroup) error {
generatedMultusConfig, err := m.GenerateConfig()
if err != nil {
return logging.Errorf("failed to generated the multus configuration: %v", err)
}
logging.Verbosef("Generated MultusCNI config: %s", generatedMultusConfig)
multusConfigFile, err := m.PersistMultusConfig(generatedMultusConfig)
if err != nil {
return logging.Errorf("failed to persist the multus configuration: %v", err)
}
wg.Add(1)
go func() {
defer wg.Done()
if err := m.monitorPluginConfiguration(ctx); err != nil {
_ = logging.Errorf("error watching file: %v", err)
}
logging.Verbosef("ConfigWatcher done")
logging.Verbosef("Delete old config @ %v", multusConfigFile)
os.Remove(multusConfigFile)
}()
return nil
}
func (m *Manager) loadPrimaryCNIConfigFromFile() error {
primaryCNIConfigData, err := primaryCNIData(m.primaryCNIConfigPath)
if err != nil {
return logging.Errorf("failed to access the primary CNI configuration from %s: %v", m.primaryCNIConfigPath, err)
}
if err = CheckVersionCompatibility(m.multusConfig, primaryCNIConfigData); err != nil {
return err
}
return m.loadPrimaryCNIConfigurationData(primaryCNIConfigData)
}
// overrideNetworkName overrides the name of the multus configuration with the
// name of the delegated primary CNI.
func (m *Manager) overrideNetworkName() error {
name, ok := m.cniConfigData["name"]
if !ok {
return fmt.Errorf("failed to access delegate CNI plugin name")
}
networkName := name.(string)
if networkName == "" {
return fmt.Errorf("the primary CNI Configuration does not feature the network name: %v", m.cniConfigData)
}
m.multusConfig.Name = networkName
return nil
}
func (m *Manager) loadPrimaryCNIConfigurationData(primaryCNIConfigData interface{}) error {
cniConfigData := primaryCNIConfigData.(map[string]interface{})
m.cniConfigData = cniConfigData
m.multusConfig.ClusterNetwork = m.primaryCNIConfigPath
return m.multusConfig.setCapabilities(cniConfigData)
}
// GenerateConfig generates a multus configuration from its current state
func (m *Manager) GenerateConfig() (string, error) {
if err := m.loadPrimaryCNIConfigFromFile(); err != nil {
_ = logging.Errorf("failed to read the primary CNI plugin config from %s", m.primaryCNIConfigPath)
return "", nil
}
return m.multusConfig.Generate()
}
// monitorPluginConfiguration monitors the configuration file pointed
// to by the primaryCNIPluginName attribute, and re-generates the multus
// configuration whenever the primary CNI config is updated.
func (m *Manager) monitorPluginConfiguration(ctx context.Context) error {
logging.Verbosef("started to watch file %s", m.primaryCNIConfigPath)
for {
select {
case event := <-m.configWatcher.Events:
if !m.shouldRegenerateConfig(event) {
continue
}
logging.Debugf("process event: %v", event)
// if readinessIndicatorFile is removed, then restart multus
if m.readinessIndicatorFilePath != "" && m.readinessIndicatorFilePath == event.Name {
logging.Verbosef("readiness indicator file is gone. restart multus-daemon")
os.Remove(m.multusConfigFilePath)
os.Exit(2)
}
updatedConfig, err := m.GenerateConfig()
if err != nil {
_ = logging.Errorf("failed to regenerate the multus configuration: %v", err)
}
logging.Debugf("Re-generated MultusCNI config: %s", updatedConfig)
if _, err := m.PersistMultusConfig(updatedConfig); err != nil {
_ = logging.Errorf("failed to persist the multus configuration: %v", err)
}
if err := m.loadPrimaryCNIConfigFromFile(); err != nil {
_ = logging.Errorf("failed to reload the updated config: %v", err)
}
case err := <-m.configWatcher.Errors:
if err == nil {
continue
}
logging.Errorf("CNI monitoring error %v", err)
case <-ctx.Done():
logging.Verbosef("Stopped monitoring, closing channel ...")
_ = m.configWatcher.Close()
return nil
}
}
}
// PersistMultusConfig persists the provided configuration to the disc, with
// Read / Write permissions. The output file path is `<multus auto config dir>/00-multus.conf`
func (m *Manager) PersistMultusConfig(config string) (string, error) {
if _, err := os.Stat(m.multusConfigFilePath); err == nil {
logging.Debugf("Overwriting Multus CNI configuration @ %s", m.multusConfigFilePath)
} else {
logging.Debugf("Writing Multus CNI configuration @ %s", m.multusConfigFilePath)
}
return m.multusConfigFilePath, os.WriteFile(m.multusConfigFilePath, []byte(config), UserRWPermission)
}
func (m *Manager) shouldRegenerateConfig(event fsnotify.Event) bool {
// first, check the readiness indicator file existence
if event.Name == m.readinessIndicatorFilePath {
return event.Has(fsnotify.Remove) || event.Has(fsnotify.Rename)
}
// we're watching the DIR where the config sits, and the event
// does not concern the primary CNI config. Skip it.
if event.Name == m.primaryCNIConfigPath {
return event.Has(fsnotify.Write) || event.Has(fsnotify.Create)
}
logging.Debugf("skipping un-related event %v", event)
return false
}
func getPrimaryCNIPluginName(multusAutoconfigDir string) (string, error) {
masterCniConfigFileName, err := findMasterPlugin(multusAutoconfigDir, 120)
if err != nil {
return "", fmt.Errorf("failed to find the cluster master CNI plugin: %w", err)
}
return masterCniConfigFileName, nil
}
func newWatcher(cniConfigDir string, readinessIndicatorDir string) (*fsnotify.Watcher, error) {
watcher, err := fsnotify.NewWatcher()
if err != nil {
return nil, fmt.Errorf("failed to create new watcher for %q: %v", cniConfigDir, err)
}
defer func() {
// Close watcher on error
if err != nil {
watcher.Close()
}
}()
if err = watcher.Add(cniConfigDir); err != nil {
return nil, fmt.Errorf("failed to add watch on %q for cni config: %v", cniConfigDir, err)
}
// if readinessIndicatorDir is different from cniConfigDir,
if readinessIndicatorDir != "" && cniConfigDir != readinessIndicatorDir {
if err = watcher.Add(readinessIndicatorDir); err != nil {
return nil, fmt.Errorf("failed to add watch on %q for readinessIndicator: %v", readinessIndicatorDir, err)
}
}
return watcher, nil
}
func primaryCNIData(masterCNIPluginPath string) (interface{}, error) {
masterCNIConfigData, err := os.ReadFile(masterCNIPluginPath)
if err != nil {
return nil, fmt.Errorf("failed to read the cluster primary CNI config %s: %w", masterCNIPluginPath, err)
}
var cniData interface{}
if err := json.Unmarshal(masterCNIConfigData, &cniData); err != nil {
return nil, fmt.Errorf("failed to unmarshall primary CNI config: %w", err)
}
return cniData, nil
}