forked from NVIDIA/k8s-device-plugin
-
Notifications
You must be signed in to change notification settings - Fork 0
/
main.go
171 lines (150 loc) · 5.4 KB
/
main.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
/*
* Copyright (c) 2019, NVIDIA CORPORATION. All rights reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package main
import (
"flag"
"log"
"os"
"syscall"
"github.com/NVIDIA/gpu-monitoring-tools/bindings/go/nvml"
"github.com/fsnotify/fsnotify"
pluginapi "k8s.io/kubelet/pkg/apis/deviceplugin/v1beta1"
)
var migStrategyFlag = flag.String(
"mig-strategy",
"none",
"the desired strategy for exposing MIG devices on GPUs that support it\n"+
"[none | single | mixed]")
var failOnInitErrorFlag = flag.Bool(
"fail-on-init-error",
true,
"fail the plugin if an error is encountered during initialization, otherwise block indefinitely [default: true]\n")
var passDeviceSpecs = flag.Bool(
"pass-device-specs",
false,
"pass the list of DeviceSpecs to the kubelet on Allocate()")
var deviceListStrategyFlag = flag.String(
"device-list-strategy",
"envvar",
"the desired strategy for passing the device list to the underlying runtime\n"+
"[envvar | volume-mounts]")
func main() {
flag.Parse()
if *deviceListStrategyFlag != DeviceListStrategyEnvvar && *deviceListStrategyFlag != DeviceListStrategyVolumeMounts {
log.SetOutput(os.Stderr)
log.Printf("Invalid --device-list-strategy option: %v", *deviceListStrategyFlag)
os.Exit(1)
}
log.Println("Loading NVML")
if err := nvml.Init(); err != nil {
log.SetOutput(os.Stderr)
log.Printf("Failed to initialize NVML: %v.", err)
log.Printf("If this is a GPU node, did you set the docker default runtime to `nvidia`?")
log.Printf("You can check the prerequisites at: https://github.com/NVIDIA/k8s-device-plugin#prerequisites")
log.Printf("You can learn how to set the runtime at: https://github.com/NVIDIA/k8s-device-plugin#quick-start")
log.Printf("If this is not a GPU node, you should set up a toleration or nodeSelector to only deploy this plugin on GPU nodes")
if *failOnInitErrorFlag {
os.Exit(1)
}
select {}
}
defer func() { log.Println("Shutdown of NVML returned:", nvml.Shutdown()) }()
log.Println("Starting FS watcher.")
watcher, err := newFSWatcher(pluginapi.DevicePluginPath)
if err != nil {
log.SetOutput(os.Stderr)
log.Println("Failed to create FS watcher:", err)
os.Exit(1)
}
defer watcher.Close()
log.Println("Starting OS watcher.")
sigs := newOSWatcher(syscall.SIGHUP, syscall.SIGINT, syscall.SIGTERM, syscall.SIGQUIT)
var plugins []*NvidiaDevicePlugin
restart:
// If we are restarting, idempotently stop any running plugins before
// recreating them below.
for _, p := range plugins {
p.Stop()
}
log.Println("Retreiving plugins.")
migStrategy, err := NewMigStrategy(*migStrategyFlag)
if err != nil {
log.SetOutput(os.Stderr)
log.Printf("Error creating MIG strategy: %v\n", err)
os.Exit(1)
}
plugins = migStrategy.GetPlugins()
// Loop through all plugins, starting them if they have any devices
// to serve. If even one plugin fails to start properly, try
// starting them all again.
started := 0
pluginStartError := make(chan struct{})
for _, p := range plugins {
// Just continue if there are no devices to serve for plugin p.
if len(p.Devices()) == 0 {
continue
}
// Start the gRPC server for plugin p and connect it with the kubelet.
if err := p.Start(); err != nil {
log.SetOutput(os.Stderr)
log.Println("Could not contact Kubelet, retrying. Did you enable the device plugin feature gate?")
log.Printf("You can check the prerequisites at: https://github.com/NVIDIA/k8s-device-plugin#prerequisites")
log.Printf("You can learn how to set the runtime at: https://github.com/NVIDIA/k8s-device-plugin#quick-start")
close(pluginStartError)
goto events
}
started++
}
if started == 0 {
log.Println("No devices found. Waiting indefinitely.")
}
events:
// Start an infinite loop, waiting for several indicators to either log
// some messages, trigger a restart of the plugins, or exit the program.
for {
select {
// If there was an error starting any plugins, restart them all.
case <-pluginStartError:
goto restart
// Detect a kubelet restart by watching for a newly created
// 'pluginapi.KubeletSocket' file. When this occurs, restart this loop,
// restarting all of the plugins in the process.
case event := <-watcher.Events:
if event.Name == pluginapi.KubeletSocket && event.Op&fsnotify.Create == fsnotify.Create {
log.Printf("inotify: %s created, restarting.", pluginapi.KubeletSocket)
goto restart
}
// Watch for any other fs errors and log them.
case err := <-watcher.Errors:
log.Printf("inotify: %s", err)
// Watch for any signals from the OS. On SIGHUP, restart this loop,
// restarting all of the plugins in the process. On all other
// signals, exit the loop and exit the program.
case s := <-sigs:
switch s {
case syscall.SIGHUP:
log.Println("Received SIGHUP, restarting.")
goto restart
default:
log.Printf("Received signal \"%v\", shutting down.", s)
for _, p := range plugins {
p.Stop()
}
break events
}
}
}
}