-
Notifications
You must be signed in to change notification settings - Fork 4.7k
/
pod_linux.go
370 lines (317 loc) · 10.4 KB
/
pod_linux.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
// +build linux
package plugin
import (
"fmt"
"strconv"
"syscall"
sdnapi "github.com/openshift/origin/pkg/sdn/apis/network"
"github.com/openshift/origin/pkg/sdn/plugin/cniserver"
"github.com/openshift/origin/pkg/util/ipcmd"
"github.com/golang/glog"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
kerrors "k8s.io/apimachinery/pkg/util/errors"
kapi "k8s.io/kubernetes/pkg/api"
kapiv1 "k8s.io/kubernetes/pkg/api/v1"
kcontainer "k8s.io/kubernetes/pkg/kubelet/container"
knetwork "k8s.io/kubernetes/pkg/kubelet/network"
"k8s.io/kubernetes/pkg/kubelet/network/hostport"
kbandwidth "k8s.io/kubernetes/pkg/util/bandwidth"
kexec "k8s.io/kubernetes/pkg/util/exec"
"github.com/containernetworking/cni/pkg/invoke"
"github.com/containernetworking/cni/pkg/ip"
"github.com/containernetworking/cni/pkg/ipam"
"github.com/containernetworking/cni/pkg/ns"
cnitypes "github.com/containernetworking/cni/pkg/types"
"github.com/vishvananda/netlink"
)
const (
podInterfaceName = knetwork.DefaultInterfaceName
)
// For a given container, returns host veth name, container veth MAC, and pod IP
func getVethInfo(netns, containerIfname string) (string, string, string, error) {
var (
peerIfindex int
contVeth netlink.Link
err error
podIP string
)
containerNs, err := ns.GetNS(netns)
if err != nil {
return "", "", "", fmt.Errorf("failed to get container netns: %v", err)
}
defer containerNs.Close()
err = containerNs.Do(func(ns.NetNS) error {
contVeth, err = netlink.LinkByName(containerIfname)
if err != nil {
return err
}
peerIfindex = contVeth.Attrs().ParentIndex
addrs, err := netlink.AddrList(contVeth, syscall.AF_INET)
if err != nil {
return fmt.Errorf("failed to get container IP addresses: %v", err)
}
if len(addrs) == 0 {
return fmt.Errorf("container had no addresses")
}
podIP = addrs[0].IP.String()
return nil
})
if err != nil {
return "", "", "", fmt.Errorf("failed to inspect container interface: %v", err)
}
hostVeth, err := netlink.LinkByIndex(peerIfindex)
if err != nil {
return "", "", "", fmt.Errorf("failed to get host veth: %v", err)
}
return hostVeth.Attrs().Name, contVeth.Attrs().HardwareAddr.String(), podIP, nil
}
// Adds a macvlan interface to a container, if requested, for use with the egress router feature
func maybeAddMacvlan(pod *kapi.Pod, netns string) error {
val, ok := pod.Annotations[sdnapi.AssignMacvlanAnnotation]
if !ok || val != "true" {
return nil
}
privileged := false
for _, container := range append(pod.Spec.Containers, pod.Spec.InitContainers...) {
if container.SecurityContext != nil && container.SecurityContext.Privileged != nil && *container.SecurityContext.Privileged {
privileged = true
break
}
}
if !privileged {
return fmt.Errorf("pod has %q annotation but is not privileged", sdnapi.AssignMacvlanAnnotation)
}
// Find interface with the default route
var defIface netlink.Link
routes, err := netlink.RouteList(nil, netlink.FAMILY_V4)
if err != nil {
return fmt.Errorf("failed to read routes: %v", err)
}
for _, r := range routes {
if r.Dst == nil {
defIface, err = netlink.LinkByIndex(r.LinkIndex)
if err != nil {
return fmt.Errorf("failed to get default route interface: %v", err)
}
}
}
if defIface == nil {
return fmt.Errorf("failed to find default route interface")
}
podNs, err := ns.GetNS(netns)
if err != nil {
return fmt.Errorf("could not open netns %q", netns)
}
defer podNs.Close()
err = netlink.LinkAdd(&netlink.Macvlan{
LinkAttrs: netlink.LinkAttrs{
MTU: defIface.Attrs().MTU,
Name: "macvlan0",
ParentIndex: defIface.Attrs().Index,
Namespace: netlink.NsFd(podNs.Fd()),
},
Mode: netlink.MACVLAN_MODE_PRIVATE,
})
if err != nil {
return fmt.Errorf("failed to create macvlan interface: %v", err)
}
return podNs.Do(func(netns ns.NetNS) error {
l, err := netlink.LinkByName("macvlan0")
if err != nil {
return fmt.Errorf("failed to find macvlan interface: %v", err)
}
err = netlink.LinkSetUp(l)
if err != nil {
return fmt.Errorf("failed to set macvlan interface up: %v", err)
}
return nil
})
}
func createIPAMArgs(netnsPath string, action cniserver.CNICommand, id string) *invoke.Args {
return &invoke.Args{
Command: string(action),
ContainerID: id,
NetNS: netnsPath,
IfName: podInterfaceName,
Path: "/opt/cni/bin",
}
}
// Run CNI IPAM allocation for the container and return the allocated IP address
func (m *podManager) ipamAdd(netnsPath string, id string) (*cnitypes.Result, error) {
if netnsPath == "" {
return nil, fmt.Errorf("netns required for CNI_ADD")
}
args := createIPAMArgs(netnsPath, cniserver.CNI_ADD, id)
result, err := invoke.ExecPluginWithResult("/opt/cni/bin/host-local", m.ipamConfig, args)
if err != nil {
return nil, fmt.Errorf("failed to run CNI IPAM ADD: %v", err)
}
if result.IP4 == nil {
return nil, fmt.Errorf("failed to obtain IP address from CNI IPAM")
}
return result, nil
}
// Run CNI IPAM release for the container
func (m *podManager) ipamDel(id string) error {
args := createIPAMArgs("", cniserver.CNI_DEL, id)
err := invoke.ExecPluginWithoutResult("/opt/cni/bin/host-local", m.ipamConfig, args)
if err != nil {
return fmt.Errorf("failed to run CNI IPAM DEL: %v", err)
}
return nil
}
func setupPodBandwidth(ovs *ovsController, pod *kapi.Pod, hostVeth string) error {
ingressVal, egressVal, err := kbandwidth.ExtractPodBandwidthResources(pod.Annotations)
if err != nil {
return fmt.Errorf("failed to parse pod bandwidth: %v", err)
}
ingressBPS := int64(-1)
egressBPS := int64(-1)
if ingressVal != nil {
ingressBPS = ingressVal.Value()
// FIXME: doesn't seem possible to do this with the netlink library?
itx := ipcmd.NewTransaction(kexec.New(), hostVeth)
itx.SetLink("qlen", "1000")
err = itx.EndTransaction()
if err != nil {
return err
}
}
if egressVal != nil {
egressBPS = egressVal.Value()
}
return ovs.SetPodBandwidth(hostVeth, ingressBPS, egressBPS)
}
func vnidToString(vnid uint32) string {
return strconv.FormatUint(uint64(vnid), 10)
}
// podIsExited returns true if the pod is exited (all containers inside are exited).
func podIsExited(p *kcontainer.Pod) bool {
for _, c := range p.Containers {
if c.State != kcontainer.ContainerStateExited {
return false
}
}
return true
}
// Set up all networking (host/container veth, OVS flows, IPAM, loopback, etc)
func (m *podManager) setup(req *cniserver.PodRequest) (*cnitypes.Result, *runningPod, error) {
pod, err := m.kClient.Core().Pods(req.PodNamespace).Get(req.PodName, metav1.GetOptions{})
if err != nil {
return nil, nil, err
}
ipamResult, err := m.ipamAdd(req.Netns, req.SandboxID)
if err != nil {
return nil, nil, fmt.Errorf("failed to run IPAM for %v: %v", req.SandboxID, err)
}
podIP := ipamResult.IP4.IP.IP
// Release any IPAM allocations and hostports if the setup failed
var success bool
defer func() {
if !success {
m.ipamDel(req.SandboxID)
if err := m.hostportSyncer.SyncHostports(TUN, m.getRunningPods()); err != nil {
glog.Warningf("failed syncing hostports: %v", err)
}
}
}()
// Open any hostports the pod wants
var v1Pod kapiv1.Pod
if err := kapiv1.Convert_api_Pod_To_v1_Pod(pod, &v1Pod, nil); err != nil {
return nil, nil, err
}
podPortMapping := hostport.ConstructPodPortMapping(&v1Pod, podIP)
if err := m.hostportSyncer.OpenPodHostportsAndSync(podPortMapping, TUN, m.getRunningPods()); err != nil {
return nil, nil, err
}
var hostVethName, contVethMac string
err = ns.WithNetNSPath(req.Netns, func(hostNS ns.NetNS) error {
hostVeth, contVeth, err := ip.SetupVeth(podInterfaceName, int(m.mtu), hostNS)
if err != nil {
return fmt.Errorf("failed to create container veth: %v", err)
}
// Force a consistent MAC address based on the IP address
if err := ip.SetHWAddrByIP(podInterfaceName, podIP, nil); err != nil {
return fmt.Errorf("failed to set pod interface MAC address: %v", err)
}
// refetch to get hardware address and other properties
contVeth, err = netlink.LinkByIndex(contVeth.Attrs().Index)
if err != nil {
return fmt.Errorf("failed to fetch container veth: %v", err)
}
// Clear out gateway to prevent ConfigureIface from adding the cluster
// subnet via the gateway
ipamResult.IP4.Gateway = nil
if err = ipam.ConfigureIface(podInterfaceName, ipamResult); err != nil {
return fmt.Errorf("failed to configure container IPAM: %v", err)
}
lo, err := netlink.LinkByName("lo")
if err == nil {
err = netlink.LinkSetUp(lo)
}
if err != nil {
return fmt.Errorf("failed to configure container loopback: %v", err)
}
hostVethName = hostVeth.Attrs().Name
contVethMac = contVeth.Attrs().HardwareAddr.String()
return nil
})
if err != nil {
return nil, nil, err
}
vnid, err := m.policy.GetVNID(req.PodNamespace)
if err != nil {
return nil, nil, err
}
if err := maybeAddMacvlan(pod, req.Netns); err != nil {
return nil, nil, err
}
ofport, err := m.ovs.SetUpPod(hostVethName, podIP.String(), contVethMac, req.SandboxID, vnid)
if err != nil {
return nil, nil, err
}
if err := setupPodBandwidth(m.ovs, pod, hostVethName); err != nil {
return nil, nil, err
}
m.policy.EnsureVNIDRules(vnid)
success = true
return ipamResult, &runningPod{podPortMapping: podPortMapping, vnid: vnid, ofport: ofport}, nil
}
// Update OVS flows when something (like the pod's namespace VNID) changes
func (m *podManager) update(req *cniserver.PodRequest) (uint32, error) {
vnid, err := m.policy.GetVNID(req.PodNamespace)
if err != nil {
return 0, err
}
if err := m.ovs.UpdatePod(req.SandboxID, vnid); err != nil {
return 0, err
}
return vnid, nil
}
// Clean up all pod networking (clear OVS flows, release IPAM lease, remove host/container veth)
func (m *podManager) teardown(req *cniserver.PodRequest) error {
errList := []error{}
netnsValid := true
if err := ns.IsNSorErr(req.Netns); err != nil {
if _, ok := err.(ns.NSPathNotExistErr); ok {
glog.V(3).Infof("teardown called on already-destroyed pod %s/%s; only cleaning up IPAM", req.PodNamespace, req.PodName)
netnsValid = false
}
}
if netnsValid {
hostVethName, _, podIP, err := getVethInfo(req.Netns, podInterfaceName)
if err != nil {
return err
}
if err := m.ovs.TearDownPod(hostVethName, podIP); err != nil {
errList = append(errList, err)
}
}
if err := m.ipamDel(req.SandboxID); err != nil {
errList = append(errList, err)
}
if err := m.hostportSyncer.SyncHostports(TUN, m.getRunningPods()); err != nil {
errList = append(errList, err)
}
return kerrors.NewAggregate(errList)
}