Skip to content

Commit a2f3a96

Browse files
manuelbuilmanuelbuil
committed
Integrate k3s and multus
Signed-off-by: Manuel Buil <[email protected]>
1 parent d8907ce commit a2f3a96

File tree

10 files changed

+155
-9
lines changed

10 files changed

+155
-9
lines changed

‎docs/adrs/multus.md

Lines changed: 49 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,49 @@
1+
# Record architecture decisions
2+
3+
Date: 2024-02-14
4+
5+
## Status
6+
7+
Approved
8+
9+
## Context
10+
11+
### Multus
12+
13+
Multus is a CNI multiplexer that allows pods to have multiple network interfaces. We have users that are operating K3s + Multus but it is not super obvious how to configure it to work with K3s and how to add the additional pieces needed (e.g. IPAM or additional CNI plugins). We could facilitate this by creating an integration with Multus.
14+
15+
We we will wait a bit to include whereabouts. That project is using very old dependencies which will creep in CVEs
16+
17+
### Design suggestion
18+
19+
Add multus to the k3s-charts repo. That multus chart will consume the tarball we generate in rke2-charts, i.e. both rke2 and k3s will use the same chart with minimal diffs (e.g. the Chart name will be k3s-multus instead of rke2-multus).
20+
21+
Then, multus will be consumed as traefik:
22+
* The chart gets downloaded with `make download`
23+
* The chart tarball gets embedded in k3s binary with `go generate` and included in `pkg/static/zz_generated_bindata.go`
24+
* The HelmChart manifest pointing to the chart tarball gets embedded in k3s binary with `go generate` and included in `pkg/deploy/zz_generated_bindata.go`
25+
26+
K3s will include a new `--multus` boolean flag. When that flag is true, we would leave the HelmChart manifest installing multus and whereabouts.
27+
28+
The multus chart will install a daemonset that:
29+
* deploys the necessary binaries (multus and common CNI plugins) in each node
30+
* generates the correct CNI plugin
31+
* Installs the required CRDs
32+
33+
It sucks a bit that the daemonset stays dormant forever after doing the job instead of just dying, but the alternatives are worse
34+
35+
### Limitations
36+
37+
The multus and cni-plugins images do not support ARM architecture. At this first release, that architecture is not supported
38+
39+
40+
## Alternatives
41+
42+
* K3s creates a job that picks the multus and whereabouts CNI plugins from the `image-build-cni-plugins` and copies them to each node. However, configuring jobs to run on each node is not that easy and very error prone. Therefore, we decided to reject this idea
43+
44+
* K3s includes the multus and whereabouts CNI plugins as part of its multi-exec cni binary. However, the whereabouts binary is using very old dependencies which would creep in CVEs. Moreover, the size of the K3s binary would increase more than 10%, something not acceptable for a something that the vast majority of K3s users will not enable
45+
46+
47+
## Decision
48+
49+
YES

‎manifests/multus.yaml

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
---
2+
apiVersion: helm.cattle.io/v1
3+
kind: HelmChart
4+
metadata:
5+
name: multus
6+
namespace: kube-system
7+
spec:
8+
chart: https://%{KUBERNETES_API}%/static/charts/multus-4.0.201+upv4.0.2-build2024020801.tgz
9+
valuesContent: |-
10+
config:
11+
cni_conf:
12+
confDir: /var/lib/rancher/k3s/agent/etc/cni/net.d
13+
binDir: %{DATA_DIR}%
14+
kubeconfig: /var/lib/rancher/k3s/agent/etc/cni/net.d/multus.d/multus.kubeconfig

‎pkg/cli/cmds/server.go

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -105,6 +105,7 @@ type Server struct {
105105
EtcdS3Folder string
106106
EtcdS3Timeout time.Duration
107107
EtcdS3Insecure bool
108+
Multus bool
108109
ServiceLBNamespace string
109110
}
110111

@@ -489,6 +490,11 @@ var ServerFlags = []cli.Flag{
489490
Usage: "(experimental/components) Enable embedded distributed container registry; requires use of embedded containerd",
490491
Destination: &ServerConfig.EmbeddedRegistry,
491492
},
493+
&cli.BoolFlag{
494+
Name: "multus",
495+
Usage: "(experimental/networking) Enable multus",
496+
Destination: &ServerConfig.Multus,
497+
},
492498
NodeNameFlag,
493499
WithNodeIDFlag,
494500
NodeLabels,

‎pkg/cli/server/server.go

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -173,6 +173,7 @@ func run(app *cli.Context, cfg *cmds.Server, leaderControllers server.CustomCont
173173
serverConfig.ControlConfig.EncryptSecrets = cfg.EncryptSecrets
174174
serverConfig.ControlConfig.EtcdExposeMetrics = cfg.EtcdExposeMetrics
175175
serverConfig.ControlConfig.EtcdDisableSnapshots = cfg.EtcdDisableSnapshots
176+
serverConfig.ControlConfig.Multus = cfg.Multus
176177
serverConfig.ControlConfig.VLevel = cmds.LogConfig.VLevel
177178
serverConfig.ControlConfig.VModule = cmds.LogConfig.VModule
178179

@@ -396,6 +397,11 @@ func run(app *cli.Context, cfg *cmds.Server, leaderControllers server.CustomCont
396397
serverConfig.ControlConfig.Disables["ccm"] = true
397398
}
398399

400+
if !serverConfig.ControlConfig.Multus {
401+
serverConfig.ControlConfig.Skips["multus"] = true
402+
serverConfig.ControlConfig.Disables["multus"] = true
403+
}
404+
399405
tlsMinVersionArg := getArgValueFromList("tls-min-version", serverConfig.ControlConfig.ExtraAPIArgs)
400406
serverConfig.ControlConfig.TLSMinVersion, err = kubeapiserverflag.TLSVersion(tlsMinVersionArg)
401407
if err != nil {

‎pkg/daemons/config/types.go

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -157,6 +157,7 @@ type CriticalControlArgs struct {
157157
FlannelIPv6Masq bool `cli:"flannel-ipv6-masq"`
158158
FlannelExternalIP bool `cli:"flannel-external-ip"`
159159
EgressSelectorMode string `cli:"egress-selector-mode"`
160+
Multus bool `cli:"multus"`
160161
ServiceIPRange *net.IPNet `cli:"service-cidr"`
161162
ServiceIPRanges []*net.IPNet `cli:"service-cidr"`
162163
}

‎pkg/deploy/zz_generated_bindata.go

Lines changed: 23 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎pkg/server/server.go

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@ import (
44
"context"
55
"fmt"
66
"os"
7+
"os/exec"
78
"path"
89
"path/filepath"
910
"runtime/debug"
@@ -279,6 +280,13 @@ func stageFiles(ctx context.Context, sc *Context, controlConfig *config.Control)
279280
dnsIPFamilyPolicy = "RequireDualStack"
280281
}
281282

283+
// Find the /var/lib/rancher/k3s/data/${SHA}/bin/ directory. Same procedure we use in pkg/agent/config/config.go
284+
hostLocal, err := exec.LookPath("host-local")
285+
if err != nil {
286+
return errors.Wrap(err, "failed to find host-local")
287+
}
288+
CNIBinDir := filepath.Dir(hostLocal)
289+
282290
templateVars := map[string]string{
283291
"%{CLUSTER_DNS}%": controlConfig.ClusterDNS.String(),
284292
"%{CLUSTER_DNS_LIST}%": fmt.Sprintf("[%s]", util.JoinIPs(controlConfig.ClusterDNSs)),
@@ -288,13 +296,15 @@ func stageFiles(ctx context.Context, sc *Context, controlConfig *config.Control)
288296
"%{SYSTEM_DEFAULT_REGISTRY}%": registryTemplate(controlConfig.SystemDefaultRegistry),
289297
"%{SYSTEM_DEFAULT_REGISTRY_RAW}%": controlConfig.SystemDefaultRegistry,
290298
"%{PREFERRED_ADDRESS_TYPES}%": addrTypesPrioTemplate(controlConfig.FlannelExternalIP),
299+
"%{DATA_DIR}%": CNIBinDir,
291300
}
292301

293302
skip := controlConfig.Skips
294303
if !skip["traefik"] && isHelmChartTraefikV1(sc) {
295304
logrus.Warn("Skipping Traefik v2 deployment due to existing Traefik v1 installation")
296305
skip["traefik"] = true
297306
}
307+
logrus.Infof("MANU - templateVars: %v", templateVars)
298308
if err := deploy.Stage(dataDir, templateVars, skip); err != nil {
299309
return err
300310
}

‎pkg/static/zz_generated_bindata.go

Lines changed: 27 additions & 4 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎scripts/airgap/image-list-multus.txt

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
docker.io/rancher/hardened-multus-cni:v4.0.2-build20240208
2+
docker.io/rancher/hardened-cni-plugins:v1.4.0-build20240122
3+
docker.io/rancher/mirrored-library-busybox:1.36.1

‎scripts/package-airgap

Lines changed: 16 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -5,12 +5,23 @@ cd $(dirname $0)/..
55

66
. ./scripts/version.sh
77

8+
function createTarball() {
9+
images=$(cat "${1}")
10+
xargs -n1 docker pull <<< "${images}"
11+
docker save ${images} -o dist/artifacts/${2}-${ARCH}.tar
12+
zstd --no-progress -T0 -16 -f --long=25 dist/artifacts/${2}-${ARCH}.tar -o dist/artifacts/${2}-${ARCH}.tar.zst
13+
pigz -v -c dist/artifacts/${2}-${ARCH}.tar > dist/artifacts/${2}-${ARCH}.tar.gz
14+
}
15+
16+
817
airgap_image_file='scripts/airgap/image-list.txt'
9-
images=$(cat "${airgap_image_file}")
10-
xargs -n1 docker pull <<< "${images}"
11-
docker save ${images} -o dist/artifacts/k3s-airgap-images-${ARCH}.tar
12-
zstd --no-progress -T0 -16 -f --long=25 dist/artifacts/k3s-airgap-images-${ARCH}.tar -o dist/artifacts/k3s-airgap-images-${ARCH}.tar.zst
13-
pigz -v -c dist/artifacts/k3s-airgap-images-${ARCH}.tar > dist/artifacts/k3s-airgap-images-${ARCH}.tar.gz
18+
multus_airgap_image_file='scripts/airgap/image-list-multus.txt'
19+
createTarball ${airgap_image_file} "k3s-airgap-images"
20+
# multus and cni-plugins image do not support arm yet
21+
if [ ${ARCH} != arm ]; then
22+
createTarball ${multus_airgap_image_file} "multus-airgap-images"
23+
fi
1424
if [ ${ARCH} = amd64 ]; then
1525
cp "${airgap_image_file}" dist/artifacts/k3s-images.txt
26+
cp "${multus_airgap_image_file}" dist/artifacts/multus-images.txt
1627
fi

0 commit comments

Comments
 (0)