Skip to content

Commit 3238880

Browse files
bastjanbastjan
authored
Provide user data secrets to ignition template (#41)
* Provide user data secrets to ignition template * Use user data secret in template if available Only uses the ignition url and CA from upstream, this helps keeping the ignition compatible with older RHCOS images by controlling the ignition version used. * Add secret JSON decode sample to test
1 parent 513d735 commit 3238880

7 files changed

Lines changed: 124 additions & 20 deletions

File tree

‎Makefile‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,8 @@ MAKEFLAGS += --no-builtin-variables
1111
PROJECT_ROOT_DIR = .
1212
include Makefile.vars.mk
1313

14+
JSONNET_FILES ?= $(shell find . -type f -not -path './vendor/*' \( -name '*.*jsonnet' -or -name '*.libsonnet' \))
15+
1416
.PHONY: help
1517
help: ## Show this help
1618
@grep -E -h '\s##\s' $(MAKEFILE_LIST) | sort | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-20s\033[0m %s\n", $$1, $$2}'
@@ -37,6 +39,7 @@ generate: ## Generate e.g. CRD, RBAC etc.
3739
.PHONY: fmt
3840
fmt: ## Run go fmt against code
3941
go fmt ./...
42+
go tool github.com/google/go-jsonnet/cmd/jsonnetfmt -i -- $(JSONNET_FILES)
4043

4144
.PHONY: vet
4245
vet: ## Run go vet against code

‎api/cloudscale/provider/v1beta1/cloudscaleprovider_types.go‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -29,8 +29,17 @@ type CloudscaleMachineProviderSpec struct {
2929
// The Jsonnet template has access to the following variables:
3030
// - std.extVar('context').machine: the Machine object. The name can be accessed via std.extVar('context').machine.metadata.name for example.
3131
// - std.extVar('context').data: all keys from the UserDataSecret. For example, std.extVar('context').data.foo will access the value of the key foo.
32+
// - std.extVar('context').secrets: all secrets matching UserDataSecretSelector. For example, std.extVar('context').secrets[0].metadata.name will access the name of the first secret.
33+
// Also see UserDataSecretSelector.
3234
// +optional
3335
UserDataSecret *corev1.LocalObjectReference `json:"userDataSecret,omitempty"`
36+
// UserDataSecretSelector allows passing secrets with the matching selector into the user data Jsonnet context.
37+
// Only secrets in the same namespace as the controller are considered.
38+
// +optional
39+
// `null` means no secrets are passed.
40+
// And empty selector means all secrets in the namespace are passed.
41+
UserDataSecretSelector *metav1.LabelSelector `json:"userDataSecretSelector,omitempty"`
42+
3443
// TokenSecret is a reference to the secret with the cloudscale API token.
3544
// The secret must contain a key named token.
3645
// If no token is provided, the operator will try to use the default token from CLOUDSCALE_API_TOKEN.

‎api/cloudscale/provider/v1beta1/zz_generated.deepcopy.go‎

Lines changed: 5 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎go.mod‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -124,3 +124,5 @@ require (
124124
sigs.k8s.io/randfill v1.0.0 // indirect
125125
sigs.k8s.io/structured-merge-diff/v6 v6.3.0 // indirect
126126
)
127+
128+
tool github.com/google/go-jsonnet/cmd/jsonnetfmt

‎pkg/machine/actuator.go‎

Lines changed: 20 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@ import (
1313
machinecontroller "github.com/openshift/machine-api-operator/pkg/controller/machine"
1414
corev1 "k8s.io/api/core/v1"
1515
"k8s.io/apimachinery/pkg/api/equality"
16+
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
1617
"k8s.io/utils/ptr"
1718
"sigs.k8s.io/controller-runtime/pkg/client"
1819
"sigs.k8s.io/controller-runtime/pkg/log"
@@ -477,7 +478,23 @@ func (a *Actuator) loadAndRenderUserDataSecret(ctx context.Context, mctx *machin
477478
data[k] = string(v)
478479
}
479480

480-
jvm, err := jsonnetVMWithContext(mctx.machine, data)
481+
var userDataSecrets corev1.SecretList
482+
if mctx.spec.UserDataSecretSelector != nil {
483+
sel, err := metav1.LabelSelectorAsSelector(mctx.spec.UserDataSecretSelector)
484+
if err != nil {
485+
return "", fmt.Errorf("failed to parse UserDataSecretSelector: %w", err)
486+
}
487+
if err := a.k8sClient.List(
488+
ctx,
489+
&userDataSecrets,
490+
client.InNamespace(mctx.machine.Namespace),
491+
client.MatchingLabelsSelector{Selector: sel},
492+
); err != nil {
493+
return "", fmt.Errorf("failed to list secrets in namespace %q: %w", mctx.machine.Namespace, err)
494+
}
495+
}
496+
497+
jvm, err := jsonnetVMWithContext(mctx.machine, data, userDataSecrets)
481498
if err != nil {
482499
return "", fmt.Errorf("userData: failed to create jsonnet VM: %w", err)
483500
}
@@ -494,10 +511,11 @@ func (a *Actuator) loadAndRenderUserDataSecret(ctx context.Context, mctx *machin
494511
return compacted.String(), nil
495512
}
496513

497-
func jsonnetVMWithContext(machine *machinev1beta1.Machine, data map[string]string) (*jsonnet.VM, error) {
514+
func jsonnetVMWithContext(machine *machinev1beta1.Machine, data map[string]string, userDataSecrets corev1.SecretList) (*jsonnet.VM, error) {
498515
jcr, err := json.Marshal(map[string]any{
499516
"machine": machine,
500517
"data": data,
518+
"secrets": userDataSecrets.Items,
501519
})
502520
if err != nil {
503521
return nil, fmt.Errorf("unable to marshal jsonnet context: %w", err)

‎pkg/machine/actuator_test.go‎

Lines changed: 35 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -42,8 +42,30 @@ func Test_Actuator_Create_ComplexMachineE2E(t *testing.T) {
4242
},
4343
},
4444
}
45+
appUserDataSecret := &corev1.Secret{
46+
ObjectMeta: metav1.ObjectMeta{
47+
Name: "user-data-managed",
48+
Labels: map[string]string{
49+
"test.com/user-data-secret": "",
50+
},
51+
},
52+
Data: map[string][]byte{
53+
"userData": []byte("{\"ignition\": {}}"),
54+
},
55+
}
56+
unrelatedSecret := &corev1.Secret{
57+
ObjectMeta: metav1.ObjectMeta{
58+
Name: "unrelated-secret",
59+
},
60+
Data: map[string][]byte{
61+
"foo": []byte("bar"),
62+
},
63+
}
4564
providerSpec := csv1beta1.CloudscaleMachineProviderSpec{
46-
UserDataSecret: &corev1.LocalObjectReference{Name: "app-user-data"},
65+
UserDataSecret: &corev1.LocalObjectReference{Name: "app-user-data"},
66+
UserDataSecretSelector: &metav1.LabelSelector{
67+
MatchLabels: appUserDataSecret.Labels,
68+
},
4769
TokenSecret: &corev1.LocalObjectReference{Name: "cloudscale-token"},
4870
BaseDomain: "cluster.example.com",
4971
Zone: "rma1",
@@ -81,11 +103,20 @@ func Test_Actuator_Create_ComplexMachineE2E(t *testing.T) {
81103
},
82104
Data: map[string][]byte{
83105
"ignitionCA": []byte("CADATA"),
84-
"userData": []byte("{ca: std.extVar('context').data.ignitionCA}"),
106+
"userData": []byte(`
107+
{
108+
ca: std.extVar('context').data.ignitionCA,
109+
udsecrets: std.map(function(s) [
110+
s.metadata.name,
111+
std.parseJson(std.decodeUTF8(std.base64DecodeBytes(s.data.userData))),
112+
],
113+
std.extVar('context').secrets)
114+
}
115+
`),
85116
},
86117
}
87118

88-
c := newFakeClient(t, machine, tokenSecret, userDataSecret)
119+
c := newFakeClient(t, machine, tokenSecret, userDataSecret, appUserDataSecret, unrelatedSecret)
89120
ss := csmock.NewMockServerService(ctrl)
90121
sgs := csmock.NewMockServerGroupService(ctrl)
91122
actuator := newActuator(c, ss, sgs)
@@ -145,7 +176,7 @@ func Test_Actuator_Create_ComplexMachineE2E(t *testing.T) {
145176
SSHKeys: []string{},
146177
UseIPV6: providerSpec.UseIPV6,
147178
ServerGroups: []string{"created-server-group-uuid"},
148-
UserData: "{\"ca\":\"CADATA\"}",
179+
UserData: "{\"ca\":\"CADATA\",\"udsecrets\":[[\"user-data-managed\",{\"ignition\":{}}]]}",
149180
}),
150181
).DoAndReturn(cloudscaleServerFromServerRequest(func(s *cloudscale.Server) {
151182
s.UUID = "created-server-uuid"

‎pkg/machine/userdata/userdata.jsonnet‎

Lines changed: 50 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -1,36 +1,72 @@
11
local context = std.extVar('context');
22

3-
{
3+
local isMaster = std.objectHas(std.get(context.machine.metadata, 'labels', {}), 'node-role.kubernetes.io/master');
4+
5+
// Tries to load user data from a secret specific to the machineset.
6+
// The variable contains `null` if the secret is not found.
7+
// The secret is expected to be named `<machineset>-user-data-managed`.
8+
// The machine set name is read from the machine labels.
9+
local userData =
10+
local machineSet =
11+
std.get(
12+
std.get(context.machine.metadata, 'labels', {}),
13+
'machine.openshift.io/cluster-api-machineset'
14+
);
15+
if machineSet != null && std.get(context, 'secrets') != null then
16+
local secretName = std.trace("Looking for '%s-user-data-managed'" % machineSet, '%s-user-data-managed' % machineSet);
17+
local uds = std.filter(function(s) s.metadata.name == secretName, context.secrets);
18+
if std.length(uds) == 1 && std.objectHas(uds[0].data, 'userData') then
19+
std.trace("Found user data secret for machineset '%s'" % machineSet,
20+
std.parseJson(std.decodeUTF8(std.base64DecodeBytes(uds[0].data.userData))))
21+
else
22+
std.trace("No user data secret found for machineset '%s'" % machineSet, null)
23+
;
24+
25+
local ignition = {
426
ignition: {
527
version: '3.1.0',
628
config: {
7-
merge: [ {
8-
source: 'https://%s:22623/config/%s' % [ context.data.ignitionHost, std.get(context.data, 'ignitionConfigName', 'worker') ],
9-
} ],
29+
local sources = if userData != null then
30+
// Use the user data source URL from the secret if available
31+
std.map(function(s) { source: s.source }, userData.ignition.config.merge)
32+
else
33+
// Fallback to the default upstream source
34+
std.trace('no upstream sources, falling back to default', [{
35+
source: 'https://%s:22623/config/%s' % [context.data.ignitionHost, if isMaster then 'master' else 'worker'],
36+
}]),
37+
merge: sources,
1038
},
1139
security: {
1240
tls: {
13-
certificateAuthorities: [ {
14-
source: 'data:text/plain;charset=utf-8;base64,%s' % [ std.base64(context.data.ignitionCA) ],
15-
} ],
41+
local certificateAuthorities = if userData != null then
42+
// Use the CA from the user data secret if available
43+
std.map(function(ca) { source: ca.source }, userData.ignition.security.tls.certificateAuthorities)
44+
else
45+
// Fallback to the default CA
46+
std.trace('no upstream certificateAuthorities, falling back to default', [{
47+
source: 'data:text/plain;charset=utf-8;base64,%s' % [std.base64(context.data.ignitionCA)],
48+
}]),
49+
certificateAuthorities: certificateAuthorities,
1650
},
1751
},
1852
},
19-
systemd: {
20-
units: [ {
53+
systemd+: {
54+
units+: [{
2155
name: 'cloudscale-hostkeys.service',
2256
enabled: true,
2357
contents: "[Unit]\nDescription=Print SSH Public Keys to tty\nAfter=sshd-keygen.target\n\n[Install]\nWantedBy=multi-user.target\n\n[Service]\nType=oneshot\nStandardOutput=tty\nTTYPath=/dev/ttyS0\nExecStart=/bin/sh -c \"echo '-----BEGIN SSH HOST KEY KEYS-----'; cat /etc/ssh/ssh_host_*key.pub; echo '-----END SSH HOST KEY KEYS-----'\"",
24-
} ],
58+
}],
2559
},
26-
storage: {
27-
files: [ {
60+
storage+: {
61+
files+: [{
2862
filesystem: 'root',
2963
path: '/etc/hostname',
3064
mode: 420,
3165
contents: {
3266
source: 'data:,%s' % context.machine.metadata.name,
3367
},
34-
} ],
68+
}],
3569
},
36-
}
70+
};
71+
72+
std.trace('Rendered ignition: %s' % std.manifestJson(ignition), ignition)

0 commit comments

Comments
 (0)