Skip to content

Commit 6e9975e

Browse files
committed
feat: private networking
1 parent ae0bdba commit 6e9975e

50 files changed

Lines changed: 2710 additions & 445 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

Makefile

Lines changed: 12 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -125,17 +125,21 @@ generate-e2e-cni: ## Regenerate Cilium CNI manifest from Helm chart
125125
generate-e2e-ccm: ## Regenerate cloudscale CCM manifest
126126
@CCM_VERSION=$(CCM_VERSION) hack/generate-e2e-ccm.sh
127127

128+
E2E_CLUSTER_TEMPLATES := cluster-template \
129+
cluster-template-ha \
130+
cluster-template-upgrades \
131+
cluster-template-md-remediation \
132+
cluster-template-byo-network \
133+
cluster-template-public-lb-private-nodes \
134+
cluster-template-private \
135+
cluster-template-fip
136+
128137
.PHONY: generate-e2e-templates
129138
generate-e2e-templates: $(KUSTOMIZE) generate-e2e-cni generate-e2e-ccm ## Generate e2e cluster templates using kustomize overlays
130139
@mkdir -p $(E2E_TEMPLATES)/main
131-
@echo "Generating cluster-template.yaml..."
132-
@"$(KUSTOMIZE)" build --load-restrictor LoadRestrictionsNone $(E2E_TEMPLATES)/cluster-template > $(E2E_TEMPLATES)/main/cluster-template.yaml
133-
@echo "Generating cluster-template-ha.yaml..."
134-
@"$(KUSTOMIZE)" build --load-restrictor LoadRestrictionsNone $(E2E_TEMPLATES)/cluster-template-ha > $(E2E_TEMPLATES)/main/cluster-template-ha.yaml
135-
@echo "Generating cluster-template-upgrades.yaml..."
136-
@"$(KUSTOMIZE)" build --load-restrictor LoadRestrictionsNone $(E2E_TEMPLATES)/cluster-template-upgrades > $(E2E_TEMPLATES)/main/cluster-template-upgrades.yaml
137-
@echo "Generating cluster-template-md-remediation.yaml..."
138-
@"$(KUSTOMIZE)" build --load-restrictor LoadRestrictionsNone $(E2E_TEMPLATES)/cluster-template-md-remediation > $(E2E_TEMPLATES)/main/cluster-template-md-remediation.yaml
140+
@$(foreach tmpl,$(E2E_CLUSTER_TEMPLATES),\
141+
echo "Generating $(tmpl).yaml..." && \
142+
"$(KUSTOMIZE)" build --load-restrictor LoadRestrictionsNone $(E2E_TEMPLATES)/$(tmpl) > $(E2E_TEMPLATES)/main/$(tmpl).yaml &&) true
139143
@echo "Templates generated successfully."
140144

141145
.PHONY: generate-e2e-config

README.md

Lines changed: 46 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -8,8 +8,9 @@ for [cloudscale.ch](https://www.cloudscale.ch).
88

99
## Features
1010

11-
- **CloudscaleCluster**: Network, Subnet, Load Balancer management
12-
- **CloudscaleMachine**: Server provisioning with cloud-init
11+
- **CloudscaleCluster**: Multi-network management (managed or BYO), Load Balancer (public or private VIP), Floating IP
12+
support
13+
- **CloudscaleMachine**: Server provisioning with cloud-init and configurable network interfaces
1314
- **CloudscaleMachineTemplate**: Immutable machine templates for KubeadmControlPlane/MachineDeployment
1415

1516
## Prerequisites
@@ -42,6 +43,9 @@ clusterctl generate cluster my-cluster \
4243
| kubectl apply -f -
4344
```
4445

46+
This uses the default template (public nodes, managed network). See [Cluster Templates](#cluster-templates) for other
47+
network topologies.
48+
4549
Watch the cluster come up:
4650

4751
```bash
@@ -50,15 +54,41 @@ clusterctl describe cluster my-cluster
5054

5155
## Environment Variables
5256

53-
| Variable | Description | Example |
54-
|-------------------------------------------|--------------------------------|-----------------------------------|
55-
| `CLOUDSCALE_API_TOKEN` | cloudscale.ch API token | `abc123...` |
56-
| `CLOUDSCALE_SSH_PUBLIC_KEY` | SSH public key added to nodes | `ssh-ed25519 AAAA...` |
57-
| `CLOUDSCALE_REGION` | cloudscale.ch region | `lpg` or `rma` |
58-
| `CLOUDSCALE_MACHINE_IMAGE` | Server image for nodes | `custom:ubuntu-2404-kube-v1.xx.x` |
59-
| `CLOUDSCALE_CONTROL_PLANE_MACHINE_FLAVOR` | Flavor for control plane nodes | `flex-4-2` |
60-
| `CLOUDSCALE_WORKER_MACHINE_FLAVOR` | Flavor for worker nodes | `flex-4-2` |
61-
| `CLOUDSCALE_ROOT_VOLUME_SIZE` | Root volume size in GB | `50` |
57+
| Variable | Description | Example |
58+
|-------------------------------------------|-------------------------------------------|-----------------------------------|
59+
| `CLOUDSCALE_API_TOKEN` | cloudscale.ch API token | `abc123...` |
60+
| `CLOUDSCALE_SSH_PUBLIC_KEY` | SSH public key added to nodes | `ssh-ed25519 AAAA...` |
61+
| `CLOUDSCALE_REGION` | cloudscale.ch region | `lpg` or `rma` |
62+
| `CLOUDSCALE_MACHINE_IMAGE` | Server image for nodes | `custom:ubuntu-2404-kube-v1.xx.x` |
63+
| `CLOUDSCALE_CONTROL_PLANE_MACHINE_FLAVOR` | Flavor for control plane nodes | `flex-4-2` |
64+
| `CLOUDSCALE_WORKER_MACHINE_FLAVOR` | Flavor for worker nodes | `flex-4-2` |
65+
| `CLOUDSCALE_ROOT_VOLUME_SIZE` | Root volume size in GB | `50` |
66+
| `CLOUDSCALE_NETWORK_UUID` | Existing cloudscale.ch network UUID (BYO) | `2db69ba3-...` |
67+
68+
> **Note:** `CLOUDSCALE_NETWORK_UUID` is required by the `fip`, `private`, `public-lb-private-nodes`, and `byo-network`
69+
> template flavors. It is not needed for the default template.
70+
71+
## Cluster Templates
72+
73+
CAPCS ships several cluster templates for different network topologies. Use `clusterctl generate cluster` with the
74+
`--flavor` flag to select one:
75+
76+
```bash
77+
clusterctl generate cluster my-cluster \
78+
--kubernetes-version v1.32.0 \
79+
--control-plane-machine-count 1 \
80+
--worker-machine-count 2 \
81+
--flavor <flavor-name> \
82+
| kubectl apply -f -
83+
```
84+
85+
| Flavor | Network | CP Endpoint | Node Connectivity | Extra Env Vars | Notes |
86+
|---------------------------|---------------------------|--------------------------|-------------------|---------------------------|----------------------|
87+
| *(default)* | Managed (`10.100.0.0/24`) | Public LB (DualStack) | Public + cluster || |
88+
| `fip` | BYO | Floating IP (IPv4) | Public + cluster | `CLOUDSCALE_NETWORK_UUID` | No load balancer |
89+
| `private` | BYO + NAT | Private LB (cluster VIP) | Private only | `CLOUDSCALE_NETWORK_UUID` | Requires NAT gateway |
90+
| `public-lb-private-nodes` | BYO + NAT | Public LB | Private only | `CLOUDSCALE_NETWORK_UUID` | Requires NAT gateway |
91+
| `byo-network` | BYO | Public LB (DualStack) | Public + cluster | `CLOUDSCALE_NETWORK_UUID` | |
6292

6393
## Development
6494

@@ -92,14 +122,16 @@ filtering and are split into suites of increasing cost, scheduled accordingly:
92122
| Cluster upgrade | `upgrade` | Rolling K8s version upgrade (v1.34 → v1.35) | < 10 min | Weekly | `test-e2e-upgrade` |
93123
| Self-hosted | `self-hosted` | clusterctl move (pivot) to workload cluster. Requires container image in public registry | < 15 min | Weekly | `test-e2e-self-hosted` |
94124
| MD remediation | `md-remediation` | MachineHealthCheck auto-replacement of unhealthy workers | < 10 min | Weekly | `test-e2e-md-remediation` |
125+
| BYO networking | `byo-networking` | BYO network: public-LB + private-nodes and floating-IP variants | < 10 min | Weekly | `test-e2e` |
95126
| Conformance (fast) | `conformance` | K8s conformance, skip Serial tests | < 60 min | Weekly | `test-e2e-conformance-fast` |
96127
| Conformance (full) | `conformance` | Full K8s conformance including Serial tests | < 120 min | Biweekly | `test-e2e-conformance` |
97128

98129
Durations are approximate from a real CI run; conformance varies with cluster size.
99130

100131
**Why this split?** The single-CP lifecycle test is the cheapest smoke test and runs
101132
nightly to catch regressions early. HA, upgrade, self-hosted, and remediation tests are more
102-
resource-intensive and run weekly. Full K8s conformance is the most expensive and runs biweekly
133+
resource-intensive and run weekly. Private networking tests require `CLOUDSCALE_NETWORK_UUID` to be set and are
134+
skipped otherwise. Full K8s conformance is the most expensive and runs biweekly
103135
(1st + 15th of month). All suites can be triggered manually via the `test-e2e.yml` workflow
104136
dispatch. E2E tests share a concurrency group so only one suite runs at a time.
105137

@@ -143,6 +175,8 @@ kustomize_substitutions:
143175
CLOUDSCALE_WORKER_MACHINE_FLAVOR: "flex-4-2"
144176
CLOUDSCALE_MACHINE_IMAGE: "IMAGE_NAME"
145177
CLOUDSCALE_ROOT_VOLUME_SIZE: "50"
178+
# Required for BYO network flavors (fip, private, public-lb-private-nodes, byo-network):
179+
# CLOUDSCALE_NETWORK_UUID: "UUID_HERE"
146180
extra_args:
147181
cloudscale:
148182
- "--zap-log-level=5"

api/v1beta2/cloudscalecluster_types.go

Lines changed: 110 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,16 @@ const (
2828
ClusterFinalizer = "cloudscalecluster.infrastructure.cluster.x-k8s.io"
2929
)
3030

31+
// IPFamily represents an IP family configuration.
32+
// +kubebuilder:validation:Enum=IPv4;IPv6;DualStack
33+
type IPFamily string
34+
35+
const (
36+
IPFamilyIPv4 IPFamily = "IPv4"
37+
IPFamilyIPv6 IPFamily = "IPv6"
38+
IPFamilyDualStack IPFamily = "DualStack"
39+
)
40+
3141
// CloudscaleClusterSpec defines the desired state of CloudscaleCluster
3242
type CloudscaleClusterSpec struct {
3343
// Region is the cloudscale.ch region (e.g., "rma", "lpg").
@@ -45,17 +55,27 @@ type CloudscaleClusterSpec struct {
4555
CredentialsRef CloudscaleCredentialsReference `json:"credentialsRef"`
4656

4757
// ControlPlaneEndpoint represents the endpoint to communicate with the control plane.
48-
// This is set automatically from the load balancer's VIP address.
58+
// This is set automatically from the load balancer's VIP address or floating IP.
4959
// +optional
5060
ControlPlaneEndpoint clusterv1.APIEndpoint `json:"controlPlaneEndpoint,omitzero"`
5161

52-
// Network contains network configuration for the cluster.
62+
// Networks define the private networks for this cluster.
63+
// Referenced by name from machine interface specs and LB config.
64+
// If empty, defaults to a single managed network named after the cluster.
65+
// +listType=map
66+
// +listMapKey=name
5367
// +optional
54-
Network NetworkSpec `json:"network,omitzero"`
68+
Networks []NetworkSpec `json:"networks,omitempty"`
5569

5670
// ControlPlaneLoadBalancer configures the load balancer for the control plane.
5771
// +optional
5872
ControlPlaneLoadBalancer LoadBalancerSpec `json:"controlPlaneLoadBalancer,omitzero"`
73+
74+
// FloatingIP configures a floating IP for a stable control plane endpoint.
75+
// The floating IP is assigned to the load balancer when enabled, or to a
76+
// control plane server when the load balancer is disabled.
77+
// +optional
78+
FloatingIP *FloatingIPSpec `json:"floatingIP,omitempty"`
5979
}
6080

6181
// CloudscaleCredentialsReference references a Secret containing the API token.
@@ -69,28 +89,43 @@ type CloudscaleCredentialsReference struct {
6989
Namespace string `json:"namespace,omitempty"`
7090
}
7191

72-
// NetworkSpec defines the network configuration.
92+
// NetworkSpec defines a private network for the cluster.
93+
// Exactly one of UUID or CIDR must be specified.
7394
type NetworkSpec struct {
74-
// CIDR is the CIDR block for the private network subnet.
75-
// +kubebuilder:default="10.0.0.0/24"
95+
// Name identifies this network within the cluster.
96+
// Used to reference this network from machine interface specs and LB config.
97+
// +kubebuilder:validation:Required
98+
// +kubebuilder:validation:Pattern=`^[a-z0-9]([a-z0-9-]*[a-z0-9])?$`
99+
// +kubebuilder:validation:MaxLength=63
100+
Name string `json:"name"`
101+
102+
// UUID references an existing cloudscale.ch network (BYO).
103+
// The network is not deleted on cluster teardown.
104+
// Mutually exclusive with CIDR.
105+
// +optional
106+
UUID string `json:"uuid,omitempty"`
107+
108+
// CIDR defines the subnet for a controller-managed network.
109+
// The network and subnet are created and deleted by CAPCS.
110+
// Mutually exclusive with UUID.
76111
// +optional
77112
CIDR string `json:"cidr,omitempty"`
78113

79114
// GatewayAddress is the gateway IP address for the subnet.
80-
// By default, no gateway is configured on the private network subnet. This ensures
81-
// that outbound internet traffic uses the public network interface, which is required
82-
// for the Cloud Controller Manager to reach the cloudscale.ch API.
115+
// Only applicable when CIDR is set (managed network).
116+
// By default, no gateway is configured on the subnet. This ensures
117+
// that outbound internet traffic uses the public network interface.
83118
// Set this to a specific IP address (e.g., "10.0.0.1") only if you have configured
84119
// a NAT gateway or similar infrastructure on the private network.
85120
// +optional
86-
GatewayAddress *string `json:"gatewayAddress,omitempty"`
121+
GatewayAddress string `json:"gatewayAddress,omitempty"`
87122
}
88123

89124
// LoadBalancerSpec defines the load balancer configuration for the control plane.
90125
type LoadBalancerSpec struct {
91126
// Enabled controls whether a load balancer is created for the control plane.
92127
// Set to false for external control planes (e.g., hosted control plane) where the endpoint
93-
// is provided externally.
128+
// is provided externally, or when using a floating IP without a load balancer.
94129
// +kubebuilder:default=true
95130
// +optional
96131
Enabled *bool `json:"enabled,omitempty"`
@@ -113,6 +148,17 @@ type LoadBalancerSpec struct {
113148
// +optional
114149
APIServerPort int32 `json:"apiServerPort,omitempty"`
115150

151+
// Network places the LB VIP on a private network (internal LB).
152+
// References spec.networks[].name. Omit for a public LB.
153+
// +optional
154+
Network string `json:"network,omitempty"`
155+
156+
// IPFamily specifies the IP family for the LB VIP address(es).
157+
// +kubebuilder:validation:Enum=IPv4;IPv6;DualStack
158+
// +kubebuilder:default=DualStack
159+
// +optional
160+
IPFamily IPFamily `json:"ipFamily,omitempty"`
161+
116162
// HealthMonitor configures the load balancer health monitor.
117163
// +optional
118164
HealthMonitor HealthMonitorSpec `json:"healthMonitor,omitempty"`
@@ -149,19 +195,40 @@ type HealthMonitorSpec struct {
149195
DownThreshold int `json:"downThreshold,omitempty"`
150196
}
151197

198+
// FloatingIPSpec configures a floating IP for the control plane endpoint.
199+
// Exactly one of IPFamily or UUID must be specified.
200+
type FloatingIPSpec struct {
201+
// IPFamily creates a new floating IP with this IP version.
202+
// A floating IP is a single address, so DualStack is not valid here.
203+
// Mutually exclusive with UUID.
204+
// +kubebuilder:validation:Enum=IPv4;IPv6
205+
// +optional
206+
IPFamily *IPFamily `json:"ipFamily,omitempty"`
207+
208+
// UUID references an existing floating IP (BYO).
209+
// The floating IP is not deleted on cluster teardown.
210+
// Mutually exclusive with IPFamily.
211+
// +optional
212+
UUID string `json:"uuid,omitempty"`
213+
}
214+
152215
// CloudscaleClusterStatus defines the observed state of CloudscaleCluster.
153216
type CloudscaleClusterStatus struct {
154217
// Initialization contains v1beta2 initialization tracking.
155218
// +optional
156219
Initialization *ClusterInitializationStatus `json:"initialization,omitempty"`
157220

158-
// NetworkID is the cloudscale.ch network UUID.
221+
// Networks track the status of each network defined in spec.networks.
222+
// +listType=map
223+
// +listMapKey=name
159224
// +optional
160-
NetworkID string `json:"networkID,omitempty"`
225+
Networks []NetworkStatus `json:"networks,omitempty"`
161226

162-
// SubnetID is the cloudscale.ch subnet UUID.
227+
// FloatingIPHREF is the cloudscale.ch floating IP HREF.
228+
// The cloudscale API identifies floating IPs by HREF (e.g. "/v1/floating-ips/192.0.2.0/32"),
229+
// not by UUID like other resources.
163230
// +optional
164-
SubnetID string `json:"subnetID,omitempty"`
231+
FloatingIPHREF string `json:"floatingIPHREF,omitempty"`
165232

166233
// LoadBalancerID is the cloudscale.ch load balancer UUID.
167234
// +optional
@@ -184,20 +251,30 @@ type CloudscaleClusterStatus struct {
184251
LoadBalancerMemberIDs []string `json:"loadBalancerMemberIDs,omitempty"`
185252

186253
// conditions represent the current state of the CloudscaleCluster resource.
187-
// Each condition has a unique type and reflects the status of a specific aspect of the resource.
188-
//
189-
// Standard condition types include:
190-
// - "Available": the resource is fully functional
191-
// - "Progressing": the resource is being created or updated
192-
// - "Degraded": the resource failed to reach or maintain its desired state
193-
//
194-
// The status of each condition is one of True, False, or Unknown.
195254
// +listType=map
196255
// +listMapKey=type
197256
// +optional
198257
Conditions []metav1.Condition `json:"conditions,omitempty"`
199258
}
200259

260+
// NetworkStatus tracks the provisioned state of a single network.
261+
type NetworkStatus struct {
262+
// Name matches the logical name from spec.networks[].name.
263+
Name string `json:"name"`
264+
265+
// NetworkID is the cloudscale.ch network UUID.
266+
// +optional
267+
NetworkID string `json:"networkID,omitempty"`
268+
269+
// SubnetID is the cloudscale.ch subnet UUID.
270+
// +optional
271+
SubnetID string `json:"subnetID,omitempty"`
272+
273+
// Managed indicates whether CAPCS manages this network's lifecycle.
274+
// false for BYO networks (referenced by UUID), true for CAPCS-created networks (defined by CIDR).
275+
Managed bool `json:"managed"`
276+
}
277+
201278
// ClusterInitializationStatus contains v1beta2 initialization tracking for CloudscaleCluster.
202279
type ClusterInitializationStatus struct {
203280
// Provisioned indicates that all cluster infrastructure has been provisioned.
@@ -206,6 +283,16 @@ type ClusterInitializationStatus struct {
206283
Provisioned *bool `json:"provisioned,omitempty"`
207284
}
208285

286+
// GetNetworkStatus returns the NetworkStatus for the given network name, or nil if not found.
287+
func (s *CloudscaleClusterStatus) GetNetworkStatus(name string) *NetworkStatus {
288+
for i := range s.Networks {
289+
if s.Networks[i].Name == name {
290+
return &s.Networks[i]
291+
}
292+
}
293+
return nil
294+
}
295+
209296
// +kubebuilder:object:root=true
210297
// +kubebuilder:subresource:status
211298
// +kubebuilder:resource:path=cloudscaleclusters,scope=Namespaced,categories=cluster-api

api/v1beta2/cloudscalemachine_types.go

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -58,6 +58,35 @@ type CloudscaleMachineSpec struct {
5858
// N.B.: Only **up to 4 machines** can be placed in the same server group.
5959
// +optional
6060
ServerGroup *ServerGroupSpec `json:"serverGroup,omitempty"`
61+
62+
// Interfaces define the network interfaces to attach to the server.
63+
// When omitted, the controller defaults to the first cluster network and a public interface
64+
// at runtime (cross-resource resolution that the webhook cannot do).
65+
// +listType=atomic
66+
// +optional
67+
Interfaces []InterfaceSpec `json:"interfaces,omitempty"`
68+
}
69+
70+
// InterfaceSpec defines a network interface to attach to a server.
71+
// Exactly one of Type or Network must be specified.
72+
type InterfaceSpec struct {
73+
// Type is "public" for a public internet interface.
74+
// Mutually exclusive with Network.
75+
// +kubebuilder:validation:Enum=public
76+
// +optional
77+
Type string `json:"type,omitempty"`
78+
79+
// Network references a named network from CloudscaleCluster.spec.networks.
80+
// Mutually exclusive with Type.
81+
// +optional
82+
Network string `json:"network,omitempty"`
83+
84+
// IPFamily controls IPv4/IPv6 for a public interface.
85+
// Only valid when Type is "public".
86+
// Maps to the cloudscale API's per-server use_ipv6 setting.
87+
// +kubebuilder:validation:Enum=IPv4;IPv6;DualStack
88+
// +optional
89+
IPFamily *IPFamily `json:"ipFamily,omitempty"`
6190
}
6291

6392
// ServerGroupSpec configures server group placement for anti-affinity.

0 commit comments

Comments
 (0)