diff --git a/.github/workflows/e2e-weekly.yml b/.github/workflows/e2e-weekly.yml index 6b32b21..97583d6 100644 --- a/.github/workflows/e2e-weekly.yml +++ b/.github/workflows/e2e-weekly.yml @@ -37,10 +37,11 @@ jobs: - name: Run weekly e2e tests env: CLOUDSCALE_API_TOKEN: ${{ secrets.CLOUDSCALE_API_TOKEN }} + CLOUDSCALE_NETWORK_UUID: ${{ secrets.CLOUDSCALE_NETWORK_UUID }} TAG: e2e-weekly-${{ github.sha }} run: | make test-e2e \ - GINKGO_LABEL_FILTER="ha || upgrade || self-hosted || kcp-remediation || conformance" \ + GINKGO_LABEL_FILTER="ha || upgrade || self-hosted || kcp-remediation || conformance || byo-networking" \ KUBETEST_CONFIGURATION=./data/kubetest/conformance-fast.yaml - name: Install regctl diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 24f7d2b..ab008cf 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -53,8 +53,9 @@ jobs: env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | + ls dist/cluster-template*.yaml >/dev/null || { echo "No cluster templates found in dist/"; exit 1; } gh release create "$TAG" \ --generate-notes \ dist/infrastructure-components.yaml \ dist/metadata.yaml \ - dist/cluster-template.yaml + dist/cluster-template*.yaml diff --git a/.github/workflows/test-e2e.yml b/.github/workflows/test-e2e.yml index 14cc546..00df2c9 100644 --- a/.github/workflows/test-e2e.yml +++ b/.github/workflows/test-e2e.yml @@ -18,6 +18,7 @@ on: - test-e2e-upgrade - test-e2e-self-hosted - test-e2e-md-remediation + - test-e2e-byo-networking - test-e2e-conformance - test-e2e-conformance-fast @@ -53,6 +54,7 @@ jobs: - name: Run e2e tests env: CLOUDSCALE_API_TOKEN: ${{ secrets.CLOUDSCALE_API_TOKEN }} + CLOUDSCALE_NETWORK_UUID: ${{ secrets.CLOUDSCALE_NETWORK_UUID }} TAG: e2e-manual-${{ github.sha }} TEST_TARGET: ${{ github.event.inputs.test_target }} run: make $TEST_TARGET diff --git a/Makefile b/Makefile index 524e5bc..91dd129 100644 --- a/Makefile +++ b/Makefile @@ -125,17 +125,20 @@ generate-e2e-cni: ## Regenerate Cilium CNI manifest from Helm chart generate-e2e-ccm: ## Regenerate cloudscale CCM manifest @CCM_VERSION=$(CCM_VERSION) hack/generate-e2e-ccm.sh +E2E_CLUSTER_TEMPLATES := cluster-template \ + cluster-template-ha \ + cluster-template-upgrades \ + cluster-template-md-remediation \ + cluster-template-byo-network \ + cluster-template-public-lb-private-nodes \ + cluster-template-fip + .PHONY: generate-e2e-templates generate-e2e-templates: $(KUSTOMIZE) generate-e2e-cni generate-e2e-ccm ## Generate e2e cluster templates using kustomize overlays @mkdir -p $(E2E_TEMPLATES)/main - @echo "Generating cluster-template.yaml..." - @"$(KUSTOMIZE)" build --load-restrictor LoadRestrictionsNone $(E2E_TEMPLATES)/cluster-template > $(E2E_TEMPLATES)/main/cluster-template.yaml - @echo "Generating cluster-template-ha.yaml..." - @"$(KUSTOMIZE)" build --load-restrictor LoadRestrictionsNone $(E2E_TEMPLATES)/cluster-template-ha > $(E2E_TEMPLATES)/main/cluster-template-ha.yaml - @echo "Generating cluster-template-upgrades.yaml..." - @"$(KUSTOMIZE)" build --load-restrictor LoadRestrictionsNone $(E2E_TEMPLATES)/cluster-template-upgrades > $(E2E_TEMPLATES)/main/cluster-template-upgrades.yaml - @echo "Generating cluster-template-md-remediation.yaml..." - @"$(KUSTOMIZE)" build --load-restrictor LoadRestrictionsNone $(E2E_TEMPLATES)/cluster-template-md-remediation > $(E2E_TEMPLATES)/main/cluster-template-md-remediation.yaml + @$(foreach tmpl,$(E2E_CLUSTER_TEMPLATES),\ + echo "Generating $(tmpl).yaml..." && \ + "$(KUSTOMIZE)" build --load-restrictor LoadRestrictionsNone $(E2E_TEMPLATES)/$(tmpl) > $(E2E_TEMPLATES)/main/$(tmpl).yaml &&) true @echo "Templates generated successfully." .PHONY: generate-e2e-config @@ -222,6 +225,19 @@ test-e2e-md-remediation: $(GINKGO) generate-e2e-templates generate-e2e-config do -e2e.skip-resource-cleanup=$(SKIP_RESOURCE_CLEANUP) \ -e2e.use-existing-cluster=$(USE_EXISTING_CLUSTER) +.PHONY: test-e2e-byo-networking +test-e2e-byo-networking: $(GINKGO) generate-e2e-templates generate-e2e-config docker-build ## Run BYO networking e2e tests + $(GINKGO) -v --trace --tags=e2e \ + --nodes=$(GINKGO_NODES) \ + --label-filter="byo-networking" \ + --timeout=90m \ + --output-dir="$(E2E_ARTIFACTS_FOLDER)" --junit-report="junit.e2e_byo_networking.xml" \ + ./test/e2e -- \ + -e2e.config=$(E2E_CONF_FILE) \ + -e2e.artifacts-folder=$(E2E_ARTIFACTS_FOLDER) \ + -e2e.skip-resource-cleanup=$(SKIP_RESOURCE_CLEANUP) \ + -e2e.use-existing-cluster=$(USE_EXISTING_CLUSTER) + .PHONY: test-e2e-conformance test-e2e-conformance: $(GINKGO) generate-e2e-templates generate-e2e-config docker-build ## Run K8s conformance e2e tests $(GINKGO) -v --trace --tags=e2e \ @@ -298,9 +314,9 @@ build-installer: manifests generate kustomize ## Generate a consolidated YAML wi "$(KUSTOMIZE)" build config/default > dist/infrastructure-components.yaml .PHONY: release-manifests -release-manifests: build-installer ## Build all release artifacts into dist/ (infrastructure-components.yaml, metadata.yaml, cluster-template.yaml). +release-manifests: build-installer ## Build all release artifacts into dist/ (infrastructure-components.yaml, metadata.yaml, cluster templates). cp metadata.yaml dist/metadata.yaml - cp templates/cluster-template.yaml dist/cluster-template.yaml + cp templates/cluster-template*.yaml dist/ ##@ Deployment diff --git a/README.md b/README.md index 2f7e5e5..04b84fa 100644 --- a/README.md +++ b/README.md @@ -8,8 +8,9 @@ for [cloudscale.ch](https://www.cloudscale.ch). ## Features -- **CloudscaleCluster**: Network, Subnet, Load Balancer management -- **CloudscaleMachine**: Server provisioning with cloud-init +- **CloudscaleCluster**: Multi-network management (managed or BYO), Load Balancer (public or private VIP), Floating IP + support +- **CloudscaleMachine**: Server provisioning with cloud-init and configurable network interfaces - **CloudscaleMachineTemplate**: Immutable machine templates for KubeadmControlPlane/MachineDeployment ## Prerequisites @@ -42,6 +43,9 @@ clusterctl generate cluster my-cluster \ | kubectl apply -f - ``` +This uses the default template (public nodes, managed network). See [Cluster Templates](#cluster-templates) for other +network topologies. + Watch the cluster come up: ```bash @@ -50,15 +54,40 @@ clusterctl describe cluster my-cluster ## Environment Variables -| Variable | Description | Example | -|-------------------------------------------|--------------------------------|-----------------------------------| -| `CLOUDSCALE_API_TOKEN` | cloudscale.ch API token | `abc123...` | -| `CLOUDSCALE_SSH_PUBLIC_KEY` | SSH public key added to nodes | `ssh-ed25519 AAAA...` | -| `CLOUDSCALE_REGION` | cloudscale.ch region | `lpg` or `rma` | -| `CLOUDSCALE_MACHINE_IMAGE` | Server image for nodes | `custom:ubuntu-2404-kube-v1.xx.x` | -| `CLOUDSCALE_CONTROL_PLANE_MACHINE_FLAVOR` | Flavor for control plane nodes | `flex-4-2` | -| `CLOUDSCALE_WORKER_MACHINE_FLAVOR` | Flavor for worker nodes | `flex-4-2` | -| `CLOUDSCALE_ROOT_VOLUME_SIZE` | Root volume size in GB | `50` | +| Variable | Description | Example | +|-------------------------------------------|-------------------------------------------|-----------------------------------| +| `CLOUDSCALE_API_TOKEN` | cloudscale.ch API token | `abc123...` | +| `CLOUDSCALE_SSH_PUBLIC_KEY` | SSH public key added to nodes | `ssh-ed25519 AAAA...` | +| `CLOUDSCALE_REGION` | cloudscale.ch region | `lpg` or `rma` | +| `CLOUDSCALE_MACHINE_IMAGE` | Server image for nodes | `custom:ubuntu-2404-kube-v1.xx.x` | +| `CLOUDSCALE_CONTROL_PLANE_MACHINE_FLAVOR` | Flavor for control plane nodes | `flex-4-2` | +| `CLOUDSCALE_WORKER_MACHINE_FLAVOR` | Flavor for worker nodes | `flex-4-2` | +| `CLOUDSCALE_ROOT_VOLUME_SIZE` | Root volume size in GB | `50` | +| `CLOUDSCALE_NETWORK_UUID` | Existing cloudscale.ch network UUID (BYO) | `2db69ba3-...` | + +> **Note:** `CLOUDSCALE_NETWORK_UUID` is required by the `fip`, `public-lb-private-nodes`, and `byo-network` +> template flavors. It is not needed for the default template. + +## Cluster Templates + +CAPCS ships several cluster templates for different network topologies. Use `clusterctl generate cluster` with the +`--flavor` flag to select one: + +```bash +clusterctl generate cluster my-cluster \ + --kubernetes-version v1.32.0 \ + --control-plane-machine-count 1 \ + --worker-machine-count 2 \ + --flavor \ + | kubectl apply -f - +``` + +| Flavor | Network | CP Endpoint | Node Connectivity | Extra Env Vars | Notes | +|---------------------------|---------------------------|-----------------------|-------------------|---------------------------|----------------------| +| *(default)* | Managed (`10.100.0.0/24`) | Public LB (DualStack) | Public + cluster | — | | +| `fip` | BYO | Floating IP (IPv4) | Public + cluster | `CLOUDSCALE_NETWORK_UUID` | | +| `public-lb-private-nodes` | BYO + NAT | Public LB | Private only | `CLOUDSCALE_NETWORK_UUID` | Requires NAT gateway | +| `byo-network` | BYO | Public LB (DualStack) | Public + cluster | `CLOUDSCALE_NETWORK_UUID` | | ## Development @@ -92,6 +121,7 @@ filtering and are split into suites of increasing cost, scheduled accordingly: | Cluster upgrade | `upgrade` | Rolling K8s version upgrade (v1.34 → v1.35) | < 10 min | Weekly | `test-e2e-upgrade` | | Self-hosted | `self-hosted` | clusterctl move (pivot) to workload cluster. Requires container image in public registry | < 15 min | Weekly | `test-e2e-self-hosted` | | MD remediation | `md-remediation` | MachineHealthCheck auto-replacement of unhealthy workers | < 10 min | Weekly | `test-e2e-md-remediation` | +| BYO networking | `byo-networking` | BYO network: public-LB + private-nodes and floating-IP variants | < 10 min | Weekly | `test-e2e-byo-networking` | | Conformance (fast) | `conformance` | K8s conformance, skip Serial tests | < 60 min | Weekly | `test-e2e-conformance-fast` | | Conformance (full) | `conformance` | Full K8s conformance including Serial tests | < 120 min | Biweekly | `test-e2e-conformance` | @@ -99,7 +129,8 @@ Durations are approximate from a real CI run; conformance varies with cluster si **Why this split?** The single-CP lifecycle test is the cheapest smoke test and runs nightly to catch regressions early. HA, upgrade, self-hosted, and remediation tests are more -resource-intensive and run weekly. Full K8s conformance is the most expensive and runs biweekly +resource-intensive and run weekly. Private networking tests require `CLOUDSCALE_NETWORK_UUID` to be set and are +skipped otherwise. Full K8s conformance is the most expensive and runs biweekly (1st + 15th of month). All suites can be triggered manually via the `test-e2e.yml` workflow dispatch. E2E tests share a concurrency group so only one suite runs at a time. @@ -143,6 +174,8 @@ kustomize_substitutions: CLOUDSCALE_WORKER_MACHINE_FLAVOR: "flex-4-2" CLOUDSCALE_MACHINE_IMAGE: "IMAGE_NAME" CLOUDSCALE_ROOT_VOLUME_SIZE: "50" + # Required for BYO network flavors (fip, public-lb-private-nodes, byo-network): + # CLOUDSCALE_NETWORK_UUID: "UUID_HERE" extra_args: cloudscale: - "--zap-log-level=5" diff --git a/api/v1beta2/cloudscalecluster_types.go b/api/v1beta2/cloudscalecluster_types.go index e8bef8e..bd25eae 100644 --- a/api/v1beta2/cloudscalecluster_types.go +++ b/api/v1beta2/cloudscalecluster_types.go @@ -28,6 +28,16 @@ const ( ClusterFinalizer = "cloudscalecluster.infrastructure.cluster.x-k8s.io" ) +// IPFamily represents an IP family configuration. +// +kubebuilder:validation:Enum=IPv4;IPv6;DualStack +type IPFamily string + +const ( + IPFamilyIPv4 IPFamily = "IPv4" + IPFamilyIPv6 IPFamily = "IPv6" + IPFamilyDualStack IPFamily = "DualStack" +) + // CloudscaleClusterSpec defines the desired state of CloudscaleCluster type CloudscaleClusterSpec struct { // Region is the cloudscale.ch region (e.g., "rma", "lpg"). @@ -45,17 +55,32 @@ type CloudscaleClusterSpec struct { CredentialsRef CloudscaleCredentialsReference `json:"credentialsRef"` // ControlPlaneEndpoint represents the endpoint to communicate with the control plane. - // This is set automatically from the load balancer's VIP address. + // This is set automatically from the load balancer's VIP address or floating IP. // +optional ControlPlaneEndpoint clusterv1.APIEndpoint `json:"controlPlaneEndpoint,omitzero"` - // Network contains network configuration for the cluster. + // Networks define the private networks for this cluster. + // Referenced by name from machine interface specs and LB config. + // If empty, defaults to a single managed network named after the cluster. + // +listType=map + // +listMapKey=name // +optional - Network NetworkSpec `json:"network,omitzero"` + Networks []NetworkSpec `json:"networks,omitempty"` // ControlPlaneLoadBalancer configures the load balancer for the control plane. // +optional ControlPlaneLoadBalancer LoadBalancerSpec `json:"controlPlaneLoadBalancer,omitzero"` + + // FloatingIP configures a floating IP for a stable control plane endpoint. + // When the load balancer is enabled (recommended), the floating IP is assigned + // to the LB, providing a stable IP that survives LB recreation. + // When using a BYO floating IP without a load balancer, the user must + // configure a dummy interface on the control plane servers (see cloudscale.ch docs). + // Managed floating IPs require the load balancer to be enabled. + // Floating IPs cannot be attached to a load balancer with a private VIP + // (i.e. one whose ControlPlaneLoadBalancer.Network is set). + // +optional + FloatingIP *FloatingIPSpec `json:"floatingIP,omitempty"` } // CloudscaleCredentialsReference references a Secret containing the API token. @@ -69,28 +94,43 @@ type CloudscaleCredentialsReference struct { Namespace string `json:"namespace,omitempty"` } -// NetworkSpec defines the network configuration. +// NetworkSpec defines a private network for the cluster. +// Exactly one of UUID or CIDR must be specified. type NetworkSpec struct { - // CIDR is the CIDR block for the private network subnet. - // +kubebuilder:default="10.0.0.0/24" + // Name identifies this network within the cluster. + // Used to reference this network from machine interface specs and LB config. + // +kubebuilder:validation:Required + // +kubebuilder:validation:Pattern=`^[a-z0-9]([a-z0-9-]*[a-z0-9])?$` + // +kubebuilder:validation:MaxLength=63 + Name string `json:"name"` + + // UUID references an existing cloudscale.ch network (BYO). + // The network is not deleted on cluster teardown. + // Mutually exclusive with CIDR. + // +optional + UUID string `json:"uuid,omitempty"` + + // CIDR defines the subnet for a controller-managed network. + // The network and subnet are created and deleted by CAPCS. + // Mutually exclusive with UUID. // +optional CIDR string `json:"cidr,omitempty"` // GatewayAddress is the gateway IP address for the subnet. - // By default, no gateway is configured on the private network subnet. This ensures - // that outbound internet traffic uses the public network interface, which is required - // for the Cloud Controller Manager to reach the cloudscale.ch API. + // Only applicable when CIDR is set (managed network). + // By default, no gateway is configured on the subnet. This ensures + // that outbound internet traffic uses the public network interface. // Set this to a specific IP address (e.g., "10.0.0.1") only if you have configured // a NAT gateway or similar infrastructure on the private network. // +optional - GatewayAddress *string `json:"gatewayAddress,omitempty"` + GatewayAddress string `json:"gatewayAddress,omitempty"` } // LoadBalancerSpec defines the load balancer configuration for the control plane. type LoadBalancerSpec struct { // Enabled controls whether a load balancer is created for the control plane. // Set to false for external control planes (e.g., hosted control plane) where the endpoint - // is provided externally. + // is provided externally, or when using a floating IP without a load balancer. // +kubebuilder:default=true // +optional Enabled *bool `json:"enabled,omitempty"` @@ -113,6 +153,19 @@ type LoadBalancerSpec struct { // +optional APIServerPort int32 `json:"apiServerPort,omitempty"` + // Network places the LB VIP on a private network (internal LB). + // References spec.networks[].name. Omit for a public LB. + // When multiple networks are defined this field is required so the LB + // pool members can be registered against a specific subnet. + // +optional + Network string `json:"network,omitempty"` + + // IPFamily specifies the IP family for the LB VIP address(es). + // +kubebuilder:validation:Enum=IPv4;IPv6;DualStack + // +kubebuilder:default=DualStack + // +optional + IPFamily IPFamily `json:"ipFamily,omitempty"` + // HealthMonitor configures the load balancer health monitor. // +optional HealthMonitor HealthMonitorSpec `json:"healthMonitor,omitempty"` @@ -149,19 +202,39 @@ type HealthMonitorSpec struct { DownThreshold int `json:"downThreshold,omitempty"` } +// FloatingIPSpec configures a floating IP for the control plane endpoint. +// Exactly one of IPFamily or IP must be specified. +type FloatingIPSpec struct { + // IPFamily creates a new floating IP with this IP version. + // A floating IP is a single address, so DualStack is not valid here. + // Mutually exclusive with IP. + // +kubebuilder:validation:Enum=IPv4;IPv6 + // +optional + IPFamily *IPFamily `json:"ipFamily,omitempty"` + + // IP references an existing floating IP (BYO) by its address. + // cloudscale.ch identifies floating IPs by their IP address rather than a UUID. + // The floating IP is not deleted on cluster teardown. + // Mutually exclusive with IPFamily. + // +optional + IP string `json:"ip,omitempty"` +} + // CloudscaleClusterStatus defines the observed state of CloudscaleCluster. type CloudscaleClusterStatus struct { // Initialization contains v1beta2 initialization tracking. // +optional Initialization *ClusterInitializationStatus `json:"initialization,omitempty"` - // NetworkID is the cloudscale.ch network UUID. + // Networks track the status of each network defined in spec.networks. + // +listType=map + // +listMapKey=name // +optional - NetworkID string `json:"networkID,omitempty"` + Networks []NetworkStatus `json:"networks,omitempty"` - // SubnetID is the cloudscale.ch subnet UUID. + // FloatingIP is the cloudscale.ch floating IP. // +optional - SubnetID string `json:"subnetID,omitempty"` + FloatingIP string `json:"floatingIP,omitempty"` // LoadBalancerID is the cloudscale.ch load balancer UUID. // +optional @@ -184,20 +257,35 @@ type CloudscaleClusterStatus struct { LoadBalancerMemberIDs []string `json:"loadBalancerMemberIDs,omitempty"` // conditions represent the current state of the CloudscaleCluster resource. - // Each condition has a unique type and reflects the status of a specific aspect of the resource. - // - // Standard condition types include: - // - "Available": the resource is fully functional - // - "Progressing": the resource is being created or updated - // - "Degraded": the resource failed to reach or maintain its desired state - // - // The status of each condition is one of True, False, or Unknown. // +listType=map // +listMapKey=type // +optional Conditions []metav1.Condition `json:"conditions,omitempty"` } +// NetworkStatus tracks the provisioned state of a single network. +type NetworkStatus struct { + // Name matches the logical name from spec.networks[].name. + Name string `json:"name"` + + // NetworkID is the cloudscale.ch network UUID. + // +optional + NetworkID string `json:"networkID,omitempty"` + + // SubnetID is the cloudscale.ch subnet UUID. + // +optional + SubnetID string `json:"subnetID,omitempty"` + + // CIDR is the subnet CIDR block. + // Set from spec for managed networks or discovered from the API for BYO networks. + // +optional + CIDR string `json:"cidr,omitempty"` + + // Managed indicates whether CAPCS manages this network's lifecycle. + // false for BYO networks (referenced by UUID), true for CAPCS-created networks (defined by CIDR). + Managed bool `json:"managed"` +} + // ClusterInitializationStatus contains v1beta2 initialization tracking for CloudscaleCluster. type ClusterInitializationStatus struct { // Provisioned indicates that all cluster infrastructure has been provisioned. @@ -206,6 +294,16 @@ type ClusterInitializationStatus struct { Provisioned *bool `json:"provisioned,omitempty"` } +// GetNetworkStatus returns the NetworkStatus for the given network name, or nil if not found. +func (s *CloudscaleClusterStatus) GetNetworkStatus(name string) *NetworkStatus { + for i := range s.Networks { + if s.Networks[i].Name == name { + return &s.Networks[i] + } + } + return nil +} + // +kubebuilder:object:root=true // +kubebuilder:subresource:status // +kubebuilder:resource:path=cloudscaleclusters,scope=Namespaced,categories=cluster-api diff --git a/api/v1beta2/cloudscalemachine_types.go b/api/v1beta2/cloudscalemachine_types.go index 409e96c..3301c1e 100644 --- a/api/v1beta2/cloudscalemachine_types.go +++ b/api/v1beta2/cloudscalemachine_types.go @@ -58,6 +58,37 @@ type CloudscaleMachineSpec struct { // N.B.: Only **up to 4 machines** can be placed in the same server group. // +optional ServerGroup *ServerGroupSpec `json:"serverGroup,omitempty"` + + // Interfaces define the network interfaces to attach to the server. + // When omitted, the controller defaults to the first cluster network and a public interface + // at runtime (cross-resource resolution that the webhook cannot do). + // +listType=atomic + // +optional + Interfaces []InterfaceSpec `json:"interfaces,omitempty"` +} + +// InterfaceSpec defines a network interface to attach to a server. +// Exactly one of Type or Network must be specified. +type InterfaceSpec struct { + // Type is "public" for a public internet interface. + // Mutually exclusive with Network. + // +kubebuilder:validation:Enum=public + // +optional + Type string `json:"type,omitempty"` + + // Network references a named network from CloudscaleCluster.spec.networks. + // Mutually exclusive with Type. + // +optional + Network string `json:"network,omitempty"` + + // IPFamily controls IPv4/IPv6 for a public interface. + // Only valid when Type is "public". + // Maps to the cloudscale API's per-server use_ipv6 setting: + // - IPv4: use_ipv6=false (IPv4 only) + // - DualStack: use_ipv6=true (IPv4 + IPv6) + // +kubebuilder:validation:Enum=IPv4;DualStack + // +optional + IPFamily *IPFamily `json:"ipFamily,omitempty"` } // ServerGroupSpec configures server group placement for anti-affinity. diff --git a/api/v1beta2/condition_types.go b/api/v1beta2/condition_types.go index db984a0..d82e359 100644 --- a/api/v1beta2/condition_types.go +++ b/api/v1beta2/condition_types.go @@ -30,6 +30,10 @@ const ( // True when the pause annotation is present on the resource or parent cluster. PausedCondition = "Paused" + // FloatingIPReadyCondition indicates whether the floating IP has been provisioned and assigned. + // Only applicable to CloudscaleCluster when spec.floatingIP is set. + FloatingIPReadyCondition = "FloatingIPReady" + // DeletingCondition indicates the resource is being deleted. // True when DeletionTimestamp is set on the resource. DeletingCondition = "Deleting" @@ -60,6 +64,18 @@ const ( // LoadBalancerDeletingReason indicates the load balancer is being deleted. LoadBalancerDeletingReason = "LoadBalancerDeleting" + + // FloatingIPProvisionedReason indicates the floating IP has been successfully provisioned and assigned. + FloatingIPProvisionedReason = "FloatingIPProvisioned" + + // FloatingIPDisabledReason indicates no floating IP is configured. + FloatingIPDisabledReason = "FloatingIPDisabled" + + // FloatingIPErrorReason indicates an error occurred during floating IP operations. + FloatingIPErrorReason = "FloatingIPError" + + // FloatingIPDeletingReason indicates the floating IP is being deleted. + FloatingIPDeletingReason = "FloatingIPDeleting" ) // Condition reasons for CloudscaleMachine. diff --git a/api/v1beta2/zz_generated.deepcopy.go b/api/v1beta2/zz_generated.deepcopy.go index dbe65e2..e54ff44 100644 --- a/api/v1beta2/zz_generated.deepcopy.go +++ b/api/v1beta2/zz_generated.deepcopy.go @@ -91,8 +91,17 @@ func (in *CloudscaleClusterSpec) DeepCopyInto(out *CloudscaleClusterSpec) { *out = *in out.CredentialsRef = in.CredentialsRef out.ControlPlaneEndpoint = in.ControlPlaneEndpoint - in.Network.DeepCopyInto(&out.Network) + if in.Networks != nil { + in, out := &in.Networks, &out.Networks + *out = make([]NetworkSpec, len(*in)) + copy(*out, *in) + } in.ControlPlaneLoadBalancer.DeepCopyInto(&out.ControlPlaneLoadBalancer) + if in.FloatingIP != nil { + in, out := &in.FloatingIP, &out.FloatingIP + *out = new(FloatingIPSpec) + (*in).DeepCopyInto(*out) + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CloudscaleClusterSpec. @@ -113,6 +122,11 @@ func (in *CloudscaleClusterStatus) DeepCopyInto(out *CloudscaleClusterStatus) { *out = new(ClusterInitializationStatus) (*in).DeepCopyInto(*out) } + if in.Networks != nil { + in, out := &in.Networks, &out.Networks + *out = make([]NetworkStatus, len(*in)) + copy(*out, *in) + } if in.LoadBalancerMemberIDs != nil { in, out := &in.LoadBalancerMemberIDs, &out.LoadBalancerMemberIDs *out = make([]string, len(*in)) @@ -231,6 +245,13 @@ func (in *CloudscaleMachineSpec) DeepCopyInto(out *CloudscaleMachineSpec) { *out = new(ServerGroupSpec) **out = **in } + if in.Interfaces != nil { + in, out := &in.Interfaces, &out.Interfaces + *out = make([]InterfaceSpec, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CloudscaleMachineSpec. @@ -415,6 +436,26 @@ func (in *ClusterInitializationStatus) DeepCopy() *ClusterInitializationStatus { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *FloatingIPSpec) DeepCopyInto(out *FloatingIPSpec) { + *out = *in + if in.IPFamily != nil { + in, out := &in.IPFamily, &out.IPFamily + *out = new(IPFamily) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new FloatingIPSpec. +func (in *FloatingIPSpec) DeepCopy() *FloatingIPSpec { + if in == nil { + return nil + } + out := new(FloatingIPSpec) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *HealthMonitorSpec) DeepCopyInto(out *HealthMonitorSpec) { *out = *in @@ -430,6 +471,26 @@ func (in *HealthMonitorSpec) DeepCopy() *HealthMonitorSpec { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *InterfaceSpec) DeepCopyInto(out *InterfaceSpec) { + *out = *in + if in.IPFamily != nil { + in, out := &in.IPFamily, &out.IPFamily + *out = new(IPFamily) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new InterfaceSpec. +func (in *InterfaceSpec) DeepCopy() *InterfaceSpec { + if in == nil { + return nil + } + out := new(InterfaceSpec) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *LoadBalancerSpec) DeepCopyInto(out *LoadBalancerSpec) { *out = *in @@ -474,11 +535,6 @@ func (in *MachineInitializationStatus) DeepCopy() *MachineInitializationStatus { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *NetworkSpec) DeepCopyInto(out *NetworkSpec) { *out = *in - if in.GatewayAddress != nil { - in, out := &in.GatewayAddress, &out.GatewayAddress - *out = new(string) - **out = **in - } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NetworkSpec. @@ -491,6 +547,21 @@ func (in *NetworkSpec) DeepCopy() *NetworkSpec { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *NetworkStatus) DeepCopyInto(out *NetworkStatus) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NetworkStatus. +func (in *NetworkStatus) DeepCopy() *NetworkStatus { + if in == nil { + return nil + } + out := new(NetworkStatus) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *NodeInfo) DeepCopyInto(out *NodeInfo) { *out = *in diff --git a/config/crd/bases/infrastructure.cluster.x-k8s.io_cloudscaleclusters.yaml b/config/crd/bases/infrastructure.cluster.x-k8s.io_cloudscaleclusters.yaml index 15ab908..8ebf791 100644 --- a/config/crd/bases/infrastructure.cluster.x-k8s.io_cloudscaleclusters.yaml +++ b/config/crd/bases/infrastructure.cluster.x-k8s.io_cloudscaleclusters.yaml @@ -61,7 +61,7 @@ spec: controlPlaneEndpoint: description: |- ControlPlaneEndpoint represents the endpoint to communicate with the control plane. - This is set automatically from the load balancer's VIP address. + This is set automatically from the load balancer's VIP address or floating IP. minProperties: 1 properties: host: @@ -101,7 +101,7 @@ spec: description: |- Enabled controls whether a load balancer is created for the control plane. Set to false for external control planes (e.g., hosted control plane) where the endpoint - is provided externally. + is provided externally, or when using a floating IP without a load balancer. type: boolean flavor: default: lb-standard @@ -139,6 +139,26 @@ spec: minimum: 1 type: integer type: object + ipFamily: + allOf: + - enum: + - IPv4 + - IPv6 + - DualStack + - enum: + - IPv4 + - IPv6 + - DualStack + default: DualStack + description: IPFamily specifies the IP family for the LB VIP address(es). + type: string + network: + description: |- + Network places the LB VIP on a private network (internal LB). + References spec.networks[].name. Omit for a public LB. + When multiple networks are defined this field is required so the LB + pool members can be registered against a specific subnet. + type: string type: object credentialsRef: description: CredentialsRef references the Secret containing the cloudscale.ch @@ -154,23 +174,84 @@ spec: required: - name type: object - network: - description: Network contains network configuration for the cluster. + floatingIP: + description: |- + FloatingIP configures a floating IP for a stable control plane endpoint. + When the load balancer is enabled (recommended), the floating IP is assigned + to the LB, providing a stable IP that survives LB recreation. + When using a BYO floating IP without a load balancer, the user must + configure a dummy interface on the control plane servers (see cloudscale.ch docs). + Managed floating IPs require the load balancer to be enabled. + Floating IPs cannot be attached to a load balancer with a private VIP + (i.e. one whose ControlPlaneLoadBalancer.Network is set). properties: - cidr: - default: 10.0.0.0/24 - description: CIDR is the CIDR block for the private network subnet. + ip: + description: |- + IP references an existing floating IP (BYO) by its address. + cloudscale.ch identifies floating IPs by their IP address rather than a UUID. + The floating IP is not deleted on cluster teardown. + Mutually exclusive with IPFamily. type: string - gatewayAddress: + ipFamily: + allOf: + - enum: + - IPv4 + - IPv6 + - DualStack + - enum: + - IPv4 + - IPv6 description: |- - GatewayAddress is the gateway IP address for the subnet. - By default, no gateway is configured on the private network subnet. This ensures - that outbound internet traffic uses the public network interface, which is required - for the Cloud Controller Manager to reach the cloudscale.ch API. - Set this to a specific IP address (e.g., "10.0.0.1") only if you have configured - a NAT gateway or similar infrastructure on the private network. + IPFamily creates a new floating IP with this IP version. + A floating IP is a single address, so DualStack is not valid here. + Mutually exclusive with IP. type: string type: object + networks: + description: |- + Networks define the private networks for this cluster. + Referenced by name from machine interface specs and LB config. + If empty, defaults to a single managed network named after the cluster. + items: + description: |- + NetworkSpec defines a private network for the cluster. + Exactly one of UUID or CIDR must be specified. + properties: + cidr: + description: |- + CIDR defines the subnet for a controller-managed network. + The network and subnet are created and deleted by CAPCS. + Mutually exclusive with UUID. + type: string + gatewayAddress: + description: |- + GatewayAddress is the gateway IP address for the subnet. + Only applicable when CIDR is set (managed network). + By default, no gateway is configured on the subnet. This ensures + that outbound internet traffic uses the public network interface. + Set this to a specific IP address (e.g., "10.0.0.1") only if you have configured + a NAT gateway or similar infrastructure on the private network. + type: string + name: + description: |- + Name identifies this network within the cluster. + Used to reference this network from machine interface specs and LB config. + maxLength: 63 + pattern: ^[a-z0-9]([a-z0-9-]*[a-z0-9])?$ + type: string + uuid: + description: |- + UUID references an existing cloudscale.ch network (BYO). + The network is not deleted on cluster teardown. + Mutually exclusive with CIDR. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map region: description: Region is the cloudscale.ch region (e.g., "rma", "lpg"). enum: @@ -190,16 +271,8 @@ spec: description: status defines the observed state of CloudscaleCluster properties: conditions: - description: |- - conditions represent the current state of the CloudscaleCluster resource. - Each condition has a unique type and reflects the status of a specific aspect of the resource. - - Standard condition types include: - - "Available": the resource is fully functional - - "Progressing": the resource is being created or updated - - "Degraded": the resource failed to reach or maintain its desired state - - The status of each condition is one of True, False, or Unknown. + description: conditions represent the current state of the CloudscaleCluster + resource. items: description: Condition contains details for one aspect of the current state of this API Resource. @@ -258,6 +331,9 @@ spec: x-kubernetes-list-map-keys: - type x-kubernetes-list-type: map + floatingIP: + description: FloatingIP is the cloudscale.ch floating IP. + type: string initialization: description: Initialization contains v1beta2 initialization tracking. properties: @@ -288,12 +364,40 @@ spec: description: LoadBalancerPoolID is the cloudscale.ch load balancer pool UUID for the API server. type: string - networkID: - description: NetworkID is the cloudscale.ch network UUID. - type: string - subnetID: - description: SubnetID is the cloudscale.ch subnet UUID. - type: string + networks: + description: Networks track the status of each network defined in + spec.networks. + items: + description: NetworkStatus tracks the provisioned state of a single + network. + properties: + cidr: + description: |- + CIDR is the subnet CIDR block. + Set from spec for managed networks or discovered from the API for BYO networks. + type: string + managed: + description: |- + Managed indicates whether CAPCS manages this network's lifecycle. + false for BYO networks (referenced by UUID), true for CAPCS-created networks (defined by CIDR). + type: boolean + name: + description: Name matches the logical name from spec.networks[].name. + type: string + networkID: + description: NetworkID is the cloudscale.ch network UUID. + type: string + subnetID: + description: SubnetID is the cloudscale.ch subnet UUID. + type: string + required: + - managed + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map type: object required: - spec diff --git a/config/crd/bases/infrastructure.cluster.x-k8s.io_cloudscalemachines.yaml b/config/crd/bases/infrastructure.cluster.x-k8s.io_cloudscalemachines.yaml index 5a6bf65..fe271f4 100644 --- a/config/crd/bases/infrastructure.cluster.x-k8s.io_cloudscalemachines.yaml +++ b/config/crd/bases/infrastructure.cluster.x-k8s.io_cloudscalemachines.yaml @@ -67,6 +67,47 @@ spec: image slug (e.g., "custom:ubuntu-foo"), or custom image UUID. minLength: 1 type: string + interfaces: + description: |- + Interfaces define the network interfaces to attach to the server. + When omitted, the controller defaults to the first cluster network and a public interface + at runtime (cross-resource resolution that the webhook cannot do). + items: + description: |- + InterfaceSpec defines a network interface to attach to a server. + Exactly one of Type or Network must be specified. + properties: + ipFamily: + allOf: + - enum: + - IPv4 + - IPv6 + - DualStack + - enum: + - IPv4 + - DualStack + description: |- + IPFamily controls IPv4/IPv6 for a public interface. + Only valid when Type is "public". + Maps to the cloudscale API's per-server use_ipv6 setting: + - IPv4: use_ipv6=false (IPv4 only) + - DualStack: use_ipv6=true (IPv4 + IPv6) + type: string + network: + description: |- + Network references a named network from CloudscaleCluster.spec.networks. + Mutually exclusive with Type. + type: string + type: + description: |- + Type is "public" for a public internet interface. + Mutually exclusive with Network. + enum: + - public + type: string + type: object + type: array + x-kubernetes-list-type: atomic providerID: description: |- ProviderID is the unique identifier as specified by the cloud provider. diff --git a/config/crd/bases/infrastructure.cluster.x-k8s.io_cloudscalemachinetemplates.yaml b/config/crd/bases/infrastructure.cluster.x-k8s.io_cloudscalemachinetemplates.yaml index c7b565a..65a8944 100644 --- a/config/crd/bases/infrastructure.cluster.x-k8s.io_cloudscalemachinetemplates.yaml +++ b/config/crd/bases/infrastructure.cluster.x-k8s.io_cloudscalemachinetemplates.yaml @@ -58,6 +58,47 @@ spec: image UUID. minLength: 1 type: string + interfaces: + description: |- + Interfaces define the network interfaces to attach to the server. + When omitted, the controller defaults to the first cluster network and a public interface + at runtime (cross-resource resolution that the webhook cannot do). + items: + description: |- + InterfaceSpec defines a network interface to attach to a server. + Exactly one of Type or Network must be specified. + properties: + ipFamily: + allOf: + - enum: + - IPv4 + - IPv6 + - DualStack + - enum: + - IPv4 + - DualStack + description: |- + IPFamily controls IPv4/IPv6 for a public interface. + Only valid when Type is "public". + Maps to the cloudscale API's per-server use_ipv6 setting: + - IPv4: use_ipv6=false (IPv4 only) + - DualStack: use_ipv6=true (IPv4 + IPv6) + type: string + network: + description: |- + Network references a named network from CloudscaleCluster.spec.networks. + Mutually exclusive with Type. + type: string + type: + description: |- + Type is "public" for a public internet interface. + Mutually exclusive with Network. + enum: + - public + type: string + type: object + type: array + x-kubernetes-list-type: atomic providerID: description: |- ProviderID is the unique identifier as specified by the cloud provider. diff --git a/internal/cloudscale/client.go b/internal/cloudscale/client.go index 69002bb..390f268 100644 --- a/internal/cloudscale/client.go +++ b/internal/cloudscale/client.go @@ -35,6 +35,7 @@ type Client struct { LoadBalancerPoolMembers LoadBalancerPoolMemberService LoadBalancerListeners LoadBalancerListenerService LoadBalancerHealthMonitors LoadBalancerHealthMonitorService + FloatingIPs FloatingIPService Flavors FlavorService } @@ -54,6 +55,7 @@ func NewClient(token string) *Client { LoadBalancerPoolMembers: sdkClient.LoadBalancerPoolMembers, LoadBalancerListeners: sdkClient.LoadBalancerListeners, LoadBalancerHealthMonitors: sdkClient.LoadBalancerHealthMonitors, + FloatingIPs: sdkClient.FloatingIPs, Flavors: sdkClient.Flavors, } } diff --git a/internal/cloudscale/flavors_test.go b/internal/cloudscale/flavors_test.go new file mode 100644 index 0000000..b954017 --- /dev/null +++ b/internal/cloudscale/flavors_test.go @@ -0,0 +1,75 @@ +package cloudscale + +import ( + "testing" + + cloudscalesdk "github.com/cloudscale-ch/cloudscale-go-sdk/v8" + . "github.com/onsi/gomega" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" +) + +func TestFlavorInfo(t *testing.T) { + g := NewWithT(t) + flavors := []cloudscalesdk.Flavor{ + { + Slug: "small", + VCPUCount: 2, + MemoryGB: 4, + }, + { + Slug: "gpu-large", + VCPUCount: 8, + MemoryGB: 32, + GPU: &cloudscalesdk.FlavorGPU{ + Count: 1, + }, + }, + } + + fi := NewFlavorInfo(flavors) + + t.Run("IsValidFlavor", func(t *testing.T) { + g.Expect(fi.IsValidFlavor("small")).To(BeTrue()) + g.Expect(fi.IsValidFlavor("gpu-large")).To(BeTrue()) + g.Expect(fi.IsValidFlavor("non-existent")).To(BeFalse()) + }) + + t.Run("GetFlavor", func(t *testing.T) { + f := fi.GetFlavor("small") + g.Expect(f).NotTo(BeNil()) + g.Expect(f.Slug).To(Equal("small")) + + f = fi.GetFlavor("non-existent") + g.Expect(f).To(BeNil()) + }) + + t.Run("GetCapacity", func(t *testing.T) { + // Test standard flavor + capSmall, err := fi.GetCapacity("small", 20) + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(capSmall[corev1.ResourceCPU]).To(Equal(resource.MustParse("2"))) + g.Expect(capSmall[corev1.ResourceMemory]).To(Equal(resource.MustParse("4Gi"))) + g.Expect(capSmall[ResourceNvidiaGPU]).To(BeZero()) + + // Test GPU flavor + capGPU, err := fi.GetCapacity("gpu-large", 100) + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(capGPU[corev1.ResourceCPU]).To(Equal(resource.MustParse("8"))) + g.Expect(capGPU[corev1.ResourceMemory]).To(Equal(resource.MustParse("32Gi"))) + g.Expect(capGPU[ResourceNvidiaGPU]).To(Equal(resource.MustParse("1"))) + + // Test unknown flavor + capUnknown, err := fi.GetCapacity("unknown", 20) + g.Expect(err).To(HaveOccurred()) + g.Expect(capUnknown).To(BeNil()) + g.Expect(err.Error()).To(ContainSubstring("unknown flavor: unknown")) + }) + + t.Run("GetAllFlavors", func(t *testing.T) { + slugs := fi.GetAllFlavors() + g.Expect(slugs).To(HaveLen(2)) + g.Expect(slugs).To(ContainElement("small")) + g.Expect(slugs).To(ContainElement("gpu-large")) + }) +} diff --git a/internal/cloudscale/services.go b/internal/cloudscale/services.go index e4dedf9..e6290a0 100644 --- a/internal/cloudscale/services.go +++ b/internal/cloudscale/services.go @@ -96,6 +96,14 @@ type LoadBalancerHealthMonitorService interface { Update(ctx context.Context, id string, req *cloudscalesdk.LoadBalancerHealthMonitorRequest) error } +type FloatingIPService interface { + Create(ctx context.Context, req *cloudscalesdk.FloatingIPCreateRequest) (*cloudscalesdk.FloatingIP, error) + Get(ctx context.Context, id string) (*cloudscalesdk.FloatingIP, error) + List(ctx context.Context, modifiers ...cloudscalesdk.ListRequestModifier) ([]cloudscalesdk.FloatingIP, error) + Update(ctx context.Context, id string, req *cloudscalesdk.FloatingIPUpdateRequest) error + Delete(ctx context.Context, id string) error +} + type FlavorService interface { List(ctx context.Context) ([]cloudscalesdk.Flavor, error) } diff --git a/internal/controller/cloudscale_services.go b/internal/controller/cloudscale_services.go index 4b53f23..f1835bb 100644 --- a/internal/controller/cloudscale_services.go +++ b/internal/controller/cloudscale_services.go @@ -17,7 +17,8 @@ type getListService[T any] interface { } // ensureResource checks if a resource exists by its status ID, or adopts one found by tags. -// Returns the resolved ID and nil on success. An empty ID means the caller should create the resource. +// Returns the resource object and its ID on success. A nil resource and empty ID means +// the caller should create the resource. func ensureResource[T any]( ctx context.Context, clusterScope *scope.ClusterScope, @@ -26,30 +27,31 @@ func ensureResource[T any]( svc getListService[T], extractUUID func(T) string, tags cloudscalesdk.TagMap, -) (string, error) { +) (*T, string, error) { if currentID != "" { - _, err := svc.Get(ctx, currentID) + resource, err := svc.Get(ctx, currentID) if err == nil { - return currentID, nil + return resource, currentID, nil } if !cloudscale.IsNotFound(err) { - return "", fmt.Errorf("getting %s: %w", resourceName, err) + return nil, "", fmt.Errorf("getting %s: %w", resourceName, err) } // Resource was deleted externally, fall through to list/recreate } items, err := svc.List(ctx, cloudscalesdk.WithTagFilter(tags)) if err != nil { - return "", fmt.Errorf("listing %ss: %w", resourceName, err) + return nil, "", fmt.Errorf("listing %ss: %w", resourceName, err) } if len(items) > 1 { - return "", fmt.Errorf("found %d %ss matching tag filter, expected at most 1", len(items), resourceName) + return nil, "", fmt.Errorf("found %d %ss matching tag filter, expected at most 1", len(items), resourceName) } if len(items) == 1 { - uuid := extractUUID(items[0]) + item := items[0] + uuid := extractUUID(item) clusterScope.Info("Found existing "+resourceName+" by tag", "id", uuid) - return uuid, nil + return &item, uuid, nil } - return "", nil + return nil, "", nil } diff --git a/internal/controller/cloudscale_services_test.go b/internal/controller/cloudscale_services_test.go index 4c1431b..df601bd 100644 --- a/internal/controller/cloudscale_services_test.go +++ b/internal/controller/cloudscale_services_test.go @@ -82,10 +82,12 @@ func TestEnsureResource_ExistingID_Found(t *testing.T) { }, } - id, err := ensureResource(context.Background(), testClusterScope(), "existing-123", "test resource", svc, extractTestUUID, testTags) + resource, id, err := ensureResource(context.Background(), testClusterScope(), "existing-123", "test resource", svc, extractTestUUID, testTags) g.Expect(err).ToNot(HaveOccurred()) g.Expect(id).To(Equal("existing-123")) + g.Expect(resource).ToNot(BeNil()) + g.Expect(resource.UUID).To(Equal("existing-123")) } func TestEnsureResource_ExistingID_NotFound_FallsThrough(t *testing.T) { @@ -100,10 +102,11 @@ func TestEnsureResource_ExistingID_NotFound_FallsThrough(t *testing.T) { }, } - id, err := ensureResource(context.Background(), testClusterScope(), "deleted-123", "test resource", svc, extractTestUUID, testTags) + resource, id, err := ensureResource(context.Background(), testClusterScope(), "deleted-123", "test resource", svc, extractTestUUID, testTags) g.Expect(err).ToNot(HaveOccurred()) g.Expect(id).To(Equal(""), "should return empty ID so caller creates the resource") + g.Expect(resource).To(BeNil()) } func TestEnsureResource_ExistingID_GetError(t *testing.T) { @@ -115,7 +118,7 @@ func TestEnsureResource_ExistingID_GetError(t *testing.T) { }, } - _, err := ensureResource(context.Background(), testClusterScope(), "existing-123", "test resource", svc, extractTestUUID, testTags) + _, _, err := ensureResource(context.Background(), testClusterScope(), "existing-123", "test resource", svc, extractTestUUID, testTags) g.Expect(err).To(HaveOccurred()) g.Expect(err.Error()).To(ContainSubstring("api connection error")) @@ -130,10 +133,12 @@ func TestEnsureResource_NoID_ListFindsOne(t *testing.T) { }, } - id, err := ensureResource(context.Background(), testClusterScope(), "", "test resource", svc, extractTestUUID, testTags) + resource, id, err := ensureResource(context.Background(), testClusterScope(), "", "test resource", svc, extractTestUUID, testTags) g.Expect(err).ToNot(HaveOccurred()) g.Expect(id).To(Equal("adopted-123")) + g.Expect(resource).ToNot(BeNil()) + g.Expect(resource.UUID).To(Equal("adopted-123")) } func TestEnsureResource_NoID_ListFindsMultiple(t *testing.T) { @@ -148,7 +153,7 @@ func TestEnsureResource_NoID_ListFindsMultiple(t *testing.T) { }, } - _, err := ensureResource(context.Background(), testClusterScope(), "", "test resource", svc, extractTestUUID, testTags) + _, _, err := ensureResource(context.Background(), testClusterScope(), "", "test resource", svc, extractTestUUID, testTags) g.Expect(err).To(HaveOccurred()) g.Expect(err.Error()).To(ContainSubstring("found 2 test resources matching tag filter")) @@ -163,10 +168,11 @@ func TestEnsureResource_NoID_ListFindsNone(t *testing.T) { }, } - id, err := ensureResource(context.Background(), testClusterScope(), "", "test resource", svc, extractTestUUID, testTags) + resource, id, err := ensureResource(context.Background(), testClusterScope(), "", "test resource", svc, extractTestUUID, testTags) g.Expect(err).ToNot(HaveOccurred()) g.Expect(id).To(Equal(""), "should return empty ID so caller creates the resource") + g.Expect(resource).To(BeNil()) } func TestEnsureResource_NoID_ListError(t *testing.T) { @@ -178,7 +184,7 @@ func TestEnsureResource_NoID_ListError(t *testing.T) { }, } - _, err := ensureResource(context.Background(), testClusterScope(), "", "test resource", svc, extractTestUUID, testTags) + _, _, err := ensureResource(context.Background(), testClusterScope(), "", "test resource", svc, extractTestUUID, testTags) g.Expect(err).To(HaveOccurred()) g.Expect(err.Error()).To(ContainSubstring("list api error")) diff --git a/internal/controller/cloudscalecluster_controller.go b/internal/controller/cloudscalecluster_controller.go index 37b8f34..1f6b058 100644 --- a/internal/controller/cloudscalecluster_controller.go +++ b/internal/controller/cloudscalecluster_controller.go @@ -147,6 +147,10 @@ func (r *CloudscaleClusterReconciler) reconcileNormal(ctx context.Context, clust return result, nil } + if err := r.reconcileFloatingIP(ctx, clusterScope); err != nil { + return ctrl.Result{}, fmt.Errorf("reconciling floating IP: %w", err) + } + // Mark infrastructure as provisioned when all resources exist if clusterScope.CloudscaleCluster.Status.Initialization == nil { clusterScope.CloudscaleCluster.Status.Initialization = &infrastructurev1beta2.ClusterInitializationStatus{} @@ -154,9 +158,6 @@ func (r *CloudscaleClusterReconciler) reconcileNormal(ctx context.Context, clust provisioned := r.isInfrastructureProvisioned(clusterScope) clusterScope.CloudscaleCluster.Status.Initialization.Provisioned = ptr.To(provisioned) - // Set Ready condition based on all sub-conditions - r.setReadyCondition(clusterScope) - return ctrl.Result{}, nil } @@ -170,7 +171,12 @@ func (r *CloudscaleClusterReconciler) reconcileDelete(ctx context.Context, clust // Set Deleting condition r.setCondition(clusterScope, infrastructurev1beta2.DeletingCondition, metav1.ConditionTrue, infrastructurev1beta2.DeletingReason, "Deleting infrastructure resources") - // Delete load balancer first (it depends on the subnet) + // Delete floating IP first (it may be assigned to the LB or a server) + if err := r.deleteFloatingIP(ctx, clusterScope); err != nil { + return ctrl.Result{}, fmt.Errorf("deleting floating IP: %w", err) + } + + // Delete load balancer (it depends on the subnet) if err := r.deleteLoadBalancer(ctx, clusterScope); err != nil { return ctrl.Result{}, fmt.Errorf("deleting load balancer: %w", err) } @@ -192,13 +198,18 @@ func (r *CloudscaleClusterReconciler) reconcileDelete(ctx context.Context, clust } // isInfrastructureProvisioned returns true if all cluster infrastructure is ready. -// This includes Network, Subnet, Load Balancer (with pool and listener) if enabled, and Control Plane Endpoint. +// This includes all Networks+Subnets, Load Balancer (with pool and listener) if enabled, +// Floating IP if configured, and Control Plane Endpoint. func (r *CloudscaleClusterReconciler) isInfrastructureProvisioned(clusterScope *scope.ClusterScope) bool { - // Network and Subnet must exist - if clusterScope.CloudscaleCluster.Status.NetworkID == "" || - clusterScope.CloudscaleCluster.Status.SubnetID == "" { + // All networks must have both network and subnet IDs + if len(clusterScope.CloudscaleCluster.Status.Networks) == 0 { return false } + for _, ns := range clusterScope.CloudscaleCluster.Status.Networks { + if ns.NetworkID == "" || ns.SubnetID == "" { + return false + } + } // Load balancer, pool, and listener must exist (if LB is enabled) if ptr.Deref(clusterScope.CloudscaleCluster.Spec.ControlPlaneLoadBalancer.Enabled, true) { @@ -209,7 +220,14 @@ func (r *CloudscaleClusterReconciler) isInfrastructureProvisioned(clusterScope * } } - // Control plane endpoint must be set (from LB VIP or externally) + // Floating IP must be provisioned if configured + if clusterScope.CloudscaleCluster.Spec.FloatingIP != nil { + if clusterScope.CloudscaleCluster.Status.FloatingIP == "" { + return false + } + } + + // Control plane endpoint must be set (from LB VIP, floating IP, or externally) if clusterScope.CloudscaleCluster.Spec.ControlPlaneEndpoint.Host == "" { return false } @@ -223,6 +241,7 @@ func (r *CloudscaleClusterReconciler) setReadyCondition(clusterScope *scope.Clus subConditions := []string{ infrastructurev1beta2.NetworkReadyCondition, infrastructurev1beta2.LoadBalancerReadyCondition, + infrastructurev1beta2.FloatingIPReadyCondition, } for _, condType := range subConditions { diff --git a/internal/controller/cloudscalecluster_controller_test.go b/internal/controller/cloudscalecluster_controller_test.go index 738e5a0..a4765c5 100644 --- a/internal/controller/cloudscalecluster_controller_test.go +++ b/internal/controller/cloudscalecluster_controller_test.go @@ -95,8 +95,9 @@ func TestCloudscaleClusterReconciler_IsInfrastructureProvisioned_LBEnabledAllRes }, }, Status: infrastructurev1beta2.CloudscaleClusterStatus{ - NetworkID: "network-123", - SubnetID: "subnet-123", + Networks: []infrastructurev1beta2.NetworkStatus{{ + Name: "test", NetworkID: "network-123", SubnetID: "subnet-123", Managed: true, + }}, LoadBalancerID: "lb-123", LoadBalancerPoolID: "pool-123", LoadBalancerListenerID: "listener-123", @@ -123,8 +124,9 @@ func TestCloudscaleClusterReconciler_IsInfrastructureProvisioned_LBEnabledMissin }, }, Status: infrastructurev1beta2.CloudscaleClusterStatus{ - NetworkID: "network-123", - SubnetID: "subnet-123", + Networks: []infrastructurev1beta2.NetworkStatus{{ + Name: "test", NetworkID: "network-123", SubnetID: "subnet-123", Managed: true, + }}, // LB resources missing }, }, @@ -149,8 +151,9 @@ func TestCloudscaleClusterReconciler_IsInfrastructureProvisioned_LBDisabledExter }, }, Status: infrastructurev1beta2.CloudscaleClusterStatus{ - NetworkID: "network-123", - SubnetID: "subnet-123", + Networks: []infrastructurev1beta2.NetworkStatus{{ + Name: "test", NetworkID: "network-123", SubnetID: "subnet-123", Managed: true, + }}, // No LB resources needed }, }, @@ -172,8 +175,9 @@ func TestCloudscaleClusterReconciler_IsInfrastructureProvisioned_LBDisabledNoEnd // ControlPlaneEndpoint not set }, Status: infrastructurev1beta2.CloudscaleClusterStatus{ - NetworkID: "network-123", - SubnetID: "subnet-123", + Networks: []infrastructurev1beta2.NetworkStatus{{ + Name: "test", NetworkID: "network-123", SubnetID: "subnet-123", Managed: true, + }}, }, }, } @@ -206,6 +210,12 @@ func TestCloudscaleClusterReconciler_SetReadyCondition_AllTrue(t *testing.T) { Reason: "Provisioned", ObservedGeneration: 1, }, + { + Type: infrastructurev1beta2.FloatingIPReadyCondition, + Status: metav1.ConditionTrue, + Reason: "Provisioned", + ObservedGeneration: 1, + }, }, }, }, diff --git a/internal/controller/cloudscalecluster_floatingip.go b/internal/controller/cloudscalecluster_floatingip.go new file mode 100644 index 0000000..164fb80 --- /dev/null +++ b/internal/controller/cloudscalecluster_floatingip.go @@ -0,0 +1,285 @@ +/* +Copyright 2026 cloudscale.ch. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package controller + +import ( + "context" + "fmt" + + cloudscalesdk "github.com/cloudscale-ch/cloudscale-go-sdk/v8" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/utils/ptr" + clusterv1 "sigs.k8s.io/cluster-api/api/core/v1beta2" + "sigs.k8s.io/controller-runtime/pkg/client" + + infrastructurev1beta2 "github.com/cloudscale-ch/cluster-api-provider-cloudscale/api/v1beta2" + "github.com/cloudscale-ch/cluster-api-provider-cloudscale/internal/cloudscale" + "github.com/cloudscale-ch/cluster-api-provider-cloudscale/internal/scope" +) + +// reconcileFloatingIP ensures the floating IP exists and is assigned to the correct target. +// When no floating IP is configured, this sets the condition to true and returns. +func (r *CloudscaleClusterReconciler) reconcileFloatingIP(ctx context.Context, clusterScope *scope.ClusterScope) (reterr error) { + fipSpec := clusterScope.CloudscaleCluster.Spec.FloatingIP + if fipSpec == nil { + r.setCondition(clusterScope, infrastructurev1beta2.FloatingIPReadyCondition, metav1.ConditionTrue, infrastructurev1beta2.FloatingIPDisabledReason, "") + return nil + } + + defer func() { + if reterr != nil { + r.setCondition(clusterScope, infrastructurev1beta2.FloatingIPReadyCondition, metav1.ConditionFalse, infrastructurev1beta2.FloatingIPErrorReason, reterr.Error()) + } else { + r.setCondition(clusterScope, infrastructurev1beta2.FloatingIPReadyCondition, metav1.ConditionTrue, infrastructurev1beta2.FloatingIPProvisionedReason, "") + } + }() + + // BYO floating IP: just look it up and use its address + if fipSpec.IP != "" { + return r.reconcileBYOFloatingIP(ctx, clusterScope, fipSpec.IP) + } + + // Managed floating IP: create if needed, then assign + return r.reconcileManagedFloatingIP(ctx, clusterScope, fipSpec) +} + +func (r *CloudscaleClusterReconciler) reconcileBYOFloatingIP(ctx context.Context, clusterScope *scope.ClusterScope, ip string) error { + fip, err := clusterScope.CloudscaleClient.FloatingIPs.Get(ctx, ip) + if err != nil { + return fmt.Errorf("getting BYO floating IP %s: %w", ip, err) + } + + // fip.Region is nil for global FIPs, which are valid for any cluster region. + if fip.Region != nil && fip.Region.Slug != clusterScope.CloudscaleCluster.Spec.Region { + return fmt.Errorf("BYO floating IP %s is in region %q, expected region %q", ip, fip.Region.Slug, clusterScope.CloudscaleCluster.Spec.Region) + } + + clusterScope.CloudscaleCluster.Status.FloatingIP = fip.IP() + r.setControlPlaneEndpointFromFIP(clusterScope, fip) + return r.ensureFloatingIPAssignment(ctx, clusterScope, fip) +} + +func (r *CloudscaleClusterReconciler) reconcileManagedFloatingIP(ctx context.Context, clusterScope *scope.ClusterScope, fipSpec *infrastructurev1beta2.FloatingIPSpec) error { + tags := clusterOwnershipTags(clusterScope.CloudscaleCluster) + + clusterScope.Info("reconcile managed floating IP") + + // Check if the floating IP already exists (by status ID or by tags) + fip, id, err := ensureResource(ctx, clusterScope, + clusterScope.CloudscaleCluster.Status.FloatingIP, + "floating IP", + clusterScope.CloudscaleClient.FloatingIPs, + func(fip cloudscalesdk.FloatingIP) string { return fip.IP() }, + tags, + ) + if err != nil { + return err + } + clusterScope.CloudscaleCluster.Status.FloatingIP = id + + if id != "" { + // Existing floating IP: ensure it's assigned to the right target and set endpoint + if err := r.ensureFloatingIPAssignment(ctx, clusterScope, fip); err != nil { + return err + } + r.setControlPlaneEndpointFromFIP(clusterScope, fip) + return nil + } + + // Create new floating IP + ipVersion := 4 + if fipSpec.IPFamily != nil && *fipSpec.IPFamily == infrastructurev1beta2.IPFamilyIPv6 { + ipVersion = 6 + } + + req := &cloudscalesdk.FloatingIPCreateRequest{ + IPVersion: ipVersion, + RegionalResourceRequest: cloudscalesdk.RegionalResourceRequest{ + Region: clusterScope.CloudscaleCluster.Spec.Region, + }, + TaggedResourceRequest: cloudscalesdk.TaggedResourceRequest{ + Tags: ptr.To(tags), + }, + } + + // Assign to LB or CP server. + // If the target is not ready yet, create the FIP unassigned anyway. + // On the next reconcile, ensureResource finds it by tag and + // ensureFloatingIPAssignment attaches it to the correct target. + target, err := r.getFloatingIPTarget(ctx, clusterScope) + if err != nil { + clusterScope.Info("no target available yet", "err", err) + } + if target.lbUUID != "" { + req.LoadBalancer = target.lbUUID + } else if target.serverUUID != "" { + req.Server = target.serverUUID + } + + clusterScope.Info("Creating floating IP", "ipVersion", ipVersion, "target", target) + fip, err = clusterScope.CloudscaleClient.FloatingIPs.Create(ctx, req) + if err != nil { + return fmt.Errorf("creating floating IP: %w", err) + } + + ip := fip.IP() + clusterScope.CloudscaleCluster.Status.FloatingIP = ip + clusterScope.Info("Created floating IP", "network", fip.Network, "ip", ip) + r.recorder.Eventf(clusterScope.CloudscaleCluster, nil, corev1.EventTypeNormal, "FloatingIPCreated", "CreateFloatingIP", + "Created floating IP %s", ip) + + r.setControlPlaneEndpointFromFIP(clusterScope, fip) + return nil +} + +type floatingIPTarget struct { + lbUUID string + serverUUID string +} + +func (t floatingIPTarget) String() string { + if t.lbUUID != "" { + return "lb:" + t.lbUUID + } + return "server:" + t.serverUUID +} + +// getFloatingIPTarget returns the target to assign the floating IP to. +// If LB is enabled, targets the LB. Otherwise, targets the first ready CP server. +func (r *CloudscaleClusterReconciler) getFloatingIPTarget(ctx context.Context, clusterScope *scope.ClusterScope) (floatingIPTarget, error) { + if ptr.Deref(clusterScope.CloudscaleCluster.Spec.ControlPlaneLoadBalancer.Enabled, true) { + lbID := clusterScope.CloudscaleCluster.Status.LoadBalancerID + if lbID == "" { + return floatingIPTarget{}, fmt.Errorf("waiting for load balancer to be provisioned") + } + return floatingIPTarget{lbUUID: lbID}, nil + } + + // LB disabled (BYO FIP without LB): find the first ready CP server. + // The user is responsible for configuring a dummy interface with the FIP address + // on their control plane servers (see cloudscale.ch docs). + machineList := &infrastructurev1beta2.CloudscaleMachineList{} + if err := r.List(ctx, machineList, + client.InNamespace(clusterScope.CloudscaleCluster.Namespace), + client.MatchingLabels{ + clusterv1.ClusterNameLabel: clusterScope.Cluster.Name, + clusterv1.MachineControlPlaneLabel: "", + }, + ); err != nil { + return floatingIPTarget{}, fmt.Errorf("listing CP machines: %w", err) + } + + for _, machine := range machineList.Items { + if machine.Status.ServerID != "" { + return floatingIPTarget{serverUUID: machine.Status.ServerID}, nil + } + } + + return floatingIPTarget{}, fmt.Errorf("waiting for a control plane server to be provisioned") +} + +// ensureFloatingIPAssignment verifies the FIP is assigned to the correct target and reassigns if needed. +func (r *CloudscaleClusterReconciler) ensureFloatingIPAssignment(ctx context.Context, clusterScope *scope.ClusterScope, fip *cloudscalesdk.FloatingIP) error { + target, err := r.getFloatingIPTarget(ctx, clusterScope) + if err != nil { + // Target not ready yet, leave current assignment + return nil + } + + needsUpdate := false + updateReq := &cloudscalesdk.FloatingIPUpdateRequest{} + + if target.lbUUID != "" && (fip.LoadBalancer == nil || fip.LoadBalancer.UUID != target.lbUUID) { + updateReq.LoadBalancer = target.lbUUID + needsUpdate = true + } else if target.serverUUID != "" && (fip.Server == nil || fip.Server.UUID != target.serverUUID) { + updateReq.Server = target.serverUUID + needsUpdate = true + } + + if needsUpdate { + floatingIP := clusterScope.CloudscaleCluster.Status.FloatingIP + clusterScope.Info("Reassigning floating IP", "ip", floatingIP, "target", target) + if err := clusterScope.CloudscaleClient.FloatingIPs.Update(ctx, floatingIP, updateReq); err != nil { + return fmt.Errorf("updating floating IP assignment: %w", err) + } + r.recorder.Eventf(clusterScope.CloudscaleCluster, nil, corev1.EventTypeNormal, "FloatingIPReassigned", "UpdateFloatingIP", + "Reassigned floating IP %s to %s", floatingIP, target) + } + + return nil +} + +func (r *CloudscaleClusterReconciler) setControlPlaneEndpointFromFIP(clusterScope *scope.ClusterScope, fip *cloudscalesdk.FloatingIP) { + if clusterScope.CloudscaleCluster.Spec.ControlPlaneEndpoint.Host != "" { + return + } + + floatingIP := fip.IP() + + apiServerPort := clusterScope.CloudscaleCluster.Spec.ControlPlaneLoadBalancer.APIServerPort + clusterScope.CloudscaleCluster.Spec.ControlPlaneEndpoint.Host = floatingIP + clusterScope.CloudscaleCluster.Spec.ControlPlaneEndpoint.Port = apiServerPort + clusterScope.Info("Set control plane endpoint from floating IP", + "endpoint", floatingIP, "port", apiServerPort) + r.recorder.Eventf(clusterScope.CloudscaleCluster, nil, corev1.EventTypeNormal, "ControlPlaneSet", "SetControlPlaneEndpoint", + "Control plane endpoint set to %s:%d (floating IP)", floatingIP, apiServerPort) +} + +// deleteFloatingIP deletes the floating IP if it's managed. +// BYO floating IPs are left untouched. +func (r *CloudscaleClusterReconciler) deleteFloatingIP(ctx context.Context, clusterScope *scope.ClusterScope) (reterr error) { + fipSpec := clusterScope.CloudscaleCluster.Spec.FloatingIP + if fipSpec == nil { + return nil + } + + // BYO floating IPs are not deleted; skip before registering the defer + // so the condition is not set to "Deleting" for an untouched resource. + if fipSpec.IP != "" { + clusterScope.Info("Skipping BYO floating IP deletion", "ip", fipSpec.IP) + return nil + } + + defer func() { + if reterr != nil { + r.setCondition(clusterScope, infrastructurev1beta2.FloatingIPReadyCondition, metav1.ConditionFalse, infrastructurev1beta2.FloatingIPErrorReason, fmt.Sprintf("Failed to delete floating IP: %v", reterr)) + } else { + r.setCondition(clusterScope, infrastructurev1beta2.FloatingIPReadyCondition, metav1.ConditionFalse, infrastructurev1beta2.FloatingIPDeletingReason, "Floating IP has been deleted") + } + }() + + floatingIP := clusterScope.CloudscaleCluster.Status.FloatingIP + if floatingIP == "" { + return nil + } + + clusterScope.Info("Deleting floating IP", "id", floatingIP) + if err := clusterScope.CloudscaleClient.FloatingIPs.Delete(ctx, floatingIP); err != nil { + if !cloudscale.IsNotFound(err) { + return fmt.Errorf("deleting floating IP: %w", err) + } + clusterScope.Info("Floating IP already deleted", "id", floatingIP) + } + + r.recorder.Eventf(clusterScope.CloudscaleCluster, nil, corev1.EventTypeNormal, "FloatingIPDeleted", "DeleteFloatingIP", + "Deleted floating IP %s", floatingIP) + clusterScope.CloudscaleCluster.Status.FloatingIP = "" + + return nil +} diff --git a/internal/controller/cloudscalecluster_floatingip_test.go b/internal/controller/cloudscalecluster_floatingip_test.go new file mode 100644 index 0000000..4222bd0 --- /dev/null +++ b/internal/controller/cloudscalecluster_floatingip_test.go @@ -0,0 +1,929 @@ +/* +Copyright 2026 cloudscale.ch. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package controller + +import ( + "context" + "fmt" + "testing" + + cloudscalesdk "github.com/cloudscale-ch/cloudscale-go-sdk/v8" + "github.com/go-logr/logr" + . "github.com/onsi/gomega" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/tools/events" + "k8s.io/utils/ptr" + clusterv1 "sigs.k8s.io/cluster-api/api/core/v1beta2" + "sigs.k8s.io/cluster-api/util/conditions" + "sigs.k8s.io/controller-runtime/pkg/client" + + infrastructurev1beta2 "github.com/cloudscale-ch/cluster-api-provider-cloudscale/api/v1beta2" + cs "github.com/cloudscale-ch/cluster-api-provider-cloudscale/internal/cloudscale" + "github.com/cloudscale-ch/cluster-api-provider-cloudscale/internal/scope" +) + +// --- Test helpers --- + +func newFIPTestClusterScope(fipService cs.FloatingIPService) *scope.ClusterScope { + return &scope.ClusterScope{ + Logger: logr.Discard(), + Cluster: &clusterv1.Cluster{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-cluster", + Namespace: "default", + }, + }, + CloudscaleCluster: &infrastructurev1beta2.CloudscaleCluster{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-cluster", + Namespace: "default", + }, + Spec: infrastructurev1beta2.CloudscaleClusterSpec{ + Region: "rma", + Zone: "rma1", + ControlPlaneLoadBalancer: infrastructurev1beta2.LoadBalancerSpec{ + Enabled: ptr.To(true), + APIServerPort: 6443, + }, + }, + }, + CloudscaleClient: &cs.Client{ + FloatingIPs: fipService, + }, + } +} + +func newFIPTestReconciler(objs ...client.Object) *CloudscaleClusterReconciler { + return &CloudscaleClusterReconciler{ + Client: newTestFakeClient(objs...), + recorder: events.NewFakeRecorder(10), + } +} + +// --- reconcileFloatingIP orchestrator tests --- + +func TestReconcileFloatingIP_Disabled(t *testing.T) { + g := NewWithT(t) + + clusterScope := newFIPTestClusterScope(&mockFloatingIPService{}) + // No FloatingIP spec = disabled + r := newFIPTestReconciler() + + err := r.reconcileFloatingIP(context.Background(), clusterScope) + + g.Expect(err).ToNot(HaveOccurred()) + cond := conditions.Get(clusterScope.CloudscaleCluster, infrastructurev1beta2.FloatingIPReadyCondition) + g.Expect(cond).ToNot(BeNil()) + g.Expect(cond.Status).To(Equal(metav1.ConditionTrue)) + g.Expect(cond.Reason).To(Equal(infrastructurev1beta2.FloatingIPDisabledReason)) +} + +func TestReconcileFloatingIP_BYO(t *testing.T) { + g := NewWithT(t) + + fipService := &mockFloatingIPService{ + getFn: func(ctx context.Context, id string) (*cloudscalesdk.FloatingIP, error) { + return &cloudscalesdk.FloatingIP{ + Network: "1.2.3.4/32", + }, nil + }, + } + + clusterScope := newFIPTestClusterScope(fipService) + clusterScope.CloudscaleCluster.Spec.FloatingIP = &infrastructurev1beta2.FloatingIPSpec{ + IP: "1.2.3.4", + } + + r := newFIPTestReconciler() + + err := r.reconcileFloatingIP(context.Background(), clusterScope) + + g.Expect(err).ToNot(HaveOccurred()) + g.Expect(clusterScope.CloudscaleCluster.Status.FloatingIP).To(Equal("1.2.3.4")) + cond := conditions.Get(clusterScope.CloudscaleCluster, infrastructurev1beta2.FloatingIPReadyCondition) + g.Expect(cond).ToNot(BeNil()) + g.Expect(cond.Status).To(Equal(metav1.ConditionTrue)) + g.Expect(cond.Reason).To(Equal(infrastructurev1beta2.FloatingIPProvisionedReason)) +} + +func TestReconcileFloatingIP_ErrorSetsConditionFalse(t *testing.T) { + g := NewWithT(t) + + fipService := &mockFloatingIPService{ + getFn: func(ctx context.Context, id string) (*cloudscalesdk.FloatingIP, error) { + return nil, fmt.Errorf("api error") + }, + } + + clusterScope := newFIPTestClusterScope(fipService) + clusterScope.CloudscaleCluster.Spec.FloatingIP = &infrastructurev1beta2.FloatingIPSpec{ + IP: "1.2.3.4", + } + + r := newFIPTestReconciler() + + err := r.reconcileFloatingIP(context.Background(), clusterScope) + + g.Expect(err).To(HaveOccurred()) + cond := conditions.Get(clusterScope.CloudscaleCluster, infrastructurev1beta2.FloatingIPReadyCondition) + g.Expect(cond).ToNot(BeNil()) + g.Expect(cond.Status).To(Equal(metav1.ConditionFalse)) + g.Expect(cond.Reason).To(Equal(infrastructurev1beta2.FloatingIPErrorReason)) +} + +// --- reconcileBYOFloatingIP tests --- + +func TestReconcileBYOFloatingIP_RefetchesAndKeepsAssignmentWhenCached(t *testing.T) { + g := NewWithT(t) + + getCalled := false + fipService := &mockFloatingIPService{ + getFn: func(ctx context.Context, id string) (*cloudscalesdk.FloatingIP, error) { + getCalled = true + return &cloudscalesdk.FloatingIP{ + Network: "1.2.3.4/32", + LoadBalancer: &cloudscalesdk.LoadBalancerStub{UUID: "lb-uuid"}, + }, nil + }, + updateFn: func(ctx context.Context, id string, req *cloudscalesdk.FloatingIPUpdateRequest) error { + t.Fatal("Update must not fire when FIP is already assigned to the LB") + return nil + }, + } + + clusterScope := newFIPTestClusterScope(fipService) + clusterScope.CloudscaleCluster.Status.FloatingIP = "1.2.3.4" + clusterScope.CloudscaleCluster.Status.LoadBalancerID = "lb-uuid" + clusterScope.CloudscaleCluster.Spec.ControlPlaneEndpoint.Host = "1.2.3.4" + + r := newFIPTestReconciler() + + err := r.reconcileBYOFloatingIP(context.Background(), clusterScope, "7.7.7.7") + + g.Expect(err).ToNot(HaveOccurred()) + g.Expect(getCalled).To(BeTrue(), "BYO FIP must be refetched so the assignment can be verified") +} + +func TestReconcileBYOFloatingIP_FetchesAndSetsStatus(t *testing.T) { + g := NewWithT(t) + + var capturedUpdateID string + var capturedUpdateReq *cloudscalesdk.FloatingIPUpdateRequest + fipService := &mockFloatingIPService{ + getFn: func(ctx context.Context, id string) (*cloudscalesdk.FloatingIP, error) { + g.Expect(id).To(Equal("7.7.7.7")) + return &cloudscalesdk.FloatingIP{ + Network: "5.6.7.8/32", + }, nil + }, + updateFn: func(ctx context.Context, id string, req *cloudscalesdk.FloatingIPUpdateRequest) error { + capturedUpdateID = id + capturedUpdateReq = req + return nil + }, + } + + clusterScope := newFIPTestClusterScope(fipService) + clusterScope.CloudscaleCluster.Status.LoadBalancerID = "lb-x" + r := newFIPTestReconciler() + + err := r.reconcileBYOFloatingIP(context.Background(), clusterScope, "7.7.7.7") + + g.Expect(err).ToNot(HaveOccurred()) + g.Expect(clusterScope.CloudscaleCluster.Status.FloatingIP).To(Equal("5.6.7.8")) + g.Expect(clusterScope.CloudscaleCluster.Spec.ControlPlaneEndpoint.Host).To(Equal("5.6.7.8")) + g.Expect(clusterScope.CloudscaleCluster.Spec.ControlPlaneEndpoint.Port).To(Equal(int32(6443))) + g.Expect(capturedUpdateID).To(Equal("5.6.7.8"), "BYO FIP must be assigned to the LB via Update") + g.Expect(capturedUpdateReq).ToNot(BeNil()) + g.Expect(capturedUpdateReq.LoadBalancer).To(Equal("lb-x")) +} + +func TestReconcileBYOFloatingIP_GetError(t *testing.T) { + g := NewWithT(t) + + fipService := &mockFloatingIPService{ + getFn: func(ctx context.Context, id string) (*cloudscalesdk.FloatingIP, error) { + return nil, fmt.Errorf("not found") + }, + } + + clusterScope := newFIPTestClusterScope(fipService) + r := newFIPTestReconciler() + + err := r.reconcileBYOFloatingIP(context.Background(), clusterScope, "7.7.7.7") + + g.Expect(err).To(HaveOccurred()) + g.Expect(err.Error()).To(ContainSubstring("getting BYO floating IP")) +} + +func TestReconcileBYOFloatingIP_RegionMismatchErrors(t *testing.T) { + g := NewWithT(t) + + fipService := &mockFloatingIPService{ + getFn: func(ctx context.Context, id string) (*cloudscalesdk.FloatingIP, error) { + return &cloudscalesdk.FloatingIP{ + Network: "1.2.3.4/32", + Region: &cloudscalesdk.RegionStub{Slug: "lpg"}, + }, nil + }, + } + + clusterScope := newFIPTestClusterScope(fipService) + clusterScope.CloudscaleCluster.Spec.FloatingIP = &infrastructurev1beta2.FloatingIPSpec{ + IP: "1.2.3.4", + } + + r := newFIPTestReconciler() + + err := r.reconcileFloatingIP(context.Background(), clusterScope) + + g.Expect(err).To(HaveOccurred()) + g.Expect(err.Error()).To(ContainSubstring("lpg")) + g.Expect(err.Error()).To(ContainSubstring("rma")) + g.Expect(clusterScope.CloudscaleCluster.Status.FloatingIP).To(BeEmpty()) + cond := conditions.Get(clusterScope.CloudscaleCluster, infrastructurev1beta2.FloatingIPReadyCondition) + g.Expect(cond).ToNot(BeNil()) + g.Expect(cond.Status).To(Equal(metav1.ConditionFalse)) + g.Expect(cond.Reason).To(Equal(infrastructurev1beta2.FloatingIPErrorReason)) +} + +func TestReconcileBYOFloatingIP_RegionMatches(t *testing.T) { + g := NewWithT(t) + + var capturedUpdateReq *cloudscalesdk.FloatingIPUpdateRequest + fipService := &mockFloatingIPService{ + getFn: func(ctx context.Context, id string) (*cloudscalesdk.FloatingIP, error) { + return &cloudscalesdk.FloatingIP{ + Network: "1.2.3.4/32", + Region: &cloudscalesdk.RegionStub{Slug: "rma"}, + }, nil + }, + updateFn: func(ctx context.Context, id string, req *cloudscalesdk.FloatingIPUpdateRequest) error { + capturedUpdateReq = req + return nil + }, + } + + clusterScope := newFIPTestClusterScope(fipService) + clusterScope.CloudscaleCluster.Status.LoadBalancerID = "lb-rma" + r := newFIPTestReconciler() + + err := r.reconcileBYOFloatingIP(context.Background(), clusterScope, "7.7.7.7") + + g.Expect(err).ToNot(HaveOccurred()) + g.Expect(clusterScope.CloudscaleCluster.Status.FloatingIP).To(Equal("1.2.3.4")) + g.Expect(capturedUpdateReq).ToNot(BeNil()) + g.Expect(capturedUpdateReq.LoadBalancer).To(Equal("lb-rma")) +} + +func TestReconcileBYOFloatingIP_AssignsToFirstReadyCPServer(t *testing.T) { + g := NewWithT(t) + + var capturedUpdateReq *cloudscalesdk.FloatingIPUpdateRequest + fipService := &mockFloatingIPService{ + getFn: func(ctx context.Context, id string) (*cloudscalesdk.FloatingIP, error) { + return &cloudscalesdk.FloatingIP{ + Network: "1.2.3.4/32", + }, nil + }, + updateFn: func(ctx context.Context, id string, req *cloudscalesdk.FloatingIPUpdateRequest) error { + capturedUpdateReq = req + return nil + }, + } + + cpMachine := &infrastructurev1beta2.CloudscaleMachine{ + ObjectMeta: metav1.ObjectMeta{ + Name: "cp-machine-0", + Namespace: "default", + Labels: map[string]string{ + clusterv1.ClusterNameLabel: "test-cluster", + clusterv1.MachineControlPlaneLabel: "", + }, + }, + Status: infrastructurev1beta2.CloudscaleMachineStatus{ + ServerID: "srv-x", + }, + } + + clusterScope := newFIPTestClusterScope(fipService) + clusterScope.CloudscaleCluster.Spec.ControlPlaneLoadBalancer.Enabled = ptr.To(false) + r := newFIPTestReconciler(cpMachine) + + err := r.reconcileBYOFloatingIP(context.Background(), clusterScope, "7.7.7.7") + + g.Expect(err).ToNot(HaveOccurred()) + g.Expect(capturedUpdateReq).ToNot(BeNil(), "BYO FIP must be assigned to the CP server when LB is disabled") + g.Expect(capturedUpdateReq.Server).To(Equal("srv-x")) +} + +func TestReconcileBYOFloatingIP_GlobalFIPAccepted(t *testing.T) { + g := NewWithT(t) + + fipService := &mockFloatingIPService{ + getFn: func(ctx context.Context, id string) (*cloudscalesdk.FloatingIP, error) { + return &cloudscalesdk.FloatingIP{ + Network: "9.9.9.9/32", + Region: nil, + }, nil + }, + } + + clusterScope := newFIPTestClusterScope(fipService) + r := newFIPTestReconciler() + + err := r.reconcileBYOFloatingIP(context.Background(), clusterScope, "7.7.7.7") + + g.Expect(err).ToNot(HaveOccurred()) + g.Expect(clusterScope.CloudscaleCluster.Status.FloatingIP).To(Equal("9.9.9.9")) + g.Expect(clusterScope.CloudscaleCluster.Spec.ControlPlaneEndpoint.Host).To(Equal("9.9.9.9")) +} + +// --- reconcileManagedFloatingIP tests --- + +func TestReconcileManagedFloatingIP_ExistingFIPEnsuresAssignment(t *testing.T) { + g := NewWithT(t) + + fipService := &mockFloatingIPService{ + getFn: func(ctx context.Context, id string) (*cloudscalesdk.FloatingIP, error) { + return &cloudscalesdk.FloatingIP{ + Network: "10.0.0.1/32", + LoadBalancer: &cloudscalesdk.LoadBalancerStub{UUID: "lb-uuid"}, + }, nil + }, + } + + clusterScope := newFIPTestClusterScope(fipService) + clusterScope.CloudscaleCluster.Status.FloatingIP = "10.0.0.1" + clusterScope.CloudscaleCluster.Status.LoadBalancerID = "lb-uuid" + fipSpec := &infrastructurev1beta2.FloatingIPSpec{} + + r := newFIPTestReconciler() + + err := r.reconcileManagedFloatingIP(context.Background(), clusterScope, fipSpec) + + g.Expect(err).ToNot(HaveOccurred()) + g.Expect(clusterScope.CloudscaleCluster.Spec.ControlPlaneEndpoint.Host).To(Equal("10.0.0.1")) +} + +func TestReconcileManagedFloatingIP_CreatesIPv4(t *testing.T) { + g := NewWithT(t) + + var capturedReq *cloudscalesdk.FloatingIPCreateRequest + + fipService := &mockFloatingIPService{ + listFn: func(ctx context.Context, modifiers ...cloudscalesdk.ListRequestModifier) ([]cloudscalesdk.FloatingIP, error) { + return nil, nil + }, + createFn: func(ctx context.Context, req *cloudscalesdk.FloatingIPCreateRequest) (*cloudscalesdk.FloatingIP, error) { + capturedReq = req + return &cloudscalesdk.FloatingIP{ + Network: "1.2.3.4/32", + }, nil + }, + } + + clusterScope := newFIPTestClusterScope(fipService) + clusterScope.CloudscaleCluster.Status.LoadBalancerID = "lb-uuid" + fipSpec := &infrastructurev1beta2.FloatingIPSpec{} + + r := newFIPTestReconciler() + + err := r.reconcileManagedFloatingIP(context.Background(), clusterScope, fipSpec) + + g.Expect(err).ToNot(HaveOccurred()) + g.Expect(capturedReq).ToNot(BeNil()) + g.Expect(capturedReq.IPVersion).To(Equal(4)) + g.Expect(capturedReq.LoadBalancer).To(Equal("lb-uuid")) + g.Expect(capturedReq.Region).To(Equal("rma")) + g.Expect(clusterScope.CloudscaleCluster.Status.FloatingIP).To(Equal("1.2.3.4")) +} + +func TestReconcileManagedFloatingIP_CreatesIPv6(t *testing.T) { + g := NewWithT(t) + + var capturedReq *cloudscalesdk.FloatingIPCreateRequest + + fipService := &mockFloatingIPService{ + listFn: func(ctx context.Context, modifiers ...cloudscalesdk.ListRequestModifier) ([]cloudscalesdk.FloatingIP, error) { + return nil, nil + }, + createFn: func(ctx context.Context, req *cloudscalesdk.FloatingIPCreateRequest) (*cloudscalesdk.FloatingIP, error) { + capturedReq = req + return &cloudscalesdk.FloatingIP{ + Network: "2001:db8::1/128", + }, nil + }, + } + + clusterScope := newFIPTestClusterScope(fipService) + clusterScope.CloudscaleCluster.Status.LoadBalancerID = "lb-uuid" + ipv6 := infrastructurev1beta2.IPFamilyIPv6 + fipSpec := &infrastructurev1beta2.FloatingIPSpec{ + IPFamily: &ipv6, + } + + r := newFIPTestReconciler() + + err := r.reconcileManagedFloatingIP(context.Background(), clusterScope, fipSpec) + + g.Expect(err).ToNot(HaveOccurred()) + g.Expect(capturedReq.IPVersion).To(Equal(6)) +} + +func TestReconcileManagedFloatingIP_CreateError(t *testing.T) { + g := NewWithT(t) + + fipService := &mockFloatingIPService{ + listFn: func(ctx context.Context, modifiers ...cloudscalesdk.ListRequestModifier) ([]cloudscalesdk.FloatingIP, error) { + return nil, nil + }, + createFn: func(ctx context.Context, req *cloudscalesdk.FloatingIPCreateRequest) (*cloudscalesdk.FloatingIP, error) { + return nil, fmt.Errorf("quota exceeded") + }, + } + + clusterScope := newFIPTestClusterScope(fipService) + clusterScope.CloudscaleCluster.Status.LoadBalancerID = "lb-uuid" + fipSpec := &infrastructurev1beta2.FloatingIPSpec{} + + r := newFIPTestReconciler() + + err := r.reconcileManagedFloatingIP(context.Background(), clusterScope, fipSpec) + + g.Expect(err).To(HaveOccurred()) + g.Expect(err.Error()).To(ContainSubstring("creating floating IP")) +} + +func TestReconcileManagedFloatingIP_AssignsToLBTarget(t *testing.T) { + g := NewWithT(t) + + var capturedReq *cloudscalesdk.FloatingIPCreateRequest + + fipService := &mockFloatingIPService{ + listFn: func(ctx context.Context, modifiers ...cloudscalesdk.ListRequestModifier) ([]cloudscalesdk.FloatingIP, error) { + return nil, nil + }, + createFn: func(ctx context.Context, req *cloudscalesdk.FloatingIPCreateRequest) (*cloudscalesdk.FloatingIP, error) { + capturedReq = req + return &cloudscalesdk.FloatingIP{Network: "1.2.3.4/32"}, nil + }, + } + + clusterScope := newFIPTestClusterScope(fipService) + clusterScope.CloudscaleCluster.Status.LoadBalancerID = "target-lb-uuid" + fipSpec := &infrastructurev1beta2.FloatingIPSpec{} + + r := newFIPTestReconciler() + + err := r.reconcileManagedFloatingIP(context.Background(), clusterScope, fipSpec) + + g.Expect(err).ToNot(HaveOccurred()) + g.Expect(capturedReq).ToNot(BeNil()) + g.Expect(capturedReq.LoadBalancer).To(Equal("target-lb-uuid")) + g.Expect(capturedReq.Server).To(BeEmpty()) +} + +// --- getFloatingIPTarget tests --- + +func TestGetFloatingIPTarget_LBEnabled(t *testing.T) { + g := NewWithT(t) + + clusterScope := newFIPTestClusterScope(&mockFloatingIPService{}) + clusterScope.CloudscaleCluster.Status.LoadBalancerID = "lb-uuid" + + r := newFIPTestReconciler() + + target, err := r.getFloatingIPTarget(context.Background(), clusterScope) + + g.Expect(err).ToNot(HaveOccurred()) + g.Expect(target.lbUUID).To(Equal("lb-uuid")) + g.Expect(target.serverUUID).To(BeEmpty()) +} + +func TestGetFloatingIPTarget_LBNotProvisioned(t *testing.T) { + g := NewWithT(t) + + clusterScope := newFIPTestClusterScope(&mockFloatingIPService{}) + // LB enabled but no LB ID yet + + r := newFIPTestReconciler() + + _, err := r.getFloatingIPTarget(context.Background(), clusterScope) + + g.Expect(err).To(HaveOccurred()) + g.Expect(err.Error()).To(ContainSubstring("waiting for load balancer")) +} + +func TestGetFloatingIPTarget_LBDisabled_FindsCPServer(t *testing.T) { + g := NewWithT(t) + + cpMachine := &infrastructurev1beta2.CloudscaleMachine{ + ObjectMeta: metav1.ObjectMeta{ + Name: "cp-machine-0", + Namespace: "default", + Labels: map[string]string{ + clusterv1.ClusterNameLabel: "test-cluster", + clusterv1.MachineControlPlaneLabel: "", + }, + }, + Status: infrastructurev1beta2.CloudscaleMachineStatus{ + ServerID: "cp-server-uuid", + }, + } + + clusterScope := newFIPTestClusterScope(&mockFloatingIPService{}) + clusterScope.CloudscaleCluster.Spec.ControlPlaneLoadBalancer.Enabled = ptr.To(false) + + r := newFIPTestReconciler(cpMachine) + + target, err := r.getFloatingIPTarget(context.Background(), clusterScope) + + g.Expect(err).ToNot(HaveOccurred()) + g.Expect(target.serverUUID).To(Equal("cp-server-uuid")) + g.Expect(target.lbUUID).To(BeEmpty()) +} + +func TestGetFloatingIPTarget_LBDisabled_NoCPServer(t *testing.T) { + g := NewWithT(t) + + clusterScope := newFIPTestClusterScope(&mockFloatingIPService{}) + clusterScope.CloudscaleCluster.Spec.ControlPlaneLoadBalancer.Enabled = ptr.To(false) + + r := newFIPTestReconciler() + + _, err := r.getFloatingIPTarget(context.Background(), clusterScope) + + g.Expect(err).To(HaveOccurred()) + g.Expect(err.Error()).To(ContainSubstring("waiting for a control plane server to be provisioned")) +} + +// --- ensureFloatingIPAssignment tests --- + +func TestEnsureFloatingIPAssignment_TargetNotReady(t *testing.T) { + g := NewWithT(t) + + clusterScope := newFIPTestClusterScope(&mockFloatingIPService{}) + // LB enabled but no LB ID → target not ready + + r := newFIPTestReconciler() + + fip := &cloudscalesdk.FloatingIP{Network: "1.2.3.4/32"} + + err := r.ensureFloatingIPAssignment(context.Background(), clusterScope, fip) + + g.Expect(err).ToNot(HaveOccurred()) +} + +func TestEnsureFloatingIPAssignment_LBAlreadyCorrect(t *testing.T) { + g := NewWithT(t) + + fipService := &mockFloatingIPService{ + updateFn: func(ctx context.Context, id string, req *cloudscalesdk.FloatingIPUpdateRequest) error { + t.Fatal("Update should not be called when assignment is correct") + return nil + }, + } + + clusterScope := newFIPTestClusterScope(fipService) + clusterScope.CloudscaleCluster.Status.LoadBalancerID = "lb-uuid" + + r := newFIPTestReconciler() + + fip := &cloudscalesdk.FloatingIP{ + Network: "1.2.3.4/32", + LoadBalancer: &cloudscalesdk.LoadBalancerStub{UUID: "lb-uuid"}, + } + + err := r.ensureFloatingIPAssignment(context.Background(), clusterScope, fip) + + g.Expect(err).ToNot(HaveOccurred()) +} + +func TestEnsureFloatingIPAssignment_ReassignsLB(t *testing.T) { + g := NewWithT(t) + + var capturedID string + var capturedReq *cloudscalesdk.FloatingIPUpdateRequest + + fipService := &mockFloatingIPService{ + updateFn: func(ctx context.Context, id string, req *cloudscalesdk.FloatingIPUpdateRequest) error { + capturedID = id + capturedReq = req + return nil + }, + } + + clusterScope := newFIPTestClusterScope(fipService) + clusterScope.CloudscaleCluster.Status.LoadBalancerID = "new-lb-uuid" + clusterScope.CloudscaleCluster.Status.FloatingIP = "1.2.3.4" + + r := newFIPTestReconciler() + + fip := &cloudscalesdk.FloatingIP{ + Network: "1.2.3.4/32", + LoadBalancer: &cloudscalesdk.LoadBalancerStub{UUID: "old-lb-uuid"}, + } + + err := r.ensureFloatingIPAssignment(context.Background(), clusterScope, fip) + + g.Expect(err).ToNot(HaveOccurred()) + g.Expect(capturedID).To(Equal("1.2.3.4")) + g.Expect(capturedReq.LoadBalancer).To(Equal("new-lb-uuid")) +} + +func TestEnsureFloatingIPAssignment_ReassignsServer(t *testing.T) { + g := NewWithT(t) + + var capturedReq *cloudscalesdk.FloatingIPUpdateRequest + + fipService := &mockFloatingIPService{ + updateFn: func(ctx context.Context, id string, req *cloudscalesdk.FloatingIPUpdateRequest) error { + capturedReq = req + return nil + }, + } + + cpMachine := &infrastructurev1beta2.CloudscaleMachine{ + ObjectMeta: metav1.ObjectMeta{ + Name: "cp-machine-0", + Namespace: "default", + Labels: map[string]string{ + clusterv1.ClusterNameLabel: "test-cluster", + clusterv1.MachineControlPlaneLabel: "", + }, + }, + Status: infrastructurev1beta2.CloudscaleMachineStatus{ + ServerID: "srv-new", + }, + } + + clusterScope := newFIPTestClusterScope(fipService) + clusterScope.CloudscaleCluster.Spec.ControlPlaneLoadBalancer.Enabled = ptr.To(false) + clusterScope.CloudscaleCluster.Status.FloatingIP = "1.2.3.4" + + r := newFIPTestReconciler(cpMachine) + + // FIP assigned to an old server + fip := &cloudscalesdk.FloatingIP{ + Network: "1.2.3.4/32", + Server: &cloudscalesdk.ServerStub{UUID: "srv-old"}, + } + + err := r.ensureFloatingIPAssignment(context.Background(), clusterScope, fip) + + g.Expect(err).ToNot(HaveOccurred()) + g.Expect(capturedReq).ToNot(BeNil()) + g.Expect(capturedReq.Server).To(Equal("srv-new")) + g.Expect(capturedReq.LoadBalancer).To(BeEmpty()) +} + +func TestEnsureFloatingIPAssignment_UpdateError(t *testing.T) { + g := NewWithT(t) + + fipService := &mockFloatingIPService{ + updateFn: func(ctx context.Context, id string, req *cloudscalesdk.FloatingIPUpdateRequest) error { + return fmt.Errorf("update failed") + }, + } + + clusterScope := newFIPTestClusterScope(fipService) + clusterScope.CloudscaleCluster.Status.LoadBalancerID = "lb-uuid" + clusterScope.CloudscaleCluster.Status.FloatingIP = "1.2.3.4" + + r := newFIPTestReconciler() + + fip := &cloudscalesdk.FloatingIP{ + Network: "1.2.3.4/32", + // No LB assigned — needs update + } + + err := r.ensureFloatingIPAssignment(context.Background(), clusterScope, fip) + + g.Expect(err).To(HaveOccurred()) + g.Expect(err.Error()).To(ContainSubstring("updating floating IP assignment")) +} + +// --- setControlPlaneEndpointFromFIP tests --- + +func TestSetControlPlaneEndpointFromFIP_SetsEndpoint(t *testing.T) { + g := NewWithT(t) + + clusterScope := newFIPTestClusterScope(&mockFloatingIPService{}) + + r := newFIPTestReconciler() + + fip := &cloudscalesdk.FloatingIP{Network: "10.20.30.40/32"} + + r.setControlPlaneEndpointFromFIP(clusterScope, fip) + + g.Expect(clusterScope.CloudscaleCluster.Spec.ControlPlaneEndpoint.Host).To(Equal("10.20.30.40")) + g.Expect(clusterScope.CloudscaleCluster.Spec.ControlPlaneEndpoint.Port).To(Equal(int32(6443))) +} + +func TestSetControlPlaneEndpointFromFIP_SkipsIfAlreadySet(t *testing.T) { + g := NewWithT(t) + + clusterScope := newFIPTestClusterScope(&mockFloatingIPService{}) + clusterScope.CloudscaleCluster.Spec.ControlPlaneEndpoint.Host = "existing-host" + clusterScope.CloudscaleCluster.Spec.ControlPlaneEndpoint.Port = 9999 + + r := newFIPTestReconciler() + + fip := &cloudscalesdk.FloatingIP{Network: "10.20.30.40/32"} + + r.setControlPlaneEndpointFromFIP(clusterScope, fip) + + g.Expect(clusterScope.CloudscaleCluster.Spec.ControlPlaneEndpoint.Host).To(Equal("existing-host")) + g.Expect(clusterScope.CloudscaleCluster.Spec.ControlPlaneEndpoint.Port).To(Equal(int32(9999))) +} + +// --- deleteFloatingIP tests --- + +func TestDeleteFloatingIP_NilSpec(t *testing.T) { + g := NewWithT(t) + + clusterScope := newFIPTestClusterScope(&mockFloatingIPService{}) + r := newFIPTestReconciler() + + err := r.deleteFloatingIP(context.Background(), clusterScope) + + g.Expect(err).ToNot(HaveOccurred()) +} + +func TestDeleteFloatingIP_BYOSkipsDeletionAndLeavesConditionUntouched(t *testing.T) { + g := NewWithT(t) + + fipService := &mockFloatingIPService{ + deleteFn: func(ctx context.Context, id string) error { + t.Fatal("Delete should not be called for BYO floating IPs") + return nil + }, + } + + clusterScope := newFIPTestClusterScope(fipService) + clusterScope.CloudscaleCluster.Spec.FloatingIP = &infrastructurev1beta2.FloatingIPSpec{ + IP: "9.9.9.9", + } + + r := newFIPTestReconciler() + + err := r.deleteFloatingIP(context.Background(), clusterScope) + + g.Expect(err).ToNot(HaveOccurred()) + // BYO FIPs are not deleted, and no condition should be set (the defer is + // not registered for BYO so that the condition does not falsely report + // "Floating IP has been deleted"). + cond := conditions.Get(clusterScope.CloudscaleCluster, infrastructurev1beta2.FloatingIPReadyCondition) + g.Expect(cond).To(BeNil()) +} + +func TestDeleteFloatingIP_ManagedDeletes(t *testing.T) { + g := NewWithT(t) + + var deletedID string + + fipService := &mockFloatingIPService{ + deleteFn: func(ctx context.Context, id string) error { + deletedID = id + return nil + }, + } + + clusterScope := newFIPTestClusterScope(fipService) + clusterScope.CloudscaleCluster.Spec.FloatingIP = &infrastructurev1beta2.FloatingIPSpec{} + clusterScope.CloudscaleCluster.Status.FloatingIP = "1.2.3.4" + + r := newFIPTestReconciler() + + err := r.deleteFloatingIP(context.Background(), clusterScope) + + g.Expect(err).ToNot(HaveOccurred()) + g.Expect(deletedID).To(Equal("1.2.3.4")) + g.Expect(clusterScope.CloudscaleCluster.Status.FloatingIP).To(BeEmpty()) +} + +func TestDeleteFloatingIP_NoStatusSkipsDeletion(t *testing.T) { + g := NewWithT(t) + + fipService := &mockFloatingIPService{ + deleteFn: func(ctx context.Context, id string) error { + t.Fatal("Delete should not be called when status is empty") + return nil + }, + } + + clusterScope := newFIPTestClusterScope(fipService) + clusterScope.CloudscaleCluster.Spec.FloatingIP = &infrastructurev1beta2.FloatingIPSpec{} + + r := newFIPTestReconciler() + + err := r.deleteFloatingIP(context.Background(), clusterScope) + + g.Expect(err).ToNot(HaveOccurred()) +} + +func TestDeleteFloatingIP_AlreadyDeletedSucceeds(t *testing.T) { + g := NewWithT(t) + + fipService := &mockFloatingIPService{ + deleteFn: func(ctx context.Context, id string) error { + return &cloudscalesdk.ErrorResponse{StatusCode: 404} + }, + } + + clusterScope := newFIPTestClusterScope(fipService) + clusterScope.CloudscaleCluster.Spec.FloatingIP = &infrastructurev1beta2.FloatingIPSpec{} + clusterScope.CloudscaleCluster.Status.FloatingIP = "1.2.3.4" + + r := newFIPTestReconciler() + + err := r.deleteFloatingIP(context.Background(), clusterScope) + + g.Expect(err).ToNot(HaveOccurred()) + g.Expect(clusterScope.CloudscaleCluster.Status.FloatingIP).To(BeEmpty()) +} + +func TestDeleteFloatingIP_DeleteError(t *testing.T) { + g := NewWithT(t) + + fipService := &mockFloatingIPService{ + deleteFn: func(ctx context.Context, id string) error { + return fmt.Errorf("api error") + }, + } + + clusterScope := newFIPTestClusterScope(fipService) + clusterScope.CloudscaleCluster.Spec.FloatingIP = &infrastructurev1beta2.FloatingIPSpec{} + clusterScope.CloudscaleCluster.Status.FloatingIP = "1.2.3.4" + + r := newFIPTestReconciler() + + err := r.deleteFloatingIP(context.Background(), clusterScope) + + g.Expect(err).To(HaveOccurred()) + g.Expect(err.Error()).To(ContainSubstring("deleting floating IP")) + cond := conditions.Get(clusterScope.CloudscaleCluster, infrastructurev1beta2.FloatingIPReadyCondition) + g.Expect(cond).ToNot(BeNil()) + g.Expect(cond.Status).To(Equal(metav1.ConditionFalse)) + g.Expect(cond.Reason).To(Equal(infrastructurev1beta2.FloatingIPErrorReason)) +} + +// --- Mock FloatingIPService --- + +type mockFloatingIPService struct { + createFn func(ctx context.Context, req *cloudscalesdk.FloatingIPCreateRequest) (*cloudscalesdk.FloatingIP, error) + getFn func(ctx context.Context, id string) (*cloudscalesdk.FloatingIP, error) + listFn func(ctx context.Context, modifiers ...cloudscalesdk.ListRequestModifier) ([]cloudscalesdk.FloatingIP, error) + updateFn func(ctx context.Context, id string, req *cloudscalesdk.FloatingIPUpdateRequest) error + deleteFn func(ctx context.Context, id string) error +} + +func (m *mockFloatingIPService) Create(ctx context.Context, req *cloudscalesdk.FloatingIPCreateRequest) (*cloudscalesdk.FloatingIP, error) { + if m.createFn != nil { + return m.createFn(ctx, req) + } + return nil, nil +} + +func (m *mockFloatingIPService) Get(ctx context.Context, id string) (*cloudscalesdk.FloatingIP, error) { + if m.getFn != nil { + return m.getFn(ctx, id) + } + return nil, nil +} + +func (m *mockFloatingIPService) List(ctx context.Context, modifiers ...cloudscalesdk.ListRequestModifier) ([]cloudscalesdk.FloatingIP, error) { + if m.listFn != nil { + return m.listFn(ctx, modifiers...) + } + return nil, nil +} + +func (m *mockFloatingIPService) Update(ctx context.Context, id string, req *cloudscalesdk.FloatingIPUpdateRequest) error { + if m.updateFn != nil { + return m.updateFn(ctx, id, req) + } + return nil +} + +func (m *mockFloatingIPService) Delete(ctx context.Context, id string) error { + if m.deleteFn != nil { + return m.deleteFn(ctx, id) + } + return nil +} diff --git a/internal/controller/cloudscalecluster_loadbalancer.go b/internal/controller/cloudscalecluster_loadbalancer.go index 73cd025..1eb88f3 100644 --- a/internal/controller/cloudscalecluster_loadbalancer.go +++ b/internal/controller/cloudscalecluster_loadbalancer.go @@ -19,6 +19,7 @@ package controller import ( "context" "fmt" + "net" "slices" "time" @@ -51,9 +52,6 @@ func (r *CloudscaleClusterReconciler) reconcileLoadBalancer(ctx context.Context, return ctrl.Result{}, nil } - // lbPending indicates whether the load balancer is in progress of being updated. - // This needs to be set by code checking LB (and sub-resources) status. - // It's used in the deferred function to set the right condition. var lbPending bool defer func() { @@ -110,7 +108,11 @@ func (r *CloudscaleClusterReconciler) reconcileLoadBalancer(ctx context.Context, // 6. Set the control plane endpoint from the VIP if clusterScope.CloudscaleCluster.Spec.ControlPlaneEndpoint.Host == "" { - if len(lb.VIPAddresses) > 0 { + if clusterScope.CloudscaleCluster.Spec.FloatingIP != nil { + // Floating IP is configured — the FIP reconciler will set the endpoint. + // The FIP provides a stable IP that survives LB recreation. + clusterScope.Info("Skipping control plane endpoint from LB VIP (floating IP will provide it)") + } else if len(lb.VIPAddresses) > 0 { apiServerPort := clusterScope.CloudscaleCluster.Spec.ControlPlaneLoadBalancer.APIServerPort clusterScope.CloudscaleCluster.Spec.ControlPlaneEndpoint.Host = lb.VIPAddresses[0].Address clusterScope.CloudscaleCluster.Spec.ControlPlaneEndpoint.Port = apiServerPort @@ -125,7 +127,7 @@ func (r *CloudscaleClusterReconciler) reconcileLoadBalancer(ctx context.Context, // reconcileLB ensures the load balancer exists. func (r *CloudscaleClusterReconciler) reconcileLB(ctx context.Context, clusterScope *scope.ClusterScope) error { - id, err := ensureResource(ctx, clusterScope, + _, id, err := ensureResource(ctx, clusterScope, clusterScope.CloudscaleCluster.Status.LoadBalancerID, "load balancer", clusterScope.CloudscaleClient.LoadBalancers, @@ -142,11 +144,11 @@ func (r *CloudscaleClusterReconciler) reconcileLB(ctx context.Context, clusterSc // Create new load balancer zone := clusterScope.CloudscaleCluster.Spec.Zone - flavor := clusterScope.CloudscaleCluster.Spec.ControlPlaneLoadBalancer.Flavor + lbSpec := clusterScope.CloudscaleCluster.Spec.ControlPlaneLoadBalancer req := &cloudscalesdk.LoadBalancerRequest{ Name: fmt.Sprintf("%s-cp-lb", clusterScope.CloudscaleCluster.Name), - Flavor: flavor, + Flavor: lbSpec.Flavor, ZonalResourceRequest: cloudscalesdk.ZonalResourceRequest{ Zone: zone, }, @@ -155,7 +157,19 @@ func (r *CloudscaleClusterReconciler) reconcileLB(ctx context.Context, clusterSc }, } - clusterScope.Info("Creating load balancer", "zone", zone, "flavor", flavor) + // Place LB on a private network if specified, otherwise public VIP + if lbSpec.Network != "" { + subnetID, err := lbPrivateNetworkSubnetID(clusterScope) + if err != nil { + return err + } + req.VIPAddresses = &[]cloudscalesdk.VIPAddressRequest{ + {Subnet: subnetID}, + } + clusterScope.Info("Creating load balancer with private VIP", "network", lbSpec.Network, "subnet", subnetID) + } + + clusterScope.Info("Creating load balancer", "zone", zone, "flavor", lbSpec.Flavor) lb, err := clusterScope.CloudscaleClient.LoadBalancers.Create(ctx, req) if err != nil { return fmt.Errorf("creating load balancer: %w", err) @@ -170,7 +184,7 @@ func (r *CloudscaleClusterReconciler) reconcileLB(ctx context.Context, clusterSc // reconcileLBPool ensures the load balancer pool exists. func (r *CloudscaleClusterReconciler) reconcileLBPool(ctx context.Context, clusterScope *scope.ClusterScope) error { - id, err := ensureResource(ctx, clusterScope, + _, id, err := ensureResource(ctx, clusterScope, clusterScope.CloudscaleCluster.Status.LoadBalancerPoolID, "load balancer pool", clusterScope.CloudscaleClient.LoadBalancerPools, @@ -213,7 +227,7 @@ func (r *CloudscaleClusterReconciler) reconcileLBPool(ctx context.Context, clust // reconcileLBListener ensures the load balancer listener exists. func (r *CloudscaleClusterReconciler) reconcileLBListener(ctx context.Context, clusterScope *scope.ClusterScope) error { - id, err := ensureResource(ctx, clusterScope, + _, id, err := ensureResource(ctx, clusterScope, clusterScope.CloudscaleCluster.Status.LoadBalancerListenerID, "load balancer listener", clusterScope.CloudscaleClient.LoadBalancerListeners, @@ -256,7 +270,7 @@ func (r *CloudscaleClusterReconciler) reconcileLBListener(ctx context.Context, c // reconcileLBHealthMonitor ensures the load balancer health monitor exists. // The health monitor performs TCP health checks on the API server port. func (r *CloudscaleClusterReconciler) reconcileLBHealthMonitor(ctx context.Context, clusterScope *scope.ClusterScope) error { - id, err := ensureResource(ctx, clusterScope, + _, id, err := ensureResource(ctx, clusterScope, clusterScope.CloudscaleCluster.Status.LoadBalancerHealthMonitorID, "load balancer health monitor", clusterScope.CloudscaleClient.LoadBalancerHealthMonitors, @@ -365,13 +379,31 @@ func (r *CloudscaleClusterReconciler) getDesiredLoadBalancerMembers(ctx context. return nil, fmt.Errorf("listing CloudscaleMachines: %w", err) } + // Determine which subnet to use for pool members + memberSubnetID, err := r.getPoolMemberSubnetID(clusterScope) + if err != nil { + return nil, err + } + + memberSubnetCIDR, err := r.getMemberSubnetCIDR(clusterScope, memberSubnetID) + if err != nil { + return nil, fmt.Errorf("resolving pool member subnet CIDR: %w", err) + } + + _, memberIPNet, err := net.ParseCIDR(memberSubnetCIDR) + if err != nil { + return nil, fmt.Errorf("parsing pool member subnet CIDR %q: %w", memberSubnetCIDR, err) + } + desiredList := make([]cloudscalesdk.LoadBalancerPoolMemberRequest, 0) - // Build desired pool members from machines with an internal IP + // Build desired pool members from machines with an internal IP on the member subnet. + // With multi-NIC machines, the first MachineInternalIP may belong to a different + // network than the pool's subnet, so we filter by CIDR containment. for _, machine := range machineList.Items { member := cloudscalesdk.LoadBalancerPoolMemberRequest{ Name: machine.Name, - Subnet: clusterScope.CloudscaleCluster.Status.SubnetID, + Subnet: memberSubnetID, ProtocolPort: int(clusterScope.CloudscaleCluster.Spec.ControlPlaneLoadBalancer.APIServerPort), TaggedResourceRequest: cloudscalesdk.TaggedResourceRequest{ Tags: ptr.To(clusterOwnershipTags(clusterScope.CloudscaleCluster)), @@ -380,9 +412,12 @@ func (r *CloudscaleClusterReconciler) getDesiredLoadBalancerMembers(ctx context. hasAddr := false for _, addr := range machine.Status.Addresses { if addr.Type == clusterv1.MachineInternalIP && addr.Address != "" { - hasAddr = true - member.Address = addr.Address - break + ip := net.ParseIP(addr.Address) + if ip != nil && memberIPNet.Contains(ip) { + hasAddr = true + member.Address = addr.Address + break + } } } // can't add a member without an address @@ -393,6 +428,48 @@ func (r *CloudscaleClusterReconciler) getDesiredLoadBalancerMembers(ctx context. return desiredList, nil } +// getMemberSubnetCIDR resolves the subnet UUID used for LB pool members to its CIDR. +// The CIDR is read from status (set during network reconciliation) so that BYO subnets +// are discovered once and cached, avoiding repeated API calls. +func (r *CloudscaleClusterReconciler) getMemberSubnetCIDR(clusterScope *scope.ClusterScope, subnetID string) (string, error) { + for _, ns := range clusterScope.CloudscaleCluster.Status.Networks { + if ns.SubnetID == subnetID && ns.CIDR != "" { + return ns.CIDR, nil + } + } + return "", fmt.Errorf("subnet %s has no cached CIDR in status; ensure networks are reconciled first", subnetID) +} + +// getPoolMemberSubnetID determines the subnet UUID for LB pool members. +// If the LB is on a private network, use that network's subnet. +// Otherwise (public LB), use the first network's subnet. +func (r *CloudscaleClusterReconciler) getPoolMemberSubnetID(clusterScope *scope.ClusterScope) (string, error) { + if clusterScope.CloudscaleCluster.Spec.ControlPlaneLoadBalancer.Network != "" { + return lbPrivateNetworkSubnetID(clusterScope) + } + + networks := clusterScope.CloudscaleCluster.Status.Networks + if len(networks) == 0 { + return "", fmt.Errorf("no networks in cluster status") + } + if networks[0].SubnetID == "" { + return "", fmt.Errorf("first network %q has no subnet ID", networks[0].Name) + } + return networks[0].SubnetID, nil +} + +// lbPrivateNetworkSubnetID returns the subnet UUID of the private network that the LB +// VIP is placed on (spec.controlPlaneLoadBalancer.network). Caller must verify that +// spec.controlPlaneLoadBalancer.network is non-empty before calling. +func lbPrivateNetworkSubnetID(clusterScope *scope.ClusterScope) (string, error) { + name := clusterScope.CloudscaleCluster.Spec.ControlPlaneLoadBalancer.Network + ns := clusterScope.CloudscaleCluster.Status.GetNetworkStatus(name) + if ns == nil || ns.SubnetID == "" { + return "", fmt.Errorf("network %q not yet provisioned for LB VIP placement", name) + } + return ns.SubnetID, nil +} + func (r *CloudscaleClusterReconciler) createLoadBalancerMember(ctx context.Context, clusterScope *scope.ClusterScope, member cloudscalesdk.LoadBalancerPoolMemberRequest) error { clusterScope.V(2).Info("Creating load balancer member", "member", member) cm, err := clusterScope.CloudscaleClient.LoadBalancerPoolMembers.Create(ctx, clusterScope.CloudscaleCluster.Status.LoadBalancerPoolID, &member) diff --git a/internal/controller/cloudscalecluster_loadbalancer_test.go b/internal/controller/cloudscalecluster_loadbalancer_test.go index 6b52dbe..fa34162 100644 --- a/internal/controller/cloudscalecluster_loadbalancer_test.go +++ b/internal/controller/cloudscalecluster_loadbalancer_test.go @@ -116,6 +116,9 @@ func newTestClusterScopeWithLB(opts lbTestScopeOptions) *scope.ClusterScope { Spec: infrastructurev1beta2.CloudscaleClusterSpec{ Region: "rma", Zone: "rma1", + Networks: []infrastructurev1beta2.NetworkSpec{ + {Name: "test", CIDR: "10.0.0.0/24"}, + }, ControlPlaneLoadBalancer: infrastructurev1beta2.LoadBalancerSpec{ Enabled: &opts.lbEnabled, Algorithm: opts.algorithm, @@ -129,6 +132,11 @@ func newTestClusterScopeWithLB(opts lbTestScopeOptions) *scope.ClusterScope { }, }, }, + Status: infrastructurev1beta2.CloudscaleClusterStatus{ + Networks: []infrastructurev1beta2.NetworkStatus{ + {Name: "test", NetworkID: "net-uuid", SubnetID: "subnet-uuid", CIDR: "10.0.0.0/24", Managed: true}, + }, + }, }, CloudscaleClient: cloudscaleClient, } @@ -1146,7 +1154,7 @@ func TestReconcileLBMembers_AddsMissingMember(t *testing.T) { lbEnabled: true, }) clusterScope.CloudscaleCluster.Status.LoadBalancerPoolID = testPoolUUID - clusterScope.CloudscaleCluster.Status.SubnetID = "subnet-uuid" + clusterScope.CloudscaleCluster.Status.Networks = []infrastructurev1beta2.NetworkStatus{{Name: "test", NetworkID: "net-uuid", SubnetID: "subnet-uuid", CIDR: "10.0.0.0/24", Managed: true}} r := newTestReconcilerWithClient(k8sClient) @@ -1341,7 +1349,7 @@ func TestGetDesiredLoadBalancerMembers_SkipsMachinesWithoutIP(t *testing.T) { clusterScope := newTestClusterScopeWithLB(lbTestScopeOptions{ lbEnabled: true, }) - clusterScope.CloudscaleCluster.Status.SubnetID = "subnet-uuid" + clusterScope.CloudscaleCluster.Status.Networks = []infrastructurev1beta2.NetworkStatus{{Name: "test", NetworkID: "net-uuid", SubnetID: "subnet-uuid", CIDR: "10.0.0.0/24", Managed: true}} r := newTestReconcilerWithClient(k8sClient) @@ -1369,6 +1377,47 @@ func TestGetDesiredLoadBalancerMembers_NoMachines(t *testing.T) { g.Expect(members).To(BeEmpty()) } +func TestGetDesiredLoadBalancerMembers_PicksAddressInMemberSubnet(t *testing.T) { + g := NewWithT(t) + + machine := &infrastructurev1beta2.CloudscaleMachine{ + ObjectMeta: metav1.ObjectMeta{ + Name: "cp-machine-1", + Namespace: "default", + Labels: map[string]string{ + clusterv1.ClusterNameLabel: "test-cluster", + clusterv1.MachineControlPlaneLabel: "", + }, + }, + Status: infrastructurev1beta2.CloudscaleMachineStatus{ + Addresses: []clusterv1.MachineAddress{ + {Type: clusterv1.MachineInternalIP, Address: "192.168.1.5"}, // wrong network + {Type: clusterv1.MachineInternalIP, Address: "10.0.0.15"}, // correct network + {Type: clusterv1.MachineExternalIP, Address: "185.1.2.3"}, + }, + }, + } + + k8sClient := newFakeClientForLB(machine) + clusterScope := newTestClusterScopeWithLB(lbTestScopeOptions{ + lbEnabled: true, + }) + clusterScope.CloudscaleCluster.Spec.Networks = []infrastructurev1beta2.NetworkSpec{ + {Name: "main", CIDR: "10.0.0.0/24"}, + } + clusterScope.CloudscaleCluster.Status.Networks = []infrastructurev1beta2.NetworkStatus{ + {Name: "main", NetworkID: "net-uuid", SubnetID: "subnet-uuid", CIDR: "10.0.0.0/24", Managed: true}, + } + + r := newTestReconcilerWithClient(k8sClient) + + members, err := r.getDesiredLoadBalancerMembers(context.Background(), clusterScope) + + g.Expect(err).ToNot(HaveOccurred()) + g.Expect(members).To(HaveLen(1)) + g.Expect(members[0].Address).To(Equal("10.0.0.15")) +} + // ============================================================================ // Tests for createLoadBalancerMember / deleteLoadBalancerMember // ============================================================================ diff --git a/internal/controller/cloudscalecluster_network.go b/internal/controller/cloudscalecluster_network.go index dd4d631..969500a 100644 --- a/internal/controller/cloudscalecluster_network.go +++ b/internal/controller/cloudscalecluster_network.go @@ -18,6 +18,7 @@ package controller import ( "context" + "errors" "fmt" cloudscalesdk "github.com/cloudscale-ch/cloudscale-go-sdk/v8" @@ -30,8 +31,8 @@ import ( "github.com/cloudscale-ch/cluster-api-provider-cloudscale/internal/scope" ) -// reconcileNetwork orchestrates network and subnet provisioning. -// A single NetworkReadyCondition covers both resources. +// reconcileNetwork orchestrates network and subnet provisioning for all networks +// defined in spec.networks. A single NetworkReadyCondition covers all networks. func (r *CloudscaleClusterReconciler) reconcileNetwork(ctx context.Context, clusterScope *scope.ClusterScope) (reterr error) { defer func() { if reterr != nil { @@ -41,136 +42,211 @@ func (r *CloudscaleClusterReconciler) reconcileNetwork(ctx context.Context, clus } }() - if err := r.reconcileNetworkResource(ctx, clusterScope); err != nil { - return fmt.Errorf("reconciling network: %w", err) + if len(clusterScope.CloudscaleCluster.Spec.Networks) == 0 { + return fmt.Errorf("no networks defined in spec") } - if err := r.reconcileSubnet(ctx, clusterScope); err != nil { - return fmt.Errorf("reconciling subnet: %w", err) + for _, netSpec := range clusterScope.CloudscaleCluster.Spec.Networks { + if netSpec.UUID != "" { + if err := r.reconcileBYONetwork(ctx, clusterScope, netSpec); err != nil { + return fmt.Errorf("reconciling BYO network %q: %w", netSpec.Name, err) + } + } else { + if err := r.reconcileManagedNetwork(ctx, clusterScope, netSpec); err != nil { + return fmt.Errorf("reconciling managed network %q: %w", netSpec.Name, err) + } + } } return nil } -// reconcileNetworkResource ensures the network exists. -func (r *CloudscaleClusterReconciler) reconcileNetworkResource(ctx context.Context, clusterScope *scope.ClusterScope) error { - id, err := ensureResource(ctx, clusterScope, - clusterScope.CloudscaleCluster.Status.NetworkID, - "network", - clusterScope.CloudscaleClient.Networks, - func(n cloudscalesdk.Network) string { return n.UUID }, - clusterOwnershipTags(clusterScope.CloudscaleCluster), - ) - if err != nil { - return err - } - clusterScope.CloudscaleCluster.Status.NetworkID = id - if id != "" { +// reconcileBYONetwork validates a BYO network exists and discovers its subnet. +// The subnet is discovered once and cached in status. Subsequent reconciles +// short-circuit if the network and subnet IDs are already populated. +// This is intentional: BYO networks are managed externally, so CAPCS does not +// re-verify them. If the network/subnet is reconfigured externally, the next +// machine creation will fail at the cloudscale API level. +func (r *CloudscaleClusterReconciler) reconcileBYONetwork(ctx context.Context, clusterScope *scope.ClusterScope, netSpec infrastructurev1beta2.NetworkSpec) error { + ns := clusterScope.CloudscaleCluster.Status.GetNetworkStatus(netSpec.Name) + if ns != nil && ns.NetworkID != "" && ns.SubnetID != "" && ns.CIDR != "" { return nil } - // Create new network - clusterScope.Info("Creating network") - - network, err := clusterScope.CloudscaleClient.Networks.Create(ctx, &cloudscalesdk.NetworkCreateRequest{ - Name: clusterScope.Name(), - AutoCreateIPV4Subnet: ptr.To(false), - ZonalResourceRequest: cloudscalesdk.ZonalResourceRequest{ - Zone: clusterScope.CloudscaleCluster.Spec.Zone, - }, - TaggedResourceRequest: cloudscalesdk.TaggedResourceRequest{ - Tags: ptr.To(clusterOwnershipTags(clusterScope.CloudscaleCluster)), - }, - }) + network, err := clusterScope.CloudscaleClient.Networks.Get(ctx, netSpec.UUID) if err != nil { - return fmt.Errorf("creating network: %w", err) + return fmt.Errorf("getting BYO network %s: %w", netSpec.UUID, err) } - clusterScope.CloudscaleCluster.Status.NetworkID = network.UUID - clusterScope.Info("Created network", "networkID", network.UUID) - r.recorder.Eventf(clusterScope.CloudscaleCluster, nil, corev1.EventTypeNormal, "NetworkCreated", "CreateNetwork", - "Created network %s in zone %s", network.UUID, clusterScope.CloudscaleCluster.Spec.Zone) + if network.Zone.Slug != clusterScope.CloudscaleCluster.Spec.Zone { + return fmt.Errorf("BYO network %s is in zone %q, expected zone %q", netSpec.UUID, network.Zone.Slug, clusterScope.CloudscaleCluster.Spec.Zone) + } + + if len(network.Subnets) == 0 { + return fmt.Errorf("BYO network %s has no subnets", netSpec.UUID) + } + + r.setNetworkStatus(clusterScope, netSpec.Name, network.UUID, network.Subnets[0].UUID, network.Subnets[0].CIDR, false) + clusterScope.Info("Discovered BYO network", "name", netSpec.Name, "networkID", network.UUID, "subnetID", network.Subnets[0].UUID, "cidr", network.Subnets[0].CIDR) return nil } -// reconcileSubnet ensures the subnet exists within the network. -func (r *CloudscaleClusterReconciler) reconcileSubnet(ctx context.Context, clusterScope *scope.ClusterScope) error { - if clusterScope.CloudscaleCluster.Status.NetworkID == "" { - return fmt.Errorf("network must be created before subnet") +// reconcileManagedNetwork ensures a managed network and its subnet exist. +func (r *CloudscaleClusterReconciler) reconcileManagedNetwork(ctx context.Context, clusterScope *scope.ClusterScope, netSpec infrastructurev1beta2.NetworkSpec) error { + ns := clusterScope.CloudscaleCluster.Status.GetNetworkStatus(netSpec.Name) + + // Reconcile the network resource + var networkID string + if ns != nil { + networkID = ns.NetworkID } - id, err := ensureResource(ctx, clusterScope, - clusterScope.CloudscaleCluster.Status.SubnetID, - "subnet", - clusterScope.CloudscaleClient.Subnets, - func(s cloudscalesdk.Subnet) string { return s.UUID }, - clusterOwnershipTags(clusterScope.CloudscaleCluster), + tags := r.networkTags(clusterScope, netSpec.Name) + + _, resolvedNetworkID, err := ensureResource(ctx, clusterScope, + networkID, + fmt.Sprintf("network/%s", netSpec.Name), + clusterScope.CloudscaleClient.Networks, + func(n cloudscalesdk.Network) string { return n.UUID }, + tags, ) if err != nil { return err } - clusterScope.CloudscaleCluster.Status.SubnetID = id - if id != "" { - return nil + + if resolvedNetworkID == "" { + // Create new network + clusterScope.Info("Creating network", "name", netSpec.Name) + network, err := clusterScope.CloudscaleClient.Networks.Create(ctx, &cloudscalesdk.NetworkCreateRequest{ + Name: netSpec.Name, + AutoCreateIPV4Subnet: ptr.To(false), + ZonalResourceRequest: cloudscalesdk.ZonalResourceRequest{ + Zone: clusterScope.CloudscaleCluster.Spec.Zone, + }, + TaggedResourceRequest: cloudscalesdk.TaggedResourceRequest{ + Tags: ptr.To(tags), + }, + }) + if err != nil { + return fmt.Errorf("creating network: %w", err) + } + resolvedNetworkID = network.UUID + clusterScope.Info("Created network", "name", netSpec.Name, "networkID", network.UUID) + r.recorder.Eventf(clusterScope.CloudscaleCluster, nil, corev1.EventTypeNormal, "NetworkCreated", "CreateNetwork", + "Created network %s (%s) in zone %s", netSpec.Name, network.UUID, clusterScope.CloudscaleCluster.Spec.Zone) } - // Create new subnet - // GatewayAddress is defaulted by the webhook (empty string = no gateway) - spec := &clusterScope.CloudscaleCluster.Spec.Network - clusterScope.Info("Creating subnet", "cidr", spec.CIDR, "gateway", *spec.GatewayAddress) - - subnet, err := clusterScope.CloudscaleClient.Subnets.Create(ctx, &cloudscalesdk.SubnetCreateRequest{ - Network: clusterScope.CloudscaleCluster.Status.NetworkID, - CIDR: spec.CIDR, - GatewayAddress: *spec.GatewayAddress, - TaggedResourceRequest: cloudscalesdk.TaggedResourceRequest{ - Tags: ptr.To(clusterOwnershipTags(clusterScope.CloudscaleCluster)), - }, - }) + // Reconcile the subnet + var subnetID string + if ns != nil { + subnetID = ns.SubnetID + } + + _, resolvedSubnetID, err := ensureResource(ctx, clusterScope, + subnetID, + fmt.Sprintf("subnet/%s", netSpec.Name), + clusterScope.CloudscaleClient.Subnets, + func(s cloudscalesdk.Subnet) string { return s.UUID }, + tags, + ) if err != nil { - return fmt.Errorf("creating subnet: %w", err) + return err } - clusterScope.CloudscaleCluster.Status.SubnetID = subnet.UUID - clusterScope.Info("Created subnet", "subnetID", subnet.UUID) - r.recorder.Eventf(clusterScope.CloudscaleCluster, nil, corev1.EventTypeNormal, "SubnetCreated", "CreateSubnet", - "Created subnet %s with CIDR %s", subnet.UUID, clusterScope.CloudscaleCluster.Spec.Network.CIDR) + if resolvedSubnetID == "" { + // Create new subnet + clusterScope.Info("Creating subnet", "name", netSpec.Name, "cidr", netSpec.CIDR, "gateway", netSpec.GatewayAddress) + subnet, err := clusterScope.CloudscaleClient.Subnets.Create(ctx, &cloudscalesdk.SubnetCreateRequest{ + Network: resolvedNetworkID, + CIDR: netSpec.CIDR, + GatewayAddress: netSpec.GatewayAddress, + TaggedResourceRequest: cloudscalesdk.TaggedResourceRequest{ + Tags: ptr.To(tags), + }, + }) + if err != nil { + return fmt.Errorf("creating subnet: %w", err) + } + resolvedSubnetID = subnet.UUID + clusterScope.Info("Created subnet", "name", netSpec.Name, "subnetID", subnet.UUID) + r.recorder.Eventf(clusterScope.CloudscaleCluster, nil, corev1.EventTypeNormal, "SubnetCreated", "CreateSubnet", + "Created subnet %s (%s) with CIDR %s", netSpec.Name, subnet.UUID, netSpec.CIDR) + } + r.setNetworkStatus(clusterScope, netSpec.Name, resolvedNetworkID, resolvedSubnetID, netSpec.CIDR, true) return nil } -// deleteNetwork deletes the network. Subnets are cascade-deleted by the cloudscale.ch API -// when their parent network is deleted, so only the network needs explicit deletion. +// deleteNetwork deletes all managed networks. BYO networks are left untouched. +// Subnets are cascade-deleted by the cloudscale.ch API when their parent network is deleted. +// On partial failure, successfully deleted networks are removed from status so that +// only undeleted networks remain for the next reconcile attempt. func (r *CloudscaleClusterReconciler) deleteNetwork(ctx context.Context, clusterScope *scope.ClusterScope) (reterr error) { defer func() { if reterr != nil { r.setCondition(clusterScope, infrastructurev1beta2.NetworkReadyCondition, metav1.ConditionFalse, infrastructurev1beta2.NetworkErrorReason, fmt.Sprintf("Failed to delete network: %v", reterr)) } else { - r.setCondition(clusterScope, infrastructurev1beta2.NetworkReadyCondition, metav1.ConditionFalse, infrastructurev1beta2.NetworkDeletingReason, "Network has been deleted") + r.setCondition(clusterScope, infrastructurev1beta2.NetworkReadyCondition, metav1.ConditionFalse, infrastructurev1beta2.NetworkDeletingReason, "Networks have been deleted") } }() - if clusterScope.CloudscaleCluster.Status.NetworkID == "" { - return nil - } + var remaining []infrastructurev1beta2.NetworkStatus + var errs []error + + for _, ns := range clusterScope.CloudscaleCluster.Status.Networks { + if !ns.Managed { + clusterScope.Info("Skipping BYO network deletion", "name", ns.Name, "networkID", ns.NetworkID) + remaining = append(remaining, ns) + continue + } - networkID := clusterScope.CloudscaleCluster.Status.NetworkID - clusterScope.Info("Deleting network", "networkID", networkID) + if ns.NetworkID == "" { + continue + } - if err := clusterScope.CloudscaleClient.Networks.Delete(ctx, networkID); err != nil { - // Ignore 404 - network was already deleted externally - if !cloudscale.IsNotFound(err) { - return fmt.Errorf("deleting network: %w", err) + clusterScope.Info("Deleting network", "name", ns.Name, "networkID", ns.NetworkID) + if err := clusterScope.CloudscaleClient.Networks.Delete(ctx, ns.NetworkID); err != nil { + if !cloudscale.IsNotFound(err) { + remaining = append(remaining, ns) + errs = append(errs, fmt.Errorf("deleting network %s: %w", ns.Name, err)) + continue + } + clusterScope.Info("Network already deleted", "name", ns.Name, "networkID", ns.NetworkID) } - clusterScope.Info("Network already deleted", "networkID", networkID) + + r.recorder.Eventf(clusterScope.CloudscaleCluster, nil, corev1.EventTypeNormal, "NetworkDeleted", "DeleteNetwork", + "Deleted network %s (%s)", ns.Name, ns.NetworkID) } - r.recorder.Eventf(clusterScope.CloudscaleCluster, nil, corev1.EventTypeNormal, "NetworkDeleted", "DeleteNetwork", - "Deleted network %s", networkID) + clusterScope.CloudscaleCluster.Status.Networks = remaining + return errors.Join(errs...) +} - // Clear both IDs (subnet is cascade-deleted with the network) - clusterScope.CloudscaleCluster.Status.NetworkID = "" - clusterScope.CloudscaleCluster.Status.SubnetID = "" - return nil +// setNetworkStatus updates or appends the network status entry for the given name. +func (r *CloudscaleClusterReconciler) setNetworkStatus(clusterScope *scope.ClusterScope, name, networkID, subnetID, cidr string, managed bool) { + for i, ns := range clusterScope.CloudscaleCluster.Status.Networks { + if ns.Name == name { + clusterScope.CloudscaleCluster.Status.Networks[i].NetworkID = networkID + clusterScope.CloudscaleCluster.Status.Networks[i].SubnetID = subnetID + clusterScope.CloudscaleCluster.Status.Networks[i].CIDR = cidr + clusterScope.CloudscaleCluster.Status.Networks[i].Managed = managed + return + } + } + clusterScope.CloudscaleCluster.Status.Networks = append(clusterScope.CloudscaleCluster.Status.Networks, infrastructurev1beta2.NetworkStatus{ + Name: name, + NetworkID: networkID, + SubnetID: subnetID, + CIDR: cidr, + Managed: managed, + }) +} + +// networkTags returns the tags for a specific named network, combining cluster ownership with network name. +func (r *CloudscaleClusterReconciler) networkTags(clusterScope *scope.ClusterScope, networkName string) cloudscalesdk.TagMap { + tags := cloudscalesdk.TagMap{ + infrastructurev1beta2.NameCloudscaleProviderOwned + clusterScope.Cluster.Name: networkName, + } + return tags } diff --git a/internal/controller/cloudscalecluster_network_test.go b/internal/controller/cloudscalecluster_network_test.go index dd9f5f2..a86f86e 100644 --- a/internal/controller/cloudscalecluster_network_test.go +++ b/internal/controller/cloudscalecluster_network_test.go @@ -38,7 +38,6 @@ const netUUID = "net-uuid-123" // --- Test helpers --- func newTestClusterScope(networkService cs.NetworkService, subnetService cs.SubnetService) *scope.ClusterScope { - defaultGateway := "" return &scope.ClusterScope{ Logger: logr.Discard(), Cluster: &clusterv1.Cluster{ @@ -55,9 +54,11 @@ func newTestClusterScope(networkService cs.NetworkService, subnetService cs.Subn Spec: infrastructurev1beta2.CloudscaleClusterSpec{ Region: "rma", Zone: "rma1", - Network: infrastructurev1beta2.NetworkSpec{ - CIDR: "10.0.0.0/24", - GatewayAddress: &defaultGateway, + Networks: []infrastructurev1beta2.NetworkSpec{ + { + Name: "test", + CIDR: "10.0.0.0/24", + }, }, }, }, @@ -101,9 +102,11 @@ func TestReconcileNetwork_CreatesBothResources(t *testing.T) { err := r.reconcileNetwork(context.Background(), clusterScope) g.Expect(err).ToNot(HaveOccurred()) - g.Expect(clusterScope.CloudscaleCluster.Status.NetworkID).To(Equal(netUUID)) - g.Expect(clusterScope.CloudscaleCluster.Status.SubnetID).To(Equal("subnet-uuid-123")) - g.Expect(capturedNetReq.Name).To(Equal("test-cluster")) + ns := clusterScope.CloudscaleCluster.Status.GetNetworkStatus("test") + g.Expect(ns).ToNot(BeNil()) + g.Expect(ns.NetworkID).To(Equal(netUUID)) + g.Expect(ns.SubnetID).To(Equal("subnet-uuid-123")) + g.Expect(capturedNetReq.Name).To(Equal("test")) g.Expect(capturedNetReq.Zone).To(Equal("rma1")) g.Expect(capturedNetReq.AutoCreateIPV4Subnet).ToNot(BeNil()) g.Expect(*capturedNetReq.AutoCreateIPV4Subnet).To(BeFalse()) @@ -135,16 +138,19 @@ func TestReconcileNetwork_SkipsIfBothExist(t *testing.T) { } clusterScope := newTestClusterScope(networkService, subnetService) - clusterScope.CloudscaleCluster.Status.NetworkID = "existing-net" - clusterScope.CloudscaleCluster.Status.SubnetID = "existing-subnet" + clusterScope.CloudscaleCluster.Status.Networks = []infrastructurev1beta2.NetworkStatus{ + {Name: "test", NetworkID: "existing-net", SubnetID: "existing-subnet", Managed: true}, + } r := newTestReconciler() err := r.reconcileNetwork(context.Background(), clusterScope) g.Expect(err).ToNot(HaveOccurred()) - g.Expect(clusterScope.CloudscaleCluster.Status.NetworkID).To(Equal("existing-net")) - g.Expect(clusterScope.CloudscaleCluster.Status.SubnetID).To(Equal("existing-subnet")) + ns := clusterScope.CloudscaleCluster.Status.GetNetworkStatus("test") + g.Expect(ns).ToNot(BeNil()) + g.Expect(ns.NetworkID).To(Equal("existing-net")) + g.Expect(ns.SubnetID).To(Equal("existing-subnet")) } func TestReconcileNetwork_NetworkErrorStopsSubnet(t *testing.T) { @@ -169,7 +175,7 @@ func TestReconcileNetwork_NetworkErrorStopsSubnet(t *testing.T) { g.Expect(err).To(HaveOccurred()) g.Expect(err.Error()).To(ContainSubstring("api error")) - g.Expect(clusterScope.CloudscaleCluster.Status.SubnetID).To(BeEmpty()) + g.Expect(clusterScope.CloudscaleCluster.Status.GetNetworkStatus("test")).To(BeNil()) } func TestReconcileNetwork_SubnetErrorSurfaced(t *testing.T) { @@ -193,12 +199,11 @@ func TestReconcileNetwork_SubnetErrorSurfaced(t *testing.T) { g.Expect(err).To(HaveOccurred()) g.Expect(err.Error()).To(ContainSubstring("subnet api error")) - g.Expect(clusterScope.CloudscaleCluster.Status.NetworkID).To(Equal("net-uuid")) } -// --- Network sub-resource tests --- +// --- Managed network sub-resource tests (via reconcileNetwork orchestrator) --- -func TestReconcileNetworkResource_FindsByTag(t *testing.T) { +func TestReconcileNetwork_FindsByTag(t *testing.T) { g := NewWithT(t) networkService := &mockNetworkService{ @@ -212,17 +217,31 @@ func TestReconcileNetworkResource_FindsByTag(t *testing.T) { return nil, nil }, } + subnetService := &mockSubnetService{ + listFn: func(ctx context.Context, modifiers ...cloudscale.ListRequestModifier) ([]cloudscale.Subnet, error) { + return []cloudscale.Subnet{ + {UUID: "found-subnet-uuid", CIDR: "10.0.0.0/24"}, + }, nil + }, + createFn: func(ctx context.Context, req *cloudscale.SubnetCreateRequest) (*cloudscale.Subnet, error) { + t.Fatal("Create should not be called when subnet is found by tag") + return nil, nil + }, + } - clusterScope := newTestClusterScope(networkService, &mockSubnetService{}) + clusterScope := newTestClusterScope(networkService, subnetService) r := newTestReconciler() - err := r.reconcileNetworkResource(context.Background(), clusterScope) + err := r.reconcileNetwork(context.Background(), clusterScope) g.Expect(err).ToNot(HaveOccurred()) - g.Expect(clusterScope.CloudscaleCluster.Status.NetworkID).To(Equal("found-net-uuid")) + ns := clusterScope.CloudscaleCluster.Status.GetNetworkStatus("test") + g.Expect(ns).ToNot(BeNil()) + g.Expect(ns.NetworkID).To(Equal("found-net-uuid")) + g.Expect(ns.SubnetID).To(Equal("found-subnet-uuid")) } -func TestReconcileNetworkResource_ErrorsOnMultiple(t *testing.T) { +func TestReconcileNetwork_ErrorsOnMultipleNetworks(t *testing.T) { g := NewWithT(t) networkService := &mockNetworkService{ @@ -237,13 +256,13 @@ func TestReconcileNetworkResource_ErrorsOnMultiple(t *testing.T) { clusterScope := newTestClusterScope(networkService, &mockSubnetService{}) r := newTestReconciler() - err := r.reconcileNetworkResource(context.Background(), clusterScope) + err := r.reconcileNetwork(context.Background(), clusterScope) g.Expect(err).To(HaveOccurred()) - g.Expect(err.Error()).To(ContainSubstring("found 2 networks matching tag filter")) + g.Expect(err.Error()).To(ContainSubstring("found 2 network/tests matching tag filter")) } -func TestReconcileNetworkResource_RecreatesIfDeletedExternally(t *testing.T) { +func TestReconcileNetwork_RecreatesIfDeletedExternally(t *testing.T) { g := NewWithT(t) var created bool @@ -260,24 +279,38 @@ func TestReconcileNetworkResource_RecreatesIfDeletedExternally(t *testing.T) { return &cloudscale.Network{UUID: "new-net-uuid", Name: req.Name}, nil }, } + subnetService := &mockSubnetService{ + createFn: func(ctx context.Context, req *cloudscale.SubnetCreateRequest) (*cloudscale.Subnet, error) { + return &cloudscale.Subnet{UUID: "new-subnet-uuid", CIDR: req.CIDR}, nil + }, + } - clusterScope := newTestClusterScope(networkService, &mockSubnetService{}) - clusterScope.CloudscaleCluster.Status.NetworkID = "deleted-net-uuid" + clusterScope := newTestClusterScope(networkService, subnetService) + clusterScope.CloudscaleCluster.Status.Networks = []infrastructurev1beta2.NetworkStatus{ + {Name: "test", NetworkID: "deleted-net-uuid", Managed: true}, + } r := newTestReconciler() - err := r.reconcileNetworkResource(context.Background(), clusterScope) + err := r.reconcileNetwork(context.Background(), clusterScope) g.Expect(err).ToNot(HaveOccurred()) g.Expect(created).To(BeTrue(), "Should create a new network when old one was deleted") - g.Expect(clusterScope.CloudscaleCluster.Status.NetworkID).To(Equal("new-net-uuid")) + ns := clusterScope.CloudscaleCluster.Status.GetNetworkStatus("test") + g.Expect(ns).ToNot(BeNil()) + g.Expect(ns.NetworkID).To(Equal("new-net-uuid")) } -// --- Subnet sub-resource tests --- +// --- Subnet sub-resource tests (via reconcileNetwork orchestrator) --- -func TestReconcileSubnet_FindsByTag(t *testing.T) { +func TestReconcileNetwork_SubnetFindsByTag(t *testing.T) { g := NewWithT(t) + networkService := &mockNetworkService{ + createFn: func(ctx context.Context, req *cloudscale.NetworkCreateRequest) (*cloudscale.Network, error) { + return &cloudscale.Network{UUID: netUUID, Name: req.Name}, nil + }, + } subnetService := &mockSubnetService{ listFn: func(ctx context.Context, modifiers ...cloudscale.ListRequestModifier) ([]cloudscale.Subnet, error) { return []cloudscale.Subnet{ @@ -290,20 +323,34 @@ func TestReconcileSubnet_FindsByTag(t *testing.T) { }, } - clusterScope := newTestClusterScope(&mockNetworkService{}, subnetService) - clusterScope.CloudscaleCluster.Status.NetworkID = netUUID + clusterScope := newTestClusterScope(networkService, subnetService) + // Pre-populate network status so the network part is resolved via Get + clusterScope.CloudscaleCluster.Status.Networks = []infrastructurev1beta2.NetworkStatus{ + {Name: "test", NetworkID: netUUID, Managed: true}, + } + // The networkService Get should return the existing network + networkService.getFn = func(ctx context.Context, id string) (*cloudscale.Network, error) { + return &cloudscale.Network{UUID: id}, nil + } r := newTestReconciler() - err := r.reconcileSubnet(context.Background(), clusterScope) + err := r.reconcileNetwork(context.Background(), clusterScope) g.Expect(err).ToNot(HaveOccurred()) - g.Expect(clusterScope.CloudscaleCluster.Status.SubnetID).To(Equal("found-subnet-uuid")) + ns := clusterScope.CloudscaleCluster.Status.GetNetworkStatus("test") + g.Expect(ns).ToNot(BeNil()) + g.Expect(ns.SubnetID).To(Equal("found-subnet-uuid")) } -func TestReconcileSubnet_ErrorsOnMultiple(t *testing.T) { +func TestReconcileNetwork_SubnetErrorsOnMultiple(t *testing.T) { g := NewWithT(t) + networkService := &mockNetworkService{ + getFn: func(ctx context.Context, id string) (*cloudscale.Network, error) { + return &cloudscale.Network{UUID: id}, nil + }, + } subnetService := &mockSubnetService{ listFn: func(ctx context.Context, modifiers ...cloudscale.ListRequestModifier) ([]cloudscale.Subnet, error) { return []cloudscale.Subnet{ @@ -313,22 +360,29 @@ func TestReconcileSubnet_ErrorsOnMultiple(t *testing.T) { }, } - clusterScope := newTestClusterScope(&mockNetworkService{}, subnetService) - clusterScope.CloudscaleCluster.Status.NetworkID = netUUID + clusterScope := newTestClusterScope(networkService, subnetService) + clusterScope.CloudscaleCluster.Status.Networks = []infrastructurev1beta2.NetworkStatus{ + {Name: "test", NetworkID: netUUID, Managed: true}, + } r := newTestReconciler() - err := r.reconcileSubnet(context.Background(), clusterScope) + err := r.reconcileNetwork(context.Background(), clusterScope) g.Expect(err).To(HaveOccurred()) - g.Expect(err.Error()).To(ContainSubstring("found 2 subnets matching tag filter")) + g.Expect(err.Error()).To(ContainSubstring("found 2 subnet/tests matching tag filter")) } -func TestReconcileSubnet_RecreatesIfDeletedExternally(t *testing.T) { +func TestReconcileNetwork_SubnetRecreatesIfDeletedExternally(t *testing.T) { g := NewWithT(t) var created bool + networkService := &mockNetworkService{ + getFn: func(ctx context.Context, id string) (*cloudscale.Network, error) { + return &cloudscale.Network{UUID: id}, nil + }, + } subnetService := &mockSubnetService{ getFn: func(ctx context.Context, id string) (*cloudscale.Subnet, error) { return nil, &cloudscale.ErrorResponse{StatusCode: 404} @@ -342,24 +396,32 @@ func TestReconcileSubnet_RecreatesIfDeletedExternally(t *testing.T) { }, } - clusterScope := newTestClusterScope(&mockNetworkService{}, subnetService) - clusterScope.CloudscaleCluster.Status.NetworkID = netUUID - clusterScope.CloudscaleCluster.Status.SubnetID = "deleted-subnet-uuid" + clusterScope := newTestClusterScope(networkService, subnetService) + clusterScope.CloudscaleCluster.Status.Networks = []infrastructurev1beta2.NetworkStatus{ + {Name: "test", NetworkID: netUUID, SubnetID: "deleted-subnet-uuid", Managed: true}, + } r := newTestReconciler() - err := r.reconcileSubnet(context.Background(), clusterScope) + err := r.reconcileNetwork(context.Background(), clusterScope) g.Expect(err).ToNot(HaveOccurred()) g.Expect(created).To(BeTrue(), "Should create a new subnet when old one was deleted") - g.Expect(clusterScope.CloudscaleCluster.Status.SubnetID).To(Equal("new-subnet-uuid")) + ns := clusterScope.CloudscaleCluster.Status.GetNetworkStatus("test") + g.Expect(ns).ToNot(BeNil()) + g.Expect(ns.SubnetID).To(Equal("new-subnet-uuid")) } -func TestReconcileSubnet_CustomCIDR(t *testing.T) { +func TestReconcileNetwork_CustomCIDR(t *testing.T) { g := NewWithT(t) var capturedReq *cloudscale.SubnetCreateRequest + networkService := &mockNetworkService{ + createFn: func(ctx context.Context, req *cloudscale.NetworkCreateRequest) (*cloudscale.Network, error) { + return &cloudscale.Network{UUID: netUUID, Name: req.Name}, nil + }, + } subnetService := &mockSubnetService{ createFn: func(ctx context.Context, req *cloudscale.SubnetCreateRequest) (*cloudscale.Subnet, error) { capturedReq = req @@ -367,23 +429,27 @@ func TestReconcileSubnet_CustomCIDR(t *testing.T) { }, } - clusterScope := newTestClusterScope(&mockNetworkService{}, subnetService) - clusterScope.CloudscaleCluster.Status.NetworkID = netUUID - clusterScope.CloudscaleCluster.Spec.Network.CIDR = "192.168.0.0/16" + clusterScope := newTestClusterScope(networkService, subnetService) + clusterScope.CloudscaleCluster.Spec.Networks[0].CIDR = "192.168.0.0/16" r := newTestReconciler() - err := r.reconcileSubnet(context.Background(), clusterScope) + err := r.reconcileNetwork(context.Background(), clusterScope) g.Expect(err).ToNot(HaveOccurred()) g.Expect(capturedReq.CIDR).To(Equal("192.168.0.0/16")) } -func TestReconcileSubnet_ExplicitGateway(t *testing.T) { +func TestReconcileNetwork_ExplicitGateway(t *testing.T) { g := NewWithT(t) var capturedReq *cloudscale.SubnetCreateRequest + networkService := &mockNetworkService{ + createFn: func(ctx context.Context, req *cloudscale.NetworkCreateRequest) (*cloudscale.Network, error) { + return &cloudscale.Network{UUID: netUUID, Name: req.Name}, nil + }, + } subnetService := &mockSubnetService{ createFn: func(ctx context.Context, req *cloudscale.SubnetCreateRequest) (*cloudscale.Subnet, error) { capturedReq = req @@ -391,36 +457,20 @@ func TestReconcileSubnet_ExplicitGateway(t *testing.T) { }, } - clusterScope := newTestClusterScope(&mockNetworkService{}, subnetService) - clusterScope.CloudscaleCluster.Status.NetworkID = netUUID - gateway := "10.0.0.254" - clusterScope.CloudscaleCluster.Spec.Network.GatewayAddress = &gateway + clusterScope := newTestClusterScope(networkService, subnetService) + clusterScope.CloudscaleCluster.Spec.Networks[0].GatewayAddress = "10.0.0.254" r := newTestReconciler() - err := r.reconcileSubnet(context.Background(), clusterScope) + err := r.reconcileNetwork(context.Background(), clusterScope) g.Expect(err).ToNot(HaveOccurred()) g.Expect(capturedReq.GatewayAddress).To(Equal("10.0.0.254")) } -func TestReconcileSubnet_FailsIfNoNetwork(t *testing.T) { - g := NewWithT(t) - - clusterScope := newTestClusterScope(&mockNetworkService{}, &mockSubnetService{}) - clusterScope.CloudscaleCluster.Status.NetworkID = "" - - r := newTestReconciler() - - err := r.reconcileSubnet(context.Background(), clusterScope) - - g.Expect(err).To(HaveOccurred()) - g.Expect(err.Error()).To(ContainSubstring("network must be created before subnet")) -} - // --- Delete tests --- -func TestDeleteNetwork_DeletesNetworkAndClearsBothIDs(t *testing.T) { +func TestDeleteNetwork_DeletesNetworkAndClearsStatus(t *testing.T) { g := NewWithT(t) var deletedID string @@ -433,8 +483,9 @@ func TestDeleteNetwork_DeletesNetworkAndClearsBothIDs(t *testing.T) { } clusterScope := newTestClusterScope(networkService, &mockSubnetService{}) - clusterScope.CloudscaleCluster.Status.NetworkID = "net-to-delete" - clusterScope.CloudscaleCluster.Status.SubnetID = "subnet-to-cascade" + clusterScope.CloudscaleCluster.Status.Networks = []infrastructurev1beta2.NetworkStatus{ + {Name: "test", NetworkID: "net-to-delete", SubnetID: "subnet-to-cascade", Managed: true}, + } r := newTestReconciler() @@ -442,8 +493,7 @@ func TestDeleteNetwork_DeletesNetworkAndClearsBothIDs(t *testing.T) { g.Expect(err).ToNot(HaveOccurred()) g.Expect(deletedID).To(Equal("net-to-delete")) - g.Expect(clusterScope.CloudscaleCluster.Status.NetworkID).To(BeEmpty()) - g.Expect(clusterScope.CloudscaleCluster.Status.SubnetID).To(BeEmpty()) + g.Expect(clusterScope.CloudscaleCluster.Status.Networks).To(BeNil()) } func TestDeleteNetwork_SkipsIfNoNetwork(t *testing.T) { @@ -475,16 +525,304 @@ func TestDeleteNetwork_IgnoresAlreadyDeleted(t *testing.T) { } clusterScope := newTestClusterScope(networkService, &mockSubnetService{}) - clusterScope.CloudscaleCluster.Status.NetworkID = "already-deleted-net" - clusterScope.CloudscaleCluster.Status.SubnetID = "already-deleted-subnet" + clusterScope.CloudscaleCluster.Status.Networks = []infrastructurev1beta2.NetworkStatus{ + {Name: "test", NetworkID: "already-deleted-net", SubnetID: "already-deleted-subnet", Managed: true}, + } + + r := newTestReconciler() + + err := r.deleteNetwork(context.Background(), clusterScope) + + g.Expect(err).ToNot(HaveOccurred()) + g.Expect(clusterScope.CloudscaleCluster.Status.Networks).To(BeNil()) +} + +func TestDeleteNetwork_SkipsBYONetwork(t *testing.T) { + g := NewWithT(t) + + networkService := &mockNetworkService{ + deleteFn: func(ctx context.Context, id string) error { + t.Fatal("Delete should not be called for BYO networks") + return nil + }, + } + + clusterScope := newTestClusterScope(networkService, &mockSubnetService{}) + clusterScope.CloudscaleCluster.Status.Networks = []infrastructurev1beta2.NetworkStatus{ + {Name: "byo", NetworkID: "byo-net", SubnetID: "byo-subnet", Managed: false}, + } r := newTestReconciler() err := r.deleteNetwork(context.Background(), clusterScope) g.Expect(err).ToNot(HaveOccurred()) - g.Expect(clusterScope.CloudscaleCluster.Status.NetworkID).To(BeEmpty()) - g.Expect(clusterScope.CloudscaleCluster.Status.SubnetID).To(BeEmpty()) + // BYO networks are preserved in status even during deletion + g.Expect(clusterScope.CloudscaleCluster.Status.Networks).To(HaveLen(1)) + g.Expect(clusterScope.CloudscaleCluster.Status.Networks[0].Name).To(Equal("byo")) + g.Expect(clusterScope.CloudscaleCluster.Status.Networks[0].Managed).To(BeFalse()) +} + +func TestDeleteNetwork_PartialFailureKeepsFailedInStatus(t *testing.T) { + g := NewWithT(t) + + var deletedIDs []string + + networkService := &mockNetworkService{ + deleteFn: func(ctx context.Context, id string) error { + if id == "net-2" { + return fmt.Errorf("api timeout") + } + deletedIDs = append(deletedIDs, id) + return nil + }, + } + + clusterScope := newTestClusterScope(networkService, &mockSubnetService{}) + clusterScope.CloudscaleCluster.Status.Networks = []infrastructurev1beta2.NetworkStatus{ + {Name: "net-a", NetworkID: "net-1", Managed: true}, + {Name: "net-b", NetworkID: "net-2", Managed: true}, + {Name: "net-c", NetworkID: "net-3", Managed: true}, + } + + r := newTestReconciler() + + err := r.deleteNetwork(context.Background(), clusterScope) + + g.Expect(err).To(HaveOccurred()) + g.Expect(err.Error()).To(ContainSubstring("deleting network net-b")) + + // net-1 and net-3 were successfully deleted + g.Expect(deletedIDs).To(ConsistOf("net-1", "net-3")) + + // Only the failed network remains in status + g.Expect(clusterScope.CloudscaleCluster.Status.Networks).To(HaveLen(1)) + g.Expect(clusterScope.CloudscaleCluster.Status.Networks[0].Name).To(Equal("net-b")) + g.Expect(clusterScope.CloudscaleCluster.Status.Networks[0].NetworkID).To(Equal("net-2")) +} + +func TestDeleteNetwork_PartialFailurePreservesBYO(t *testing.T) { + g := NewWithT(t) + + var deletedIDs []string + + networkService := &mockNetworkService{ + deleteFn: func(ctx context.Context, id string) error { + if id == "managed-net-2" { + return fmt.Errorf("api timeout") + } + deletedIDs = append(deletedIDs, id) + return nil + }, + } + + clusterScope := newTestClusterScope(networkService, &mockSubnetService{}) + clusterScope.CloudscaleCluster.Status.Networks = []infrastructurev1beta2.NetworkStatus{ + {Name: "managed-a", NetworkID: "managed-net-1", Managed: true}, + {Name: "managed-b", NetworkID: "managed-net-2", Managed: true}, + {Name: "byo-net", NetworkID: "byo-net-uuid", SubnetID: "byo-subnet-uuid", Managed: false}, + } + + r := newTestReconciler() + + err := r.deleteNetwork(context.Background(), clusterScope) + + g.Expect(err).To(HaveOccurred()) + g.Expect(err.Error()).To(ContainSubstring("deleting network managed-b")) + + // Both the failed managed network and the BYO network are preserved + g.Expect(clusterScope.CloudscaleCluster.Status.Networks).To(HaveLen(2)) + g.Expect(clusterScope.CloudscaleCluster.Status.Networks[0].Name).To(Equal("managed-b")) + g.Expect(clusterScope.CloudscaleCluster.Status.Networks[1].Name).To(Equal("byo-net")) + g.Expect(clusterScope.CloudscaleCluster.Status.Networks[1].Managed).To(BeFalse()) +} + +// --- BYO network tests --- + +func TestReconcileNetwork_BYOCachedShortCircuits(t *testing.T) { + g := NewWithT(t) + + networkService := &mockNetworkService{ + getFn: func(ctx context.Context, id string) (*cloudscale.Network, error) { + t.Fatal("Get should not be called when BYO status is cached") + return nil, nil + }, + } + + clusterScope := newTestClusterScope(networkService, &mockSubnetService{}) + clusterScope.CloudscaleCluster.Spec.Networks = []infrastructurev1beta2.NetworkSpec{ + {Name: "byo-net", UUID: "byo-uuid"}, + } + clusterScope.CloudscaleCluster.Status.Networks = []infrastructurev1beta2.NetworkStatus{ + {Name: "byo-net", NetworkID: "byo-uuid", SubnetID: "byo-subnet-uuid", CIDR: "10.0.0.0/24", Managed: false}, + } + + r := newTestReconciler() + + err := r.reconcileNetwork(context.Background(), clusterScope) + + g.Expect(err).ToNot(HaveOccurred()) + ns := clusterScope.CloudscaleCluster.Status.GetNetworkStatus("byo-net") + g.Expect(ns).ToNot(BeNil()) + g.Expect(ns.NetworkID).To(Equal("byo-uuid")) + g.Expect(ns.SubnetID).To(Equal("byo-subnet-uuid")) +} + +func TestReconcileNetwork_BYOReDiscoversWhenCIDRMissing(t *testing.T) { + g := NewWithT(t) + + networkService := &mockNetworkService{ + getFn: func(ctx context.Context, id string) (*cloudscale.Network, error) { + g.Expect(id).To(Equal("byo-uuid")) + return &cloudscale.Network{ + UUID: "byo-uuid", + ZonalResource: cloudscale.ZonalResource{ + Zone: cloudscale.ZoneStub{Slug: "rma1"}, + }, + Subnets: []cloudscale.SubnetStub{ + {UUID: "byo-subnet-uuid", CIDR: "192.168.0.0/24"}, + }, + }, nil + }, + } + + clusterScope := newTestClusterScope(networkService, &mockSubnetService{}) + clusterScope.CloudscaleCluster.Spec.Networks = []infrastructurev1beta2.NetworkSpec{ + {Name: "byo-net", UUID: "byo-uuid"}, + } + // CIDR is missing — simulates stale/upgrade status + clusterScope.CloudscaleCluster.Status.Networks = []infrastructurev1beta2.NetworkStatus{ + {Name: "byo-net", NetworkID: "byo-uuid", SubnetID: "byo-subnet-uuid", Managed: false}, + } + + r := newTestReconciler() + + err := r.reconcileNetwork(context.Background(), clusterScope) + + g.Expect(err).ToNot(HaveOccurred()) + ns := clusterScope.CloudscaleCluster.Status.GetNetworkStatus("byo-net") + g.Expect(ns).ToNot(BeNil()) + g.Expect(ns.NetworkID).To(Equal("byo-uuid")) + g.Expect(ns.SubnetID).To(Equal("byo-subnet-uuid")) + g.Expect(ns.CIDR).To(Equal("192.168.0.0/24")) + g.Expect(ns.Managed).To(BeFalse()) +} + +func TestReconcileNetwork_BYOFetchesAndSetsStatus(t *testing.T) { + g := NewWithT(t) + + networkService := &mockNetworkService{ + getFn: func(ctx context.Context, id string) (*cloudscale.Network, error) { + g.Expect(id).To(Equal("byo-uuid")) + return &cloudscale.Network{ + UUID: "byo-uuid", + ZonalResource: cloudscale.ZonalResource{ + Zone: cloudscale.ZoneStub{Slug: "rma1"}, + }, + Subnets: []cloudscale.SubnetStub{ + {UUID: "discovered-subnet-uuid"}, + }, + }, nil + }, + } + + clusterScope := newTestClusterScope(networkService, &mockSubnetService{}) + clusterScope.CloudscaleCluster.Spec.Networks = []infrastructurev1beta2.NetworkSpec{ + {Name: "byo-net", UUID: "byo-uuid"}, + } + + r := newTestReconciler() + + err := r.reconcileNetwork(context.Background(), clusterScope) + + g.Expect(err).ToNot(HaveOccurred()) + ns := clusterScope.CloudscaleCluster.Status.GetNetworkStatus("byo-net") + g.Expect(ns).ToNot(BeNil()) + g.Expect(ns.NetworkID).To(Equal("byo-uuid")) + g.Expect(ns.SubnetID).To(Equal("discovered-subnet-uuid")) + g.Expect(ns.Managed).To(BeFalse()) +} + +func TestReconcileNetwork_BYOGetError(t *testing.T) { + g := NewWithT(t) + + networkService := &mockNetworkService{ + getFn: func(ctx context.Context, id string) (*cloudscale.Network, error) { + return nil, fmt.Errorf("network not found") + }, + } + + clusterScope := newTestClusterScope(networkService, &mockSubnetService{}) + clusterScope.CloudscaleCluster.Spec.Networks = []infrastructurev1beta2.NetworkSpec{ + {Name: "byo-net", UUID: "byo-uuid"}, + } + + r := newTestReconciler() + + err := r.reconcileNetwork(context.Background(), clusterScope) + + g.Expect(err).To(HaveOccurred()) + g.Expect(err.Error()).To(ContainSubstring("network not found")) +} + +func TestReconcileNetwork_BYOZoneMismatchErrors(t *testing.T) { + g := NewWithT(t) + + networkService := &mockNetworkService{ + getFn: func(ctx context.Context, id string) (*cloudscale.Network, error) { + return &cloudscale.Network{ + UUID: "byo-uuid", + ZonalResource: cloudscale.ZonalResource{ + Zone: cloudscale.ZoneStub{Slug: "lpg1"}, + }, + Subnets: []cloudscale.SubnetStub{ + {UUID: "discovered-subnet-uuid"}, + }, + }, nil + }, + } + + clusterScope := newTestClusterScope(networkService, &mockSubnetService{}) + clusterScope.CloudscaleCluster.Spec.Networks = []infrastructurev1beta2.NetworkSpec{ + {Name: "byo-net", UUID: "byo-uuid"}, + } + + r := newTestReconciler() + + err := r.reconcileNetwork(context.Background(), clusterScope) + + g.Expect(err).To(HaveOccurred()) + g.Expect(err.Error()).To(ContainSubstring("lpg1")) + g.Expect(err.Error()).To(ContainSubstring("rma1")) + g.Expect(clusterScope.CloudscaleCluster.Status.GetNetworkStatus("byo-net")).To(BeNil()) +} + +func TestReconcileNetwork_BYONoSubnetsErrors(t *testing.T) { + g := NewWithT(t) + + networkService := &mockNetworkService{ + getFn: func(ctx context.Context, id string) (*cloudscale.Network, error) { + return &cloudscale.Network{ + UUID: "byo-uuid", + ZonalResource: cloudscale.ZonalResource{ + Zone: cloudscale.ZoneStub{Slug: "rma1"}, + }, + Subnets: []cloudscale.SubnetStub{}, + }, nil + }, + } + + clusterScope := newTestClusterScope(networkService, &mockSubnetService{}) + clusterScope.CloudscaleCluster.Spec.Networks = []infrastructurev1beta2.NetworkSpec{ + {Name: "byo-net", UUID: "byo-uuid"}, + } + + r := newTestReconciler() + + err := r.reconcileNetwork(context.Background(), clusterScope) + + g.Expect(err).To(HaveOccurred()) + g.Expect(err.Error()).To(ContainSubstring("has no subnets")) } // --- Mock services --- diff --git a/internal/controller/cloudscalecluster_reconcile_test.go b/internal/controller/cloudscalecluster_reconcile_test.go index 51c4531..55ba6ea 100644 --- a/internal/controller/cloudscalecluster_reconcile_test.go +++ b/internal/controller/cloudscalecluster_reconcile_test.go @@ -39,7 +39,6 @@ import ( // reconcileTestScope builds a ClusterScope with both network and LB services wired up. func reconcileTestScope(opts reconcileTestOpts) *scope.ClusterScope { - defaultGateway := "" return &scope.ClusterScope{ Logger: logr.Discard(), Cluster: &clusterv1.Cluster{ @@ -57,9 +56,11 @@ func reconcileTestScope(opts reconcileTestOpts) *scope.ClusterScope { Spec: infrastructurev1beta2.CloudscaleClusterSpec{ Region: "rma", Zone: "rma1", - Network: infrastructurev1beta2.NetworkSpec{ - CIDR: "10.0.0.0/24", - GatewayAddress: &defaultGateway, + Networks: []infrastructurev1beta2.NetworkSpec{ + { + Name: "test", + CIDR: "10.0.0.0/24", + }, }, ControlPlaneLoadBalancer: infrastructurev1beta2.LoadBalancerSpec{ Enabled: ptr.To(opts.lbEnabled), @@ -160,8 +161,9 @@ func TestReconcileNormal_FullyProvisionedCluster(t *testing.T) { mocks := defaultMocks() clusterScope := reconcileTestScope(mocks) - clusterScope.CloudscaleCluster.Status.NetworkID = "net-123" - clusterScope.CloudscaleCluster.Status.SubnetID = "subnet-123" + clusterScope.CloudscaleCluster.Status.Networks = []infrastructurev1beta2.NetworkStatus{ + {Name: "test", NetworkID: "net-123", SubnetID: "subnet-123", Managed: true}, + } clusterScope.CloudscaleCluster.Status.LoadBalancerID = "lb-123" clusterScope.CloudscaleCluster.Status.LoadBalancerPoolID = "pool-123" clusterScope.CloudscaleCluster.Status.LoadBalancerListenerID = "listener-123" @@ -228,8 +230,9 @@ func TestReconcileNormal_LBErrorStopsReconciliation(t *testing.T) { clusterScope := reconcileTestScope(mocks) // Network is already provisioned - clusterScope.CloudscaleCluster.Status.NetworkID = "net-123" - clusterScope.CloudscaleCluster.Status.SubnetID = "subnet-123" + clusterScope.CloudscaleCluster.Status.Networks = []infrastructurev1beta2.NetworkStatus{ + {Name: "test", NetworkID: "net-123", SubnetID: "subnet-123", Managed: true}, + } r := newTestReconciler() @@ -250,8 +253,9 @@ func TestReconcileNormal_LBPendingReturnsRequeue(t *testing.T) { } clusterScope := reconcileTestScope(mocks) - clusterScope.CloudscaleCluster.Status.NetworkID = "net-123" - clusterScope.CloudscaleCluster.Status.SubnetID = "subnet-123" + clusterScope.CloudscaleCluster.Status.Networks = []infrastructurev1beta2.NetworkStatus{ + {Name: "test", NetworkID: "net-123", SubnetID: "subnet-123", Managed: true}, + } clusterScope.CloudscaleCluster.Status.LoadBalancerID = "lb-123" r := newTestReconciler() @@ -271,8 +275,9 @@ func TestReconcileNormal_LBDisabledSetsProvisioned(t *testing.T) { mocks.lbEnabled = false clusterScope := reconcileTestScope(mocks) - clusterScope.CloudscaleCluster.Status.NetworkID = "net-123" - clusterScope.CloudscaleCluster.Status.SubnetID = "subnet-123" + clusterScope.CloudscaleCluster.Status.Networks = []infrastructurev1beta2.NetworkStatus{ + {Name: "test", NetworkID: "net-123", SubnetID: "subnet-123", Managed: true}, + } clusterScope.CloudscaleCluster.Spec.ControlPlaneEndpoint = clusterv1.APIEndpoint{ Host: "external-cp.example.com", Port: 6443, @@ -312,8 +317,9 @@ func TestReconcileDelete_SuccessfulDeletion(t *testing.T) { } clusterScope := reconcileTestScope(mocks) - clusterScope.CloudscaleCluster.Status.NetworkID = "net-123" - clusterScope.CloudscaleCluster.Status.SubnetID = "subnet-123" + clusterScope.CloudscaleCluster.Status.Networks = []infrastructurev1beta2.NetworkStatus{ + {Name: "test", NetworkID: "net-123", SubnetID: "subnet-123", Managed: true}, + } clusterScope.CloudscaleCluster.Status.LoadBalancerID = "lb-123" clusterScope.CloudscaleCluster.Status.LoadBalancerPoolID = "pool-123" clusterScope.CloudscaleCluster.Status.LoadBalancerListenerID = "listener-123" @@ -334,8 +340,7 @@ func TestReconcileDelete_SuccessfulDeletion(t *testing.T) { g.Expect(clusterScope.CloudscaleCluster.Status.LoadBalancerPoolID).To(BeEmpty()) g.Expect(clusterScope.CloudscaleCluster.Status.LoadBalancerListenerID).To(BeEmpty()) // Verify network status cleared - g.Expect(clusterScope.CloudscaleCluster.Status.NetworkID).To(BeEmpty()) - g.Expect(clusterScope.CloudscaleCluster.Status.SubnetID).To(BeEmpty()) + g.Expect(clusterScope.CloudscaleCluster.Status.Networks).To(BeNil()) // Verify finalizer removed g.Expect(clusterScope.CloudscaleCluster.Finalizers).ToNot(ContainElement(infrastructurev1beta2.ClusterFinalizer)) @@ -368,7 +373,9 @@ func TestReconcileDelete_LBDeleteErrorStopsDeletion(t *testing.T) { clusterScope := reconcileTestScope(mocks) clusterScope.CloudscaleCluster.Status.LoadBalancerID = "lb-123" - clusterScope.CloudscaleCluster.Status.NetworkID = "net-123" + clusterScope.CloudscaleCluster.Status.Networks = []infrastructurev1beta2.NetworkStatus{ + {Name: "test", NetworkID: "net-123", Managed: true}, + } clusterScope.CloudscaleCluster.Finalizers = []string{infrastructurev1beta2.ClusterFinalizer} r := newTestReconciler() @@ -398,7 +405,9 @@ func TestReconcileDelete_NetworkDeleteErrorStopsDeletion(t *testing.T) { clusterScope := reconcileTestScope(mocks) clusterScope.CloudscaleCluster.Status.LoadBalancerID = "lb-123" - clusterScope.CloudscaleCluster.Status.NetworkID = "net-123" + clusterScope.CloudscaleCluster.Status.Networks = []infrastructurev1beta2.NetworkStatus{ + {Name: "test", NetworkID: "net-123", Managed: true}, + } clusterScope.CloudscaleCluster.Finalizers = []string{infrastructurev1beta2.ClusterFinalizer} r := newTestReconciler() @@ -432,7 +441,9 @@ func TestReconcileDelete_LBDisabledSkipsLBDeletion(t *testing.T) { } clusterScope := reconcileTestScope(mocks) - clusterScope.CloudscaleCluster.Status.NetworkID = "net-123" + clusterScope.CloudscaleCluster.Status.Networks = []infrastructurev1beta2.NetworkStatus{ + {Name: "test", NetworkID: "net-123", Managed: true}, + } clusterScope.CloudscaleCluster.Finalizers = []string{infrastructurev1beta2.ClusterFinalizer} r := newTestReconciler() diff --git a/internal/controller/cloudscalemachine_server.go b/internal/controller/cloudscalemachine_server.go index 1181e21..e7fd681 100644 --- a/internal/controller/cloudscalemachine_server.go +++ b/internal/controller/cloudscalemachine_server.go @@ -33,6 +33,10 @@ import ( "github.com/cloudscale-ch/cluster-api-provider-cloudscale/internal/scope" ) +// InterfaceTypePublic is the cloudscale.ch SDK value for a public network interface, +// used both as the Server.Interfaces[].Type and as InterfaceRequest.Network. +const InterfaceTypePublic = "public" + // ServerStatus represents the status of a cloudscale.ch server. type ServerStatus string @@ -126,6 +130,12 @@ func (r *CloudscaleMachineReconciler) reconcileServer(ctx context.Context, machi return ctrl.Result{}, fmt.Errorf("getting bootstrap data: %w", err) } + // Build network interfaces + interfaces, ipFamily, err := r.buildInterfaceRequests(machineScope) + if err != nil { + return ctrl.Result{}, fmt.Errorf("building interface requests: %w", err) + } + // Build server request req := &cloudscalesdk.ServerRequest{ Name: machineScope.Name(), @@ -136,7 +146,8 @@ func (r *CloudscaleMachineReconciler) reconcileServer(ctx context.Context, machi Tags: r.machineCreateTags(machineScope), }, UserData: bootstrapData, - Interfaces: r.buildInterfaceRequests(machineScope), + Interfaces: interfaces, + UseIPV6: ipFamilyToUseIPV6(ipFamily), // sending nil doesn't work, we need to explicitly send an empty slice SSHKeys: []string{}, } @@ -208,7 +219,7 @@ func (r *CloudscaleMachineReconciler) updateMachineFromServer(machineScope *scop addresses := make([]clusterv1.MachineAddress, 0, len(server.Interfaces)*2) for _, iface := range server.Interfaces { addressType := clusterv1.MachineInternalIP - if iface.Type == "public" { + if iface.Type == InterfaceTypePublic { addressType = clusterv1.MachineExternalIP } for _, addr := range iface.Addresses { @@ -271,17 +282,60 @@ func (r *CloudscaleMachineReconciler) machineLookupTag(machineScope *scope.Machi } // buildInterfaceRequests constructs the network interfaces for server creation. -// All machines get both a private network interface and a public network interface. -// The public interface is required for machines to reach the API server endpoint. -func (r *CloudscaleMachineReconciler) buildInterfaceRequests(machineScope *scope.MachineScope) *[]cloudscalesdk.InterfaceRequest { - interfaces := []cloudscalesdk.InterfaceRequest{ - { - Network: machineScope.CloudscaleCluster.Status.NetworkID, - }, - { - Network: "public", - }, +// If spec.interfaces is empty, defaults to the first cluster network + a public interface +// (runtime cross-resource resolution that the webhook cannot do). +// Returns the interface requests and the IPFamily from the public interface (if any). +func (r *CloudscaleMachineReconciler) buildInterfaceRequests(machineScope *scope.MachineScope) (*[]cloudscalesdk.InterfaceRequest, *infrastructurev1beta2.IPFamily, error) { + ifaceSpecs := machineScope.CloudscaleMachine.Spec.Interfaces + + // Runtime default: first cluster network + public DualStack interface + if len(ifaceSpecs) == 0 { + if len(machineScope.CloudscaleCluster.Status.Networks) == 0 { + return nil, nil, fmt.Errorf("cluster has no networks provisioned yet") + } + firstNetwork := machineScope.CloudscaleCluster.Status.Networks[0] + return &[]cloudscalesdk.InterfaceRequest{ + {Network: InterfaceTypePublic}, + {Network: firstNetwork.NetworkID}, + }, nil, nil } - return &interfaces + // Build from spec + reqs := make([]cloudscalesdk.InterfaceRequest, 0, len(ifaceSpecs)) + var ipFamily *infrastructurev1beta2.IPFamily + for _, iface := range ifaceSpecs { + switch { + case iface.Type == InterfaceTypePublic: + reqs = append(reqs, cloudscalesdk.InterfaceRequest{Network: InterfaceTypePublic}) + ipFamily = iface.IPFamily + case iface.Network != "": + ns := machineScope.CloudscaleCluster.Status.GetNetworkStatus(iface.Network) + if ns == nil { + return nil, nil, fmt.Errorf("network %q not found in cluster status", iface.Network) + } + if ns.NetworkID == "" { + return nil, nil, fmt.Errorf("network %q not yet provisioned", iface.Network) + } + reqs = append(reqs, cloudscalesdk.InterfaceRequest{Network: ns.NetworkID}) + default: + return nil, nil, fmt.Errorf("interface must have either type or network set") + } + } + + return &reqs, ipFamily, nil +} + +// ipFamilyToUseIPV6 maps an IPFamily value to the cloudscale API's use_ipv6 server-level setting. +func ipFamilyToUseIPV6(ipFamily *infrastructurev1beta2.IPFamily) *bool { + if ipFamily == nil { + return nil + } + switch *ipFamily { + case infrastructurev1beta2.IPFamilyDualStack: + return ptr.To(true) + case infrastructurev1beta2.IPFamilyIPv4: + return ptr.To(false) + default: + return nil + } } diff --git a/internal/controller/cloudscalemachine_server_test.go b/internal/controller/cloudscalemachine_server_test.go index f9eef2c..d97a84c 100644 --- a/internal/controller/cloudscalemachine_server_test.go +++ b/internal/controller/cloudscalemachine_server_test.go @@ -150,8 +150,9 @@ func newTestMachineScopeWithServer(serverService cs.ServerService) *scope.Machin Zone: "rma1", }, Status: infrastructurev1beta2.CloudscaleClusterStatus{ - NetworkID: "net-uuid-123", - SubnetID: "subnet-uuid-123", + Networks: []infrastructurev1beta2.NetworkStatus{ + {Name: "test", NetworkID: "net-uuid-123", SubnetID: "subnet-uuid-123", Managed: true}, + }, }, }, CloudscaleMachine: cloudscaleMachine, @@ -608,3 +609,278 @@ func TestReconcileServer_NoServerGroupWhenStatusEmpty(t *testing.T) { g.Expect(capturedReq).ToNot(BeNil()) g.Expect(capturedReq.ServerGroups).To(BeNil()) } + +// --- buildInterfaceRequests tests --- + +func TestBuildInterfaceRequests_DefaultsToPublicPlusFirstNetwork(t *testing.T) { + g := NewWithT(t) + + machineScope := newTestMachineScopeWithServer(&mockServerService{}) + // No interfaces in spec → uses runtime defaults + + r := &CloudscaleMachineReconciler{} + + reqs, ipFamily, err := r.buildInterfaceRequests(machineScope) + + g.Expect(err).ToNot(HaveOccurred()) + g.Expect(reqs).ToNot(BeNil()) + g.Expect(*reqs).To(HaveLen(2)) + g.Expect((*reqs)[0].Network).To(Equal(InterfaceTypePublic)) + g.Expect((*reqs)[1].Network).To(Equal("net-uuid-123")) + g.Expect(ipFamily).To(BeNil(), "runtime default path should not return ipFamily") +} + +func TestBuildInterfaceRequests_NoClusterNetworksErrors(t *testing.T) { + g := NewWithT(t) + + machineScope := newTestMachineScopeWithServer(&mockServerService{}) + machineScope.CloudscaleCluster.Status.Networks = nil + + r := &CloudscaleMachineReconciler{} + + _, _, err := r.buildInterfaceRequests(machineScope) + + g.Expect(err).To(HaveOccurred()) + g.Expect(err.Error()).To(ContainSubstring("no networks provisioned")) +} + +func TestBuildInterfaceRequests_PublicInterface(t *testing.T) { + g := NewWithT(t) + + machineScope := newTestMachineScopeWithServer(&mockServerService{}) + machineScope.CloudscaleMachine.Spec.Interfaces = []infrastructurev1beta2.InterfaceSpec{ + {Type: "public"}, + } + + r := &CloudscaleMachineReconciler{} + + reqs, _, err := r.buildInterfaceRequests(machineScope) + + g.Expect(err).ToNot(HaveOccurred()) + g.Expect(*reqs).To(HaveLen(1)) + g.Expect((*reqs)[0].Network).To(Equal(InterfaceTypePublic)) +} + +func TestBuildInterfaceRequests_NamedNetworkFound(t *testing.T) { + g := NewWithT(t) + + machineScope := newTestMachineScopeWithServer(&mockServerService{}) + machineScope.CloudscaleMachine.Spec.Interfaces = []infrastructurev1beta2.InterfaceSpec{ + {Network: "test"}, + } + + r := &CloudscaleMachineReconciler{} + + reqs, _, err := r.buildInterfaceRequests(machineScope) + + g.Expect(err).ToNot(HaveOccurred()) + g.Expect(*reqs).To(HaveLen(1)) + g.Expect((*reqs)[0].Network).To(Equal("net-uuid-123")) +} + +func TestBuildInterfaceRequests_NamedNetworkNotFound(t *testing.T) { + g := NewWithT(t) + + machineScope := newTestMachineScopeWithServer(&mockServerService{}) + machineScope.CloudscaleMachine.Spec.Interfaces = []infrastructurev1beta2.InterfaceSpec{ + {Network: "nonexistent"}, + } + + r := &CloudscaleMachineReconciler{} + + _, _, err := r.buildInterfaceRequests(machineScope) + + g.Expect(err).To(HaveOccurred()) + g.Expect(err.Error()).To(ContainSubstring("not found in cluster status")) +} + +func TestBuildInterfaceRequests_NamedNetworkNotProvisioned(t *testing.T) { + g := NewWithT(t) + + machineScope := newTestMachineScopeWithServer(&mockServerService{}) + machineScope.CloudscaleCluster.Status.Networks = []infrastructurev1beta2.NetworkStatus{ + {Name: "test", NetworkID: "", SubnetID: "subnet-uuid-123", Managed: true}, + } + machineScope.CloudscaleMachine.Spec.Interfaces = []infrastructurev1beta2.InterfaceSpec{ + {Network: "test"}, + } + + r := &CloudscaleMachineReconciler{} + + _, _, err := r.buildInterfaceRequests(machineScope) + + g.Expect(err).To(HaveOccurred()) + g.Expect(err.Error()).To(ContainSubstring("not yet provisioned")) +} + +func TestBuildInterfaceRequests_MixedPublicAndNetwork(t *testing.T) { + g := NewWithT(t) + + machineScope := newTestMachineScopeWithServer(&mockServerService{}) + machineScope.CloudscaleMachine.Spec.Interfaces = []infrastructurev1beta2.InterfaceSpec{ + {Network: "test"}, + {Type: "public"}, + } + + r := &CloudscaleMachineReconciler{} + + reqs, _, err := r.buildInterfaceRequests(machineScope) + + g.Expect(err).ToNot(HaveOccurred()) + g.Expect(*reqs).To(HaveLen(2)) + g.Expect((*reqs)[0].Network).To(Equal("net-uuid-123")) + g.Expect((*reqs)[1].Network).To(Equal(InterfaceTypePublic)) +} + +func TestBuildInterfaceRequests_InvalidInterfaceErrors(t *testing.T) { + g := NewWithT(t) + + machineScope := newTestMachineScopeWithServer(&mockServerService{}) + machineScope.CloudscaleMachine.Spec.Interfaces = []infrastructurev1beta2.InterfaceSpec{ + {}, // neither type nor network + } + + r := &CloudscaleMachineReconciler{} + + _, _, err := r.buildInterfaceRequests(machineScope) + + g.Expect(err).To(HaveOccurred()) + g.Expect(err.Error()).To(ContainSubstring("must have either type or network")) +} + +func TestBuildInterfaceRequests_ReturnsIPFamilyFromPublicInterface(t *testing.T) { + g := NewWithT(t) + + dualStack := infrastructurev1beta2.IPFamilyDualStack + machineScope := newTestMachineScopeWithServer(&mockServerService{}) + machineScope.CloudscaleMachine.Spec.Interfaces = []infrastructurev1beta2.InterfaceSpec{ + {Network: "test"}, + {Type: "public", IPFamily: &dualStack}, + } + + r := &CloudscaleMachineReconciler{} + + _, ipFamily, err := r.buildInterfaceRequests(machineScope) + + g.Expect(err).ToNot(HaveOccurred()) + g.Expect(ipFamily).ToNot(BeNil()) + g.Expect(*ipFamily).To(Equal(infrastructurev1beta2.IPFamilyDualStack)) +} + +func TestBuildInterfaceRequests_NilIPFamilyWhenNoPublicInterface(t *testing.T) { + g := NewWithT(t) + + machineScope := newTestMachineScopeWithServer(&mockServerService{}) + machineScope.CloudscaleMachine.Spec.Interfaces = []infrastructurev1beta2.InterfaceSpec{ + {Network: "test"}, + } + + r := &CloudscaleMachineReconciler{} + + _, ipFamily, err := r.buildInterfaceRequests(machineScope) + + g.Expect(err).ToNot(HaveOccurred()) + g.Expect(ipFamily).To(BeNil()) +} + +// --- ipFamilyToUseIPV6 tests --- + +func TestIPFamilyToUseIPV6_DualStack(t *testing.T) { + g := NewWithT(t) + dualStack := infrastructurev1beta2.IPFamilyDualStack + result := ipFamilyToUseIPV6(&dualStack) + g.Expect(result).ToNot(BeNil()) + g.Expect(*result).To(BeTrue()) +} + +func TestIPFamilyToUseIPV6_IPv4(t *testing.T) { + g := NewWithT(t) + ipv4 := infrastructurev1beta2.IPFamilyIPv4 + result := ipFamilyToUseIPV6(&ipv4) + g.Expect(result).ToNot(BeNil()) + g.Expect(*result).To(BeFalse()) +} + +func TestIPFamilyToUseIPV6_Nil(t *testing.T) { + g := NewWithT(t) + result := ipFamilyToUseIPV6(nil) + g.Expect(result).To(BeNil()) +} + +// --- UseIPV6 integration via reconcileServer --- + +func TestReconcileServer_SetsUseIPV6DualStack(t *testing.T) { + g := NewWithT(t) + var capturedReq *cloudscalesdk.ServerRequest + + serverService := &mockServerService{ + listFn: func(ctx context.Context, modifiers ...cloudscalesdk.ListRequestModifier) ([]cloudscalesdk.Server, error) { + return nil, nil + }, + createFn: func(ctx context.Context, req *cloudscalesdk.ServerRequest) (*cloudscalesdk.Server, error) { + capturedReq = req + return &cloudscalesdk.Server{ + UUID: "server-uuid-ipv6", + Name: req.Name, + Status: "running", + ZonalResource: cloudscalesdk.ZonalResource{Zone: cloudscalesdk.ZoneStub{Slug: "rma1"}}, + }, nil + }, + } + + dualStack := infrastructurev1beta2.IPFamilyDualStack + machineScope := newTestMachineScopeWithServer(serverService) + machineScope.CloudscaleMachine.Spec.Interfaces = []infrastructurev1beta2.InterfaceSpec{ + {Network: "test"}, + {Type: "public", IPFamily: &dualStack}, + } + + r := &CloudscaleMachineReconciler{ + recorder: events.NewFakeRecorder(10), + } + + _, err := r.reconcileServer(context.Background(), machineScope) + + g.Expect(err).ToNot(HaveOccurred()) + g.Expect(capturedReq).ToNot(BeNil()) + g.Expect(capturedReq.UseIPV6).ToNot(BeNil()) + g.Expect(*capturedReq.UseIPV6).To(BeTrue()) +} + +func TestReconcileServer_SetsUseIPV6IPv4Only(t *testing.T) { + g := NewWithT(t) + var capturedReq *cloudscalesdk.ServerRequest + + serverService := &mockServerService{ + listFn: func(ctx context.Context, modifiers ...cloudscalesdk.ListRequestModifier) ([]cloudscalesdk.Server, error) { + return nil, nil + }, + createFn: func(ctx context.Context, req *cloudscalesdk.ServerRequest) (*cloudscalesdk.Server, error) { + capturedReq = req + return &cloudscalesdk.Server{ + UUID: "server-uuid-ipv4", + Name: req.Name, + Status: "running", + ZonalResource: cloudscalesdk.ZonalResource{Zone: cloudscalesdk.ZoneStub{Slug: "rma1"}}, + }, nil + }, + } + + ipv4 := infrastructurev1beta2.IPFamilyIPv4 + machineScope := newTestMachineScopeWithServer(serverService) + machineScope.CloudscaleMachine.Spec.Interfaces = []infrastructurev1beta2.InterfaceSpec{ + {Network: "test"}, + {Type: "public", IPFamily: &ipv4}, + } + + r := &CloudscaleMachineReconciler{ + recorder: events.NewFakeRecorder(10), + } + + _, err := r.reconcileServer(context.Background(), machineScope) + + g.Expect(err).ToNot(HaveOccurred()) + g.Expect(capturedReq).ToNot(BeNil()) + g.Expect(capturedReq.UseIPV6).ToNot(BeNil()) + g.Expect(*capturedReq.UseIPV6).To(BeFalse()) +} diff --git a/internal/controller/cloudscalemachine_servergroup_test.go b/internal/controller/cloudscalemachine_servergroup_test.go index 145741d..4506be4 100644 --- a/internal/controller/cloudscalemachine_servergroup_test.go +++ b/internal/controller/cloudscalemachine_servergroup_test.go @@ -138,8 +138,9 @@ func newTestMachineScopeWithServerGroup(serverGroupService cs.ServerGroupService Zone: "rma1", }, Status: infrastructurev1beta2.CloudscaleClusterStatus{ - NetworkID: "net-uuid-123", - SubnetID: "subnet-uuid-123", + Networks: []infrastructurev1beta2.NetworkStatus{ + {Name: "test", NetworkID: "net-uuid-123", SubnetID: "subnet-uuid-123", Managed: true}, + }, }, }, CloudscaleMachine: cloudscaleMachine, diff --git a/internal/scope/cluster_test.go b/internal/scope/cluster_test.go index 8d865bc..9f2973a 100644 --- a/internal/scope/cluster_test.go +++ b/internal/scope/cluster_test.go @@ -236,7 +236,7 @@ func TestClusterScope_Close(t *testing.T) { g.Expect(err).ToNot(HaveOccurred()) // Modify status to verify patch happens - scope.CloudscaleCluster.Status.NetworkID = "patched-network-id" + scope.CloudscaleCluster.Status.Networks = []infrastructurev1beta2.NetworkStatus{{Name: "test", NetworkID: "patched-network-id", Managed: true}} err = scope.Close(context.Background()) g.Expect(err).ToNot(HaveOccurred()) @@ -248,5 +248,6 @@ func TestClusterScope_Close(t *testing.T) { Namespace: cloudscaleCluster.Namespace, }, updated) g.Expect(err).ToNot(HaveOccurred()) - g.Expect(updated.Status.NetworkID).To(Equal("patched-network-id")) + g.Expect(updated.Status.Networks).To(HaveLen(1)) + g.Expect(updated.Status.Networks[0].NetworkID).To(Equal("patched-network-id")) } diff --git a/internal/webhook/v1beta2/cloudscalecluster_webhook.go b/internal/webhook/v1beta2/cloudscalecluster_webhook.go index fd3925e..0029599 100644 --- a/internal/webhook/v1beta2/cloudscalecluster_webhook.go +++ b/internal/webhook/v1beta2/cloudscalecluster_webhook.go @@ -65,21 +65,19 @@ const defaultSubnetCIDR = "10.0.0.0/24" func (d *CloudscaleClusterCustomDefaulter) Default(_ context.Context, cluster *infrastructurev1beta2.CloudscaleCluster) error { cloudscaleclusterlog.Info("Defaulting for CloudscaleCluster", "name", cluster.GetName()) - // Default network zone to region's default zone if not set + // Default zone to region's default zone if not set if cluster.Spec.Zone == "" { cluster.Spec.Zone = d.RegionInfo.GetDefaultZoneForRegion(cluster.Spec.Region) } - // Default network CIDR if not set - if cluster.Spec.Network.CIDR == "" { - cluster.Spec.Network.CIDR = defaultSubnetCIDR - } - - // Default gateway address to empty string (no gateway) - // This ensures outbound internet traffic uses the public interface, - // which is required for CCM to reach the cloudscale.ch API. - if cluster.Spec.Network.GatewayAddress == nil { - cluster.Spec.Network.GatewayAddress = ptr.To("") + // Default networks: if empty, create one managed network named after the cluster + if len(cluster.Spec.Networks) == 0 { + cluster.Spec.Networks = []infrastructurev1beta2.NetworkSpec{ + { + Name: cluster.Name, + CIDR: defaultSubnetCIDR, + }, + } } // Default load balancer settings @@ -95,8 +93,8 @@ func (d *CloudscaleClusterCustomDefaulter) Default(_ context.Context, cluster *i if cluster.Spec.ControlPlaneLoadBalancer.APIServerPort == 0 { cluster.Spec.ControlPlaneLoadBalancer.APIServerPort = 6443 } - if cluster.Spec.ControlPlaneLoadBalancer.Algorithm == "" { - cluster.Spec.ControlPlaneLoadBalancer.Algorithm = "round_robin" + if cluster.Spec.ControlPlaneLoadBalancer.IPFamily == "" { + cluster.Spec.ControlPlaneLoadBalancer.IPFamily = infrastructurev1beta2.IPFamilyDualStack } if cluster.Spec.ControlPlaneLoadBalancer.HealthMonitor.DelayS == 0 { @@ -112,6 +110,12 @@ func (d *CloudscaleClusterCustomDefaulter) Default(_ context.Context, cluster *i cluster.Spec.ControlPlaneLoadBalancer.HealthMonitor.DownThreshold = 3 } + // Default floating IP: if set but both fields empty, default to IPv4 + if cluster.Spec.FloatingIP != nil && cluster.Spec.FloatingIP.IPFamily == nil && cluster.Spec.FloatingIP.IP == "" { + ipv4 := infrastructurev1beta2.IPFamilyIPv4 + cluster.Spec.FloatingIP.IPFamily = &ipv4 + } + return nil } @@ -142,15 +146,27 @@ func (v *CloudscaleClusterCustomValidator) ValidateCreate(_ context.Context, clu } } - // Validate gateway address is within CIDR if specified - if cluster.Spec.Network.GatewayAddress != nil && *cluster.Spec.Network.GatewayAddress != "" { - allErrs = append(allErrs, validateGatewayInCIDR( - cluster.Spec.Network.CIDR, - *cluster.Spec.Network.GatewayAddress, - field.NewPath("spec", "network", "gatewayAddress"), + // Validate networks + allErrs = append(allErrs, validateNetworks(cluster.Spec.Networks, field.NewPath("spec", "networks"))...) + + // Validate LB network reference + if cluster.Spec.ControlPlaneLoadBalancer.Network != "" { + allErrs = append(allErrs, validateNetworkReference( + cluster.Spec.ControlPlaneLoadBalancer.Network, + cluster.Spec.Networks, + field.NewPath("spec", "controlPlaneLoadBalancer", "network"), )...) } + // Validate floating IP + if cluster.Spec.FloatingIP != nil { + allErrs = append(allErrs, validateFloatingIP(cluster.Spec.FloatingIP, field.NewPath("spec", "floatingIP"))...) + } + + allErrs = append(allErrs, validateFloatingIPRequiresLBOrBYO(cluster)...) + allErrs = append(allErrs, validateFloatingIPRequiresPublicLB(cluster)...) + allErrs = append(allErrs, validateLBPoolMemberNetworkResolvable(cluster)...) + if len(allErrs) > 0 { return nil, apierrors.NewInvalid( schema.GroupKind{Group: infrastructurev1beta2.SchemeGroupVersion.Group, Kind: "CloudscaleCluster"}, @@ -173,19 +189,18 @@ func (v *CloudscaleClusterCustomValidator) ValidateUpdate(_ context.Context, old "field is immutable after cluster creation")) } - // Network zone is immutable + // Zone is immutable if newCluster.Spec.Zone != oldCluster.Spec.Zone { allErrs = append(allErrs, field.Forbidden( field.NewPath("spec", "zone"), "field is immutable after cluster creation")) } - // Network CIDR is immutable - if newCluster.Spec.Network.CIDR != oldCluster.Spec.Network.CIDR { - allErrs = append(allErrs, field.Forbidden( - field.NewPath("spec", "network", "cidr"), - "field is immutable after cluster creation")) - } + // Network immutability: existing networks cannot be modified or removed + allErrs = append(allErrs, validateNetworkImmutability(oldCluster.Spec.Networks, newCluster.Spec.Networks, field.NewPath("spec", "networks"))...) + + // Validate new networks (new entries must still pass creation validation) + allErrs = append(allErrs, validateNetworks(newCluster.Spec.Networks, field.NewPath("spec", "networks"))...) // LoadBalancer Enabled is immutable if ptr.Deref(newCluster.Spec.ControlPlaneLoadBalancer.Enabled, true) != ptr.Deref(oldCluster.Spec.ControlPlaneLoadBalancer.Enabled, true) { @@ -194,6 +209,31 @@ func (v *CloudscaleClusterCustomValidator) ValidateUpdate(_ context.Context, old "field is immutable after cluster creation")) } + // LB network is immutable once set + if oldCluster.Spec.ControlPlaneLoadBalancer.Network != "" && + newCluster.Spec.ControlPlaneLoadBalancer.Network != oldCluster.Spec.ControlPlaneLoadBalancer.Network { + allErrs = append(allErrs, field.Forbidden( + field.NewPath("spec", "controlPlaneLoadBalancer", "network"), + "field is immutable once set")) + } + + // Other LB fields are immutable: they're baked into the LB at creation + // and changing them post-create would silently diverge from the live LB. + allErrs = append(allErrs, validateLBImmutability( + &oldCluster.Spec.ControlPlaneLoadBalancer, + &newCluster.Spec.ControlPlaneLoadBalancer, + field.NewPath("spec", "controlPlaneLoadBalancer"), + )...) + + // Validate LB network reference (for new or existing) + if newCluster.Spec.ControlPlaneLoadBalancer.Network != "" { + allErrs = append(allErrs, validateNetworkReference( + newCluster.Spec.ControlPlaneLoadBalancer.Network, + newCluster.Spec.Networks, + field.NewPath("spec", "controlPlaneLoadBalancer", "network"), + )...) + } + // ControlPlaneEndpoint is immutable once set if oldCluster.Spec.ControlPlaneEndpoint.Host != "" { if newCluster.Spec.ControlPlaneEndpoint.Host != oldCluster.Spec.ControlPlaneEndpoint.Host { @@ -208,21 +248,18 @@ func (v *CloudscaleClusterCustomValidator) ValidateUpdate(_ context.Context, old } } - // GatewayAddress is immutable - oldGateway := "" - newGateway := "" - if oldCluster.Spec.Network.GatewayAddress != nil { - oldGateway = *oldCluster.Spec.Network.GatewayAddress - } - if newCluster.Spec.Network.GatewayAddress != nil { - newGateway = *newCluster.Spec.Network.GatewayAddress - } - if oldGateway != newGateway { - allErrs = append(allErrs, field.Forbidden( - field.NewPath("spec", "network", "gatewayAddress"), - "field is immutable after cluster creation")) + // FloatingIP is immutable once set + allErrs = append(allErrs, validateFloatingIPImmutability(oldCluster.Spec.FloatingIP, newCluster.Spec.FloatingIP, field.NewPath("spec", "floatingIP"))...) + + // Validate floating IP if set + if newCluster.Spec.FloatingIP != nil { + allErrs = append(allErrs, validateFloatingIP(newCluster.Spec.FloatingIP, field.NewPath("spec", "floatingIP"))...) } + allErrs = append(allErrs, validateFloatingIPRequiresLBOrBYO(newCluster)...) + allErrs = append(allErrs, validateFloatingIPRequiresPublicLB(newCluster)...) + allErrs = append(allErrs, validateLBPoolMemberNetworkResolvable(newCluster)...) + if len(allErrs) > 0 { return nil, apierrors.NewInvalid( schema.GroupKind{Group: infrastructurev1beta2.SchemeGroupVersion.Group, Kind: "CloudscaleCluster"}, @@ -238,6 +275,265 @@ func (v *CloudscaleClusterCustomValidator) ValidateDelete(_ context.Context, obj return nil, nil } +// validateNetworks validates the network list. +func validateNetworks(networks []infrastructurev1beta2.NetworkSpec, fldPath *field.Path) field.ErrorList { + var allErrs field.ErrorList + + names := make(map[string]bool) + for i, netSpec := range networks { + netPath := fldPath.Index(i) + + // Unique names + if names[netSpec.Name] { + allErrs = append(allErrs, field.Duplicate(netPath.Child("name"), netSpec.Name)) + } + names[netSpec.Name] = true + + // Exactly one of UUID or CIDR + hasUUID := netSpec.UUID != "" + hasCIDR := netSpec.CIDR != "" + if hasUUID == hasCIDR { + allErrs = append(allErrs, field.Invalid(netPath, netSpec.Name, + "exactly one of uuid or cidr must be specified")) + } + + // Validate CIDR format + if hasCIDR { + if _, _, err := net.ParseCIDR(netSpec.CIDR); err != nil { + allErrs = append(allErrs, field.Invalid(netPath.Child("cidr"), netSpec.CIDR, + fmt.Sprintf("invalid CIDR: %v", err))) + } + } + + // GatewayAddress only valid with CIDR + if netSpec.GatewayAddress != "" && !hasCIDR { + allErrs = append(allErrs, field.Invalid(netPath.Child("gatewayAddress"), netSpec.GatewayAddress, + "gatewayAddress can only be set when cidr is specified")) + } + + // Validate gateway is within CIDR + if netSpec.GatewayAddress != "" && hasCIDR { + allErrs = append(allErrs, validateGatewayInCIDR( + netSpec.CIDR, + netSpec.GatewayAddress, + netPath.Child("gatewayAddress"), + )...) + } + } + + return allErrs +} + +// validateNetworkReference validates that a network name references a defined network. +func validateNetworkReference(networkName string, networks []infrastructurev1beta2.NetworkSpec, fldPath *field.Path) field.ErrorList { + for _, n := range networks { + if n.Name == networkName { + return nil + } + } + return field.ErrorList{ + field.NotFound(fldPath, networkName), + } +} + +// validateNetworkImmutability checks that existing networks are not modified or removed. +// Errors reference the network's index in the *new* list so users can locate the offending +// entry in their updated manifest, even after a reorder. +func validateNetworkImmutability(oldNetworks, newNetworks []infrastructurev1beta2.NetworkSpec, fldPath *field.Path) field.ErrorList { + var allErrs field.ErrorList + + newByName := make(map[string]*infrastructurev1beta2.NetworkSpec, len(newNetworks)) + newIndexByName := make(map[string]int, len(newNetworks)) + for i := range newNetworks { + newByName[newNetworks[i].Name] = &newNetworks[i] + newIndexByName[newNetworks[i].Name] = i + } + + for _, oldNet := range oldNetworks { + newNet, exists := newByName[oldNet.Name] + if !exists { + allErrs = append(allErrs, field.Forbidden( + fldPath, + fmt.Sprintf("removing network %q is not allowed", oldNet.Name))) + continue + } + + newIdx := newIndexByName[oldNet.Name] + newPath := fldPath.Index(newIdx) + + if newNet.CIDR != oldNet.CIDR { + allErrs = append(allErrs, field.Forbidden( + newPath.Child("cidr"), + "field is immutable after cluster creation")) + } + + if newNet.UUID != oldNet.UUID { + allErrs = append(allErrs, field.Forbidden( + newPath.Child("uuid"), + "field is immutable after cluster creation")) + } + + if newNet.GatewayAddress != oldNet.GatewayAddress { + allErrs = append(allErrs, field.Forbidden( + newPath.Child("gatewayAddress"), + "field is immutable after cluster creation")) + } + } + + return allErrs +} + +// validateFloatingIPRequiresLBOrBYO rejects managed floating IPs when the load balancer is disabled. +// cloudscale.ch floating IPs require a dummy interface on the target server. +// With a BYO FIP, the user knows the address upfront and can configure +// the dummy interface in KubeadmControlPlane preKubeadmCommands. +// With a managed FIP, the address isn't known until creation, +// so the dummy interface can't be pre-configured. +func validateFloatingIPRequiresLBOrBYO(cluster *infrastructurev1beta2.CloudscaleCluster) field.ErrorList { + var allErrs field.ErrorList + + if cluster.Spec.FloatingIP != nil && + cluster.Spec.FloatingIP.IP == "" && + !ptr.Deref(cluster.Spec.ControlPlaneLoadBalancer.Enabled, true) { + allErrs = append(allErrs, field.Invalid( + field.NewPath("spec", "floatingIP"), + "", + "managed floating IP requires the load balancer to be enabled; use a BYO floating IP if you need a floating IP without a load balancer")) + } + + return allErrs +} + +// validateLBImmutability forbids changes to LB fields that are baked into the LB at creation. +// Algorithm, Flavor, APIServerPort, IPFamily and the HealthMonitor settings cannot be reissued +// to an existing cloudscale.ch LB, so changing them in spec would silently lie to the user. +func validateLBImmutability(oldLB, newLB *infrastructurev1beta2.LoadBalancerSpec, fldPath *field.Path) field.ErrorList { + var allErrs field.ErrorList + + forbidIfChanged := func(child string, oldV, newV any) { + if oldV != newV { + allErrs = append(allErrs, field.Forbidden( + fldPath.Child(child), + "field is immutable after cluster creation")) + } + } + + forbidIfChanged("algorithm", oldLB.Algorithm, newLB.Algorithm) + forbidIfChanged("flavor", oldLB.Flavor, newLB.Flavor) + forbidIfChanged("apiServerPort", oldLB.APIServerPort, newLB.APIServerPort) + forbidIfChanged("ipFamily", oldLB.IPFamily, newLB.IPFamily) + + hmPath := fldPath.Child("healthMonitor") + hmForbid := func(child string, oldV, newV int) { + if oldV != newV { + allErrs = append(allErrs, field.Forbidden( + hmPath.Child(child), + "field is immutable after cluster creation")) + } + } + hmForbid("delayS", oldLB.HealthMonitor.DelayS, newLB.HealthMonitor.DelayS) + hmForbid("timeoutS", oldLB.HealthMonitor.TimeoutS, newLB.HealthMonitor.TimeoutS) + hmForbid("upThreshold", oldLB.HealthMonitor.UpThreshold, newLB.HealthMonitor.UpThreshold) + hmForbid("downThreshold", oldLB.HealthMonitor.DownThreshold, newLB.HealthMonitor.DownThreshold) + + return allErrs +} + +// validateFloatingIPRequiresPublicLB rejects a floating IP attached to a load balancer +// that uses an internal-network VIP. cloudscale.ch floating IPs only attach to public LBs. +func validateFloatingIPRequiresPublicLB(cluster *infrastructurev1beta2.CloudscaleCluster) field.ErrorList { + var allErrs field.ErrorList + + if cluster.Spec.FloatingIP != nil && + ptr.Deref(cluster.Spec.ControlPlaneLoadBalancer.Enabled, true) && + cluster.Spec.ControlPlaneLoadBalancer.Network != "" { + allErrs = append(allErrs, field.Invalid( + field.NewPath("spec", "floatingIP"), + "", + "floating IPs cannot be attached to a load balancer with a private VIP; use a public load balancer or remove the floating IP")) + } + + return allErrs +} + +// validateLBPoolMemberNetworkResolvable requires controlPlaneLoadBalancer.network to be set +// when there are multiple networks and the LB is public. Without an explicit network the +// controller would default the LB pool members' subnet to networks[0], which silently +// breaks clusters whose machines join a different network. +func validateLBPoolMemberNetworkResolvable(cluster *infrastructurev1beta2.CloudscaleCluster) field.ErrorList { + var allErrs field.ErrorList + + if !ptr.Deref(cluster.Spec.ControlPlaneLoadBalancer.Enabled, true) { + return nil + } + if cluster.Spec.ControlPlaneLoadBalancer.Network != "" { + return nil + } + if len(cluster.Spec.Networks) <= 1 { + return nil + } + + allErrs = append(allErrs, field.Required( + field.NewPath("spec", "controlPlaneLoadBalancer", "network"), + "must be set to one of spec.networks[].name when multiple networks are defined; the load balancer pool members need an explicit subnet to attach to")) + + return allErrs +} + +// validateFloatingIP validates the floating IP spec. +func validateFloatingIP(fip *infrastructurev1beta2.FloatingIPSpec, fldPath *field.Path) field.ErrorList { + var allErrs field.ErrorList + + hasIPFamily := fip.IPFamily != nil + hasIP := fip.IP != "" + + if hasIPFamily == hasIP { + allErrs = append(allErrs, field.Invalid(fldPath, "", + "exactly one of ipFamily or ip must be specified")) + } + + if hasIP && net.ParseIP(fip.IP) == nil { + allErrs = append(allErrs, field.Invalid(fldPath.Child("ip"), fip.IP, + "must be a valid IP address")) + } + + return allErrs +} + +// validateFloatingIPImmutability checks that the floating IP config is immutable once set. +func validateFloatingIPImmutability(oldFIP, newFIP *infrastructurev1beta2.FloatingIPSpec, fldPath *field.Path) field.ErrorList { + var allErrs field.ErrorList + + if oldFIP == nil && newFIP == nil { + return nil + } + + // Cannot add or remove floating IP after creation. + // After the both-nil early return above, at least one is non-nil. + if oldFIP == nil { + allErrs = append(allErrs, field.Forbidden(fldPath, + "floating IP cannot be added after cluster creation")) + return allErrs + } + if newFIP == nil { + allErrs = append(allErrs, field.Forbidden(fldPath, + "floating IP cannot be removed after cluster creation")) + return allErrs + } + + // Cannot switch between managed and BYO + if oldFIP.IP != newFIP.IP { + allErrs = append(allErrs, field.Forbidden(fldPath.Child("ip"), + "field is immutable once set")) + } + if ptr.Deref(oldFIP.IPFamily, "") != ptr.Deref(newFIP.IPFamily, "") { + allErrs = append(allErrs, field.Forbidden(fldPath.Child("ipFamily"), + "field is immutable once set")) + } + + return allErrs +} + // validateGatewayInCIDR validates that the gateway address is within the specified CIDR. func validateGatewayInCIDR(cidr, gateway string, fldPath *field.Path) field.ErrorList { var allErrs field.ErrorList diff --git a/internal/webhook/v1beta2/cloudscalecluster_webhook_test.go b/internal/webhook/v1beta2/cloudscalecluster_webhook_test.go index 7ec7058..411a7b3 100644 --- a/internal/webhook/v1beta2/cloudscalecluster_webhook_test.go +++ b/internal/webhook/v1beta2/cloudscalecluster_webhook_test.go @@ -82,40 +82,30 @@ func TestClusterDefaulting_ExplicitZoneNotOverridden(t *testing.T) { g.Expect(obj.Spec.Zone).To(Equal(ZoneRma1)) } -func TestClusterDefaulting_CIDR(t *testing.T) { +func TestClusterDefaulting_NetworksDefaultToClusterName(t *testing.T) { g := NewWithT(t) obj, _, _, defaulter := newClusterWebhookTestObjects() - obj.Spec.Network.CIDR = "" - - g.Expect(defaulter.Default(ctx, obj)).To(Succeed()) - g.Expect(obj.Spec.Network.CIDR).To(Equal(defaultSubnetCIDR)) -} - -func TestClusterDefaulting_ExplicitCIDRNotOverridden(t *testing.T) { - g := NewWithT(t) - obj, _, _, defaulter := newClusterWebhookTestObjects() - obj.Spec.Network.CIDR = "10.1.0.0/16" - - g.Expect(defaulter.Default(ctx, obj)).To(Succeed()) - g.Expect(obj.Spec.Network.CIDR).To(Equal("10.1.0.0/16")) -} - -func TestClusterDefaulting_GatewayToEmptyString(t *testing.T) { - g := NewWithT(t) - obj, _, _, defaulter := newClusterWebhookTestObjects() - obj.Spec.Network.GatewayAddress = nil + obj.Name = "my-cluster" + obj.Spec.Region = RegionRma g.Expect(defaulter.Default(ctx, obj)).To(Succeed()) - g.Expect(obj.Spec.Network.GatewayAddress).To(Equal(ptr.To(""))) + g.Expect(obj.Spec.Networks).To(HaveLen(1)) + g.Expect(obj.Spec.Networks[0].Name).To(Equal("my-cluster")) + g.Expect(obj.Spec.Networks[0].CIDR).To(Equal(defaultSubnetCIDR)) } -func TestClusterDefaulting_ExplicitGatewayNotOverridden(t *testing.T) { +func TestClusterDefaulting_ExplicitNetworksNotOverridden(t *testing.T) { g := NewWithT(t) obj, _, _, defaulter := newClusterWebhookTestObjects() - obj.Spec.Network.GatewayAddress = ptr.To("10.0.0.1") + obj.Spec.Region = RegionRma + obj.Spec.Networks = []infrastructurev1beta2.NetworkSpec{ + {Name: "custom", CIDR: "10.1.0.0/16"}, + } g.Expect(defaulter.Default(ctx, obj)).To(Succeed()) - g.Expect(obj.Spec.Network.GatewayAddress).To(Equal(ptr.To("10.0.0.1"))) + g.Expect(obj.Spec.Networks).To(HaveLen(1)) + g.Expect(obj.Spec.Networks[0].Name).To(Equal("custom")) + g.Expect(obj.Spec.Networks[0].CIDR).To(Equal("10.1.0.0/16")) } func TestClusterDefaulting_LBEnabledToTrue(t *testing.T) { @@ -163,6 +153,15 @@ func TestClusterDefaulting_APIServerPort(t *testing.T) { g.Expect(obj.Spec.ControlPlaneLoadBalancer.APIServerPort).To(Equal(int32(6443))) } +func TestClusterDefaulting_LBIPFamily(t *testing.T) { + g := NewWithT(t) + obj, _, _, defaulter := newClusterWebhookTestObjects() + obj.Spec.ControlPlaneLoadBalancer.IPFamily = "" + + g.Expect(defaulter.Default(ctx, obj)).To(Succeed()) + g.Expect(obj.Spec.ControlPlaneLoadBalancer.IPFamily).To(Equal(infrastructurev1beta2.IPFamilyDualStack)) +} + func TestClusterDefaulting_HealthMonitorFields(t *testing.T) { g := NewWithT(t) obj, _, _, defaulter := newClusterWebhookTestObjects() @@ -189,20 +188,44 @@ func TestClusterDefaulting_ExplicitHealthMonitorNotOverridden(t *testing.T) { g.Expect(obj.Spec.ControlPlaneLoadBalancer.HealthMonitor.DownThreshold).To(Equal(8)) } +func TestClusterDefaulting_FloatingIPDefaultsToIPv4(t *testing.T) { + g := NewWithT(t) + obj, _, _, defaulter := newClusterWebhookTestObjects() + obj.Spec.FloatingIP = &infrastructurev1beta2.FloatingIPSpec{} + + g.Expect(defaulter.Default(ctx, obj)).To(Succeed()) + g.Expect(obj.Spec.FloatingIP.IPFamily).To(Equal(ptr.To(infrastructurev1beta2.IPFamilyIPv4))) +} + +func TestClusterDefaulting_FloatingIPExplicitNotOverridden(t *testing.T) { + g := NewWithT(t) + obj, _, _, defaulter := newClusterWebhookTestObjects() + obj.Spec.FloatingIP = &infrastructurev1beta2.FloatingIPSpec{ + IP: "1.2.3.4", + } + + g.Expect(defaulter.Default(ctx, obj)).To(Succeed()) + g.Expect(obj.Spec.FloatingIP.IPFamily).To(BeNil()) + g.Expect(obj.Spec.FloatingIP.IP).To(Equal("1.2.3.4")) +} + func TestClusterDefaulting_AllDefaultsApplied(t *testing.T) { g := NewWithT(t) obj, _, _, defaulter := newClusterWebhookTestObjects() + obj.Name = "test-cluster" obj.Spec.Region = RegionRma g.Expect(defaulter.Default(ctx, obj)).To(Succeed()) g.Expect(obj.Spec.Zone).To(Equal(ZoneRma1)) - g.Expect(obj.Spec.Network.CIDR).To(Equal(defaultSubnetCIDR)) - g.Expect(obj.Spec.Network.GatewayAddress).To(Equal(ptr.To(""))) + g.Expect(obj.Spec.Networks).To(HaveLen(1)) + g.Expect(obj.Spec.Networks[0].Name).To(Equal("test-cluster")) + g.Expect(obj.Spec.Networks[0].CIDR).To(Equal(defaultSubnetCIDR)) g.Expect(obj.Spec.ControlPlaneLoadBalancer.Enabled).To(Equal(ptr.To(true))) g.Expect(obj.Spec.ControlPlaneLoadBalancer.Algorithm).To(Equal("round_robin")) g.Expect(obj.Spec.ControlPlaneLoadBalancer.Flavor).To(Equal("lb-standard")) g.Expect(obj.Spec.ControlPlaneLoadBalancer.APIServerPort).To(Equal(int32(6443))) + g.Expect(obj.Spec.ControlPlaneLoadBalancer.IPFamily).To(Equal(infrastructurev1beta2.IPFamilyDualStack)) g.Expect(obj.Spec.ControlPlaneLoadBalancer.HealthMonitor.DelayS).To(Equal(5)) g.Expect(obj.Spec.ControlPlaneLoadBalancer.HealthMonitor.TimeoutS).To(Equal(3)) g.Expect(obj.Spec.ControlPlaneLoadBalancer.HealthMonitor.UpThreshold).To(Equal(2)) @@ -218,6 +241,9 @@ func TestClusterValidateCreate_ValidCluster(t *testing.T) { obj, _, validator, _ := newClusterWebhookTestObjects() obj.Spec.Region = RegionRma obj.Spec.Zone = ZoneRma1 + obj.Spec.Networks = []infrastructurev1beta2.NetworkSpec{ + {Name: "main", CIDR: defaultSubnetCIDR}, + } _, err := validator.ValidateCreate(ctx, obj) g.Expect(err).NotTo(HaveOccurred()) @@ -255,13 +281,57 @@ func TestClusterValidateCreate_EmptyZone(t *testing.T) { g.Expect(err).NotTo(HaveOccurred()) } +func TestClusterValidateCreate_NetworkMustHaveUUIDOrCIDR(t *testing.T) { + g := NewWithT(t) + obj, _, validator, _ := newClusterWebhookTestObjects() + obj.Spec.Region = RegionRma + obj.Spec.Zone = ZoneRma1 + obj.Spec.Networks = []infrastructurev1beta2.NetworkSpec{ + {Name: "bad"}, // neither UUID nor CIDR + } + + _, err := validator.ValidateCreate(ctx, obj) + g.Expect(err).To(HaveOccurred()) + g.Expect(err.Error()).To(ContainSubstring("exactly one of uuid or cidr")) +} + +func TestClusterValidateCreate_NetworkBothUUIDAndCIDR(t *testing.T) { + g := NewWithT(t) + obj, _, validator, _ := newClusterWebhookTestObjects() + obj.Spec.Region = RegionRma + obj.Spec.Zone = ZoneRma1 + obj.Spec.Networks = []infrastructurev1beta2.NetworkSpec{ + {Name: "bad", UUID: "some-uuid", CIDR: "10.0.0.0/24"}, + } + + _, err := validator.ValidateCreate(ctx, obj) + g.Expect(err).To(HaveOccurred()) + g.Expect(err.Error()).To(ContainSubstring("exactly one of uuid or cidr")) +} + +func TestClusterValidateCreate_DuplicateNetworkNames(t *testing.T) { + g := NewWithT(t) + obj, _, validator, _ := newClusterWebhookTestObjects() + obj.Spec.Region = RegionRma + obj.Spec.Zone = ZoneRma1 + obj.Spec.Networks = []infrastructurev1beta2.NetworkSpec{ + {Name: "dup", CIDR: "10.0.0.0/24"}, + {Name: "dup", CIDR: "10.1.0.0/24"}, + } + + _, err := validator.ValidateCreate(ctx, obj) + g.Expect(err).To(HaveOccurred()) + g.Expect(err.Error()).To(ContainSubstring("Duplicate")) +} + func TestClusterValidateCreate_GatewayWithinCIDR(t *testing.T) { g := NewWithT(t) obj, _, validator, _ := newClusterWebhookTestObjects() obj.Spec.Region = RegionRma obj.Spec.Zone = ZoneRma1 - obj.Spec.Network.CIDR = defaultSubnetCIDR - obj.Spec.Network.GatewayAddress = ptr.To("10.0.0.1") + obj.Spec.Networks = []infrastructurev1beta2.NetworkSpec{ + {Name: "main", CIDR: defaultSubnetCIDR, GatewayAddress: "10.0.0.1"}, + } _, err := validator.ValidateCreate(ctx, obj) g.Expect(err).NotTo(HaveOccurred()) @@ -272,12 +342,13 @@ func TestClusterValidateCreate_GatewayOutsideCIDR(t *testing.T) { obj, _, validator, _ := newClusterWebhookTestObjects() obj.Spec.Region = RegionRma obj.Spec.Zone = ZoneRma1 - obj.Spec.Network.CIDR = defaultSubnetCIDR - obj.Spec.Network.GatewayAddress = ptr.To("192.168.1.1") + obj.Spec.Networks = []infrastructurev1beta2.NetworkSpec{ + {Name: "main", CIDR: defaultSubnetCIDR, GatewayAddress: "192.168.1.1"}, + } _, err := validator.ValidateCreate(ctx, obj) g.Expect(err).To(HaveOccurred()) - g.Expect(err.Error()).To(ContainSubstring("spec.network.gatewayAddress")) + g.Expect(err.Error()).To(ContainSubstring("gatewayAddress")) } func TestClusterValidateCreate_InvalidGatewayIP(t *testing.T) { @@ -285,31 +356,175 @@ func TestClusterValidateCreate_InvalidGatewayIP(t *testing.T) { obj, _, validator, _ := newClusterWebhookTestObjects() obj.Spec.Region = RegionRma obj.Spec.Zone = ZoneRma1 - obj.Spec.Network.CIDR = defaultSubnetCIDR - obj.Spec.Network.GatewayAddress = ptr.To("notanip") + obj.Spec.Networks = []infrastructurev1beta2.NetworkSpec{ + {Name: "main", CIDR: defaultSubnetCIDR, GatewayAddress: "notanip"}, + } + + _, err := validator.ValidateCreate(ctx, obj) + g.Expect(err).To(HaveOccurred()) + g.Expect(err.Error()).To(ContainSubstring("gatewayAddress")) +} + +func TestClusterValidateCreate_GatewayOnBYONetworkRejected(t *testing.T) { + g := NewWithT(t) + obj, _, validator, _ := newClusterWebhookTestObjects() + obj.Spec.Region = RegionRma + obj.Spec.Zone = ZoneRma1 + obj.Spec.Networks = []infrastructurev1beta2.NetworkSpec{ + {Name: "byo", UUID: "some-uuid", GatewayAddress: "10.0.0.1"}, + } + + _, err := validator.ValidateCreate(ctx, obj) + g.Expect(err).To(HaveOccurred()) + g.Expect(err.Error()).To(ContainSubstring("gatewayAddress")) +} + +func TestClusterValidateCreate_LBNetworkReference(t *testing.T) { + g := NewWithT(t) + obj, _, validator, _ := newClusterWebhookTestObjects() + obj.Spec.Region = RegionRma + obj.Spec.Zone = ZoneRma1 + obj.Spec.Networks = []infrastructurev1beta2.NetworkSpec{ + {Name: "main", CIDR: defaultSubnetCIDR}, + } + obj.Spec.ControlPlaneLoadBalancer.Network = "main" + + _, err := validator.ValidateCreate(ctx, obj) + g.Expect(err).NotTo(HaveOccurred()) +} + +func TestClusterValidateCreate_PublicLBWithMultipleNetworksRequiresExplicitNetwork(t *testing.T) { + g := NewWithT(t) + obj, _, validator, _ := newClusterWebhookTestObjects() + obj.Spec.Region = RegionRma + obj.Spec.Zone = ZoneRma1 + obj.Spec.Networks = []infrastructurev1beta2.NetworkSpec{ + {Name: "main", CIDR: defaultSubnetCIDR}, + {Name: "aux", CIDR: "10.1.0.0/24"}, + } + // Public LB (Network == ""), multiple networks → ambiguous which subnet + // pool members should attach to. + obj.Spec.ControlPlaneLoadBalancer.Network = "" + + _, err := validator.ValidateCreate(ctx, obj) + g.Expect(err).To(HaveOccurred()) + g.Expect(err.Error()).To(ContainSubstring("controlPlaneLoadBalancer.network")) +} + +func TestClusterValidateCreate_LBNetworkReferenceInvalid(t *testing.T) { + g := NewWithT(t) + obj, _, validator, _ := newClusterWebhookTestObjects() + obj.Spec.Region = RegionRma + obj.Spec.Zone = ZoneRma1 + obj.Spec.Networks = []infrastructurev1beta2.NetworkSpec{ + {Name: "main", CIDR: defaultSubnetCIDR}, + } + obj.Spec.ControlPlaneLoadBalancer.Network = "nonexistent" _, err := validator.ValidateCreate(ctx, obj) g.Expect(err).To(HaveOccurred()) - g.Expect(err.Error()).To(ContainSubstring("spec.network.gatewayAddress")) + g.Expect(err.Error()).To(ContainSubstring("controlPlaneLoadBalancer.network")) } -func TestClusterValidateCreate_EmptyGatewayString(t *testing.T) { +func TestClusterValidateCreate_FloatingIPValid(t *testing.T) { g := NewWithT(t) obj, _, validator, _ := newClusterWebhookTestObjects() obj.Spec.Region = RegionRma obj.Spec.Zone = ZoneRma1 - obj.Spec.Network.GatewayAddress = ptr.To("") + obj.Spec.FloatingIP = &infrastructurev1beta2.FloatingIPSpec{ + IPFamily: ptr.To(infrastructurev1beta2.IPFamilyIPv4), + } _, err := validator.ValidateCreate(ctx, obj) g.Expect(err).NotTo(HaveOccurred()) } -func TestClusterValidateCreate_NilGateway(t *testing.T) { +func TestClusterValidateCreate_FloatingIPWithPrivateLBRejected(t *testing.T) { + g := NewWithT(t) + obj, _, validator, _ := newClusterWebhookTestObjects() + obj.Spec.Region = RegionRma + obj.Spec.Zone = ZoneRma1 + obj.Spec.Networks = []infrastructurev1beta2.NetworkSpec{ + {Name: "main", CIDR: defaultSubnetCIDR}, + } + obj.Spec.ControlPlaneLoadBalancer.Network = "main" + obj.Spec.FloatingIP = &infrastructurev1beta2.FloatingIPSpec{ + IPFamily: ptr.To(infrastructurev1beta2.IPFamilyIPv4), + } + + _, err := validator.ValidateCreate(ctx, obj) + g.Expect(err).To(HaveOccurred()) + g.Expect(err.Error()).To(ContainSubstring("floatingIP")) + g.Expect(err.Error()).To(ContainSubstring("private")) +} + +func TestClusterValidateCreate_FloatingIPBothFieldsInvalid(t *testing.T) { + g := NewWithT(t) + obj, _, validator, _ := newClusterWebhookTestObjects() + obj.Spec.Region = RegionRma + obj.Spec.Zone = ZoneRma1 + obj.Spec.FloatingIP = &infrastructurev1beta2.FloatingIPSpec{ + IPFamily: ptr.To(infrastructurev1beta2.IPFamilyIPv4), + IP: "1.2.3.4", + } + + _, err := validator.ValidateCreate(ctx, obj) + g.Expect(err).To(HaveOccurred()) + g.Expect(err.Error()).To(ContainSubstring("floatingIP")) + g.Expect(err.Error()).To(ContainSubstring("ipFamily")) + g.Expect(err.Error()).To(ContainSubstring("ip")) +} + +func TestClusterValidateCreate_BYOFloatingIPInvalidIP(t *testing.T) { + g := NewWithT(t) + obj, _, validator, _ := newClusterWebhookTestObjects() + obj.Spec.Region = RegionRma + obj.Spec.Zone = ZoneRma1 + obj.Spec.FloatingIP = &infrastructurev1beta2.FloatingIPSpec{ + IP: "not-an-ip", + } + + _, err := validator.ValidateCreate(ctx, obj) + g.Expect(err).To(HaveOccurred()) + g.Expect(err.Error()).To(ContainSubstring("floatingIP.ip")) +} + +func TestClusterValidateCreate_FloatingIPNeitherFieldInvalid(t *testing.T) { g := NewWithT(t) obj, _, validator, _ := newClusterWebhookTestObjects() obj.Spec.Region = RegionRma obj.Spec.Zone = ZoneRma1 - obj.Spec.Network.GatewayAddress = nil + obj.Spec.FloatingIP = &infrastructurev1beta2.FloatingIPSpec{} + + _, err := validator.ValidateCreate(ctx, obj) + g.Expect(err).To(HaveOccurred()) + g.Expect(err.Error()).To(ContainSubstring("floatingIP")) +} + +func TestClusterValidateCreate_ManagedFloatingIPWithoutLBRejected(t *testing.T) { + g := NewWithT(t) + obj, _, validator, _ := newClusterWebhookTestObjects() + obj.Spec.Region = RegionRma + obj.Spec.Zone = ZoneRma1 + obj.Spec.ControlPlaneLoadBalancer.Enabled = ptr.To(false) + obj.Spec.FloatingIP = &infrastructurev1beta2.FloatingIPSpec{ + IPFamily: ptr.To(infrastructurev1beta2.IPFamilyIPv4), + } + + _, err := validator.ValidateCreate(ctx, obj) + g.Expect(err).To(HaveOccurred()) + g.Expect(err.Error()).To(ContainSubstring("managed floating IP")) +} + +func TestClusterValidateCreate_BYOFloatingIPWithoutLBAllowed(t *testing.T) { + g := NewWithT(t) + obj, _, validator, _ := newClusterWebhookTestObjects() + obj.Spec.Region = RegionRma + obj.Spec.Zone = ZoneRma1 + obj.Spec.ControlPlaneLoadBalancer.Enabled = ptr.To(false) + obj.Spec.FloatingIP = &infrastructurev1beta2.FloatingIPSpec{ + IP: "1.2.3.4", + } _, err := validator.ValidateCreate(ctx, obj) g.Expect(err).NotTo(HaveOccurred()) @@ -325,17 +540,21 @@ func setupUpdateTestObjects() ( validator CloudscaleClusterCustomValidator, ) { obj, oldObj, validator, _ = newClusterWebhookTestObjects() + networks := []infrastructurev1beta2.NetworkSpec{ + {Name: "main", CIDR: defaultSubnetCIDR}, + } + oldObj.Spec.Region = RegionRma oldObj.Spec.Zone = ZoneRma1 - oldObj.Spec.Network.CIDR = defaultSubnetCIDR + oldObj.Spec.Networks = networks oldObj.Spec.ControlPlaneLoadBalancer.Enabled = ptr.To(true) - oldObj.Spec.Network.GatewayAddress = ptr.To("") obj.Spec.Region = RegionRma obj.Spec.Zone = ZoneRma1 - obj.Spec.Network.CIDR = defaultSubnetCIDR + obj.Spec.Networks = []infrastructurev1beta2.NetworkSpec{ + {Name: "main", CIDR: defaultSubnetCIDR}, + } obj.Spec.ControlPlaneLoadBalancer.Enabled = ptr.To(true) - obj.Spec.Network.GatewayAddress = ptr.To("") return } @@ -367,14 +586,57 @@ func TestClusterValidateUpdate_ZoneChange(t *testing.T) { g.Expect(err.Error()).To(ContainSubstring("spec.zone")) } -func TestClusterValidateUpdate_CIDRChange(t *testing.T) { +func TestClusterValidateUpdate_NetworkCIDRChange(t *testing.T) { + g := NewWithT(t) + obj, oldObj, validator := setupUpdateTestObjects() + obj.Spec.Networks[0].CIDR = "10.1.0.0/24" + + _, err := validator.ValidateUpdate(ctx, oldObj, obj) + g.Expect(err).To(HaveOccurred()) + g.Expect(err.Error()).To(ContainSubstring("cidr")) +} + +func TestClusterValidateUpdate_NetworkReorderedCIDRChangedReportsNewIndex(t *testing.T) { g := NewWithT(t) obj, oldObj, validator := setupUpdateTestObjects() - obj.Spec.Network.CIDR = "10.1.0.0/24" + oldObj.Spec.Networks = []infrastructurev1beta2.NetworkSpec{ + {Name: "net-a", CIDR: "10.1.0.0/24"}, + {Name: "net-b", CIDR: "10.2.0.0/24"}, + } + // Swap order in new object and change net-a's CIDR. + obj.Spec.Networks = []infrastructurev1beta2.NetworkSpec{ + {Name: "net-b", CIDR: "10.2.0.0/24"}, + {Name: "net-a", CIDR: "10.9.0.0/24"}, + } _, err := validator.ValidateUpdate(ctx, oldObj, obj) g.Expect(err).To(HaveOccurred()) - g.Expect(err.Error()).To(ContainSubstring("spec.network.cidr")) + g.Expect(err.Error()).To(ContainSubstring("spec.networks[1].cidr"), + "error message must point at net-a's new index (1), not its old index (0)") +} + +func TestClusterValidateUpdate_NetworkRemoved(t *testing.T) { + g := NewWithT(t) + obj, oldObj, validator := setupUpdateTestObjects() + obj.Spec.Networks = nil + + _, err := validator.ValidateUpdate(ctx, oldObj, obj) + g.Expect(err).To(HaveOccurred()) + g.Expect(err.Error()).To(ContainSubstring("removing network")) +} + +func TestClusterValidateUpdate_NetworkAdded(t *testing.T) { + g := NewWithT(t) + obj, oldObj, validator := setupUpdateTestObjects() + obj.Spec.Networks = append(obj.Spec.Networks, infrastructurev1beta2.NetworkSpec{ + Name: "extra", CIDR: "10.1.0.0/24", + }) + // Multiple networks now require an explicit LB pool-member network. Pin to "main" (matches old). + oldObj.Spec.ControlPlaneLoadBalancer.Network = "main" + obj.Spec.ControlPlaneLoadBalancer.Network = "main" + + _, err := validator.ValidateUpdate(ctx, oldObj, obj) + g.Expect(err).NotTo(HaveOccurred()) } func TestClusterValidateUpdate_LBEnabledChange(t *testing.T) { @@ -387,6 +649,17 @@ func TestClusterValidateUpdate_LBEnabledChange(t *testing.T) { g.Expect(err.Error()).To(ContainSubstring("spec.controlPlaneLoadBalancer.enabled")) } +func TestClusterValidateUpdate_LBNetworkImmutable(t *testing.T) { + g := NewWithT(t) + obj, oldObj, validator := setupUpdateTestObjects() + oldObj.Spec.ControlPlaneLoadBalancer.Network = "main" + obj.Spec.ControlPlaneLoadBalancer.Network = "other" + + _, err := validator.ValidateUpdate(ctx, oldObj, obj) + g.Expect(err).To(HaveOccurred()) + g.Expect(err.Error()).To(ContainSubstring("controlPlaneLoadBalancer.network")) +} + func TestClusterValidateUpdate_EndpointHostChange(t *testing.T) { g := NewWithT(t) obj, oldObj, validator := setupUpdateTestObjects() @@ -419,48 +692,113 @@ func TestClusterValidateUpdate_EndpointSetWhenEmpty(t *testing.T) { g.Expect(err).NotTo(HaveOccurred()) } -func TestClusterValidateUpdate_GatewayChange(t *testing.T) { +func TestClusterValidateUpdate_FloatingIPCannotBeAdded(t *testing.T) { g := NewWithT(t) obj, oldObj, validator := setupUpdateTestObjects() - oldObj.Spec.Network.GatewayAddress = ptr.To("") - obj.Spec.Network.GatewayAddress = ptr.To("10.0.0.1") + obj.Spec.FloatingIP = &infrastructurev1beta2.FloatingIPSpec{ + IPFamily: ptr.To(infrastructurev1beta2.IPFamilyIPv4), + } _, err := validator.ValidateUpdate(ctx, oldObj, obj) g.Expect(err).To(HaveOccurred()) - g.Expect(err.Error()).To(ContainSubstring("spec.network.gatewayAddress")) + g.Expect(err.Error()).To(ContainSubstring("floatingIP")) } -func TestClusterValidateUpdate_GatewayNilToValue(t *testing.T) { +func TestClusterValidateUpdate_FloatingIPCannotBeRemoved(t *testing.T) { g := NewWithT(t) obj, oldObj, validator := setupUpdateTestObjects() - oldObj.Spec.Network.GatewayAddress = nil - obj.Spec.Network.GatewayAddress = ptr.To("10.0.0.1") + oldObj.Spec.FloatingIP = &infrastructurev1beta2.FloatingIPSpec{ + IPFamily: ptr.To(infrastructurev1beta2.IPFamilyIPv4), + } + obj.Spec.FloatingIP = nil _, err := validator.ValidateUpdate(ctx, oldObj, obj) g.Expect(err).To(HaveOccurred()) - g.Expect(err.Error()).To(ContainSubstring("spec.network.gatewayAddress")) -} - -func TestClusterValidateUpdate_GatewayNilToNil(t *testing.T) { - g := NewWithT(t) - obj, oldObj, validator := setupUpdateTestObjects() - oldObj.Spec.Network.GatewayAddress = nil - obj.Spec.Network.GatewayAddress = nil - - _, err := validator.ValidateUpdate(ctx, oldObj, obj) - g.Expect(err).NotTo(HaveOccurred()) -} - -func TestClusterValidateUpdate_MutableFieldsChange(t *testing.T) { - g := NewWithT(t) - obj, oldObj, validator := setupUpdateTestObjects() - obj.Spec.ControlPlaneLoadBalancer.Algorithm = "least_connections" - obj.Spec.ControlPlaneLoadBalancer.Flavor = "lb-premium" - obj.Spec.ControlPlaneLoadBalancer.APIServerPort = 8443 - obj.Spec.ControlPlaneLoadBalancer.HealthMonitor.DelayS = 10 + g.Expect(err.Error()).To(ContainSubstring("floatingIP")) +} + +func TestClusterValidateUpdate_LBFieldsImmutable(t *testing.T) { + cases := []struct { + name string + mutate func(c *infrastructurev1beta2.CloudscaleCluster) + errPath string + }{ + { + name: "Algorithm", + mutate: func(c *infrastructurev1beta2.CloudscaleCluster) { + c.Spec.ControlPlaneLoadBalancer.Algorithm = "least_connections" + }, + errPath: "controlPlaneLoadBalancer.algorithm", + }, + { + name: "Flavor", + mutate: func(c *infrastructurev1beta2.CloudscaleCluster) { + c.Spec.ControlPlaneLoadBalancer.Flavor = "lb-premium" + }, + errPath: "controlPlaneLoadBalancer.flavor", + }, + { + name: "APIServerPort", + mutate: func(c *infrastructurev1beta2.CloudscaleCluster) { c.Spec.ControlPlaneLoadBalancer.APIServerPort = 8443 }, + errPath: "controlPlaneLoadBalancer.apiServerPort", + }, + { + name: "IPFamily", + mutate: func(c *infrastructurev1beta2.CloudscaleCluster) { + c.Spec.ControlPlaneLoadBalancer.IPFamily = infrastructurev1beta2.IPFamilyIPv6 + }, + errPath: "controlPlaneLoadBalancer.ipFamily", + }, + { + name: "HealthMonitor.DelayS", + mutate: func(c *infrastructurev1beta2.CloudscaleCluster) { + c.Spec.ControlPlaneLoadBalancer.HealthMonitor.DelayS = 10 + }, + errPath: "controlPlaneLoadBalancer.healthMonitor.delayS", + }, + { + name: "HealthMonitor.TimeoutS", + mutate: func(c *infrastructurev1beta2.CloudscaleCluster) { + c.Spec.ControlPlaneLoadBalancer.HealthMonitor.TimeoutS = 10 + }, + errPath: "controlPlaneLoadBalancer.healthMonitor.timeoutS", + }, + { + name: "HealthMonitor.UpThreshold", + mutate: func(c *infrastructurev1beta2.CloudscaleCluster) { + c.Spec.ControlPlaneLoadBalancer.HealthMonitor.UpThreshold = 9 + }, + errPath: "controlPlaneLoadBalancer.healthMonitor.upThreshold", + }, + { + name: "HealthMonitor.DownThreshold", + mutate: func(c *infrastructurev1beta2.CloudscaleCluster) { + c.Spec.ControlPlaneLoadBalancer.HealthMonitor.DownThreshold = 9 + }, + errPath: "controlPlaneLoadBalancer.healthMonitor.downThreshold", + }, + } - _, err := validator.ValidateUpdate(ctx, oldObj, obj) - g.Expect(err).NotTo(HaveOccurred()) + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + g := NewWithT(t) + obj, oldObj, validator := setupUpdateTestObjects() + // Seed health monitor + IPFamily on old so changes are visible. + oldObj.Spec.ControlPlaneLoadBalancer.Algorithm = "round_robin" + oldObj.Spec.ControlPlaneLoadBalancer.Flavor = "lb-standard" + oldObj.Spec.ControlPlaneLoadBalancer.APIServerPort = 6443 + oldObj.Spec.ControlPlaneLoadBalancer.IPFamily = infrastructurev1beta2.IPFamilyDualStack + oldObj.Spec.ControlPlaneLoadBalancer.HealthMonitor = infrastructurev1beta2.HealthMonitorSpec{ + DelayS: 5, TimeoutS: 3, UpThreshold: 2, DownThreshold: 3, + } + obj.Spec.ControlPlaneLoadBalancer = *oldObj.Spec.ControlPlaneLoadBalancer.DeepCopy() + tc.mutate(obj) + + _, err := validator.ValidateUpdate(ctx, oldObj, obj) + g.Expect(err).To(HaveOccurred()) + g.Expect(err.Error()).To(ContainSubstring(tc.errPath)) + }) + } } func TestClusterValidateUpdate_MultipleImmutableChanges(t *testing.T) { @@ -468,13 +806,13 @@ func TestClusterValidateUpdate_MultipleImmutableChanges(t *testing.T) { obj, oldObj, validator := setupUpdateTestObjects() obj.Spec.Region = "lpg" obj.Spec.Zone = "lpg1" - obj.Spec.Network.CIDR = "10.1.0.0/24" + obj.Spec.Networks[0].CIDR = "10.1.0.0/24" _, err := validator.ValidateUpdate(ctx, oldObj, obj) g.Expect(err).To(HaveOccurred()) g.Expect(err.Error()).To(ContainSubstring("spec.region")) g.Expect(err.Error()).To(ContainSubstring("spec.zone")) - g.Expect(err.Error()).To(ContainSubstring("spec.network.cidr")) + g.Expect(err.Error()).To(ContainSubstring("cidr")) } // ============================================================================ @@ -495,25 +833,25 @@ func TestClusterValidateDelete_AlwaysSucceeds(t *testing.T) { func TestValidateGatewayInCIDR_ValidGateway(t *testing.T) { g := NewWithT(t) - errs := validateGatewayInCIDR(defaultSubnetCIDR, "10.0.0.1", field.NewPath("spec", "network", "gatewayAddress")) + errs := validateGatewayInCIDR(defaultSubnetCIDR, "10.0.0.1", field.NewPath("spec", "networks", "gatewayAddress")) g.Expect(errs).To(BeEmpty()) } func TestValidateGatewayInCIDR_OutsideCIDR(t *testing.T) { g := NewWithT(t) - errs := validateGatewayInCIDR(defaultSubnetCIDR, "192.168.1.1", field.NewPath("spec", "network", "gatewayAddress")) + errs := validateGatewayInCIDR(defaultSubnetCIDR, "192.168.1.1", field.NewPath("spec", "networks", "gatewayAddress")) g.Expect(errs).To(HaveLen(1)) } func TestValidateGatewayInCIDR_InvalidIP(t *testing.T) { g := NewWithT(t) - errs := validateGatewayInCIDR(defaultSubnetCIDR, "invalid", field.NewPath("spec", "network", "gatewayAddress")) + errs := validateGatewayInCIDR(defaultSubnetCIDR, "invalid", field.NewPath("spec", "networks", "gatewayAddress")) g.Expect(errs).To(HaveLen(1)) g.Expect(errs[0].Detail).To(ContainSubstring("invalid IP")) } func TestValidateGatewayInCIDR_InvalidCIDR(t *testing.T) { g := NewWithT(t) - errs := validateGatewayInCIDR("notacidr", "10.0.0.1", field.NewPath("spec", "network", "gatewayAddress")) + errs := validateGatewayInCIDR("notacidr", "10.0.0.1", field.NewPath("spec", "networks", "gatewayAddress")) g.Expect(errs).To(BeEmpty()) } diff --git a/internal/webhook/v1beta2/cloudscalemachine_validation.go b/internal/webhook/v1beta2/cloudscalemachine_validation.go index e5d0f53..a5dfeb2 100644 --- a/internal/webhook/v1beta2/cloudscalemachine_validation.go +++ b/internal/webhook/v1beta2/cloudscalemachine_validation.go @@ -17,6 +17,7 @@ limitations under the License. package v1beta2 import ( + "reflect" "strings" "k8s.io/apimachinery/pkg/util/validation/field" @@ -25,6 +26,16 @@ import ( "github.com/cloudscale-ch/cluster-api-provider-cloudscale/internal/cloudscale" ) +// defaultInterfaceIPFamily defaults the IPFamily of public interfaces to DualStack. +func defaultInterfaceIPFamily(interfaces []infrastructurev1beta2.InterfaceSpec) { + for i := range interfaces { + if interfaces[i].Type == "public" && interfaces[i].IPFamily == nil { + dualStack := infrastructurev1beta2.IPFamilyDualStack + interfaces[i].IPFamily = &dualStack + } + } +} + // validateMachineSpec validates a CloudscaleMachineSpec at creation time. func validateMachineSpec(spec *infrastructurev1beta2.CloudscaleMachineSpec, flavorInfo *cloudscale.FlavorInfo, fldPath *field.Path) field.ErrorList { var allErrs field.ErrorList @@ -35,6 +46,7 @@ func validateMachineSpec(spec *infrastructurev1beta2.CloudscaleMachineSpec, flav "unknown flavor")) } allErrs = append(allErrs, validateTags(spec.Tags, fldPath.Child("tags"))...) + allErrs = append(allErrs, validateInterfaces(spec.Interfaces, fldPath.Child("interfaces"))...) return allErrs } @@ -74,12 +86,66 @@ func validateMachineSpecUpdate(newSpec, oldSpec *infrastructurev1beta2.Cloudscal } } + // Interfaces are immutable (changing requires server recreation) + if !reflect.DeepEqual(newSpec.Interfaces, oldSpec.Interfaces) { + allErrs = append(allErrs, field.Forbidden( + fldPath.Child("interfaces"), + "field is immutable")) + } + + // ServerGroup is immutable (changing requires server recreation) + if !reflect.DeepEqual(newSpec.ServerGroup, oldSpec.ServerGroup) { + allErrs = append(allErrs, field.Forbidden( + fldPath.Child("serverGroup"), + "field is immutable")) + } + // Tags are mutable but still validated for reserved prefix allErrs = append(allErrs, validateTags(newSpec.Tags, fldPath.Child("tags"))...) return allErrs } +// validateInterfaces validates the interface spec list. +func validateInterfaces(interfaces []infrastructurev1beta2.InterfaceSpec, fldPath *field.Path) field.ErrorList { + var allErrs field.ErrorList + + if len(interfaces) == 0 { + // Empty is valid — controller applies runtime defaults + return nil + } + + publicCount := 0 + for i, iface := range interfaces { + ifacePath := fldPath.Index(i) + + hasType := iface.Type != "" + hasNetwork := iface.Network != "" + + // Exactly one of type or network + if hasType == hasNetwork { + allErrs = append(allErrs, field.Invalid(ifacePath, "", + "exactly one of type or network must be specified")) + } + + if iface.Type == "public" { + publicCount++ + } + + // IPFamily only valid on public interfaces + if iface.IPFamily != nil && iface.Type != "public" { + allErrs = append(allErrs, field.Invalid(ifacePath.Child("ipFamily"), *iface.IPFamily, + "ipFamily can only be set on public interfaces (type: public)")) + } + } + + if publicCount > 1 { + allErrs = append(allErrs, field.TooMany(fldPath, publicCount, 1)) + } + + return allErrs +} + // validateTags rejects tags with the reserved capcs- prefix. func validateTags(tags map[string]string, fldPath *field.Path) field.ErrorList { var allErrs field.ErrorList diff --git a/internal/webhook/v1beta2/cloudscalemachine_webhook.go b/internal/webhook/v1beta2/cloudscalemachine_webhook.go index d0dc6c1..d0ccafc 100644 --- a/internal/webhook/v1beta2/cloudscalemachine_webhook.go +++ b/internal/webhook/v1beta2/cloudscalemachine_webhook.go @@ -55,6 +55,9 @@ type CloudscaleMachineCustomDefaulter struct{} // Default implements webhook.CustomDefaulter so a webhook will be registered for the Kind CloudscaleMachine. func (d *CloudscaleMachineCustomDefaulter) Default(_ context.Context, obj *infrastructurev1beta2.CloudscaleMachine) error { cloudscalemachinelog.Info("Defaulting for CloudscaleMachine", "name", obj.GetName()) + + defaultInterfaceIPFamily(obj.Spec.Interfaces) + return nil } diff --git a/internal/webhook/v1beta2/cloudscalemachine_webhook_test.go b/internal/webhook/v1beta2/cloudscalemachine_webhook_test.go index 634ff22..cb146bb 100644 --- a/internal/webhook/v1beta2/cloudscalemachine_webhook_test.go +++ b/internal/webhook/v1beta2/cloudscalemachine_webhook_test.go @@ -240,3 +240,198 @@ func TestMachineValidateDelete_AlwaysSucceeds(t *testing.T) { _, err := validator.ValidateDelete(ctx, obj) g.Expect(err).NotTo(HaveOccurred()) } + +// ============================================================================ +// Tests for validateInterfaces +// ============================================================================ + +func TestValidateInterfaces_EmptyIsValid(t *testing.T) { + g := NewWithT(t) + obj, _ := newMachineWebhookTestObjects() + validator := CloudscaleMachineCustomValidator{FlavorInfo: newTestFlavorInfo()} + + _, err := validator.ValidateCreate(ctx, obj) + g.Expect(err).NotTo(HaveOccurred()) +} + +func TestValidateInterfaces_SinglePublic(t *testing.T) { + g := NewWithT(t) + obj, _ := newMachineWebhookTestObjects() + obj.Spec.Interfaces = []infrastructurev1beta2.InterfaceSpec{ + {Type: "public"}, + } + validator := CloudscaleMachineCustomValidator{FlavorInfo: newTestFlavorInfo()} + + _, err := validator.ValidateCreate(ctx, obj) + g.Expect(err).NotTo(HaveOccurred()) +} + +func TestValidateInterfaces_SingleNetwork(t *testing.T) { + g := NewWithT(t) + obj, _ := newMachineWebhookTestObjects() + obj.Spec.Interfaces = []infrastructurev1beta2.InterfaceSpec{ + {Network: "my-network"}, + } + validator := CloudscaleMachineCustomValidator{FlavorInfo: newTestFlavorInfo()} + + _, err := validator.ValidateCreate(ctx, obj) + g.Expect(err).NotTo(HaveOccurred()) +} + +func TestValidateInterfaces_BothTypeAndNetworkSet(t *testing.T) { + g := NewWithT(t) + obj, _ := newMachineWebhookTestObjects() + obj.Spec.Interfaces = []infrastructurev1beta2.InterfaceSpec{ + {Type: "public", Network: "my-network"}, + } + validator := CloudscaleMachineCustomValidator{FlavorInfo: newTestFlavorInfo()} + + _, err := validator.ValidateCreate(ctx, obj) + g.Expect(err).To(HaveOccurred()) + g.Expect(err.Error()).To(ContainSubstring("exactly one of type or network")) +} + +func TestValidateInterfaces_NeitherTypeNorNetwork(t *testing.T) { + g := NewWithT(t) + obj, _ := newMachineWebhookTestObjects() + obj.Spec.Interfaces = []infrastructurev1beta2.InterfaceSpec{ + {}, + } + validator := CloudscaleMachineCustomValidator{FlavorInfo: newTestFlavorInfo()} + + _, err := validator.ValidateCreate(ctx, obj) + g.Expect(err).To(HaveOccurred()) + g.Expect(err.Error()).To(ContainSubstring("exactly one of type or network")) +} + +func TestValidateInterfaces_IPFamilyOnNonPublic(t *testing.T) { + g := NewWithT(t) + obj, _ := newMachineWebhookTestObjects() + dualStack := infrastructurev1beta2.IPFamilyDualStack + obj.Spec.Interfaces = []infrastructurev1beta2.InterfaceSpec{ + {Network: "my-network", IPFamily: &dualStack}, + } + validator := CloudscaleMachineCustomValidator{FlavorInfo: newTestFlavorInfo()} + + _, err := validator.ValidateCreate(ctx, obj) + g.Expect(err).To(HaveOccurred()) + g.Expect(err.Error()).To(ContainSubstring("ipFamily can only be set on public interfaces")) +} + +func TestValidateInterfaces_MultiplePublic(t *testing.T) { + g := NewWithT(t) + obj, _ := newMachineWebhookTestObjects() + obj.Spec.Interfaces = []infrastructurev1beta2.InterfaceSpec{ + {Type: "public"}, + {Type: "public"}, + } + validator := CloudscaleMachineCustomValidator{FlavorInfo: newTestFlavorInfo()} + + _, err := validator.ValidateCreate(ctx, obj) + g.Expect(err).To(HaveOccurred()) + g.Expect(err.Error()).To(ContainSubstring("Too many")) +} + +func TestValidateInterfaces_MixedValid(t *testing.T) { + g := NewWithT(t) + obj, _ := newMachineWebhookTestObjects() + dualStack := infrastructurev1beta2.IPFamilyDualStack + obj.Spec.Interfaces = []infrastructurev1beta2.InterfaceSpec{ + {Network: "my-network"}, + {Type: "public", IPFamily: &dualStack}, + } + validator := CloudscaleMachineCustomValidator{FlavorInfo: newTestFlavorInfo()} + + _, err := validator.ValidateCreate(ctx, obj) + g.Expect(err).NotTo(HaveOccurred()) +} + +func TestValidateInterfaces_UpdateImmutable(t *testing.T) { + g := NewWithT(t) + obj, oldObj := newMachineWebhookTestObjects() + oldObj.Spec.Interfaces = []infrastructurev1beta2.InterfaceSpec{ + {Type: "public"}, + } + obj.Spec.Interfaces = []infrastructurev1beta2.InterfaceSpec{ + {Network: "my-network"}, + } + validator := CloudscaleMachineCustomValidator{FlavorInfo: newTestFlavorInfo()} + + _, err := validator.ValidateUpdate(ctx, oldObj, obj) + g.Expect(err).To(HaveOccurred()) + g.Expect(err.Error()).To(ContainSubstring("spec.interfaces")) +} + +func TestValidateServerGroup_UpdateImmutable(t *testing.T) { + g := NewWithT(t) + obj, oldObj := newMachineWebhookTestObjects() + oldObj.Spec.ServerGroup = &infrastructurev1beta2.ServerGroupSpec{Name: "group-a"} + obj.Spec.ServerGroup = &infrastructurev1beta2.ServerGroupSpec{Name: "group-b"} + validator := CloudscaleMachineCustomValidator{FlavorInfo: newTestFlavorInfo()} + + _, err := validator.ValidateUpdate(ctx, oldObj, obj) + g.Expect(err).To(HaveOccurred()) + g.Expect(err.Error()).To(ContainSubstring("spec.serverGroup")) +} + +// ============================================================================ +// Tests for defaultInterfaceIPFamily +// ============================================================================ + +func TestDefaultInterfaceIPFamily_PublicNilDefaultsToDualStack(t *testing.T) { + g := NewWithT(t) + interfaces := []infrastructurev1beta2.InterfaceSpec{ + {Type: "public"}, + } + + defaultInterfaceIPFamily(interfaces) + + g.Expect(interfaces[0].IPFamily).ToNot(BeNil()) + g.Expect(*interfaces[0].IPFamily).To(Equal(infrastructurev1beta2.IPFamilyDualStack)) +} + +func TestDefaultInterfaceIPFamily_PublicExplicitNotOverridden(t *testing.T) { + g := NewWithT(t) + ipv4 := infrastructurev1beta2.IPFamilyIPv4 + interfaces := []infrastructurev1beta2.InterfaceSpec{ + {Type: "public", IPFamily: &ipv4}, + } + + defaultInterfaceIPFamily(interfaces) + + g.Expect(*interfaces[0].IPFamily).To(Equal(infrastructurev1beta2.IPFamilyIPv4)) +} + +func TestDefaultInterfaceIPFamily_NonPublicNotModified(t *testing.T) { + g := NewWithT(t) + interfaces := []infrastructurev1beta2.InterfaceSpec{ + {Network: "my-network"}, + } + + defaultInterfaceIPFamily(interfaces) + + g.Expect(interfaces[0].IPFamily).To(BeNil()) +} + +func TestDefaultInterfaceIPFamily_EmptyList(t *testing.T) { + g := NewWithT(t) + interfaces := []infrastructurev1beta2.InterfaceSpec{} + + defaultInterfaceIPFamily(interfaces) + + g.Expect(interfaces).To(BeEmpty()) +} + +func TestDefaultInterfaceIPFamily_MixedInterfaces(t *testing.T) { + g := NewWithT(t) + interfaces := []infrastructurev1beta2.InterfaceSpec{ + {Network: "my-network"}, + {Type: "public"}, + } + + defaultInterfaceIPFamily(interfaces) + + g.Expect(interfaces[0].IPFamily).To(BeNil()) + g.Expect(interfaces[1].IPFamily).ToNot(BeNil()) + g.Expect(*interfaces[1].IPFamily).To(Equal(infrastructurev1beta2.IPFamilyDualStack)) +} diff --git a/internal/webhook/v1beta2/cloudscalemachinetemplate_webhook.go b/internal/webhook/v1beta2/cloudscalemachinetemplate_webhook.go index 07793b3..c260cb1 100644 --- a/internal/webhook/v1beta2/cloudscalemachinetemplate_webhook.go +++ b/internal/webhook/v1beta2/cloudscalemachinetemplate_webhook.go @@ -56,6 +56,9 @@ type CloudscaleMachineTemplateCustomDefaulter struct{} // Default implements webhook.CustomDefaulter so a webhook will be registered for the Kind CloudscaleMachineTemplate. func (d *CloudscaleMachineTemplateCustomDefaulter) Default(_ context.Context, obj *infrastructurev1beta2.CloudscaleMachineTemplate) error { cloudscalemachinetemplatelog.Info("Defaulting for CloudscaleMachineTemplate", "name", obj.GetName()) + + defaultInterfaceIPFamily(obj.Spec.Template.Spec.Interfaces) + return nil } diff --git a/templates/cluster-template-byo-network.yaml b/templates/cluster-template-byo-network.yaml new file mode 100644 index 0000000..ca50ff2 --- /dev/null +++ b/templates/cluster-template-byo-network.yaml @@ -0,0 +1,155 @@ +apiVersion: v1 +kind: Secret +metadata: + name: "${CLUSTER_NAME}-credentials" + namespace: "${NAMESPACE}" +type: Opaque +stringData: + token: "${CLOUDSCALE_API_TOKEN}" +--- +apiVersion: cluster.x-k8s.io/v1beta2 +kind: Cluster +metadata: + name: "${CLUSTER_NAME}" + namespace: "${NAMESPACE}" + labels: + ccm: cloudscale +spec: + clusterNetwork: + pods: + cidrBlocks: ["192.168.0.0/16"] + services: + cidrBlocks: ["10.96.0.0/12"] + serviceDomain: "cluster.local" + infrastructureRef: + apiGroup: infrastructure.cluster.x-k8s.io + kind: CloudscaleCluster + name: "${CLUSTER_NAME}" + controlPlaneRef: + apiGroup: controlplane.cluster.x-k8s.io + kind: KubeadmControlPlane + name: "${CLUSTER_NAME}-control-plane" +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta2 +kind: CloudscaleCluster +metadata: + name: "${CLUSTER_NAME}" + namespace: "${NAMESPACE}" +spec: + region: "${CLOUDSCALE_REGION}" + networks: + - name: "${CLUSTER_NAME}" + uuid: "${CLOUDSCALE_NETWORK_UUID}" # BYO network + credentialsRef: + name: "${CLUSTER_NAME}-credentials" +--- +apiVersion: controlplane.cluster.x-k8s.io/v1beta2 +kind: KubeadmControlPlane +metadata: + name: "${CLUSTER_NAME}-control-plane" + namespace: "${NAMESPACE}" +spec: + replicas: ${CONTROL_PLANE_MACHINE_COUNT:=1} + version: "${KUBERNETES_VERSION}" + machineTemplate: + spec: + infrastructureRef: + apiGroup: infrastructure.cluster.x-k8s.io + kind: CloudscaleMachineTemplate + name: "${CLUSTER_NAME}-control-plane" + kubeadmConfigSpec: + initConfiguration: + nodeRegistration: + kubeletExtraArgs: + - name: cloud-provider + value: external + joinConfiguration: + nodeRegistration: + kubeletExtraArgs: + - name: cloud-provider + value: external + users: + - name: capi + groups: "adm, sudo" + shell: "/bin/bash" + sudo: "ALL=(ALL) NOPASSWD:ALL" + sshAuthorizedKeys: + - "${CLOUDSCALE_SSH_PUBLIC_KEY}" +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta2 +kind: CloudscaleMachineTemplate +metadata: + name: "${CLUSTER_NAME}-control-plane" + namespace: "${NAMESPACE}" +spec: + template: + spec: + flavor: "${CLOUDSCALE_CONTROL_PLANE_MACHINE_FLAVOR}" + image: "${CLOUDSCALE_MACHINE_IMAGE}" + rootVolumeSize: ${CLOUDSCALE_ROOT_VOLUME_SIZE} + serverGroup: + name: "${CLUSTER_NAME}-control-plane" +--- +apiVersion: cluster.x-k8s.io/v1beta2 +kind: MachineDeployment +metadata: + name: "${CLUSTER_NAME}-md-0" + namespace: "${NAMESPACE}" +spec: + clusterName: "${CLUSTER_NAME}" + replicas: ${WORKER_MACHINE_COUNT:=2} + selector: + matchLabels: + template: + spec: + clusterName: "${CLUSTER_NAME}" + version: "${KUBERNETES_VERSION}" + bootstrap: + configRef: + name: "${CLUSTER_NAME}-md-0" + apiGroup: bootstrap.cluster.x-k8s.io + kind: KubeadmConfigTemplate + infrastructureRef: + name: "${CLUSTER_NAME}-md-0" + apiGroup: infrastructure.cluster.x-k8s.io + kind: CloudscaleMachineTemplate +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta2 +kind: CloudscaleMachineTemplate +metadata: + name: "${CLUSTER_NAME}-md-0" + namespace: "${NAMESPACE}" +spec: + template: + spec: + flavor: "${CLOUDSCALE_WORKER_MACHINE_FLAVOR}" + image: "${CLOUDSCALE_MACHINE_IMAGE}" + rootVolumeSize: ${CLOUDSCALE_ROOT_VOLUME_SIZE} + serverGroup: + name: "${CLUSTER_NAME}-md-0" +--- +apiVersion: bootstrap.cluster.x-k8s.io/v1beta2 +kind: KubeadmConfigTemplate +metadata: + name: "${CLUSTER_NAME}-md-0" + namespace: "${NAMESPACE}" +spec: + template: + spec: + initConfiguration: + nodeRegistration: + kubeletExtraArgs: + - name: cloud-provider + value: external + joinConfiguration: + nodeRegistration: + kubeletExtraArgs: + - name: cloud-provider + value: external + users: + - name: capi + groups: "adm, sudo" + shell: "/bin/bash" + sudo: "ALL=(ALL) NOPASSWD:ALL" + sshAuthorizedKeys: + - "${CLOUDSCALE_SSH_PUBLIC_KEY}" diff --git a/templates/cluster-template-fip.yaml b/templates/cluster-template-fip.yaml new file mode 100644 index 0000000..d7d46a7 --- /dev/null +++ b/templates/cluster-template-fip.yaml @@ -0,0 +1,157 @@ +apiVersion: v1 +kind: Secret +metadata: + name: "${CLUSTER_NAME}-credentials" + namespace: "${NAMESPACE}" +type: Opaque +stringData: + token: "${CLOUDSCALE_API_TOKEN}" +--- +apiVersion: cluster.x-k8s.io/v1beta2 +kind: Cluster +metadata: + name: "${CLUSTER_NAME}" + namespace: "${NAMESPACE}" + labels: + ccm: cloudscale +spec: + clusterNetwork: + pods: + cidrBlocks: ["192.168.0.0/16"] + services: + cidrBlocks: ["10.96.0.0/12"] + serviceDomain: "cluster.local" + infrastructureRef: + apiGroup: infrastructure.cluster.x-k8s.io + kind: CloudscaleCluster + name: "${CLUSTER_NAME}" + controlPlaneRef: + apiGroup: controlplane.cluster.x-k8s.io + kind: KubeadmControlPlane + name: "${CLUSTER_NAME}-control-plane" +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta2 +kind: CloudscaleCluster +metadata: + name: "${CLUSTER_NAME}" + namespace: "${NAMESPACE}" +spec: + region: "${CLOUDSCALE_REGION}" + networks: + - name: "${CLUSTER_NAME}" + uuid: "${CLOUDSCALE_NETWORK_UUID}" # BYO network + floatingIP: + ipFamily: IPv4 + credentialsRef: + name: "${CLUSTER_NAME}-credentials" +--- +apiVersion: controlplane.cluster.x-k8s.io/v1beta2 +kind: KubeadmControlPlane +metadata: + name: "${CLUSTER_NAME}-control-plane" + namespace: "${NAMESPACE}" +spec: + replicas: ${CONTROL_PLANE_MACHINE_COUNT:=1} + version: "${KUBERNETES_VERSION}" + machineTemplate: + spec: + infrastructureRef: + apiGroup: infrastructure.cluster.x-k8s.io + kind: CloudscaleMachineTemplate + name: "${CLUSTER_NAME}-control-plane" + kubeadmConfigSpec: + initConfiguration: + nodeRegistration: + kubeletExtraArgs: + - name: cloud-provider + value: external + joinConfiguration: + nodeRegistration: + kubeletExtraArgs: + - name: cloud-provider + value: external + users: + - name: capi + groups: "adm, sudo" + shell: "/bin/bash" + sudo: "ALL=(ALL) NOPASSWD:ALL" + sshAuthorizedKeys: + - "${CLOUDSCALE_SSH_PUBLIC_KEY}" +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta2 +kind: CloudscaleMachineTemplate +metadata: + name: "${CLUSTER_NAME}-control-plane" + namespace: "${NAMESPACE}" +spec: + template: + spec: + flavor: "${CLOUDSCALE_CONTROL_PLANE_MACHINE_FLAVOR}" + image: "${CLOUDSCALE_MACHINE_IMAGE}" + rootVolumeSize: ${CLOUDSCALE_ROOT_VOLUME_SIZE} + serverGroup: + name: "${CLUSTER_NAME}-control-plane" +--- +apiVersion: cluster.x-k8s.io/v1beta2 +kind: MachineDeployment +metadata: + name: "${CLUSTER_NAME}-md-0" + namespace: "${NAMESPACE}" +spec: + clusterName: "${CLUSTER_NAME}" + replicas: ${WORKER_MACHINE_COUNT:=2} + selector: + matchLabels: + template: + spec: + clusterName: "${CLUSTER_NAME}" + version: "${KUBERNETES_VERSION}" + bootstrap: + configRef: + name: "${CLUSTER_NAME}-md-0" + apiGroup: bootstrap.cluster.x-k8s.io + kind: KubeadmConfigTemplate + infrastructureRef: + name: "${CLUSTER_NAME}-md-0" + apiGroup: infrastructure.cluster.x-k8s.io + kind: CloudscaleMachineTemplate +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta2 +kind: CloudscaleMachineTemplate +metadata: + name: "${CLUSTER_NAME}-md-0" + namespace: "${NAMESPACE}" +spec: + template: + spec: + flavor: "${CLOUDSCALE_WORKER_MACHINE_FLAVOR}" + image: "${CLOUDSCALE_MACHINE_IMAGE}" + rootVolumeSize: ${CLOUDSCALE_ROOT_VOLUME_SIZE} + serverGroup: + name: "${CLUSTER_NAME}-md-0" +--- +apiVersion: bootstrap.cluster.x-k8s.io/v1beta2 +kind: KubeadmConfigTemplate +metadata: + name: "${CLUSTER_NAME}-md-0" + namespace: "${NAMESPACE}" +spec: + template: + spec: + initConfiguration: + nodeRegistration: + kubeletExtraArgs: + - name: cloud-provider + value: external + joinConfiguration: + nodeRegistration: + kubeletExtraArgs: + - name: cloud-provider + value: external + users: + - name: capi + groups: "adm, sudo" + shell: "/bin/bash" + sudo: "ALL=(ALL) NOPASSWD:ALL" + sshAuthorizedKeys: + - "${CLOUDSCALE_SSH_PUBLIC_KEY}" diff --git a/templates/cluster-template-public-lb-private-nodes.yaml b/templates/cluster-template-public-lb-private-nodes.yaml new file mode 100644 index 0000000..dc3d3f5 --- /dev/null +++ b/templates/cluster-template-public-lb-private-nodes.yaml @@ -0,0 +1,159 @@ +apiVersion: v1 +kind: Secret +metadata: + name: "${CLUSTER_NAME}-credentials" + namespace: "${NAMESPACE}" +type: Opaque +stringData: + token: "${CLOUDSCALE_API_TOKEN}" +--- +apiVersion: cluster.x-k8s.io/v1beta2 +kind: Cluster +metadata: + name: "${CLUSTER_NAME}" + namespace: "${NAMESPACE}" + labels: + ccm: cloudscale +spec: + clusterNetwork: + pods: + cidrBlocks: ["192.168.0.0/16"] + services: + cidrBlocks: ["10.96.0.0/12"] + serviceDomain: "cluster.local" + infrastructureRef: + apiGroup: infrastructure.cluster.x-k8s.io + kind: CloudscaleCluster + name: "${CLUSTER_NAME}" + controlPlaneRef: + apiGroup: controlplane.cluster.x-k8s.io + kind: KubeadmControlPlane + name: "${CLUSTER_NAME}-control-plane" +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta2 +kind: CloudscaleCluster +metadata: + name: "${CLUSTER_NAME}" + namespace: "${NAMESPACE}" +spec: + region: "${CLOUDSCALE_REGION}" + networks: + - name: "${CLUSTER_NAME}" + uuid: "${CLOUDSCALE_NETWORK_UUID}" # BYO network (must have NAT for internet egress) + credentialsRef: + name: "${CLUSTER_NAME}-credentials" +--- +apiVersion: controlplane.cluster.x-k8s.io/v1beta2 +kind: KubeadmControlPlane +metadata: + name: "${CLUSTER_NAME}-control-plane" + namespace: "${NAMESPACE}" +spec: + replicas: ${CONTROL_PLANE_MACHINE_COUNT:=1} + version: "${KUBERNETES_VERSION}" + machineTemplate: + spec: + infrastructureRef: + apiGroup: infrastructure.cluster.x-k8s.io + kind: CloudscaleMachineTemplate + name: "${CLUSTER_NAME}-control-plane" + kubeadmConfigSpec: + initConfiguration: + nodeRegistration: + kubeletExtraArgs: + - name: cloud-provider + value: external + joinConfiguration: + nodeRegistration: + kubeletExtraArgs: + - name: cloud-provider + value: external + users: + - name: capi + groups: "adm, sudo" + shell: "/bin/bash" + sudo: "ALL=(ALL) NOPASSWD:ALL" + sshAuthorizedKeys: + - "${CLOUDSCALE_SSH_PUBLIC_KEY}" +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta2 +kind: CloudscaleMachineTemplate +metadata: + name: "${CLUSTER_NAME}-control-plane" + namespace: "${NAMESPACE}" +spec: + template: + spec: + flavor: "${CLOUDSCALE_CONTROL_PLANE_MACHINE_FLAVOR}" + image: "${CLOUDSCALE_MACHINE_IMAGE}" + rootVolumeSize: ${CLOUDSCALE_ROOT_VOLUME_SIZE} + serverGroup: + name: "${CLUSTER_NAME}-control-plane" + interfaces: + - network: "${CLUSTER_NAME}" +--- +apiVersion: cluster.x-k8s.io/v1beta2 +kind: MachineDeployment +metadata: + name: "${CLUSTER_NAME}-md-0" + namespace: "${NAMESPACE}" +spec: + clusterName: "${CLUSTER_NAME}" + replicas: ${WORKER_MACHINE_COUNT:=2} + selector: + matchLabels: + template: + spec: + clusterName: "${CLUSTER_NAME}" + version: "${KUBERNETES_VERSION}" + bootstrap: + configRef: + name: "${CLUSTER_NAME}-md-0" + apiGroup: bootstrap.cluster.x-k8s.io + kind: KubeadmConfigTemplate + infrastructureRef: + name: "${CLUSTER_NAME}-md-0" + apiGroup: infrastructure.cluster.x-k8s.io + kind: CloudscaleMachineTemplate +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta2 +kind: CloudscaleMachineTemplate +metadata: + name: "${CLUSTER_NAME}-md-0" + namespace: "${NAMESPACE}" +spec: + template: + spec: + flavor: "${CLOUDSCALE_WORKER_MACHINE_FLAVOR}" + image: "${CLOUDSCALE_MACHINE_IMAGE}" + rootVolumeSize: ${CLOUDSCALE_ROOT_VOLUME_SIZE} + serverGroup: + name: "${CLUSTER_NAME}-md-0" + interfaces: + - network: "${CLUSTER_NAME}" +--- +apiVersion: bootstrap.cluster.x-k8s.io/v1beta2 +kind: KubeadmConfigTemplate +metadata: + name: "${CLUSTER_NAME}-md-0" + namespace: "${NAMESPACE}" +spec: + template: + spec: + initConfiguration: + nodeRegistration: + kubeletExtraArgs: + - name: cloud-provider + value: external + joinConfiguration: + nodeRegistration: + kubeletExtraArgs: + - name: cloud-provider + value: external + users: + - name: capi + groups: "adm, sudo" + shell: "/bin/bash" + sudo: "ALL=(ALL) NOPASSWD:ALL" + sshAuthorizedKeys: + - "${CLOUDSCALE_SSH_PUBLIC_KEY}" diff --git a/templates/cluster-template.yaml b/templates/cluster-template.yaml index 9b4e84d..d075f2a 100644 --- a/templates/cluster-template.yaml +++ b/templates/cluster-template.yaml @@ -37,8 +37,9 @@ metadata: namespace: "${NAMESPACE}" spec: region: "${CLOUDSCALE_REGION}" - network: - cidr: "10.100.0.0/24" + networks: + - name: "${CLUSTER_NAME}" + cidr: "10.0.0.0/24" gatewayAddress: "" # disable gateway, use public interface for internet access credentialsRef: name: "${CLUSTER_NAME}-credentials" diff --git a/test/e2e/cloudscale_helpers.go b/test/e2e/cloudscale_helpers.go index 6b66ae7..6f33bb6 100644 --- a/test/e2e/cloudscale_helpers.go +++ b/test/e2e/cloudscale_helpers.go @@ -39,6 +39,7 @@ type resourceSnapshot struct { NetworkUUIDs map[string]bool LoadBalancerUUIDs map[string]bool ServerGroupUUIDs map[string]bool + FloatingIPHREFs map[string]bool } // takeResourceSnapshot lists all relevant cloudscale resources and records their UUIDs. @@ -48,6 +49,7 @@ func takeResourceSnapshot(ctx context.Context, client *cloudscale.Client) (*reso NetworkUUIDs: make(map[string]bool), LoadBalancerUUIDs: make(map[string]bool), ServerGroupUUIDs: make(map[string]bool), + FloatingIPHREFs: make(map[string]bool), } servers, err := client.Servers.List(ctx) @@ -59,7 +61,14 @@ func takeResourceSnapshot(ctx context.Context, client *cloudscale.Client) (*reso } // TODO: volumes list - // TODO: floating ips list + + floatingIPs, err := client.FloatingIPs.List(ctx) + if err != nil { + return nil, fmt.Errorf("listing floating IPs: %w", err) + } + for _, fip := range floatingIPs { + snap.FloatingIPHREFs[fip.HREF] = true + } networks, err := client.Networks.List(ctx) if err != nil { @@ -118,6 +127,11 @@ func checkForLeakedResources(ctx context.Context, client *cloudscale.Client, bef leaks = append(leaks, fmt.Sprintf("leaked server group: %s", uuid)) } } + for href := range after.FloatingIPHREFs { + if !before.FloatingIPHREFs[href] { + leaks = append(leaks, fmt.Sprintf("leaked floating IP: %s", href)) + } + } if len(leaks) > 0 { return fmt.Errorf("infrastructure resource leaks detected:\n%s", strings.Join(leaks, "\n")) diff --git a/test/e2e/config/cloudscale.yaml b/test/e2e/config/cloudscale.yaml index c1e9ae3..c541088 100644 --- a/test/e2e/config/cloudscale.yaml +++ b/test/e2e/config/cloudscale.yaml @@ -38,7 +38,7 @@ providers: files: - sourcePath: "../data/shared/v1beta2/metadata.yaml" - - name: cloudscale + - name: cloudscale-ch-cloudscale type: InfrastructureProvider versions: - name: v0.99.99 # dev version, always higher than any release @@ -53,6 +53,9 @@ providers: - sourcePath: "../data/infrastructure-cloudscale/main/cluster-template-ha.yaml" - sourcePath: "../data/infrastructure-cloudscale/main/cluster-template-upgrades.yaml" - sourcePath: "../data/infrastructure-cloudscale/main/cluster-template-md-remediation.yaml" + - sourcePath: "../data/infrastructure-cloudscale/main/cluster-template-byo-network.yaml" + - sourcePath: "../data/infrastructure-cloudscale/main/cluster-template-public-lb-private-nodes.yaml" + - sourcePath: "../data/infrastructure-cloudscale/main/cluster-template-fip.yaml" - sourcePath: "../data/infrastructure-cloudscale/main/metadata.yaml" targetName: metadata.yaml diff --git a/test/e2e/data/infrastructure-cloudscale/bases/cluster.yaml b/test/e2e/data/infrastructure-cloudscale/bases/cluster.yaml index 6363822..741153d 100644 --- a/test/e2e/data/infrastructure-cloudscale/bases/cluster.yaml +++ b/test/e2e/data/infrastructure-cloudscale/bases/cluster.yaml @@ -41,7 +41,8 @@ metadata: namespace: "${NAMESPACE}" spec: region: "${CLOUDSCALE_REGION}" - network: + networks: + - name: "${CLUSTER_NAME}" cidr: "${CLOUDSCALE_NETWORK_CIDR}" gatewayAddress: "" # disable gateway, use public interface for internet access controlPlaneLoadBalancer: diff --git a/test/e2e/data/infrastructure-cloudscale/cluster-template-byo-network/byo-network.yaml b/test/e2e/data/infrastructure-cloudscale/cluster-template-byo-network/byo-network.yaml new file mode 100644 index 0000000..3e4a0d9 --- /dev/null +++ b/test/e2e/data/infrastructure-cloudscale/cluster-template-byo-network/byo-network.yaml @@ -0,0 +1,10 @@ +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta2 +kind: CloudscaleCluster +metadata: + name: "${CLUSTER_NAME}" + namespace: "${NAMESPACE}" +spec: + networks: + - name: "${CLUSTER_NAME}" + uuid: "${CLOUDSCALE_NETWORK_UUID}" diff --git a/test/e2e/data/infrastructure-cloudscale/cluster-template-byo-network/kustomization.yaml b/test/e2e/data/infrastructure-cloudscale/cluster-template-byo-network/kustomization.yaml new file mode 100644 index 0000000..9fc5c47 --- /dev/null +++ b/test/e2e/data/infrastructure-cloudscale/cluster-template-byo-network/kustomization.yaml @@ -0,0 +1,9 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + - ../bases/cluster.yaml + - ../bases/md.yaml + - ../bases/ccm.yaml + - ../bases/cni.yaml +patches: + - path: byo-network.yaml diff --git a/test/e2e/data/infrastructure-cloudscale/cluster-template-fip/fip.yaml b/test/e2e/data/infrastructure-cloudscale/cluster-template-fip/fip.yaml new file mode 100644 index 0000000..82ae6d1 --- /dev/null +++ b/test/e2e/data/infrastructure-cloudscale/cluster-template-fip/fip.yaml @@ -0,0 +1,12 @@ +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta2 +kind: CloudscaleCluster +metadata: + name: "${CLUSTER_NAME}" + namespace: "${NAMESPACE}" +spec: + networks: + - name: "${CLUSTER_NAME}" + uuid: "${CLOUDSCALE_NETWORK_UUID}" + floatingIP: + ipFamily: IPv4 diff --git a/test/e2e/data/infrastructure-cloudscale/cluster-template-fip/kustomization.yaml b/test/e2e/data/infrastructure-cloudscale/cluster-template-fip/kustomization.yaml new file mode 100644 index 0000000..0c14643 --- /dev/null +++ b/test/e2e/data/infrastructure-cloudscale/cluster-template-fip/kustomization.yaml @@ -0,0 +1,9 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + - ../bases/cluster.yaml + - ../bases/md.yaml + - ../bases/ccm.yaml + - ../bases/cni.yaml +patches: + - path: fip.yaml diff --git a/test/e2e/data/infrastructure-cloudscale/cluster-template-public-lb-private-nodes/kustomization.yaml b/test/e2e/data/infrastructure-cloudscale/cluster-template-public-lb-private-nodes/kustomization.yaml new file mode 100644 index 0000000..9de5817 --- /dev/null +++ b/test/e2e/data/infrastructure-cloudscale/cluster-template-public-lb-private-nodes/kustomization.yaml @@ -0,0 +1,9 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + - ../bases/cluster.yaml + - ../bases/md.yaml + - ../bases/ccm.yaml + - ../bases/cni.yaml +patches: + - path: public-lb-private-nodes.yaml diff --git a/test/e2e/data/infrastructure-cloudscale/cluster-template-public-lb-private-nodes/public-lb-private-nodes.yaml b/test/e2e/data/infrastructure-cloudscale/cluster-template-public-lb-private-nodes/public-lb-private-nodes.yaml new file mode 100644 index 0000000..2221dd4 --- /dev/null +++ b/test/e2e/data/infrastructure-cloudscale/cluster-template-public-lb-private-nodes/public-lb-private-nodes.yaml @@ -0,0 +1,32 @@ +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta2 +kind: CloudscaleCluster +metadata: + name: "${CLUSTER_NAME}" + namespace: "${NAMESPACE}" +spec: + networks: + - name: "${CLUSTER_NAME}" + uuid: "${CLOUDSCALE_NETWORK_UUID}" +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta2 +kind: CloudscaleMachineTemplate +metadata: + name: "${CLUSTER_NAME}-control-plane" + namespace: "${NAMESPACE}" +spec: + template: + spec: + interfaces: + - network: "${CLUSTER_NAME}" +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta2 +kind: CloudscaleMachineTemplate +metadata: + name: "${CLUSTER_NAME}-md-0" + namespace: "${NAMESPACE}" +spec: + template: + spec: + interfaces: + - network: "${CLUSTER_NAME}" diff --git a/test/e2e/e2e_suite_test.go b/test/e2e/e2e_suite_test.go index d8a39a6..9683f2c 100644 --- a/test/e2e/e2e_suite_test.go +++ b/test/e2e/e2e_suite_test.go @@ -111,8 +111,15 @@ var _ = SynchronizedBeforeSuite(func() []byte { Expect(sshKey).NotTo(BeEmpty(), "CLOUDSCALE_SSH_PUBLIC_KEY environment variable is required") e2eConfig.Variables["CLOUDSCALE_SSH_PUBLIC_KEY"] = sshKey - By("Taking pre-test snapshot of cloudscale infrastructure resources") cloudscaleClient = newCloudscaleClient(apiToken) + + // Optional: BYO network for private networking tests. + // If not set, tests requiring a BYO network will be skipped. + if networkUUID := os.Getenv("CLOUDSCALE_NETWORK_UUID"); networkUUID != "" { + e2eConfig.Variables["CLOUDSCALE_NETWORK_UUID"] = networkUUID + } + + By("Taking pre-test snapshot of cloudscale infrastructure resources") var err error preTestSnapshot, err = takeResourceSnapshot(ctx, cloudscaleClient) Expect(err).NotTo(HaveOccurred(), "Failed to snapshot cloudscale resources") @@ -154,6 +161,11 @@ var _ = SynchronizedBeforeSuite(func() []byte { }) Expect(e2eConfig).NotTo(BeNil()) + // Re-inject env-only variables lost when LoadE2EConfig overwrites e2eConfig. + if networkUUID := os.Getenv("CLOUDSCALE_NETWORK_UUID"); networkUUID != "" { + e2eConfig.Variables["CLOUDSCALE_NETWORK_UUID"] = networkUUID + } + if artifactFolder == "" { artifactFolder = filepath.Join(os.TempDir(), "capcs-e2e-artifacts") } diff --git a/test/e2e/e2e_test.go b/test/e2e/e2e_test.go index 9082085..ff99146 100644 --- a/test/e2e/e2e_test.go +++ b/test/e2e/e2e_test.go @@ -39,7 +39,7 @@ var _ = Describe("Workload cluster lifecycle", Label("lifecycle"), func() { BootstrapClusterProxy: bootstrapClusterProxy, ArtifactFolder: artifactFolder, SkipCleanup: skipCleanup, - InfrastructureProvider: ptr.To("cloudscale"), + InfrastructureProvider: ptr.To("cloudscale-ch-cloudscale"), Flavor: ptr.To(""), ControlPlaneMachineCount: ptr.To[int64](1), WorkerMachineCount: ptr.To[int64](1), @@ -56,7 +56,7 @@ var _ = Describe("Workload cluster lifecycle", Label("lifecycle"), func() { BootstrapClusterProxy: bootstrapClusterProxy, ArtifactFolder: artifactFolder, SkipCleanup: skipCleanup, - InfrastructureProvider: ptr.To("cloudscale"), + InfrastructureProvider: ptr.To("cloudscale-ch-cloudscale"), Flavor: ptr.To("ha"), ControlPlaneMachineCount: ptr.To[int64](3), WorkerMachineCount: ptr.To[int64](2), @@ -66,6 +66,72 @@ var _ = Describe("Workload cluster lifecycle", Label("lifecycle"), func() { }) }) +// BYO networking tests verify cluster provisioning against a pre-existing (BYO) network. +// All contexts are skipped when CLOUDSCALE_NETWORK_UUID is not set. The BYO network must +// provide internet egress (e.g. Support-arranged NAT) for the private-nodes contexts, +// otherwise kubeadm bootstrap hangs. +var _ = Describe("BYO networking", Label("byo-networking"), func() { + BeforeEach(func() { + if _, ok := e2eConfig.Variables["CLOUDSCALE_NETWORK_UUID"]; !ok { + Skip("CLOUDSCALE_NETWORK_UUID not set, skipping BYO networking tests") + } + }) + + // With BYO network: public LB, machines dual-attached (BYO + public). + Context("With BYO network", func() { + capi_e2e.QuickStartSpec(ctx, func() capi_e2e.QuickStartSpecInput { + return capi_e2e.QuickStartSpecInput{ + E2EConfig: e2eConfig, + ClusterctlConfigPath: clusterctlConfigPath, + BootstrapClusterProxy: bootstrapClusterProxy, + ArtifactFolder: artifactFolder, + SkipCleanup: skipCleanup, + InfrastructureProvider: ptr.To("cloudscale-ch-cloudscale"), + Flavor: ptr.To("byo-network"), + ControlPlaneMachineCount: ptr.To[int64](1), + WorkerMachineCount: ptr.To[int64](1), + PostMachinesProvisioned: validateCloudscaleResources, + } + }) + }) + + // With public LB, machines attached only to the BYO network (no public interface). + Context("With public LB, private nodes", func() { + capi_e2e.QuickStartSpec(ctx, func() capi_e2e.QuickStartSpecInput { + return capi_e2e.QuickStartSpecInput{ + E2EConfig: e2eConfig, + ClusterctlConfigPath: clusterctlConfigPath, + BootstrapClusterProxy: bootstrapClusterProxy, + ArtifactFolder: artifactFolder, + SkipCleanup: skipCleanup, + InfrastructureProvider: ptr.To("cloudscale-ch-cloudscale"), + Flavor: ptr.To("public-lb-private-nodes"), + ControlPlaneMachineCount: ptr.To[int64](1), + WorkerMachineCount: ptr.To[int64](1), + PostMachinesProvisioned: validateCloudscaleResources, + } + }) + }) + + // Floating IP on a LB. + Context("With Floating IP on CP server", func() { + capi_e2e.QuickStartSpec(ctx, func() capi_e2e.QuickStartSpecInput { + return capi_e2e.QuickStartSpecInput{ + E2EConfig: e2eConfig, + ClusterctlConfigPath: clusterctlConfigPath, + BootstrapClusterProxy: bootstrapClusterProxy, + ArtifactFolder: artifactFolder, + SkipCleanup: skipCleanup, + InfrastructureProvider: ptr.To("cloudscale-ch-cloudscale"), + Flavor: ptr.To("fip"), + ControlPlaneMachineCount: ptr.To[int64](1), + WorkerMachineCount: ptr.To[int64](1), + PostMachinesProvisioned: validateCloudscaleResources, + } + }) + }) +}) + // Cluster upgrade tests verify in-place Kubernetes version upgrades by rolling // control-plane and worker nodes to new machine images. Conformance tests are skipped // (SkipConformanceTests: true) to keep runtime reasonable — the separate conformance @@ -79,7 +145,7 @@ var _ = Describe("Cluster upgrade", Label("upgrade"), func() { ArtifactFolder: artifactFolder, SkipCleanup: skipCleanup, SkipConformanceTests: true, - InfrastructureProvider: ptr.To("cloudscale"), + InfrastructureProvider: ptr.To("cloudscale-ch-cloudscale"), ControlPlaneMachineCount: ptr.To[int64](1), WorkerMachineCount: ptr.To[int64](1), } @@ -98,7 +164,7 @@ var _ = Describe("Self-hosted cluster", Label("self-hosted"), func() { BootstrapClusterProxy: bootstrapClusterProxy, ArtifactFolder: artifactFolder, SkipCleanup: skipCleanup, - InfrastructureProvider: ptr.To("cloudscale"), + InfrastructureProvider: ptr.To("cloudscale-ch-cloudscale"), SkipUpgrade: true, ControlPlaneMachineCount: ptr.To[int64](1), WorkerMachineCount: ptr.To[int64](1), @@ -122,7 +188,7 @@ var _ = Describe("MD remediation", Label("md-remediation"), func() { BootstrapClusterProxy: bootstrapClusterProxy, ArtifactFolder: artifactFolder, SkipCleanup: skipCleanup, - InfrastructureProvider: ptr.To("cloudscale"), + InfrastructureProvider: ptr.To("cloudscale-ch-cloudscale"), } }) }) @@ -138,7 +204,7 @@ var _ = Describe("Kubernetes conformance", Label("conformance"), func() { BootstrapClusterProxy: bootstrapClusterProxy, ArtifactFolder: artifactFolder, SkipCleanup: skipCleanup, - InfrastructureProvider: ptr.To("cloudscale"), + InfrastructureProvider: ptr.To("cloudscale-ch-cloudscale"), } }) }) diff --git a/test/e2e/helpers.go b/test/e2e/helpers.go index a59a049..8f7193e 100644 --- a/test/e2e/helpers.go +++ b/test/e2e/helpers.go @@ -43,14 +43,24 @@ func validateCloudscaleResources(proxy framework.ClusterProxy, namespace, cluste key := client.ObjectKey{Namespace: namespace, Name: clusterName} Expect(c.Get(ctx, key, cloudscaleCluster)).To(Succeed(), "Failed to get CloudscaleCluster") - // Validate network resources are created - Expect(cloudscaleCluster.Status.NetworkID).NotTo(BeEmpty(), "NetworkID should be set") - Expect(cloudscaleCluster.Status.SubnetID).NotTo(BeEmpty(), "SubnetID should be set") + // Validate all network resources are created + Expect(cloudscaleCluster.Status.Networks).NotTo(BeEmpty(), "At least one network should be defined in status") + for i, net := range cloudscaleCluster.Status.Networks { + Expect(net.NetworkID).NotTo(BeEmpty(), "Network %d (%s) should have NetworkID", i, net.Name) + Expect(net.SubnetID).NotTo(BeEmpty(), "Network %d (%s) should have SubnetID", i, net.Name) + } + // Validate load balancer resources (if enabled - default is true) if ptr.Deref(cloudscaleCluster.Spec.ControlPlaneLoadBalancer.Enabled, true) { Expect(cloudscaleCluster.Status.LoadBalancerID).NotTo(BeEmpty(), "LoadBalancerID should be set") Expect(cloudscaleCluster.Status.LoadBalancerPoolID).NotTo(BeEmpty(), "LoadBalancerPoolID should be set") Expect(cloudscaleCluster.Status.LoadBalancerListenerID).NotTo(BeEmpty(), "LoadBalancerListenerID should be set") + Expect(cloudscaleCluster.Status.LoadBalancerHealthMonitorID).NotTo(BeEmpty(), "LoadBalancerHealthMonitorID should be set") + } + + // Validate floating IP (if configured) + if cloudscaleCluster.Spec.FloatingIP != nil { + Expect(cloudscaleCluster.Status.FloatingIP).NotTo(BeEmpty(), "FloatingIP should be set when floating IP is configured") } // Validate provisioned status diff --git a/tilt-provider.yaml b/tilt-provider.yaml index ec00fd2..1ef842e 100644 --- a/tilt-provider.yaml +++ b/tilt-provider.yaml @@ -3,4 +3,5 @@ config: image: quay.io/cloudscalech/capcs-staging:latest label: CAPCS live_reload_deps: ["cmd", "go.mod", "go.sum", "api", "internal"] - go_main: cmd/main.go \ No newline at end of file + go_main: cmd/main.go + version: v0.99.99