Add job to ensure expected test networks are created

ci/pipelines/builder.yml

@@ -44,6 +44,9 @@ groups:
 - name: docker
   jobs:
   - build-os-image-stemcell-builder
+- name: infrastructure
+  jobs:
+  - ensure-integration-network
 
 #@yaml/text-templated-strings
 jobs:
@@ -89,6 +92,25 @@ jobs:
       get_params:
         skip_download: true
 
+#! Manually triggered job that idempotently ensures the GCP subnetwork and
+#! firewall rule consumed by deploy-director / cleanup-bats-vms / prepare-bats
+#! in the test-stemcells-ipv4 and bats jobs below exist. GCP is the source of
+#! truth — no state file is required.
+- name: ensure-integration-network
+  serial: true
+  plan:
+  - get: bosh-stemcells-ci
+  - get: bosh-integration-image
+  - task: ensure-integration-network
+    file: bosh-stemcells-ci/ci/tasks/gcp/ensure-integration-network.yml
+    image: bosh-integration-image
+    params:
+      GCP_JSON_KEY: ((gcp_json_key))
+      GCP_PROJECT_ID: ((gcp_project_id))
+      GCP_REGION: europe-north2
+      GCP_NETWORK_NAME: bosh-concourse
+      SUBNET_INT: (@= data.values.stemcell_details.subnet_int @)
+
 - name: process-high-critical-cves
   serial_groups: [log-cves]
   plan:
@@ -885,7 +907,6 @@ resource_types:
   type: registry-image
   source:
     repository: frodenas/gcs-resource
-
 #@yaml/text-templated-strings
 resources:
 - name: daily

ci/tasks/gcp/ensure-integration-network.sh

@@ -0,0 +1,112 @@
+#!/usr/bin/env bash
+set -eu -o pipefail
+
+: "${GCP_JSON_KEY:?}"
+: "${GCP_PROJECT_ID:?}"
+: "${GCP_REGION:?}"
+: "${GCP_NETWORK_NAME:?}"
+: "${SUBNET_INT:?}"
+
+echo "${GCP_JSON_KEY}" | gcloud auth activate-service-account --key-file - --project "${GCP_PROJECT_ID}"
+
+SUBNET_NAME="stemcell-builder-integration-${SUBNET_INT}"
+SUBNET_CIDR="10.100.${SUBNET_INT}.0/24"
+
+# 'bat'                 => BATS created VM tag
+# 'test-stemcells-bats' => director, and compilation VM tag
+FIREWALL_TAGS="bat,test-stemcells-bats"
+
+gcloud_stderr="$(mktemp)"
+trap 'rm -f "${gcloud_stderr}"' EXIT
+
+echo "Checking for subnet '${SUBNET_NAME}' in region '${GCP_REGION}'..."
+existing_subnet_name="$(gcloud compute networks subnets list \
+    --regions="${GCP_REGION}" \
+    --project="${GCP_PROJECT_ID}" \
+    --filter="name=('${SUBNET_NAME}')" \
+    --format='value(name)' \
+    2>"${gcloud_stderr}")" && subnet_lookup_ok=true || subnet_lookup_ok=false
+
+if ${subnet_lookup_ok}; then
+  if [[ -n "${existing_subnet_name}" ]]; then
+    current_subnet="$(gcloud compute networks subnets describe "${SUBNET_NAME}" \
+        --region="${GCP_REGION}" \
+        --project="${GCP_PROJECT_ID}" \
+        --format='csv[no-heading](network.basename(),ipCidrRange,privateIpGoogleAccess,stackType)' \
+        2>"${gcloud_stderr}")"
+    expected_subnet="${GCP_NETWORK_NAME},${SUBNET_CIDR},True,IPV4_ONLY"
+    if [[ "${current_subnet}" != "${expected_subnet}" ]]; then
+      echo "ERROR: Subnet '${SUBNET_NAME}' exists but is misconfigured."
+      echo "  Expected: ${expected_subnet}"
+      echo "  Actual:   ${current_subnet}"
+      exit 1
+    fi
+    echo "Subnet '${SUBNET_NAME}' already exists and matches expected configuration."
+  else
+    echo "Creating subnet '${SUBNET_NAME}'..."
+    gcloud compute networks subnets create "${SUBNET_NAME}" \
+      --network="${GCP_NETWORK_NAME}" \
+      --region="${GCP_REGION}" \
+      --range="${SUBNET_CIDR}" \
+      --enable-private-ip-google-access \
+      --stack-type=IPV4_ONLY \
+      --project="${GCP_PROJECT_ID}"
+    echo "Subnet '${SUBNET_NAME}' created."
+  fi
+else
+  echo "ERROR: gcloud subnet lookup failed for subnet '${SUBNET_NAME}':"
+  cat "${gcloud_stderr}" >&2
+  exit 1
+fi
+
+echo "Checking for firewall rule '${SUBNET_NAME}'..."
+current_fw="$(gcloud compute firewall-rules describe "${SUBNET_NAME}" \
+    --project="${GCP_PROJECT_ID}" \
+    --format='csv[no-heading](network.basename(),direction,allowed[0].IPProtocol,sourceRanges[0],disabled)' \
+    2>"${gcloud_stderr}")" && fw_exists=true || fw_exists=false
+
+if ${fw_exists}; then
+  expected_fw="${GCP_NETWORK_NAME},INGRESS,all,${SUBNET_CIDR},False"
+  if [[ "${current_fw}" != "${expected_fw}" ]]; then
+    echo "ERROR: Firewall rule '${SUBNET_NAME}' exists but is misconfigured."
+    echo "  Expected: ${expected_fw}"
+    echo "  Actual:   ${current_fw}"
+    exit 1
+  fi
+  # Validate target tags independently; sort before comparing since order is not deterministic
+  current_tags="$(gcloud compute firewall-rules describe "${SUBNET_NAME}" \
+    --project="${GCP_PROJECT_ID}" \
+    --format='value(targetTags.list())' \
+    2>"${gcloud_stderr}" \
+    | tr ',;' '\n' | LC_ALL=C sort | tr '\n' ',' | sed 's/,$//')" && current_tags_read=true || current_tags_read=false
+  if ! ${current_tags_read}; then
+    echo "ERROR: gcloud describe failed while reading target tags for firewall rule '${SUBNET_NAME}':"
+    cat "${gcloud_stderr}" >&2
+    exit 1
+  fi
+  expected_tags="$(printf '%s\n' ${FIREWALL_TAGS//,/ } | LC_ALL=C sort | tr '\n' ',' | sed 's/,$//')"
+  if [[ "${current_tags}" != "${expected_tags}" ]]; then
+    echo "ERROR: Firewall rule '${SUBNET_NAME}' has wrong target tags."
+    echo "  Expected: ${expected_tags}"
+    echo "  Actual:   ${current_tags}"
+    exit 1
+  fi
+  echo "Firewall rule '${SUBNET_NAME}' already exists and matches expected configuration."
+elif grep -q "was not found" "${gcloud_stderr}"; then
+  echo "Creating firewall rule '${SUBNET_NAME}'..."
+  gcloud compute firewall-rules create "${SUBNET_NAME}" \
+    --network="${GCP_NETWORK_NAME}" \
+    --project="${GCP_PROJECT_ID}" \
+    --direction=INGRESS \
+    --priority=1000 \
+    --allow=all \
+    --source-ranges="${SUBNET_CIDR}" \
+    --target-tags="${FIREWALL_TAGS}"
+  echo "Firewall rule '${SUBNET_NAME}' created."
+else
+  echo "ERROR: gcloud describe failed for firewall rule '${SUBNET_NAME}':"
+  cat "${gcloud_stderr}" >&2
+  exit 1
+fi
+
+echo "Integration network '${SUBNET_NAME}' is ready."

ci/tasks/gcp/ensure-integration-network.yml

@@ -0,0 +1,15 @@
+---
+platform: linux
+
+inputs:
+  - name: bosh-stemcells-ci
+
+params:
+  GCP_JSON_KEY:
+  GCP_PROJECT_ID:
+  GCP_REGION:
+  GCP_NETWORK_NAME:
+  SUBNET_INT:
+
+run:
+  path: bosh-stemcells-ci/ci/tasks/gcp/ensure-integration-network.sh