Support custom CA & increase debuggability

.github/workflows/csi-integration-tests.yml

@@ -117,6 +117,8 @@ jobs:
       SUBNET: '${{ matrix.subnet }}'
       CLUSTER_PREFIX: '${{ matrix.cluster_prefix }}'
       IMAGE_SOURCE: import
+      CONTROL_COUNT: 3
+      WORKER_COUNT: 2
 
     # Prevent integration tests from running in parallel. Ideally this should
     # be seuqential, but that won't work due to the following issue:

helpers/run-in-test-cluster

@@ -14,6 +14,10 @@ export WORKER_COUNT="${WORKER_COUNT-2}"
 export IMAGE="${IMAGE-quay.io/cloudscalech/cloudscale-csi-plugin:test}"
 export CCM_RELEASE_URL="https://github.com/cloudscale-ch/cloudscale-cloud-controller-manager/releases/latest/download/config.yml"
 
+CUSTOM_CA_SECRET_NAME="cloudscale-custom-ca"
+CUSTOM_CA_VOLUME_NAME="cloudscale-custom-ca"
+CUSTOM_CA_MOUNT_PATH="/etc/ssl/certs/cloudscale-custom-ca.pem"
+
 # Prepares k8test with an existing virtual env, or a newly created on
 function ensure-k8test() {
 
@@ -132,44 +136,147 @@ function import-image() {
 # List CSI-related container images on all cluster nodes using ansible ad-hoc
 function list-csi-images() {
   echo "=== CSI Images on All Nodes ==="
-#  ansible -i k8test/cluster/inventory.yml nodes -b \
-#    -m shell \
-#    -a "ctr -n k8s.io images | grep -E 'cloudscale|csi' || echo '  No CSI images found on this node'" \
-#    --timeout=30
   ansible -i k8test/cluster/inventory.yml nodes -b \
     -m shell \
     -a "ctr -n k8s.io images ls" \
     --timeout=30
   echo ""
 }
 
-# Debug a failed CSI rollout by printing pod events, status, description and node images
+# Run a diagnostic command without allowing a failure to hide subsequent output.
+function run-debug-command() {
+  if ! "$@"; then
+    echo "  (Diagnostic command failed: $*)"
+  fi
+}
+
+# Debug a failed CSI rollout. This deliberately gathers evidence only after a
+# rollout has failed so that successful deployments do not perform extra checks.
 function debug-rollout-failure() {
   local component="$1"
   local label="$2"
+  local ccm_selector
+  local pods
 
   echo ""
   echo "❌ CSI ${component} rollout failed"
   echo ""
-  echo "=== ${component} Pod Events ==="
-  kubectl -n kube-system get pods -l "${label}" -o jsonpath='{range .items[*]}{.metadata.name}{"\n"}{end}' | \
+
+  echo "=== Node Scheduling Summary ==="
+  run-debug-command kubectl get nodes -o wide
+  run-debug-command kubectl get nodes \
+    -o custom-columns='NAME:.metadata.name,READY:.status.conditions[?(@.type=="Ready")].status,UNSCHEDULABLE:.spec.unschedulable,ROLES:.metadata.labels.node-role\.kubernetes\.io/control-plane,TAINTS:.spec.taints'
+  echo ""
+
+  echo "=== ${component} Pod Scheduling Details ==="
+  if pods=$(kubectl -n kube-system get pods -l "${label}" -o jsonpath='{range .items[*]}{.metadata.name}{"\n"}{end}'); then
+    if [[ -z "$pods" ]]; then
+      echo "  (No matching pods found for ${label})"
+    else
+      while read -r pod; do
+        [[ -z "$pod" ]] && continue
+        echo "Pod: $pod"
+        run-debug-command kubectl -n kube-system get pod "$pod" \
+          -o custom-columns='NAME:.metadata.name,PHASE:.status.phase,NODE:.spec.nodeName,NODE-SELECTOR:.spec.nodeSelector,TOLERATIONS:.spec.tolerations,AFFINITY:.spec.affinity,CONDITIONS:.status.conditions'
+        echo ""
+      done <<< "$pods"
+    fi
+  else
+    echo "  (Failed to list matching pods for ${label})"
+    pods=""
+  fi
+
+  echo "=== ${component} Pod Events and Descriptions ==="
+  if [[ -n "$pods" ]]; then
     while read -r pod; do
+      [[ -z "$pod" ]] && continue
       echo "Pod: $pod"
-      kubectl -n kube-system describe pod "$pod" | grep -A 20 "Events:" || echo "  (No events found)"
+      run-debug-command kubectl -n kube-system describe pod "$pod"
       echo ""
-    done
+    done <<< "$pods"
+  fi
+
+  echo "=== Recent Cluster Events ==="
+  run-debug-command kubectl get events --all-namespaces --sort-by=.metadata.creationTimestamp
+  echo ""
+
+  echo "=== cloudscale CCM Status ==="
+  run-debug-command kubectl -n kube-system get daemonset cloudscale-cloud-controller-manager -o wide
+  run-debug-command kubectl -n kube-system describe daemonset cloudscale-cloud-controller-manager
+  run-debug-command kubectl -n kube-system get pods -o wide
+  echo ""
+
+  echo "=== cloudscale CCM Logs ==="
+  if ccm_selector=$(kubectl -n kube-system get daemonset cloudscale-cloud-controller-manager -o json | \
+    jq -r '.spec.selector.matchLabels | to_entries | map("\(.key)=\(.value)") | join(",")') && [[ -n "$ccm_selector" ]]; then
+    run-debug-command kubectl -n kube-system logs -l "$ccm_selector" \
+      --all-containers=true --prefix=true --tail=-1 --max-log-requests=20
+    echo ""
+    echo "=== cloudscale CCM Previous Logs (if available) ==="
+    run-debug-command kubectl -n kube-system logs -l "$ccm_selector" \
+      --all-containers=true --prefix=true --previous --tail=-1 --max-log-requests=20
+  else
+    echo "  (Failed to determine the cloudscale CCM pod selector)"
+  fi
+  echo ""
+
   echo "=== ${component} Pod Status ==="
-  kubectl -n kube-system get pods -l "${label}" -o wide
+  run-debug-command kubectl -n kube-system get pods -l "${label}" -o wide
   echo ""
   echo "=== ${component} Container Images ==="
-  kubectl -n kube-system get pods -l "${label}" \
+  run-debug-command kubectl -n kube-system get pods -l "${label}" \
     -o jsonpath='{range .items[*]}{.metadata.name}{"\n"}{range .status.containerStatuses[*]}  Container: {.name}, Image: {.image}, Ready: {.ready}, RestartCount: {.restartCount}{"\n"}{end}{"\n"}{end}'
   echo ""
-  echo "=== ${component} Pod Full Description ==="
-  kubectl -n kube-system describe pods -l "${label}"
-  echo ""
   echo "=== CSI Images on All Nodes ==="
-  list-csi-images
+  run-debug-command list-csi-images
+}
+
+# Create or update the Secret containing the custom cloudscale API CA. Callers
+# must only invoke this when CLOUDSCALE_CA_PEM is non-empty.
+function deploy-custom-ca-secret() (
+  if [[ -z "${CLOUDSCALE_CA_PEM:-}" ]]; then
+    return 0
+  fi
+
+  local ca_file
+  ca_file=$(mktemp)
+  trap 'rm -f "$ca_file"' EXIT
+
+  chmod 0600 "$ca_file"
+  printf '%s\n' "$CLOUDSCALE_CA_PEM" > "$ca_file"
+
+  kubectl -n kube-system create secret generic "$CUSTOM_CA_SECRET_NAME" \
+    --from-file=ca.pem="$ca_file" \
+    --dry-run=client \
+    -o yaml | kubectl apply -f -
+)
+
+# Extend a container's system trust store with the custom cloudscale API CA.
+function mount-custom-ca() {
+  if [[ -z "${CLOUDSCALE_CA_PEM:-}" ]]; then
+    return 0
+  fi
+
+  local resource="$1"
+  local container="$2"
+  local patch
+
+  patch="spec:
+  template:
+    spec:
+      containers:
+        - name: ${container}
+          volumeMounts:
+            - name: ${CUSTOM_CA_VOLUME_NAME}
+              mountPath: ${CUSTOM_CA_MOUNT_PATH}
+              subPath: ca.pem
+              readOnly: true
+      volumes:
+        - name: ${CUSTOM_CA_VOLUME_NAME}
+          secret:
+            secretName: ${CUSTOM_CA_SECRET_NAME}"
+
+  kubectl -n kube-system patch "$resource" --type=strategic --patch "$patch"
 }
 
 # Deploy CCM from official GitHub release
@@ -184,6 +291,16 @@ function deploy-ccm() {
     sed "s|https://api.cloudscale.ch/|${api_url}/|g" | \
     kubectl apply -f -
 
+  if [[ -n "${CLOUDSCALE_CA_PEM:-}" ]]; then
+    echo "Configuring custom CA for the cloudscale CCM..."
+    deploy-custom-ca-secret
+    mount-custom-ca \
+      daemonset/cloudscale-cloud-controller-manager \
+      cloudscale-cloud-controller-manager
+    kubectl -n kube-system rollout restart \
+      daemonset/cloudscale-cloud-controller-manager
+  fi
+
   echo "Waiting for CCM to be ready..."
   kubectl -n kube-system rollout status daemonset/cloudscale-cloud-controller-manager --timeout=120s
 }
@@ -246,6 +363,17 @@ function deploy-csi() {
   # Apply the manifest
   kubectl apply -n kube-system -f "$manifest_file"
 
+  if [[ -n "${CLOUDSCALE_CA_PEM:-}" ]]; then
+    echo "Configuring custom CA for the cloudscale CSI plugin..."
+    deploy-custom-ca-secret
+    mount-custom-ca \
+      statefulset/csi-cloudscale-controller \
+      csi-cloudscale-plugin
+    mount-custom-ca \
+      daemonset/csi-cloudscale-node \
+      csi-cloudscale-plugin
+  fi
+
   if [[ "$csi_installed" == "yes" ]]; then
     echo ""
     echo "CSI was already installed, triggering rollout restart..."