diff --git a/.github/workflows/csi-integration-tests.yml b/.github/workflows/csi-integration-tests.yml index 8c518b42..89a8e448 100644 --- a/.github/workflows/csi-integration-tests.yml +++ b/.github/workflows/csi-integration-tests.yml @@ -117,6 +117,8 @@ jobs: SUBNET: '${{ matrix.subnet }}' CLUSTER_PREFIX: '${{ matrix.cluster_prefix }}' IMAGE_SOURCE: import + CONTROL_COUNT: 3 + WORKER_COUNT: 2 # Prevent integration tests from running in parallel. Ideally this should # be seuqential, but that won't work due to the following issue: diff --git a/helpers/run-in-test-cluster b/helpers/run-in-test-cluster index 40a277bc..34fa9ee4 100755 --- a/helpers/run-in-test-cluster +++ b/helpers/run-in-test-cluster @@ -14,6 +14,10 @@ export WORKER_COUNT="${WORKER_COUNT-2}" export IMAGE="${IMAGE-quay.io/cloudscalech/cloudscale-csi-plugin:test}" export CCM_RELEASE_URL="https://github.com/cloudscale-ch/cloudscale-cloud-controller-manager/releases/latest/download/config.yml" +CUSTOM_CA_SECRET_NAME="cloudscale-custom-ca" +CUSTOM_CA_VOLUME_NAME="cloudscale-custom-ca" +CUSTOM_CA_MOUNT_PATH="/etc/ssl/certs/cloudscale-custom-ca.pem" + # Prepares k8test with an existing virtual env, or a newly created on function ensure-k8test() { @@ -132,10 +136,6 @@ function import-image() { # List CSI-related container images on all cluster nodes using ansible ad-hoc function list-csi-images() { echo "=== CSI Images on All Nodes ===" -# ansible -i k8test/cluster/inventory.yml nodes -b \ -# -m shell \ -# -a "ctr -n k8s.io images | grep -E 'cloudscale|csi' || echo ' No CSI images found on this node'" \ -# --timeout=30 ansible -i k8test/cluster/inventory.yml nodes -b \ -m shell \ -a "ctr -n k8s.io images ls" \ @@ -143,33 +143,140 @@ function list-csi-images() { echo "" } -# Debug a failed CSI rollout by printing pod events, status, description and node images +# Run a diagnostic command without allowing a failure to hide subsequent output. +function run-debug-command() { + if ! "$@"; then + echo " (Diagnostic command failed: $*)" + fi +} + +# Debug a failed CSI rollout. This deliberately gathers evidence only after a +# rollout has failed so that successful deployments do not perform extra checks. function debug-rollout-failure() { local component="$1" local label="$2" + local ccm_selector + local pods echo "" echo "❌ CSI ${component} rollout failed" echo "" - echo "=== ${component} Pod Events ===" - kubectl -n kube-system get pods -l "${label}" -o jsonpath='{range .items[*]}{.metadata.name}{"\n"}{end}' | \ + + echo "=== Node Scheduling Summary ===" + run-debug-command kubectl get nodes -o wide + run-debug-command kubectl get nodes \ + -o custom-columns='NAME:.metadata.name,READY:.status.conditions[?(@.type=="Ready")].status,UNSCHEDULABLE:.spec.unschedulable,ROLES:.metadata.labels.node-role\.kubernetes\.io/control-plane,TAINTS:.spec.taints' + echo "" + + echo "=== ${component} Pod Scheduling Details ===" + if pods=$(kubectl -n kube-system get pods -l "${label}" -o jsonpath='{range .items[*]}{.metadata.name}{"\n"}{end}'); then + if [[ -z "$pods" ]]; then + echo " (No matching pods found for ${label})" + else + while read -r pod; do + [[ -z "$pod" ]] && continue + echo "Pod: $pod" + run-debug-command kubectl -n kube-system get pod "$pod" \ + -o custom-columns='NAME:.metadata.name,PHASE:.status.phase,NODE:.spec.nodeName,NODE-SELECTOR:.spec.nodeSelector,TOLERATIONS:.spec.tolerations,AFFINITY:.spec.affinity,CONDITIONS:.status.conditions' + echo "" + done <<< "$pods" + fi + else + echo " (Failed to list matching pods for ${label})" + pods="" + fi + + echo "=== ${component} Pod Events and Descriptions ===" + if [[ -n "$pods" ]]; then while read -r pod; do + [[ -z "$pod" ]] && continue echo "Pod: $pod" - kubectl -n kube-system describe pod "$pod" | grep -A 20 "Events:" || echo " (No events found)" + run-debug-command kubectl -n kube-system describe pod "$pod" echo "" - done + done <<< "$pods" + fi + + echo "=== Recent Cluster Events ===" + run-debug-command kubectl get events --all-namespaces --sort-by=.metadata.creationTimestamp + echo "" + + echo "=== cloudscale CCM Status ===" + run-debug-command kubectl -n kube-system get daemonset cloudscale-cloud-controller-manager -o wide + run-debug-command kubectl -n kube-system describe daemonset cloudscale-cloud-controller-manager + run-debug-command kubectl -n kube-system get pods -o wide + echo "" + + echo "=== cloudscale CCM Logs ===" + if ccm_selector=$(kubectl -n kube-system get daemonset cloudscale-cloud-controller-manager -o json | \ + jq -r '.spec.selector.matchLabels | to_entries | map("\(.key)=\(.value)") | join(",")') && [[ -n "$ccm_selector" ]]; then + run-debug-command kubectl -n kube-system logs -l "$ccm_selector" \ + --all-containers=true --prefix=true --tail=-1 --max-log-requests=20 + echo "" + echo "=== cloudscale CCM Previous Logs (if available) ===" + run-debug-command kubectl -n kube-system logs -l "$ccm_selector" \ + --all-containers=true --prefix=true --previous --tail=-1 --max-log-requests=20 + else + echo " (Failed to determine the cloudscale CCM pod selector)" + fi + echo "" + echo "=== ${component} Pod Status ===" - kubectl -n kube-system get pods -l "${label}" -o wide + run-debug-command kubectl -n kube-system get pods -l "${label}" -o wide echo "" echo "=== ${component} Container Images ===" - kubectl -n kube-system get pods -l "${label}" \ + run-debug-command kubectl -n kube-system get pods -l "${label}" \ -o jsonpath='{range .items[*]}{.metadata.name}{"\n"}{range .status.containerStatuses[*]} Container: {.name}, Image: {.image}, Ready: {.ready}, RestartCount: {.restartCount}{"\n"}{end}{"\n"}{end}' echo "" - echo "=== ${component} Pod Full Description ===" - kubectl -n kube-system describe pods -l "${label}" - echo "" echo "=== CSI Images on All Nodes ===" - list-csi-images + run-debug-command list-csi-images +} + +# Create or update the Secret containing the custom cloudscale API CA. Callers +# must only invoke this when CLOUDSCALE_CA_PEM is non-empty. +function deploy-custom-ca-secret() ( + if [[ -z "${CLOUDSCALE_CA_PEM:-}" ]]; then + return 0 + fi + + local ca_file + ca_file=$(mktemp) + trap 'rm -f "$ca_file"' EXIT + + chmod 0600 "$ca_file" + printf '%s\n' "$CLOUDSCALE_CA_PEM" > "$ca_file" + + kubectl -n kube-system create secret generic "$CUSTOM_CA_SECRET_NAME" \ + --from-file=ca.pem="$ca_file" \ + --dry-run=client \ + -o yaml | kubectl apply -f - +) + +# Extend a container's system trust store with the custom cloudscale API CA. +function mount-custom-ca() { + if [[ -z "${CLOUDSCALE_CA_PEM:-}" ]]; then + return 0 + fi + + local resource="$1" + local container="$2" + local patch + + patch="spec: + template: + spec: + containers: + - name: ${container} + volumeMounts: + - name: ${CUSTOM_CA_VOLUME_NAME} + mountPath: ${CUSTOM_CA_MOUNT_PATH} + subPath: ca.pem + readOnly: true + volumes: + - name: ${CUSTOM_CA_VOLUME_NAME} + secret: + secretName: ${CUSTOM_CA_SECRET_NAME}" + + kubectl -n kube-system patch "$resource" --type=strategic --patch "$patch" } # Deploy CCM from official GitHub release @@ -184,6 +291,16 @@ function deploy-ccm() { sed "s|https://api.cloudscale.ch/|${api_url}/|g" | \ kubectl apply -f - + if [[ -n "${CLOUDSCALE_CA_PEM:-}" ]]; then + echo "Configuring custom CA for the cloudscale CCM..." + deploy-custom-ca-secret + mount-custom-ca \ + daemonset/cloudscale-cloud-controller-manager \ + cloudscale-cloud-controller-manager + kubectl -n kube-system rollout restart \ + daemonset/cloudscale-cloud-controller-manager + fi + echo "Waiting for CCM to be ready..." kubectl -n kube-system rollout status daemonset/cloudscale-cloud-controller-manager --timeout=120s } @@ -246,6 +363,17 @@ function deploy-csi() { # Apply the manifest kubectl apply -n kube-system -f "$manifest_file" + if [[ -n "${CLOUDSCALE_CA_PEM:-}" ]]; then + echo "Configuring custom CA for the cloudscale CSI plugin..." + deploy-custom-ca-secret + mount-custom-ca \ + statefulset/csi-cloudscale-controller \ + csi-cloudscale-plugin + mount-custom-ca \ + daemonset/csi-cloudscale-node \ + csi-cloudscale-plugin + fi + if [[ "$csi_installed" == "yes" ]]; then echo "" echo "CSI was already installed, triggering rollout restart..."