diff --git a/test/bin/c2cc_common.sh b/test/bin/c2cc_common.sh index a319e9d2a9..1a998a8005 100644 --- a/test/bin/c2cc_common.sh +++ b/test/bin/c2cc_common.sh @@ -172,6 +172,16 @@ c2cc_run_tests() { ip_family_var="--variable IP_FAMILY:${ip_family}" fi + local target_ref_var="" + if [ -n "${C2CC_TARGET_REF:-}" ]; then + target_ref_var="--variable TARGET_REF:${C2CC_TARGET_REF}" + fi + + local bootc_registry_var="" + if [ -n "${MIRROR_REGISTRY_URL:-}" ]; then + bootc_registry_var="--variable BOOTC_REGISTRY:${MIRROR_REGISTRY_URL}" + fi + local host2_ip host3_ip host2_ip=$(get_host_ip host2) || return 1 host3_ip=$(get_host_ip host3) || return 1 @@ -207,6 +217,8 @@ c2cc_run_tests() { --variable "KUBECONFIG_C:${kubeconfig_c}" \ ${foreign_cidr_var} \ ${ip_family_var} \ + ${target_ref_var} \ + ${bootc_registry_var} \ "${suites_dir}" } diff --git a/test/image-blueprints-bootc/el10/layer1-base/group1/rhel102-test-agent.containerfile b/test/image-blueprints-bootc/el10/layer1-base/group1/rhel102-test-agent.containerfile index 9441297f95..1ebbd33fd5 100644 --- a/test/image-blueprints-bootc/el10/layer1-base/group1/rhel102-test-agent.containerfile +++ b/test/image-blueprints-bootc/el10/layer1-base/group1/rhel102-test-agent.containerfile @@ -27,4 +27,3 @@ RUN dnf install -y microshift-test-agent pcp pcp-zeroconf && \ rm -vf /etc/yum.repos.d/microshift-*.repo && \ rm -rvf $USHIFT_RPM_REPO_PATH && \ dnf clean all - diff --git a/test/image-blueprints-bootc/el10/layer2-presubmit/group1/rhel102-bootc-source.containerfile b/test/image-blueprints-bootc/el10/layer2-presubmit/group1/rhel102-bootc-source.containerfile index b846b1a971..8fca234289 100644 --- a/test/image-blueprints-bootc/el10/layer2-presubmit/group1/rhel102-bootc-source.containerfile +++ b/test/image-blueprints-bootc/el10/layer2-presubmit/group1/rhel102-bootc-source.containerfile @@ -43,3 +43,10 @@ RUN firewall-offline-cmd --zone=public --add-port=22/tcp && \ # when upgrading from older ostree commits to bootc container layers RUN mkdir -p /usr/lib/systemd/system/ovsdb-server.service.d COPY --chmod=644 ./bootc-images/microshift-ovsdb-ownership.conf /usr/lib/systemd/system/ovsdb-server.service.d/microshift-ovsdb-ownership.conf + +# Fix SSH host key permissions for cross-version upgrades (RHEL 9 uses 0640 +# with ssh_keys group, RHEL 10 sshd requires 0600) +# Similar workaround is used in microshift-test-agent.sh +RUN mkdir -p /usr/lib/systemd/system/sshd.service.d && \ + printf '[Service]\nExecStartPre=/bin/bash -c "chmod 600 /etc/ssh/ssh_host_*_key 2>/dev/null || true"\n' \ + > /usr/lib/systemd/system/sshd.service.d/fix-hostkey-perms.conf diff --git a/test/image-blueprints-bootc/templates/rpm-repo-config.sh.template b/test/image-blueprints-bootc/templates/rpm-repo-config.sh.template index 298eb3ec73..ef41e32902 100644 --- a/test/image-blueprints-bootc/templates/rpm-repo-config.sh.template +++ b/test/image-blueprints-bootc/templates/rpm-repo-config.sh.template @@ -12,6 +12,16 @@ function enable_eus_repositories() { "rhel-${vmajor}-for-$(uname -m)-appstream-eus-rpms" } +function enable_ga_repositories() { + local -r vmajor="$(awk -F. '{print $1}' <<< "${VERSION_ID}")" + + dnf config-manager --set-disabled '*' + dnf config-manager --set-enabled \ + "rhel-${vmajor}-for-$(uname -m)-baseos-rpms" \ + "rhel-${vmajor}-for-$(uname -m)-appstream-rpms" +} + + # Lock the OS release version to prevent the installation of packages belonging # to a newer OS release source /etc/os-release @@ -24,6 +34,10 @@ while [ $# -gt 0 ] ; do enable_eus_repositories shift ;; + --enable-ga) + enable_ga_repositories + shift + ;; --disable-all) dnf config-manager --set-disabled '*' shift diff --git a/test/resources/c2cc.resource b/test/resources/c2cc.resource index c47526a22e..4bbd96d557 100644 --- a/test/resources/c2cc.resource +++ b/test/resources/c2cc.resource @@ -77,7 +77,7 @@ Teardown All Remote Clusters END VAR @{C2CC_REMOTE_ALIASES}= @{EMPTY} scope=SUITE ${local_conn}= Get From Dictionary ${C2CC_SSH_IDS} cluster-a - SSHLibrary.Switch Connection ${local_conn} + ${status}= Run Keyword And Return Status SSHLibrary.Switch Connection ${local_conn} Command On Cluster [Documentation] Run a shell command on the specified cluster via SSH. @@ -384,6 +384,26 @@ DNS Lookup Should Succeed ... oc exec curl-pod -n ${NAMESPACES}[${alias}] -- getent hosts ${fqdn} Should Not Be Empty ${stdout} +Verify RemoteCluster State + [Documentation] Check that all RemoteCluster CRs on this cluster have the expected state. + [Arguments] ${alias} ${expected_state} + ${stdout}= Oc On Cluster ${alias} + ... oc get remoteclusters.microshift.io -o jsonpath='{.items[*].status.state}' + Should Not Be Empty ${stdout} + @{states}= Split String ${stdout} + ${count}= Get Length ${states} + Should Be Equal As Integers ${count} 2 Expected 2 RemoteCluster states, got ${count} + FOR ${state} IN @{states} + Should Be Equal As Strings ${state} ${expected_state} + END + +Verify All RemoteClusters Healthy + [Documentation] Wait for all RemoteCluster CRs on all clusters to report Healthy. + FOR ${alias} IN cluster-a cluster-b cluster-c + Wait Until Keyword Succeeds 3m 10s + ... Verify RemoteCluster State ${alias} Healthy + END + Curl DNS From Cluster [Documentation] Curl a service by DNS name from curl-pod on the given cluster. [Arguments] ${alias} ${fqdn} ${port} diff --git a/test/scenarios-bootc/el10/presubmits/el98-src@el102-src@c2cc-upgrade-ok.sh b/test/scenarios-bootc/el10/presubmits/el98-src@el102-src@c2cc-upgrade-ok.sh new file mode 100755 index 0000000000..d083dd6e98 --- /dev/null +++ b/test/scenarios-bootc/el10/presubmits/el98-src@el102-src@c2cc-upgrade-ok.sh @@ -0,0 +1,25 @@ +#!/bin/bash + +# Sourced from scenario.sh and uses functions defined there. + +# shellcheck source=test/bin/c2cc_common.sh +source "${SCRIPTDIR}/c2cc_common.sh" + +export TEST_RANDOMIZATION=none +export TEST_EXECUTION_TIMEOUT=60m + +C2CC_TARGET_REF=rhel102-bootc-source + +scenario_create_vms() { + c2cc_create_vms rhel98-bootc-source rhel98-bootc +} + +scenario_remove_vms() { + c2cc_remove_vms +} + +scenario_run_tests() { + # shellcheck disable=SC2119 + configure_c2cc_hosts + c2cc_run_tests "suites/upgrade/upgrade-c2cc.robot" +} diff --git a/test/suites/c2cc/probe.robot b/test/suites/c2cc/probe.robot index 48c029a283..2230c590c5 100644 --- a/test/suites/c2cc/probe.robot +++ b/test/suites/c2cc/probe.robot @@ -151,19 +151,6 @@ Verify Probe Service ClusterIP ... oc get service ${PROBE_DEPLOYMENT} -n ${C2CC_NAMESPACE} -o jsonpath='{.spec.clusterIP}' Should Be Equal As Strings ${actual_ip} ${expected_ip} strip_spaces=True -Verify RemoteCluster State - [Documentation] Check that all RemoteCluster CRs on this cluster have the expected state. - [Arguments] ${alias} ${expected_state} - ${stdout}= Oc On Cluster ${alias} - ... oc get remoteclusters.microshift.io -o jsonpath='{.items[*].status.state}' - Should Not Be Empty ${stdout} - @{states}= Split String ${stdout} - ${count}= Get Length ${states} - Should Be Equal As Integers ${count} 2 Expected 2 RemoteCluster states, got ${count} - FOR ${state} IN @{states} - Should Be Equal As Strings ${state} ${expected_state} - END - Verify RemoteCluster State By Name [Documentation] Check that a specific RemoteCluster CR has the expected state. [Arguments] ${alias} ${cr_name} ${expected_state} @@ -189,10 +176,7 @@ RemoteCluster CR Name From IP Ensure All Clusters Healthy [Documentation] Pre-condition: all clusters must be Healthy before fault injection. - FOR ${alias} IN cluster-a cluster-b cluster-c - Wait Until Keyword Succeeds 3m 10s - ... Verify RemoteCluster State ${alias} Healthy - END + Verify All RemoteClusters Healthy Apply Probe Deny Policy [Documentation] Apply a NetworkPolicy that denies all ingress to the probe pod. diff --git a/test/suites/upgrade/upgrade-c2cc.robot b/test/suites/upgrade/upgrade-c2cc.robot new file mode 100644 index 0000000000..f05830873f --- /dev/null +++ b/test/suites/upgrade/upgrade-c2cc.robot @@ -0,0 +1,170 @@ +*** Settings *** +Documentation Tests RHEL 9.8 to RHEL 10.2 upgrade with C2CC enabled across 3 clusters. +... Upgrades each cluster one by one and verifies C2CC connectivity +... survives at each stage. + +Resource ../../resources/common.resource +Resource ../../resources/c2cc.resource +Resource ../../resources/microshift-host.resource +Library Collections +Library SSHLibrary + +Suite Setup Setup +Suite Teardown Teardown + +Test Tags c2cc ostree + + +*** Variables *** +${TARGET_REF} ${EMPTY} +${BOOTC_REGISTRY} ${EMPTY} +${CLUSTER_A_POD_CIDR} ${EMPTY} +${CLUSTER_A_SVC_CIDR} ${EMPTY} +${CLUSTER_A_DOMAIN} ${EMPTY} +${CLUSTER_B_POD_CIDR} ${EMPTY} +${CLUSTER_B_SVC_CIDR} ${EMPTY} +${CLUSTER_B_DOMAIN} ${EMPTY} +${KUBECONFIG_B} ${EMPTY} +${CLUSTER_C_POD_CIDR} ${EMPTY} +${CLUSTER_C_SVC_CIDR} ${EMPTY} +${CLUSTER_C_DOMAIN} ${EMPTY} +${KUBECONFIG_C} ${EMPTY} + + +*** Test Cases *** +Upgrade C2CC Clusters From RHEL9 To RHEL10 + [Documentation] Upgrades 3 C2CC-connected clusters one by one from RHEL 9.8 + ... to RHEL 10.2 and verifies health and C2CC connectivity after each upgrade. + + Verify All Clusters Healthy + Verify All RemoteClusters Healthy + Deploy Test Workloads + Verify Full C2CC Connectivity + + FOR ${alias} IN cluster-a cluster-b cluster-c + Log To Console Upgrading ${alias} to ${TARGET_REF} + Upgrade Cluster ${alias} + Verify All Clusters Healthy + Verify All RemoteClusters Healthy + Wait For Test Pods + Wait For Service Endpoints + Verify Full C2CC Connectivity + END + + [Teardown] Cleanup Test Workloads + + +*** Keywords *** +Setup + [Documentation] Register all three clusters for SSH and oc access + ... and store connection details for reconnection after reboots. + Check Required Env Variables + Should Not Be Empty ${TARGET_REF} TARGET_REF variable is required + Login MicroShift Host + Setup Kubeconfig + Logout MicroShift Host + + Register Remote Cluster cluster-a ${USHIFT_HOST} ${SSH_PORT} ${KUBECONFIG} + Register Remote Cluster cluster-b ${HOST2_IP} ${HOST2_SSH_PORT} ${KUBECONFIG_B} + Register Remote Cluster cluster-c ${HOST3_IP} ${HOST3_SSH_PORT} ${KUBECONFIG_C} + +Teardown + [Documentation] Close all connections and clean up. + Teardown All Remote Clusters + Remove Kubeconfig + +Verify All Clusters Healthy + [Documentation] Verify all clusters are running and API server is reachable. + FOR ${alias} IN cluster-a cluster-b cluster-c + ${stdout}= Oc On Cluster ${alias} oc get --raw='/readyz' + Should Be Equal As Strings ${stdout} ok strip_spaces=True + END + +Verify Full C2CC Connectivity + [Documentation] Verify pod-to-pod and pod-to-service connectivity between all cluster pairs. + @{clusters}= Create List cluster-a cluster-b cluster-c + FOR ${src} IN @{clusters} + FOR ${dst} IN @{clusters} + IF '${src}' != '${dst}' + Test Connectivity Between Clusters ${src} ${dst} pod + Test Connectivity Between Clusters ${src} ${dst} service + END + END + END + +Upgrade Cluster + [Documentation] Upgrade a specific cluster to the target bootc image + ... and verify it booted into the new deployment without rollback. + [Arguments] ${alias} + + ${initial_deploy_id}= Get Deployment Id On Cluster ${alias} + + Command On Cluster + ... ${alias} + ... printf '[[registry]]\nlocation = "${BOOTC_REGISTRY}"\ninsecure = true\n' | sudo tee /etc/containers/registries.conf.d/999-microshift-insecure-registry.conf > /dev/null + + Command On Cluster ${alias} bootc switch --quiet ${BOOTC_REGISTRY}/${TARGET_REF} + + Command On Cluster ${alias} + ... rm -f /etc/containers/registries.conf.d/999-microshift-insecure-registry.conf + + Reboot Cluster And Wait ${alias} + + ${current_deploy_id}= Get Deployment Id On Cluster ${alias} + Should Not Be Equal As Strings ${current_deploy_id} ${initial_deploy_id} + ... msg=${alias} rolled back to initial deployment + +Get Deployment Id On Cluster + [Documentation] Get the booted image digest from a specific cluster. + [Arguments] ${alias} + ${stdout}= Command On Cluster + ... ${alias} + ... bootc status --booted --json | python3 -c "import sys,json; print(json.load(sys.stdin)['status']['booted']['image']['imageDigest'])" + RETURN ${stdout} + +Reboot Cluster And Wait + [Documentation] Reboot a cluster and wait for it to come back with greenboot healthy. + [Arguments] ${alias} + + ${boot_id}= Command On Cluster ${alias} + ... cat /proc/sys/kernel/random/boot_id sudo_mode=False + + Disruptive Command On Cluster ${alias} reboot + + Wait Until Keyword Succeeds 10m 15s + ... Cluster Rebooted And Healthy ${alias} ${boot_id} + +Cluster Rebooted And Healthy + [Documentation] Verify cluster has rebooted and greenboot health check passed. + [Arguments] ${alias} ${old_boot_id} + + ${old_conn_id}= Get From Dictionary ${C2CC_SSH_IDS} ${alias} + ${status}= Run Keyword And Return Status + ... SSHLibrary.Switch Connection ${old_conn_id} + IF ${status} SSHLibrary.Close Connection + + ${host} ${port} ${kc}= Get Cluster Connection Info ${alias} + Remove Values From List ${C2CC_REMOTE_ALIASES} ${alias} + Register Remote Cluster ${alias} ${host} ${port} ${kc} + + ${new_boot_id}= Command On Cluster ${alias} + ... cat /proc/sys/kernel/random/boot_id sudo_mode=False + Should Not Be Equal As Strings ${old_boot_id} ${new_boot_id} strip_spaces=True + + ${stdout}= Command On Cluster ${alias} + ... systemctl show -p SubState greenboot-healthcheck.service --value + Should Be Equal As Strings ${stdout} exited strip_spaces=True + +Get Cluster Connection Info + [Documentation] Return host, port, and kubeconfig for a given cluster alias. + [Arguments] ${alias} + IF '${alias}' == 'cluster-a' + RETURN ${USHIFT_HOST} ${SSH_PORT} ${KUBECONFIG} + ELSE IF '${alias}' == 'cluster-b' + RETURN ${HOST2_IP} ${HOST2_SSH_PORT} ${KUBECONFIG_B} + ELSE IF '${alias}' == 'cluster-c' + RETURN ${HOST3_IP} ${HOST3_SSH_PORT} ${KUBECONFIG_C} + ELSE + Fail Unknown cluster alias: ${alias} + END +