From 4a82d26f89d55df64eae8eb27dae2a16fd0265b3 Mon Sep 17 00:00:00 2001 From: Roy Kaufman Date: Sun, 18 Jan 2026 18:12:22 +0200 Subject: [PATCH 1/6] Enable trusted_execution_cluster tests on OpenShift This commit updates the test utils to support running trusted execution cluster tests on the OpenShift platform. To execute these tests on OpenShift, the following environment variables must be configured: - REGISTRY: The repository location of the container image. - TAG: The specific tag of the container image. - CLUSTER_URL: The API URL of the target cluster. - PLATFORM: Set this to 'openshift'. Signed-off-by: Roy Kaufman --- Cargo.lock | 7 ++++++ test_utils/Cargo.toml | 1 + test_utils/src/lib.rs | 55 ++++++++++++++++++++++++------------------- 3 files changed, 39 insertions(+), 24 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 295482fd..f1d5f49c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -973,6 +973,12 @@ dependencies = [ "percent-encoding", ] +[[package]] +name = "fs_extra" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c" + [[package]] name = "futures" version = "0.3.31" @@ -3800,6 +3806,7 @@ dependencies = [ "clevis-pin-trustee-lib", "compute-pcrs-lib", "env_logger", + "fs_extra", "http 1.4.0", "ignition-config", "k8s-openapi", diff --git a/test_utils/Cargo.toml b/test_utils/Cargo.toml index 8aaf4164..82913e09 100644 --- a/test_utils/Cargo.toml +++ b/test_utils/Cargo.toml @@ -31,3 +31,4 @@ tokio = { workspace = true, features = ["process"] } tower = { version = "0.5.2", features = ["full"] } uuid.workspace = true which = "8.0" +fs_extra = "1.3.0" diff --git a/test_utils/src/lib.rs b/test_utils/src/lib.rs index 60bd4d87..91c4107a 100644 --- a/test_utils/src/lib.rs +++ b/test_utils/src/lib.rs @@ -2,6 +2,7 @@ // // SPDX-License-Identifier: MIT +use fs_extra::dir; use k8s_openapi::api::apps::v1::Deployment; use k8s_openapi::api::core::v1::{ConfigMap, Namespace}; use kube::api::DeleteParams; @@ -260,9 +261,11 @@ impl TestContext { ); let crd_temp_dir = Path::new(&self.manifests_dir).join("crd"); + let rbac_dir = workspace_root.join("config/rbac/"); + let options = dir::CopyOptions::new(); + dir::copy(rbac_dir, &self.manifests_dir, &options)?; let rbac_temp_dir = Path::new(&self.manifests_dir).join("rbac"); std::fs::create_dir_all(&crd_temp_dir)?; - std::fs::create_dir_all(&rbac_temp_dir)?; let crd_temp_dir_str = crd_temp_dir .to_str() @@ -300,7 +303,8 @@ impl TestContext { trusted_cluster_gen_path.display() )); } - + let repo = std::env::var("REGISTRY").unwrap_or_else(|_| "localhost:5000".to_string()); + let tag = std::env::var("TAG").unwrap_or_else(|_| "latest".to_string()); let manifest_gen_output = Command::new(&trusted_cluster_gen_path) .args([ "-namespace", @@ -308,21 +312,20 @@ impl TestContext { "-output-dir", &self.manifests_dir, "-image", - "localhost:5000/trusted-execution-clusters/trusted-cluster-operator:latest", + &format!("{repo}/trusted-cluster-operator:{tag}"), "-pcrs-compute-image", - "localhost:5000/trusted-execution-clusters/compute-pcrs:latest", + &format!("{repo}/compute-pcrs:{tag}"), "-trustee-image", "quay.io/trusted-execution-clusters/key-broker-service:20260106", "-register-server-image", - "localhost:5000/trusted-execution-clusters/registration-server:latest", + &format!("{repo}/registration-server:{tag}"), "-attestation-key-register-image", - "localhost:5000/trusted-execution-clusters/attestation-key-register:latest", + &format!("{repo}/attestation-key-register:{tag}"), "-approved-image", "quay.io/trusted-execution-clusters/fedora-coreos@sha256:79a0657399e6c67c7c95b8a09193d18e5675b5aa3cfb4d75ea5c8d4d53b2af74" ]) .output() .await?; - if !manifest_gen_output.status.success() { let stderr = String::from_utf8_lossy(&manifest_gen_output.stderr); return Err(anyhow::anyhow!("Failed to generate manifests: {stderr}")); @@ -395,25 +398,27 @@ impl TestContext { std::fs::write(&le_rb_dst, le_rb_content)?; test_info!(&self.test_name, "Preparing RBAC kustomization"); - let kustomization_content = format!( - r#"# SPDX-FileCopyrightText: Generated for testing -# SPDX-License-Identifier: CC0-1.0 - -namespace: {} - -resources: - - service_account.yaml - - role.yaml - - role_binding.yaml - - leader_election_role.yaml - - leader_election_role_binding.yaml -"#, - ns - ); - + let platform = std::env::var("PLATFORM").unwrap_or_else(|_| "kind".to_string()); + let kustomization_src = workspace_root.join("config/rbac/kustomization.yaml.in"); + let kustomization_content = std::fs::read_to_string(&kustomization_src)? + .replace("namespace: NAMESPACE", &format!("namespace: {}", ns)) + .replace( + "resources:", + if platform == "openshift" { + "resources:\n - scc.yaml" + } else { + "resources:" + }, + ); let temp_kustomization_path = rbac_temp_dir.join("kustomization.yaml"); std::fs::write(&temp_kustomization_path, kustomization_content)?; + let scc_openshift_rb_src = workspace_root.join("config/openshift/scc.yaml"); + let scc_openshift_rb_content = + std::fs::read_to_string(&scc_openshift_rb_src)?.replace("", &ns); + let scc_openshift_rb_dst = rbac_temp_dir.join("scc.yaml"); + std::fs::write(&scc_openshift_rb_dst, scc_openshift_rb_content)?; + kube_apply!( rbac_temp_dir_str, &self.test_name, @@ -436,7 +441,9 @@ resources: &self.test_name, "Updating CR manifest with publicTrusteeAddr" ); - let trustee_addr = format!("kbs-service.{}.svc.cluster.local:8080", ns); + let cluster_url = + std::env::var("CLUSTER_URL").unwrap_or_else(|_| "svc.cluster.local".to_string()); + let trustee_addr = format!("kbs-service.{}.{}:8080", ns, cluster_url); let cr_manifest_path = manifests_path.join("trusted_execution_cluster_cr.yaml"); let cr_content = std::fs::read_to_string(&cr_manifest_path)?; From 62b8735f16cc5bb63f7fd0c668d8e38f003b4757 Mon Sep 17 00:00:00 2001 From: Jakob Naucke Date: Thu, 22 Jan 2026 14:33:49 +0100 Subject: [PATCH 2/6] reg: Do not delete Machines based on IP Does not work for NAT-based networks. Signed-off-by: Jakob Naucke --- register-server/src/main.rs | 16 +--------------- 1 file changed, 1 insertion(+), 15 deletions(-) diff --git a/register-server/src/main.rs b/register-server/src/main.rs index 148b1974..e7102bff 100644 --- a/register-server/src/main.rs +++ b/register-server/src/main.rs @@ -143,21 +143,6 @@ async fn register_handler(remote_addr: Option) -> Result anyhow::Result<()> { - let machines: Api = Api::default_namespaced(client); - - // Check for existing machines with the same IP - let machine_list = machines.list(&Default::default()).await?; - - for existing_machine in machine_list.items { - if existing_machine.spec.registration_address == client_ip { - if let Some(name) = &existing_machine.metadata.name { - info!("Found existing machine {name} with IP {client_ip}, deleting..."); - machines.delete(name, &Default::default()).await?; - info!("Deleted existing machine: {name}"); - } - } - } - let machine_name = format!("machine-{uuid}"); let machine = Machine { metadata: ObjectMeta { @@ -171,6 +156,7 @@ async fn create_machine(client: Client, uuid: &str, client_ip: &str) -> anyhow:: status: None, }; + let machines: Api = Api::default_namespaced(client); machines.create(&Default::default(), &machine).await?; info!("Created Machine: {machine_name} with IP: {client_ip}"); Ok(()) From 0dc1526eda381c1c8f2df70227863ec6096f94d8 Mon Sep 17 00:00:00 2001 From: Jakob Naucke Date: Mon, 2 Feb 2026 13:17:12 +0100 Subject: [PATCH 3/6] rego: Update Azure attestation policy - Field is now `az-snp-vtpm` (which requires quoting due to the dashes) instead of `azsnpvtpm` - No lowercasing required Signed-off-by: Jakob Naucke --- operator/src/tpm.rego | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/operator/src/tpm.rego b/operator/src/tpm.rego index dba1c3f9..abf6ea6d 100644 --- a/operator/src/tpm.rego +++ b/operator/src/tpm.rego @@ -12,8 +12,8 @@ executables := 3 if { } # Azure SNP vTPM validation executables := 3 if { - lower(input.azsnpvtpm.tpm.pcr04) in query_reference_value("tpm_pcr4") - lower(input.azsnpvtpm.tpm.pcr14) in query_reference_value("tpm_pcr14") + input["az-snp-vtpm"].tpm.pcr04 in query_reference_value("tpm_pcr4") + input["az-snp-vtpm"].tpm.pcr14 in query_reference_value("tpm_pcr14") } default configuration := 0 From f08d4b60e22927f2532d49618582a9124ea9b288 Mon Sep 17 00:00:00 2001 From: Jakob Naucke Date: Tue, 20 Jan 2026 09:19:33 +0100 Subject: [PATCH 4/6] tests: Make Kubevirt one of multiple backends Split test_utils::virt into a module with separate virt::kubevirt. Introduce the VIRT_PROVIDER variable to select. - Reformat a bit. - Make root_key an Option for environments where machine IDs cannot be correlated to IPs. Signed-off-by: Jakob Naucke Assisted-by: Claude --- Cargo.lock | 25 ++ test_utils/Cargo.toml | 4 +- test_utils/src/lib.rs | 26 +- test_utils/src/virt.rs | 418 -------------------------------- test_utils/src/virt/kubevirt.rs | 164 +++++++++++++ test_utils/src/virt/mod.rs | 278 +++++++++++++++++++++ tests/Cargo.toml | 5 +- tests/attestation.rs | 245 ++++++------------- 8 files changed, 569 insertions(+), 596 deletions(-) delete mode 100644 test_utils/src/virt.rs create mode 100644 test_utils/src/virt/kubevirt.rs create mode 100644 test_utils/src/virt/mod.rs diff --git a/Cargo.lock b/Cargo.lock index f1d5f49c..256245aa 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -157,6 +157,17 @@ dependencies = [ "syn 2.0.110", ] +[[package]] +name = "async-trait" +version = "0.1.89" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9035ad2d096bed7955a320ee7e2230574d28fd3c3a0f186cbea1ff3c7eed5dbb" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.110", +] + [[package]] name = "atomic-waker" version = "1.1.2" @@ -181,6 +192,17 @@ dependencies = [ "warp", ] +[[package]] +name = "auto_impl" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ffdcb70bdbc4d478427380519163274ac86e52916e10f0a8889adf0f96d3fee7" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.110", +] + [[package]] name = "autocfg" version = "1.5.0" @@ -3803,6 +3825,8 @@ name = "trusted-cluster-operator-test-utils" version = "0.1.0" dependencies = [ "anyhow", + "async-trait", + "auto_impl", "clevis-pin-trustee-lib", "compute-pcrs-lib", "env_logger", @@ -3829,6 +3853,7 @@ name = "trusted-cluster-operator-tests" version = "0.1.0" dependencies = [ "anyhow", + "cfg-if", "compute-pcrs-lib", "k8s-openapi", "kube", diff --git a/test_utils/Cargo.toml b/test_utils/Cargo.toml index 82913e09..a3661779 100644 --- a/test_utils/Cargo.toml +++ b/test_utils/Cargo.toml @@ -13,8 +13,9 @@ virtualization = [] [dependencies] anyhow.workspace = true +async-trait = "0.1" +auto_impl = "1" clevis-pin-trustee-lib.workspace = true -trusted-cluster-operator-lib = { path = "../lib" } compute-pcrs-lib.workspace = true env_logger.workspace = true http.workspace = true @@ -29,6 +30,7 @@ serde_yaml.workspace = true ssh-key = { version = "0.6", features = ["rsa", "std"] } tokio = { workspace = true, features = ["process"] } tower = { version = "0.5.2", features = ["full"] } +trusted-cluster-operator-lib = { path = "../lib" } uuid.workspace = true which = "8.0" fs_extra = "1.3.0" diff --git a/test_utils/src/lib.rs b/test_utils/src/lib.rs index 91c4107a..d4826114 100644 --- a/test_utils/src/lib.rs +++ b/test_utils/src/lib.rs @@ -2,6 +2,7 @@ // // SPDX-License-Identifier: MIT +use anyhow::anyhow; use fs_extra::dir; use k8s_openapi::api::apps::v1::Deployment; use k8s_openapi::api::core::v1::{ConfigMap, Namespace}; @@ -22,6 +23,9 @@ pub mod virt; use compute_pcrs_lib::Pcr; +const PLATFORM_ENV: &str = "PLATFORM"; +const ANSI_RESET: &str = "\x1b[0m"; + pub fn compare_pcrs(actual: &[Pcr], expected: &[Pcr]) -> bool { if actual.len() != expected.len() { return false; @@ -40,8 +44,15 @@ pub fn compare_pcrs(actual: &[Pcr], expected: &[Pcr]) -> bool { macro_rules! test_info { ($test_name:expr, $($arg:tt)*) => {{ const GREEN: &str = "\x1b[32m"; - const RESET: &str = "\x1b[0m"; - println!("{}INFO{}: {}: {}", GREEN, RESET, $test_name, format!($($arg)*)); + println!("{}INFO{}: {}: {}", GREEN, ANSI_RESET, $test_name, format!($($arg)*)); + }} +} + +#[macro_export] +macro_rules! test_warn { + ($test_name:expr, $($arg:tt)*) => {{ + const YELLOW: &str = "\x1b[33m"; + println!("{}WARN{}: {}: {}", YELLOW, ANSI_RESET, $test_name, format!($($arg)*)); }} } @@ -73,6 +84,11 @@ macro_rules! kube_apply { } } +pub fn ensure_command(name: &str) -> anyhow::Result<()> { + let result = which::which(name).map(|_| ()); + result.map_err(|_| anyhow!("Command {name} not found. Please install {name} first.")) +} + static INIT: Once = Once::new(); pub struct TestContext { @@ -126,7 +142,11 @@ impl TestContext { test_info!(&self.test_name, "{}", message); } - pub async fn cleanup(&self) -> anyhow::Result<()> { + pub fn warn(&self, message: impl std::fmt::Display) { + test_warn!(&self.test_name, "{}", message); + } + + pub async fn cleanup(&self) -> Result<()> { self.cleanup_namespace().await?; self.cleanup_manifests_dir()?; Ok(()) diff --git a/test_utils/src/virt.rs b/test_utils/src/virt.rs deleted file mode 100644 index 5a76574d..00000000 --- a/test_utils/src/virt.rs +++ /dev/null @@ -1,418 +0,0 @@ -// SPDX-FileCopyrightText: Alice Frosi -// SPDX-FileCopyrightText: Jakob Naucke -// -// SPDX-License-Identifier: MIT - -use clevis_pin_trustee_lib::Key as ClevisKey; -use ignition_config::v3_5::{ - Config, Dropin, File, Ignition, IgnitionConfig, Passwd, Resource, Storage, Systemd, Unit, User, -}; -use k8s_openapi::apimachinery::pkg::util::intstr::IntOrString; -use kube::api::ObjectMeta; -use kube::{Api, Client}; -use std::collections::BTreeMap; -use std::path::Path; -use std::time::Duration; -use tokio::process::Command; -use trusted_cluster_operator_lib::virtualmachines::*; - -use super::Poller; - -pub fn generate_ssh_key_pair() -> anyhow::Result<(String, String, std::path::PathBuf)> { - use rand_core::OsRng; - use ssh_key::{Algorithm, LineEnding, PrivateKey}; - use std::fs; - use std::os::unix::fs::PermissionsExt; - use std::process::Command as StdCommand; - - let private_key = PrivateKey::random(&mut OsRng, Algorithm::Rsa { hash: None })?; - let private_key_str = private_key.to_openssh(LineEnding::LF)?.to_string(); - let public_key = private_key.public_key(); - let public_key_str = public_key.to_openssh()?; - - // Save private key to a temporary file - let temp_dir = std::env::temp_dir(); - let key_path = temp_dir.join(format!("ssh_key_{}", uuid::Uuid::new_v4())); - fs::write(&key_path, &private_key_str)?; - - // Set proper permissions (0600) for SSH key - let mut perms = fs::metadata(&key_path)?.permissions(); - perms.set_mode(0o600); - fs::set_permissions(&key_path, perms)?; - - // Add key to ssh-agent using synchronous command - let ssh_add_output = StdCommand::new("ssh-add") - .arg(key_path.to_str().unwrap()) - .output()?; - - if !ssh_add_output.status.success() { - let stderr = String::from_utf8_lossy(&ssh_add_output.stderr); - // Clean up the key file if ssh-add fails - let _ = fs::remove_file(&key_path); - return Err(anyhow::anyhow!( - "Failed to add SSH key to agent: {}", - stderr - )); - } - - Ok((private_key_str, public_key_str, key_path)) -} - -pub fn generate_ignition_config( - ssh_public_key: &str, - register_server_url: &str, - namespace: &str, -) -> serde_json::Value { - // Create the ignition configuration - let ignition = Ignition { - version: "3.6.0-experimental".to_string(), - config: Some(IgnitionConfig { - merge: Some(vec![Resource { - source: Some(register_server_url.to_string()), - compression: None, - http_headers: None, - verification: None, - }]), - replace: None, - }), - proxy: None, - security: None, - timeouts: None, - }; - - let mut user = User::new("core".to_string()); - user.ssh_authorized_keys = Some(vec![ssh_public_key.to_string()]); - let config = Config { - ignition, - kernel_arguments: None, - passwd: Some(Passwd { - users: Some(vec![user]), - groups: None, - }), - storage: Some(Storage { - directories: None, - disks: None, - files: Some(vec![File { - path: "/etc/profile.d/systemd-pager.sh".to_string(), - contents: Some(Resource { - source: Some("data:,%23%20Tell%20systemd%20to%20not%20use%20a%20pager%20when%20printing%20information%0Aexport%20SYSTEMD_PAGER%3Dcat%0A".to_string()), - compression: Some(String::new()), - http_headers: None, - verification: None, - }), - mode: Some(420), - append: None, - group: None, - overwrite: None, - user: None, - }]), - filesystems: None, - links: None, - luks: None, - raid: None, - }), - systemd: Some(Systemd { - units: Some(vec![ - Unit { - name: "zincati.service".to_string(), - enabled: Some(false), - contents: None, - dropins: None, - mask: None, - }, - Unit { - name: "serial-getty@ttyS0.service".to_string(), - enabled: None, - contents: None, - mask: None, - dropins: Some(vec![Dropin { - name: "autologin-core.conf".to_string(), - contents: Some("[Service]\n# Override Execstart in main unit\nExecStart=\n# Add new Execstart with `-` prefix to ignore failure`\nExecStart=-/usr/sbin/agetty --autologin core --noclear %I $TERM\n".to_string()), - }]), - }, - ]), - }), - }; - - let mut ignition_json = - serde_json::to_value(&config).expect("Failed to serialize ignition config"); - - // Add attestation key registration field - let attestation_url = format!( - "http://attestation-key-register.{}.svc.cluster.local:8001/register-ak", - namespace - ); - - if let Some(obj) = ignition_json.as_object_mut() { - obj.insert( - "attestation".to_string(), - serde_json::json!({ - "attestation_key": { - "registration": { - "url": attestation_url - } - } - }), - ); - } - - ignition_json -} - -/// Create a KubeVirt VirtualMachine with the specified configuration -pub async fn create_kubevirt_vm( - client: &Client, - namespace: &str, - vm_name: &str, - ssh_public_key: &str, - register_server_url: &str, - image: &str, -) -> anyhow::Result<()> { - use kube::Api; - - let ignition_config = generate_ignition_config(ssh_public_key, register_server_url, namespace); - let ignition_json = serde_json::to_string(&ignition_config)?; - - let vm = VirtualMachine { - metadata: ObjectMeta { - name: Some(vm_name.to_string()), - namespace: Some(namespace.to_string()), - ..Default::default() - }, - spec: VirtualMachineSpec { - run_strategy: Some("Always".to_string()), - template: VirtualMachineTemplate { - metadata: Some(BTreeMap::from([( - "annotations".to_string(), - serde_json::json!({"kubevirt.io/ignitiondata": ignition_json}), - )])), - spec: Some(VirtualMachineTemplateSpec { - domain: VirtualMachineTemplateSpecDomain { - features: Some(VirtualMachineTemplateSpecDomainFeatures { - smm: Some(VirtualMachineTemplateSpecDomainFeaturesSmm { - enabled: Some(true), - }), - ..Default::default() - }), - firmware: Some(VirtualMachineTemplateSpecDomainFirmware { - bootloader: Some(VirtualMachineTemplateSpecDomainFirmwareBootloader { - efi: Some(VirtualMachineTemplateSpecDomainFirmwareBootloaderEfi { - persistent: Some(true), - ..Default::default() - }), - ..Default::default() - }), - ..Default::default() - }), - devices: VirtualMachineTemplateSpecDomainDevices { - disks: Some(vec![VirtualMachineTemplateSpecDomainDevicesDisks { - name: "containerdisk".to_string(), - disk: Some(VirtualMachineTemplateSpecDomainDevicesDisksDisk { - bus: Some("virtio".to_string()), - ..Default::default() - }), - ..Default::default() - }]), - tpm: Some(VirtualMachineTemplateSpecDomainDevicesTpm { - persistent: Some(true), - ..Default::default() - }), - rng: Some(VirtualMachineTemplateSpecDomainDevicesRng {}), - ..Default::default() - }, - resources: Some(VirtualMachineTemplateSpecDomainResources { - requests: Some(BTreeMap::from([ - ( - "memory".to_string(), - IntOrString::String("4096M".to_string()), - ), - ("cpu".to_string(), IntOrString::Int(2)), - ])), - ..Default::default() - }), - ..Default::default() - }, - volumes: Some(vec![VirtualMachineTemplateSpecVolumes { - name: "containerdisk".to_string(), - container_disk: Some(VirtualMachineTemplateSpecVolumesContainerDisk { - image: image.to_string(), - image_pull_policy: Some("Always".to_string()), - ..Default::default() - }), - ..Default::default() - }]), - ..Default::default() - }), - }, - ..Default::default() - }, - ..Default::default() - }; - - let vms: Api = Api::namespaced(client.clone(), namespace); - vms.create(&Default::default(), &vm).await?; - - Ok(()) -} - -/// Wait for a KubeVirt VirtualMachine to reach Running phase -pub async fn wait_for_vm_running( - client: &Client, - namespace: &str, - vm_name: &str, - timeout_secs: u64, -) -> anyhow::Result<()> { - let api: Api = Api::namespaced(client.clone(), namespace); - - let poller = Poller::new() - .with_timeout(Duration::from_secs(timeout_secs)) - .with_interval(Duration::from_secs(5)) - .with_error_message(format!( - "VirtualMachine {} did not reach Running phase after {} seconds", - vm_name, timeout_secs - )); - - poller - .poll_async(|| { - let api = api.clone(); - let name = vm_name.to_string(); - async move { - let vm = api.get(&name).await?; - - // Check VM status phase - if let Some(status) = vm.status { - if let Some(phase) = status.printable_status { - if phase.as_str() == "Running" { - return Ok(()); - } - } - } - - Err(anyhow::anyhow!( - "VirtualMachine {} is not in Running phase yet", - name - )) - } - }) - .await -} - -pub async fn virtctl_ssh_exec( - namespace: &str, - vm_name: &str, - key_path: &Path, - command: &str, -) -> anyhow::Result { - if which::which("virtctl").is_err() { - return Err(anyhow::anyhow!( - "virtctl command not found. Please install virtctl first." - )); - } - - let _vm_target = format!("core@vmi/{}/{}", vm_name, namespace); - let full_cmd = format!( - "virtctl ssh -i {} core@vmi/{}/{} -t '-o IdentitiesOnly=yes' -t '-o StrictHostKeyChecking=no' --known-hosts /dev/null -c '{}'", - key_path.display(), - vm_name, - namespace, - command - ); - - let output = Command::new("sh").arg("-c").arg(full_cmd).output().await?; - if !output.status.success() { - let stderr = String::from_utf8_lossy(&output.stderr); - return Err(anyhow::anyhow!("virtctl ssh command failed: {}", stderr)); - } - - Ok(String::from_utf8_lossy(&output.stdout).to_string()) -} - -pub async fn wait_for_vm_ssh_ready( - namespace: &str, - vm_name: &str, - key_path: &Path, - timeout_secs: u64, -) -> anyhow::Result<()> { - wait_for_vm_ssh(namespace, vm_name, key_path, timeout_secs, true).await -} - -pub async fn wait_for_vm_ssh_unavail( - namespace: &str, - vm_name: &str, - key_path: &Path, - timeout_secs: u64, -) -> anyhow::Result<()> { - wait_for_vm_ssh(namespace, vm_name, key_path, timeout_secs, false).await -} - -async fn wait_for_vm_ssh( - namespace: &str, - vm_name: &str, - key_path: &Path, - timeout_secs: u64, - await_start: bool, -) -> anyhow::Result<()> { - let avail_prefix = if await_start { "" } else { "un" }; - let poller = Poller::new() - .with_timeout(Duration::from_secs(timeout_secs)) - .with_interval(Duration::from_secs(10)) - .with_error_message(format!( - "SSH access to VM {}/{} did not become {}available after {} seconds", - namespace, vm_name, avail_prefix, timeout_secs - )); - - poller - .poll_async(|| { - let ns = namespace.to_string(); - let vm = vm_name.to_string(); - let key = key_path.to_path_buf(); - async move { - // Try a simple command to check if SSH is ready - let result = virtctl_ssh_exec(&ns, &vm, &key, "echo ready").await; - (result.is_err() ^ await_start) - .then_some(()) - .ok_or(anyhow::anyhow!("SSH not desired state yet: {result:?}")) - } - }) - .await -} - -pub async fn verify_encrypted_root( - namespace: &str, - vm_name: &str, - key_path: &Path, - encryption_key: &[u8], -) -> anyhow::Result { - let output = virtctl_ssh_exec(namespace, vm_name, key_path, "lsblk -o NAME,TYPE -J").await?; - - // Parse JSON output - let lsblk_output: serde_json::Value = serde_json::from_str(&output)?; - - // Look for a device with name "root" and type "crypt" - let get_children = |val: &serde_json::Value| { - let children = val.get("children").and_then(|v| v.as_array()); - children.map(|v| v.to_vec()).unwrap_or_default() - }; - let devices = lsblk_output.get("blockdevices").and_then(|v| v.as_array()); - for child in devices.into_iter().flatten().flat_map(get_children) { - if get_children(&child).iter().any(|nested| { - let name = nested.get("name").and_then(|n| n.as_str()); - let dev_type = nested.get("type").and_then(|t| t.as_str()); - name == Some("root") && dev_type == Some("crypt") - }) { - let jwk: ClevisKey = serde_json::from_slice(encryption_key)?; - let key = jwk.key; - let dev = child.get("name").and_then(|n| n.as_str()).unwrap(); - let cmd = format!( - "jose jwe dec \ - -k <(jose fmt -j '{{}}' -q oct -s kty -Uq $(printf {key} | jose b64 enc -I-) -s k -Uo-) \ - -i <(sudo cryptsetup token export --token-id 0 /dev/{dev} | jose fmt -j- -Og jwe -o-) \ - | sudo cryptsetup luksOpen --test-passphrase --key-file=- /dev/{dev}", - ); - let exec = virtctl_ssh_exec(namespace, vm_name, key_path, &cmd).await; - return exec.map(|_| true); - } - } - - Ok(false) -} diff --git a/test_utils/src/virt/kubevirt.rs b/test_utils/src/virt/kubevirt.rs new file mode 100644 index 00000000..25d8fcfd --- /dev/null +++ b/test_utils/src/virt/kubevirt.rs @@ -0,0 +1,164 @@ +// SPDX-FileCopyrightText: Alice Frosi +// SPDX-FileCopyrightText: Jakob Naucke +// +// SPDX-License-Identifier: MIT + +use anyhow::{Result, anyhow}; +use k8s_openapi::apimachinery::pkg::util::intstr::IntOrString; +use kube::{Api, api::ObjectMeta}; +use std::{collections::BTreeMap, time::Duration}; +use trusted_cluster_operator_lib::{ + virtualmachineinstances::VirtualMachineInstance, virtualmachines::*, +}; + +use super::{VmBackend, VmConfig, generate_ignition, get_root_key, ssh_exec}; +use crate::{Poller, ensure_command}; + +pub struct KubevirtBackend(pub VmConfig); + +#[async_trait::async_trait] +impl VmBackend for KubevirtBackend { + async fn create_vm(&self) -> Result<()> { + let ignition_json = generate_ignition(&self.0, true); + let vm = VirtualMachine { + metadata: ObjectMeta { + name: Some(self.0.vm_name.clone()), + namespace: Some(self.0.namespace.clone()), + ..Default::default() + }, + spec: VirtualMachineSpec { + run_strategy: Some("Always".to_string()), + template: VirtualMachineTemplate { + metadata: Some(BTreeMap::from([( + "annotations".to_string(), + serde_json::json!({"kubevirt.io/ignitiondata": ignition_json}), + )])), + spec: Some(VirtualMachineTemplateSpec { + domain: VirtualMachineTemplateSpecDomain { + features: Some(VirtualMachineTemplateSpecDomainFeatures { + smm: Some(VirtualMachineTemplateSpecDomainFeaturesSmm { + enabled: Some(true), + }), + ..Default::default() + }), + firmware: Some(VirtualMachineTemplateSpecDomainFirmware { + bootloader: Some( + VirtualMachineTemplateSpecDomainFirmwareBootloader { + efi: Some( + VirtualMachineTemplateSpecDomainFirmwareBootloaderEfi { + persistent: Some(true), + ..Default::default() + }, + ), + ..Default::default() + }, + ), + ..Default::default() + }), + devices: VirtualMachineTemplateSpecDomainDevices { + disks: Some(vec![VirtualMachineTemplateSpecDomainDevicesDisks { + name: "containerdisk".to_string(), + disk: Some(VirtualMachineTemplateSpecDomainDevicesDisksDisk { + bus: Some("virtio".to_string()), + ..Default::default() + }), + ..Default::default() + }]), + tpm: Some(VirtualMachineTemplateSpecDomainDevicesTpm { + persistent: Some(true), + ..Default::default() + }), + rng: Some(VirtualMachineTemplateSpecDomainDevicesRng {}), + ..Default::default() + }, + resources: Some(VirtualMachineTemplateSpecDomainResources { + requests: Some(BTreeMap::from([ + ( + "memory".to_string(), + IntOrString::String("4096M".to_string()), + ), + ("cpu".to_string(), IntOrString::Int(2)), + ])), + ..Default::default() + }), + ..Default::default() + }, + volumes: Some(vec![VirtualMachineTemplateSpecVolumes { + name: "containerdisk".to_string(), + container_disk: Some(VirtualMachineTemplateSpecVolumesContainerDisk { + image: self.0.image.clone(), + image_pull_policy: Some("Always".to_string()), + ..Default::default() + }), + ..Default::default() + }]), + ..Default::default() + }), + }, + ..Default::default() + }, + ..Default::default() + }; + + let vms: Api = Api::namespaced(self.0.client.clone(), &self.0.namespace); + vms.create(&Default::default(), &vm).await?; + + Ok(()) + } + + async fn wait_for_running(&self, timeout_secs: u64) -> Result<()> { + let api: Api = Api::namespaced(self.0.client.clone(), &self.0.namespace); + + let poller = Poller::new() + .with_timeout(Duration::from_secs(timeout_secs)) + .with_interval(Duration::from_secs(5)) + .with_error_message(format!( + "VirtualMachine {} did not reach Running phase after {timeout_secs} seconds", + self.0.vm_name + )); + + let check_fn = || { + let api = api.clone(); + async move { + let vm = api.get(&self.0.vm_name).await?; + if let Some(status) = vm.status { + if let Some(phase) = status.printable_status { + if phase.as_str() == "Running" { + return Ok(()); + } + } + } + let vm_name = &self.0.vm_name; + let err = anyhow!("VirtualMachine {vm_name} is not in Running phase yet"); + Err(err) + } + }; + poller.poll_async(check_fn).await + } + + async fn ssh_exec(&self, command: &str) -> Result { + ensure_command("virtctl")?; + let full_cmd = format!( + "virtctl ssh -i {} core@vmi/{}/{} -t '-o IdentitiesOnly=yes' -t '-o StrictHostKeyChecking=no' --known-hosts /dev/null -c '{command}'", + self.0.ssh_private_key.display(), + self.0.vm_name, + self.0.namespace, + ); + + ssh_exec(&full_cmd).await + } + + async fn get_root_key(&self) -> Result>> { + let vmis: Api = + Api::namespaced(self.0.client.clone(), &self.0.namespace); + let vmi = vmis.get(&self.0.vm_name).await?; + let interfaces = vmi.status.unwrap().interfaces.unwrap(); + let ip = interfaces.first().unwrap().ip_address.clone().unwrap(); + get_root_key(&self.0, &ip).await.map(Some) + } + + async fn cleanup(&self) -> Result<()> { + self.0.cleanup(); + Ok(()) + } +} diff --git a/test_utils/src/virt/mod.rs b/test_utils/src/virt/mod.rs new file mode 100644 index 00000000..a085d7a3 --- /dev/null +++ b/test_utils/src/virt/mod.rs @@ -0,0 +1,278 @@ +// SPDX-FileCopyrightText: Alice Frosi +// SPDX-FileCopyrightText: Jakob Naucke +// +// SPDX-License-Identifier: MIT + +pub mod kubevirt; + +use anyhow::{Context, Result, anyhow}; +use clevis_pin_trustee_lib::Key as ClevisKey; +use k8s_openapi::api::core::v1::Secret; +use kube::{Api, Client}; +use std::{env, path::PathBuf, time::Duration}; +use tokio::process::Command; +use trusted_cluster_operator_lib::Machine; + +use super::Poller; + +/// Environment variable name for selecting the VM provider +pub const VIRT_PROVIDER_ENV: &str = "VIRT_PROVIDER"; + +#[derive(Clone)] +pub struct VmConfig { + pub client: Client, + pub namespace: String, + pub vm_name: String, + pub ssh_public_key: String, + pub ssh_private_key: PathBuf, + pub image: String, +} + +impl VmConfig { + fn cleanup(&self) { + let _ = std::fs::remove_file(&self.ssh_private_key); + } +} + +pub fn generate_ssh_key_pair() -> Result<(String, PathBuf)> { + use rand_core::OsRng; + use ssh_key::{Algorithm, LineEnding, PrivateKey}; + use std::fs; + use std::os::unix::fs::PermissionsExt; + use std::process::Command as StdCommand; + + let private_key = PrivateKey::random(&mut OsRng, Algorithm::Rsa { hash: None })?; + let private_key_str = private_key.to_openssh(LineEnding::LF)?.to_string(); + let public_key = private_key.public_key(); + let public_key_str = public_key.to_openssh()?; + + // Save private key to a temporary file + let temp_dir = env::temp_dir(); + let key_path = temp_dir.join(format!("ssh_key_{}", uuid::Uuid::new_v4())); + fs::write(&key_path, &private_key_str)?; + + // Set proper permissions (0600) for SSH key + let mut perms = fs::metadata(&key_path)?.permissions(); + perms.set_mode(0o600); + fs::set_permissions(&key_path, perms)?; + + // Add key to ssh-agent using synchronous command + let ssh_add_output = StdCommand::new("ssh-add") + .arg(key_path.to_str().unwrap()) + .output()?; + + if !ssh_add_output.status.success() { + let stderr = String::from_utf8_lossy(&ssh_add_output.stderr); + // Clean up the key file if ssh-add fails + let _ = fs::remove_file(&key_path); + return Err(anyhow!("Failed to add SSH key to agent: {stderr}")); + } + + Ok((public_key_str, key_path)) +} + +pub fn generate_ignition(config: &VmConfig, with_ak: bool) -> serde_json::Value { + use ignition_config::v3_5::*; + let register_server_url = format!( + "http://register-server.{}.svc.cluster.local:8000/ignition-clevis-pin-trustee", + config.namespace + ); + let ignition = Ignition { + version: "3.6.0-experimental".to_string(), + config: Some(IgnitionConfig { + merge: Some(vec![Resource { + source: Some(register_server_url), + ..Default::default() + }]), + ..Default::default() + }), + ..Default::default() + }; + + let mut user = User::new("core".to_string()); + user.ssh_authorized_keys = Some(vec![config.ssh_public_key.clone()]); + let mut serial_getty = Unit::new("serial-getty@ttyS0.service".to_string()); + serial_getty.dropins = Some(vec![Dropin { + name: "autologin-core.conf".to_string(), + contents: Some("[Service]\n# Override Execstart in main unit\nExecStart=\n# Add new Execstart with `-` prefix to ignore failure`\nExecStart=-/usr/sbin/agetty --autologin core --noclear %I $TERM\n".to_string()), + }]); + let mut pager = File::new("/etc/profile.d/systemd-pager.sh".to_string()); + pager.contents = Some(Resource { + source: Some("data:,%23%20Tell%20systemd%20to%20not%20use%20a%20pager%20when%20printing%20information%0Aexport%20SYSTEMD_PAGER%3Dcat%0A".to_string()), + compression: Some(String::new()), + ..Default::default() + }); + pager.mode = Some(0o644); + let ignition_config = Config { + ignition, + kernel_arguments: None, + passwd: Some(Passwd { + users: Some(vec![user]), + ..Default::default() + }), + storage: Some(Storage { + files: Some(vec![pager]), + ..Default::default() + }), + systemd: Some(Systemd { + units: Some(vec![Unit::new("zincati.service".to_string()), serial_getty]), + }), + }; + + let mut ignition_json = serde_json::to_value(&ignition_config).unwrap(); + if with_ak { + ignition_json = patch_ak(&config.namespace, ignition_json); + } + ignition_json +} + +fn patch_ak(namespace: &str, mut ignition: serde_json::Value) -> serde_json::Value { + let attestation_url = + format!("http://attestation-key-register.{namespace}.svc.cluster.local:8001/register-ak",); + let ign_json = serde_json::json!({ + "attestation_key": { + "registration": { + "url": attestation_url + } + } + }); + let obj = ignition.as_object_mut().unwrap(); + obj.insert("attestation".to_string(), ign_json); + ignition +} + +pub async fn ssh_exec(command: &str) -> Result { + let output = Command::new("sh").arg("-c").arg(command).output().await?; + if !output.status.success() { + let stderr = String::from_utf8_lossy(&output.stderr); + return Err(anyhow!("ssh command failed: {stderr}")); + } + + Ok(String::from_utf8_lossy(&output.stdout).to_string()) +} + +pub async fn get_root_key(config: &VmConfig, ip: &str) -> Result> { + let machines: Api = Api::namespaced(config.client.clone(), &config.namespace); + let list = machines.list(&Default::default()).await?; + let retrieval = |m: &&Machine| m.spec.registration_address == ip; + let err = format!("No machine found with registration IP {ip}"); + let machine = list.items.iter().find(retrieval).context(err)?; + let machine_name = machine.metadata.name.clone().unwrap(); + let secret_name = machine_name.strip_prefix("machine-").unwrap(); + + let secrets: Api = Api::namespaced(config.client.clone(), &config.namespace); + let secret = secrets.get(secret_name).await?; + Ok(secret.data.unwrap().get("root").unwrap().0.clone()) +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] +pub enum VirtProvider { + #[default] + Kubevirt, +} + +fn get_virt_provider() -> Result { + match env::var(VIRT_PROVIDER_ENV) { + Ok(val) => match val.to_lowercase().as_str() { + "kubevirt" => Ok(VirtProvider::Kubevirt), + v => Err(anyhow!( + "Unknown {VIRT_PROVIDER_ENV} '{v}'. Supported providers: kubevirt" + )), + }, + Err(env::VarError::NotPresent) => Ok(VirtProvider::default()), + Err(e) => Err(anyhow!("{e}")), + } +} + +pub fn create_backend( + client: Client, + namespace: &str, + vm_name: &str, +) -> Result> { + let provider = get_virt_provider()?; + let (public_key, key_path) = generate_ssh_key_pair()?; + let config = VmConfig { + client, + namespace: namespace.to_string(), + vm_name: vm_name.to_string(), + ssh_public_key: public_key, + ssh_private_key: key_path, + image: "quay.io/trusted-execution-clusters/fedora-coreos-kubevirt:2026-14-01".to_string(), + }; + match provider { + VirtProvider::Kubevirt => Ok(Box::new(kubevirt::KubevirtBackend(config))), + } +} + +#[async_trait::async_trait] +#[auto_impl::auto_impl(Box)] +pub trait VmBackend: Send + Sync { + async fn create_vm(&self) -> Result<()>; + async fn wait_for_running(&self, timeout_secs: u64) -> Result<()>; + async fn ssh_exec(&self, command: &str) -> Result; + async fn get_root_key(&self) -> Result>>; + async fn cleanup(&self) -> Result<()>; + + async fn wait_for_vm_ssh_ready(&self, timeout_secs: u64) -> Result<()> { + self.wait_for_vm_ssh(timeout_secs, true).await + } + + async fn wait_for_vm_ssh_unavail(&self, timeout_secs: u64) -> Result<()> { + self.wait_for_vm_ssh(timeout_secs, false).await + } + + async fn wait_for_vm_ssh(&self, timeout_secs: u64, await_start: bool) -> Result<()> { + let avail_prefix = if await_start { "" } else { "un" }; + let poller = Poller::new() + .with_timeout(Duration::from_secs(timeout_secs)) + .with_interval(Duration::from_secs(10)) + .with_error_message(format!( + "SSH access to VM did not become {}available after {} seconds", + avail_prefix, timeout_secs + )); + + let check_fn = || { + async move { + // Try a simple command to check if SSH is ready + let result = self.ssh_exec("echo ready").await; + let err = anyhow!("SSH not desired state yet: {result:?}"); + (result.is_err() ^ await_start).then_some(()).ok_or(err) + } + }; + poller.poll_async(check_fn).await + } + + async fn verify_encrypted_root(&self, encryption_key: Option<&[u8]>) -> Result { + let output = self.ssh_exec("lsblk -o NAME,TYPE -J").await?; + let lsblk_output: serde_json::Value = serde_json::from_str(&output)?; + + let get_children = |val: &serde_json::Value| { + let children = val.get("children").and_then(|v| v.as_array()); + children.map(|v| v.to_vec()).unwrap_or_default() + }; + let devices = lsblk_output.get("blockdevices").and_then(|v| v.as_array()); + for child in devices.into_iter().flatten().flat_map(get_children) { + if get_children(&child).iter().any(|nested| { + let name = nested.get("name").and_then(|n| n.as_str()); + let dev_type = nested.get("type").and_then(|t| t.as_str()); + name == Some("root") && dev_type == Some("crypt") + }) { + if encryption_key.is_none() { + return Ok(true); + } + let jwk: ClevisKey = serde_json::from_slice(encryption_key.unwrap())?; + let key = jwk.key; + let dev = child.get("name").and_then(|n| n.as_str()).unwrap(); + let cmd = format!( + "jose jwe dec \ + -k <(jose fmt -j '{{}}' -q oct -s kty -Uq $(printf {key} | jose b64 enc -I-) -s k -Uo-) \ + -i <(sudo cryptsetup token export --token-id 0 /dev/{dev} | jose fmt -j- -Og jwe -o-) \ + | sudo cryptsetup luksOpen --test-passphrase --key-file=- /dev/{dev}", + ); + return self.ssh_exec(&cmd).await.map(|_| true); + } + } + + Ok(false) + } +} diff --git a/tests/Cargo.toml b/tests/Cargo.toml index 8d6bd795..76c539c6 100644 --- a/tests/Cargo.toml +++ b/tests/Cargo.toml @@ -14,14 +14,15 @@ virtualization = [] [dependencies] anyhow.workspace = true -trusted-cluster-operator-lib = { path = "../lib" } -trusted-cluster-operator-test-utils = { path = "../test_utils" } +cfg-if = "1.0.4" compute-pcrs-lib.workspace = true k8s-openapi.workspace = true kube = { workspace = true } regex = "1" serde_json.workspace = true tokio = { workspace = true } +trusted-cluster-operator-lib = { path = "../lib" } +trusted-cluster-operator-test-utils = { path = "../test_utils" } [[test]] name = "trusted_execution_cluster" diff --git a/tests/attestation.rs b/tests/attestation.rs index fecf6aef..1145d856 100644 --- a/tests/attestation.rs +++ b/tests/attestation.rs @@ -1,90 +1,65 @@ // SPDX-FileCopyrightText: Alice Frosi +// SPDX-FileCopyrightText: Jakob Naucke // // SPDX-License-Identifier: MIT -use k8s_openapi::api::{apps::v1::Deployment, core::v1::Secret}; -use kube::Api; -use trusted_cluster_operator_lib::{Machine, virtualmachineinstances::VirtualMachineInstance}; use trusted_cluster_operator_test_utils::*; -#[cfg(feature = "virtualization")] -use trusted_cluster_operator_test_utils::virt; +cfg_if::cfg_if! { +if #[cfg(feature = "virtualization")] { +use anyhow::Result; +use k8s_openapi::api::apps::v1::Deployment; +use kube::Api; +use trusted_cluster_operator_lib::Machine; +use trusted_cluster_operator_test_utils::virt::{self, VmBackend}; + +const ENCRYPTED_ROOT_ASSERT: &str = "should have an encrypted root device (attestation failed)"; +const ENCRYPTED_ROOT_WARN: &str = "Backend reports that Machine IDs cannot be correlated to IP \ + addresses with this VIRT_PROVIDER (e.g. because of NAT). Disk \ + encryption test will only verify that the disk is encrypted, \ + not that it is encrypted with the expected key."; -#[cfg(feature = "virtualization")] struct SingleAttestationContext { - key_path: std::path::PathBuf, - root_key: Vec, + root_key: Option>, + backend: Box, } -#[cfg(feature = "virtualization")] -async fn get_root_key(vm_name: &str, test_ctx: &TestContext) -> anyhow::Result> { - let client = test_ctx.client(); - let namespace = test_ctx.namespace(); - - let vmis: Api = Api::namespaced(client.clone(), namespace); - let vmi = vmis.get(vm_name).await?; - let interfaces = vmi.status.unwrap().interfaces.unwrap(); - let ip = interfaces.first().unwrap().ip_address.clone().unwrap(); +impl SingleAttestationContext { + async fn verify_encrypted_root(&self) -> Result { + self.backend.verify_encrypted_root(self.root_key.as_deref()).await + } - let machines: Api = Api::namespaced(client.clone(), namespace); - let list = machines.list(&Default::default()).await?; - let retrieval = |m: &&Machine| m.spec.registration_address == ip; - let machine = list.items.iter().find(retrieval).unwrap(); - let machine_name = machine.metadata.name.clone().unwrap(); - let secret_name = machine_name.strip_prefix("machine-").unwrap(); - - let secrets: Api = Api::namespaced(client.clone(), namespace); - let secret = secrets.get(secret_name).await?; - Ok(secret.data.unwrap().get("root").unwrap().0.clone()) + async fn cleanup(self) -> Result<()> { + self.backend.cleanup().await + } } -#[cfg(feature = "virtualization")] impl SingleAttestationContext { - async fn new(vm_name: &str, test_ctx: &TestContext) -> anyhow::Result { + async fn new(vm_name: &str, test_ctx: &TestContext) -> Result { let client = test_ctx.client(); let namespace = test_ctx.namespace(); - - let (_private_key, public_key, key_path) = virt::generate_ssh_key_pair()?; - test_ctx.info(format!( - "Generated SSH key pair and added to ssh-agent: {:?}", - key_path - )); - - let register_server_url = format!( - "http://register-server.{}.svc.cluster.local:8000/ignition-clevis-pin-trustee", - namespace - ); - let image = "quay.io/trusted-execution-clusters/fedora-coreos-kubevirt:2026-14-01"; + let backend = virt::create_backend(client.clone(), namespace, vm_name)?; test_ctx.info(format!("Creating VM: {}", vm_name)); - virt::create_kubevirt_vm( - client, - namespace, - vm_name, - &public_key, - ®ister_server_url, - image, - ) - .await?; + backend.create_vm().await?; test_ctx.info(format!("Waiting for VM {} to reach Running state", vm_name)); - virt::wait_for_vm_running(client, namespace, vm_name, 300).await?; + backend.wait_for_running(300).await?; test_ctx.info(format!("VM {} is Running", vm_name)); test_ctx.info(format!("Waiting for SSH access to VM {}", vm_name)); - virt::wait_for_vm_ssh_ready(namespace, vm_name, &key_path, 600).await?; + backend.wait_for_vm_ssh_ready(600).await?; test_ctx.info("SSH access is ready"); - let root_key = get_root_key(vm_name, test_ctx).await?; - Ok(Self { key_path, root_key }) + let root_key = backend.get_root_key().await?; + if root_key.is_none() { + test_ctx.warn(ENCRYPTED_ROOT_WARN); + } + Ok(Self { root_key, backend }) } } -#[cfg(feature = "virtualization")] -impl Drop for SingleAttestationContext { - fn drop(&mut self) { - let _ = std::fs::remove_file(&self.key_path); - } +} } virt_test! { @@ -94,18 +69,12 @@ async fn test_attestation() -> anyhow::Result<()> { let att_ctx = SingleAttestationContext::new(vm_name, &test_ctx).await?; test_ctx.info("Verifying encrypted root device"); - let namespace = test_ctx.namespace(); - let has_encrypted_root = - virt::verify_encrypted_root(namespace, vm_name, &att_ctx.key_path, &att_ctx.root_key).await?; + let has_encrypted_root = att_ctx.verify_encrypted_root().await?; - assert!( - has_encrypted_root, - "VM should have an encrypted root device (attestation failed)" - ); + assert!(has_encrypted_root, "VM {ENCRYPTED_ROOT_ASSERT}"); test_ctx.info("Attestation successful: encrypted root device verified"); - + att_ctx.cleanup().await?; test_ctx.cleanup().await?; - Ok(()) } } @@ -115,44 +84,16 @@ async fn test_parallel_vm_attestation() -> anyhow::Result<()> { let test_ctx = setup!().await?; let client = test_ctx.client(); let namespace = test_ctx.namespace(); - test_ctx.info("Testing parallel VM attestation - launching 2 VMs simultaneously"); - // Generate SSH keys for both VMs - let (_private_key1, public_key1, key_path1) = virt::generate_ssh_key_pair()?; - let (_private_key2, public_key2, key_path2) = virt::generate_ssh_key_pair()?; - test_ctx.info("Generated SSH key pairs for both VMs"); - - let register_server_url = format!( - "http://register-server.{}.svc.cluster.local:8000/ignition-clevis-pin-trustee", - namespace - ); - let image = "quay.io/trusted-execution-clusters/fedora-coreos-kubevirt:2026-14-01"; - // Launch both VMs in parallel let vm1_name = "test-coreos-vm1"; let vm2_name = "test-coreos-vm2"; + let backend1 = virt::create_backend(client.clone(), namespace, vm1_name)?; + let backend2 = virt::create_backend(client.clone(), namespace, vm2_name)?; test_ctx.info("Creating VM1 and VM2 in parallel"); - let (vm1_result, vm2_result) = tokio::join!( - virt::create_kubevirt_vm( - client, - namespace, - vm1_name, - &public_key1, - ®ister_server_url, - image, - ), - virt::create_kubevirt_vm( - client, - namespace, - vm2_name, - &public_key2, - ®ister_server_url, - image, - ) - ); - + let (vm1_result, vm2_result) = tokio::join!(backend1.create_vm(), backend2.create_vm()); vm1_result?; vm2_result?; test_ctx.info("Both VMs created successfully"); @@ -160,8 +101,8 @@ async fn test_parallel_vm_attestation() -> anyhow::Result<()> { // Wait for both VMs to reach Running state in parallel test_ctx.info("Waiting for both VMs to reach Running state"); let (vm1_running, vm2_running) = tokio::join!( - virt::wait_for_vm_running(client, namespace, vm1_name, 300), - virt::wait_for_vm_running(client, namespace, vm2_name, 300) + backend1.wait_for_running(300), + backend2.wait_for_running(300) ); vm1_running?; @@ -171,44 +112,34 @@ async fn test_parallel_vm_attestation() -> anyhow::Result<()> { // Wait for SSH access on both VMs in parallel test_ctx.info("Waiting for SSH access on both VMs"); let (ssh1_ready, ssh2_ready) = tokio::join!( - virt::wait_for_vm_ssh_ready(namespace, vm1_name, &key_path1, 900), - virt::wait_for_vm_ssh_ready(namespace, vm2_name, &key_path2, 900) + backend1.wait_for_vm_ssh_ready(900), + backend2.wait_for_vm_ssh_ready(900) ); - ssh1_ready?; ssh2_ready?; test_ctx.info("SSH access ready on both VMs"); - let root_key1 = get_root_key(vm1_name, &test_ctx).await?; - let root_key2 = get_root_key(vm2_name, &test_ctx).await?; - // Verify attestation on both VMs in parallel + let root_key1 = backend1.get_root_key().await?; + let root_key2 = backend2.get_root_key().await?; + if root_key1.is_none() || root_key2.is_none() { + test_ctx.warn(ENCRYPTED_ROOT_WARN); + } test_ctx.info("Verifying encrypted root on both VMs"); let (vm1_encrypted, vm2_encrypted) = tokio::join!( - virt::verify_encrypted_root(namespace, vm1_name, &key_path1, &root_key1), - virt::verify_encrypted_root(namespace, vm2_name, &key_path2, &root_key2) + backend1.verify_encrypted_root(root_key1.as_deref()), + backend2.verify_encrypted_root(root_key2.as_deref()) ); - let vm1_has_encrypted_root = vm1_encrypted?; let vm2_has_encrypted_root = vm2_encrypted?; - // Clean up SSH keys - let _ = std::fs::remove_file(&key_path1); - let _ = std::fs::remove_file(&key_path2); - - assert!( - vm1_has_encrypted_root, - "VM1 should have an encrypted root device (attestation failed)" - ); - assert!( - vm2_has_encrypted_root, - "VM2 should have an encrypted root device (attestation failed)" - ); + assert!(vm1_has_encrypted_root, "VM1 {ENCRYPTED_ROOT_ASSERT}"); + assert!(vm2_has_encrypted_root, "VM2 {ENCRYPTED_ROOT_ASSERT}"); test_ctx.info("Both VMs successfully attested with encrypted root devices"); - + backend1.cleanup().await?; + backend2.cleanup().await?; test_ctx.cleanup().await?; - Ok(()) } } @@ -219,11 +150,9 @@ async fn test_vm_reboot_attestation() -> anyhow::Result<()> { test_ctx.info("Testing VM reboot - VM should successfully boot after multiple reboots"); let vm_name = "test-coreos-reboot"; let att_ctx = SingleAttestationContext::new(vm_name, &test_ctx).await?; - let namespace = test_ctx.namespace(); test_ctx.info("Verifying initial encrypted root device"); - let has_encrypted_root = - virt::verify_encrypted_root(namespace, vm_name, &att_ctx.key_path, &att_ctx.root_key).await?; + let has_encrypted_root = att_ctx.verify_encrypted_root().await?; assert!( has_encrypted_root, "VM should have encrypted root device on initial boot" @@ -236,47 +165,35 @@ async fn test_vm_reboot_attestation() -> anyhow::Result<()> { test_ctx.info(format!("Performing reboot {} of {}", i, num_reboots)); // Reboot the VM via SSH - let _reboot_result = virt::virtctl_ssh_exec( - namespace, - vm_name, - &att_ctx.key_path, - "sudo systemctl reboot", - ) - .await; + let _reboot_result = att_ctx.backend.ssh_exec("sudo systemctl reboot").await; test_ctx.info(format!("Waiting for lack of SSH access after reboot {}", i)); - virt::wait_for_vm_ssh_unavail(namespace, vm_name, &att_ctx.key_path, 30).await?; + att_ctx.backend.wait_for_vm_ssh_unavail(30).await?; test_ctx.info(format!("Waiting for SSH access after reboot {}", i)); - virt::wait_for_vm_ssh_ready(namespace, vm_name, &att_ctx.key_path, 300).await?; + att_ctx.backend.wait_for_vm_ssh_ready(300).await?; // Verify encrypted root is still present after reboot test_ctx.info(format!("Verifying encrypted root after reboot {}", i)); - let has_encrypted_root = - virt::verify_encrypted_root(namespace, vm_name, &att_ctx.key_path, &att_ctx.root_key).await?; + let has_encrypted_root = att_ctx.verify_encrypted_root().await?; assert!( has_encrypted_root, - "VM should have encrypted root device after reboot {}", - i + "VM should have encrypted root device after reboot {i}" ); test_ctx.info(format!("Reboot {}: attestation successful", i)); } test_ctx.info(format!( - "VM successfully rebooted {} times with encrypted root device maintained", - num_reboots + "VM successfully rebooted {num_reboots} times with encrypted root device maintained", )); - + att_ctx.cleanup().await?; test_ctx.cleanup().await?; - Ok(()) } } virt_test! { async fn test_vm_reboot_delete_machine() -> anyhow::Result<()> { - use trusted_cluster_operator_lib::Machine; - let test_ctx = setup!().await?; test_ctx.info("Testing Machine deletion - VM should no longer boot successfully when its Machine CRD was removed"); let vm_name = "test-coreos-delete"; @@ -289,27 +206,16 @@ async fn test_vm_reboot_delete_machine() -> anyhow::Result<()> { wait_for_resource_deleted(&machines, name, 120, 5).await?; test_ctx.info("Performing reboot, expecting missing resource"); - let _reboot_result = virt::virtctl_ssh_exec( - test_ctx.namespace(), - vm_name, - &att_ctx.key_path, - "sudo systemctl reboot", - ) - .await; + let _reboot_result = att_ctx.backend.ssh_exec("sudo systemctl reboot").await; test_ctx.info("Waiting for lack of SSH access after reboot"); - virt::wait_for_vm_ssh_unavail(test_ctx.namespace(), vm_name, &att_ctx.key_path, 30).await?; + att_ctx.backend.wait_for_vm_ssh_unavail(30).await?; test_ctx.info("Waiting for SSH access after machine removal"); - let wait = virt::wait_for_vm_ssh_ready( - test_ctx.namespace(), - vm_name, - &att_ctx.key_path, - 300, - ) - .await; + let wait = att_ctx.backend.wait_for_vm_ssh_ready(300).await; assert!(wait.is_err()); + att_ctx.cleanup().await?; test_ctx.cleanup().await?; Ok(()) } @@ -326,22 +232,16 @@ async fn test_vm_restart_operator_existing() -> anyhow::Result<()> { Api::namespaced(test_ctx.client().clone(), test_ctx.namespace()); deployments.restart("trusted-cluster-operator").await?; - let _reboot_result = virt::virtctl_ssh_exec( - test_ctx.namespace(), - vm_name, - &att_ctx.key_path, - "sudo systemctl reboot", - ) - .await; + let _reboot_result = att_ctx.backend.ssh_exec("sudo systemctl reboot").await; test_ctx.info("Waiting for lack of SSH access after reboot"); - virt::wait_for_vm_ssh_unavail(test_ctx.namespace(), vm_name, &att_ctx.key_path, 30).await?; + att_ctx.backend.wait_for_vm_ssh_unavail(30).await?; test_ctx.info("Waiting for SSH access after operator restart & reboot"); - let wait = - virt::wait_for_vm_ssh_ready(test_ctx.namespace(), vm_name, &att_ctx.key_path, 300).await; + let wait = att_ctx.backend.wait_for_vm_ssh_ready(300).await; assert!(wait.is_ok()); + att_ctx.cleanup().await?; test_ctx.cleanup().await?; Ok(()) } @@ -358,7 +258,8 @@ async fn test_vm_restart_operator_new() -> anyhow::Result<()> { deployments.restart("trusted-cluster-operator").await?; test_ctx.info("Restarted operator deployment"); - let _ = SingleAttestationContext::new(vm_name, &test_ctx).await?; + let att_ctx = SingleAttestationContext::new(vm_name, &test_ctx).await?; + att_ctx.cleanup().await?; test_ctx.cleanup().await?; Ok(()) } From edc33884c268726daa933c2f9aaefdc6746a6d80 Mon Sep 17 00:00:00 2001 From: Jakob Naucke Date: Thu, 22 Jan 2026 12:18:38 +0100 Subject: [PATCH 5/6] tests: Flexibilize environment Less double-tracking and adds flexibility for testing on cloud platforms. Environment variables: - {TEST,TRUSTEE,APPROVED}_IMAGE - TEST_NAMESPACE_PREFIX Other changes: - OpenShift service exposure - Remove CLUSTER_URL, get it from OpenShift directly - Reduce some lines & indentation in TestContext::apply_operator_manifests and break it up some - Move endpoint constants to lib - Log error upon poll timeout Signed-off-by: Jakob Naucke --- Makefile | 30 ++- attestation-key-register/src/main.rs | 6 +- go.mod | 2 + go.sum | 33 +++ lib/src/endpoints.rs | 16 ++ lib/src/kopium.rs | 2 + lib/src/lib.rs | 3 + operator/src/attestation_key_register.rs | 20 +- operator/src/register_server.rs | 22 +- operator/src/trustee.rs | 47 ++-- register-server/src/main.rs | 3 +- scripts/pre-pull-images.sh | 2 +- test_utils/src/lib.rs | 274 ++++++++++++----------- test_utils/src/timer.rs | 4 +- test_utils/src/virt/kubevirt.rs | 2 +- test_utils/src/virt/mod.rs | 41 ++-- 16 files changed, 298 insertions(+), 209 deletions(-) create mode 100644 lib/src/endpoints.rs diff --git a/Makefile b/Makefile index dde306cc..61b02feb 100644 --- a/Makefile +++ b/Makefile @@ -30,6 +30,7 @@ ATTESTATION_KEY_REGISTER_IMAGE=$(REGISTRY)/attestation-key-register:$(TAG) TRUSTEE_IMAGE ?= quay.io/trusted-execution-clusters/key-broker-service:20260106 # tagged as 2026-01-20-attestation APPROVED_IMAGE ?= quay.io/trusted-execution-clusters/fedora-coreos@sha256:79a0657399e6c67c7c95b8a09193d18e5675b5aa3cfb4d75ea5c8d4d53b2af74 +TEST_IMAGE ?= quay.io/trusted-execution-clusters/fedora-coreos-kubevirt:2026-14-01 BUILD_TYPE ?= release IMAGE_BUILD_OPTION ?= @@ -48,26 +49,26 @@ attestation-key-register: crds-rs cargo build -p attestation-key-register CRD_YAML_PATH = config/crd +CRD_WORK_PATH = config/crd/tmp RBAC_YAML_PATH = config/rbac API_PATH = api/v1alpha1 generate: $(CONTROLLER_GEN) - $(CONTROLLER_GEN) rbac:roleName=trusted-cluster-operator-role crd webhook paths="./..." \ - output:crd:artifacts:config=$(CRD_YAML_PATH) \ - output:rbac:artifacts:config=$(RBAC_YAML_PATH) + $(call controller-gen,./...,*) + $(call controller-gen,github.com/openshift/api/route/v1,*) + $(call controller-gen,github.com/openshift/api/config/v1,*_ingresses.yaml) RS_LIB_PATH = lib/src CRD_RS_PATH = $(RS_LIB_PATH)/kopium $(CRD_RS_PATH): mkdir $(CRD_RS_PATH) -YAML_PREFIX = trusted-execution-clusters.io_ -$(CRD_RS_PATH)/%.rs: $(CRD_YAML_PATH)/$(YAML_PREFIX)%.yaml $(KOPIUM) $(CRD_RS_PATH) +$(CRD_RS_PATH)/%.rs: $(CRD_YAML_PATH)/*_%.yaml $(KOPIUM) $(CRD_RS_PATH) $(KOPIUM) -f $< > $@ rustfmt $@ -crds-rs: generate +crds-rs: generate $(KOPIUM) $(CRD_RS_PATH) $(MAKE) $(shell find $(CRD_YAML_PATH) -type f \ - | sed -E 's|$(CRD_YAML_PATH)/$(YAML_PREFIX)(.*)\.yaml|$(CRD_RS_PATH)/\1.rs|') + | sed -E 's|$(CRD_YAML_PATH)/.*_(.*)\.yaml|$(CRD_RS_PATH)/\1.rs|') trusted-cluster-gen: api/trusted-cluster-gen.go go build -o $@ $< @@ -192,8 +193,10 @@ test-release: crds-rs cargo test --workspace --bins --release integration-tests: generate trusted-cluster-gen crds-rs - RUST_LOG=info cargo test --test trusted_execution_cluster --test attestation \ - --features virtualization -- --no-capture --test-threads=$(INTEGRATION_TEST_THREADS) + RUST_LOG=info REGISTRY=$(REGISTRY) TAG=$(TAG) \ + TRUSTEE_IMAGE=$(TRUSTEE_IMAGE) APPROVED_IMAGE=$(APPROVED_IMAGE) TEST_IMAGE=$(TEST_IMAGE) \ + cargo test --test trusted_execution_cluster --test attestation \ + --features virtualization -- --nocapture --test-threads=$(INTEGRATION_TEST_THREADS) $(LOCALBIN): mkdir -p $(LOCALBIN) @@ -225,3 +228,12 @@ define cargo-install-tool mv "$$(dirname $(1))/$(2)" $(1) ;\ } endef + +define controller-gen +mkdir -p $(CRD_WORK_PATH) +$(CONTROLLER_GEN) rbac:roleName=trusted-cluster-operator-role crd webhook paths=$(1) \ + output:crd:artifacts:config=$(CRD_WORK_PATH) \ + output:rbac:artifacts:config=$(RBAC_YAML_PATH) +mv $(CRD_WORK_PATH)/$(2) $(CRD_YAML_PATH)/ +rm -rf $(CRD_WORK_PATH) +endef diff --git a/attestation-key-register/src/main.rs b/attestation-key-register/src/main.rs index 165d686e..fdf6202d 100644 --- a/attestation-key-register/src/main.rs +++ b/attestation-key-register/src/main.rs @@ -10,10 +10,12 @@ use log::{error, info}; use serde::{Deserialize, Serialize}; use std::convert::Infallible; use std::net::SocketAddr; -use trusted_cluster_operator_lib::{AttestationKey, AttestationKeySpec}; use uuid::Uuid; use warp::{http::StatusCode, reply, Filter}; +use trusted_cluster_operator_lib::endpoints::ATTESTATION_KEY_REGISTER_RESOURCE; +use trusted_cluster_operator_lib::{AttestationKey, AttestationKeySpec}; + #[derive(Parser)] #[command(name = "attestation-key-register")] #[command(about = "HTTP server that accepts attestation key registrations")] @@ -138,7 +140,7 @@ async fn main() -> anyhow::Result<()> { .context("Failed to create Kubernetes client")?; let register = warp::put() - .and(warp::path("register-ak")) + .and(warp::path(ATTESTATION_KEY_REGISTER_RESOURCE)) .and(warp::body::json()) .and(with_client(client)) .and(warp::addr::remote()) diff --git a/go.mod b/go.mod index cabce670..9b757efe 100644 --- a/go.mod +++ b/go.mod @@ -16,10 +16,12 @@ require ( require ( github.com/fxamacker/cbor/v2 v2.9.0 // indirect github.com/go-logr/logr v1.4.3 // indirect + github.com/gogo/protobuf v1.3.2 // indirect github.com/json-iterator/go v1.1.12 // indirect github.com/kr/text v0.2.0 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee // indirect + github.com/openshift/api v0.0.0-20260128000234-c16ec2bcf089 // indirect github.com/x448/float16 v0.8.4 // indirect go.yaml.in/yaml/v2 v2.4.3 // indirect golang.org/x/net v0.48.0 // indirect diff --git a/go.sum b/go.sum index ae580187..9dfcd89c 100644 --- a/go.sum +++ b/go.sum @@ -10,6 +10,8 @@ github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI= github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-task/slim-sprig/v3 v3.0.0 h1:sUs3vkvUymDpBKi3qH1YSqBQk9+9D/8M2mN1vB6EwHI= github.com/go-task/slim-sprig/v3 v3.0.0/go.mod h1:W848ghGpv3Qj3dhTPRyJypKRiqCdHZiAzKg9hl15HA8= +github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= +github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= @@ -17,6 +19,8 @@ github.com/google/pprof v0.0.0-20250403155104-27863c87afa6 h1:BHT72Gu3keYf3ZEu2J github.com/google/pprof v0.0.0-20250403155104-27863c87afa6/go.mod h1:boTsfXsheKC2y+lKOCMpSfarhxDeIzfZG1jqGcPl3cA= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= +github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= +github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= @@ -31,6 +35,8 @@ github.com/onsi/ginkgo/v2 v2.27.2 h1:LzwLj0b89qtIy6SSASkzlNvX6WktqurSHwkk2ipF/Ns github.com/onsi/ginkgo/v2 v2.27.2/go.mod h1:ArE1D/XhNXBXCBkKOLkbsb2c81dQHCRcF5zwn/ykDRo= github.com/onsi/gomega v1.38.2 h1:eZCjf2xjZAqe+LeWvKb5weQ+NcPwX84kqJ0cZNxok2A= github.com/onsi/gomega v1.38.2/go.mod h1:W2MJcYxRGV63b418Ai34Ud0hEdTVXq9NW9+Sx6uXf3k= +github.com/openshift/api v0.0.0-20260128000234-c16ec2bcf089 h1:qcKLN7H1dh2wt59Knpc1J5XzCCStSeaaFyEHHilFypg= +github.com/openshift/api v0.0.0-20260128000234-c16ec2bcf089/go.mod h1:d5uzF0YN2nQQFA0jIEWzzOZ+edmo6wzlGLvx5Fhz4uY= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ= @@ -43,22 +49,49 @@ github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM= github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg= +github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= go.yaml.in/yaml/v2 v2.4.3 h1:6gvOSjQoTB3vt1l+CU+tSyi/HOjfOjRLJ4YwYZGwRO0= go.yaml.in/yaml/v2 v2.4.3/go.mod h1:zSxWcmIDjOzPXpjlTTbAsKokqkDNAVtZO0WOMiT90s8= go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc= go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.31.0 h1:HaW9xtz0+kOcWKwli0ZXy79Ix+UW/vOfmWI5QVd2tgI= golang.org/x/mod v0.31.0/go.mod h1:43JraMp9cGx1Rx3AqioxrbrhNsLl2l/iNAvuBkrezpg= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.48.0 h1:zyQRTTrjc33Lhh0fBgT/H3oZq9WuvRR5gPC70xpDiQU= golang.org/x/net v0.48.0/go.mod h1:+ndRgGjkh8FGtu1w1FGbEC31if4VrNVMuKTgcAAnQRY= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.19.0 h1:vV+1eWNmZ5geRlYjzm2adRgW2/mcpevXNg50YZtPCE4= golang.org/x/sync v0.19.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.39.0 h1:CvCKL8MeisomCi6qNZ+wbb0DN9E5AATixKsvNtMoMFk= golang.org/x/sys v0.39.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.33.0 h1:B3njUFyqtHDUI5jMn1YIr5B0IE2U0qck04r6d4KPAxE= golang.org/x/text v0.33.0/go.mod h1:LuMebE6+rBincTi9+xWTY8TztLzKHc/9C1uBCG27+q8= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.40.0 h1:yLkxfA+Qnul4cs9QA3KnlFu0lVmd8JJfoq+E41uSutA= golang.org/x/tools v0.40.0/go.mod h1:Ik/tzLRlbscWpqqMRjyWYDisX8bG13FrdXp3o4Sr9lc= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= diff --git a/lib/src/endpoints.rs b/lib/src/endpoints.rs new file mode 100644 index 00000000..cd7c7e9c --- /dev/null +++ b/lib/src/endpoints.rs @@ -0,0 +1,16 @@ +// SPDX-FileCopyrightText: Jakob Naucke +// +// SPDX-License-Identifier: MIT + +pub const TRUSTEE_SERVICE: &str = "kbs-service"; +pub const TRUSTEE_DEPLOYMENT: &str = "trustee-deployment"; +pub const TRUSTEE_PORT: i32 = 8080; +pub const REGISTER_SERVER_SERVICE: &str = "register-server"; +pub const REGISTER_SERVER_DEPLOYMENT: &str = "register-server"; +pub const REGISTER_SERVER_PORT: i32 = 8000; +pub const ATTESTATION_KEY_REGISTER_SERVICE: &str = "attestation-key-register"; +pub const ATTESTATION_KEY_REGISTER_DEPLOYMENT: &str = "attestation-key-register"; +pub const ATTESTATION_KEY_REGISTER_PORT: i32 = 8001; + +pub const REGISTER_SERVER_RESOURCE: &str = "ignition-clevis-pin-trustee"; +pub const ATTESTATION_KEY_REGISTER_RESOURCE: &str = "register-ak"; diff --git a/lib/src/kopium.rs b/lib/src/kopium.rs index f15a452d..4588066e 100644 --- a/lib/src/kopium.rs +++ b/lib/src/kopium.rs @@ -4,5 +4,7 @@ pub mod approvedimages; pub mod attestationkeys; +pub mod ingresses; pub mod machines; +pub mod routes; pub mod trustedexecutionclusters; diff --git a/lib/src/lib.rs b/lib/src/lib.rs index 33a79304..aedcb9e2 100644 --- a/lib/src/lib.rs +++ b/lib/src/lib.rs @@ -3,6 +3,7 @@ // SPDX-License-Identifier: MIT pub mod conditions; +pub mod endpoints; pub mod reference_values; mod kopium; @@ -10,7 +11,9 @@ mod kopium; mod vendor_kopium; pub use kopium::approvedimages::*; pub use kopium::attestationkeys::*; +pub use kopium::ingresses as openshift_ingresses; pub use kopium::machines::*; +pub use kopium::routes; pub use kopium::trustedexecutionclusters::*; pub use vendor_kopium::virtualmachineinstances; pub use vendor_kopium::virtualmachines; diff --git a/operator/src/attestation_key_register.rs b/operator/src/attestation_key_register.rs index 5f56fa54..7a326537 100644 --- a/operator/src/attestation_key_register.rs +++ b/operator/src/attestation_key_register.rs @@ -26,10 +26,10 @@ use kube::{ use log::info; use serde_json::json; use std::{collections::BTreeMap, sync::Arc}; -use trusted_cluster_operator_lib::{ - AttestationKey, AttestationKeyStatus, Machine, conditions::ATTESTATION_KEY_MACHINE_APPROVE, - update_status, -}; + +use trusted_cluster_operator_lib::conditions::ATTESTATION_KEY_MACHINE_APPROVE; +use trusted_cluster_operator_lib::endpoints::*; +use trusted_cluster_operator_lib::{AttestationKey, AttestationKeyStatus, Machine, update_status}; use crate::conditions::attestation_key_approved_condition; use crate::trustee; @@ -44,13 +44,12 @@ pub async fn create_attestation_key_register_deployment( owner_reference: OwnerReference, image: &str, ) -> Result<()> { - let name = "attestation-key-register"; let app_label = "attestation-key-register"; let labels = BTreeMap::from([("app".to_string(), app_label.to_string())]); let deployment = Deployment { metadata: ObjectMeta { - name: Some(name.to_string()), + name: Some(ATTESTATION_KEY_REGISTER_DEPLOYMENT.to_string()), owner_references: Some(vec![owner_reference]), ..Default::default() }, @@ -68,15 +67,15 @@ pub async fn create_attestation_key_register_deployment( spec: Some(PodSpec { service_account_name: Some("trusted-cluster-operator".to_string()), containers: vec![Container { - name: name.to_string(), + name: ATTESTATION_KEY_REGISTER_DEPLOYMENT.to_string(), image: Some(image.to_string()), ports: Some(vec![ContainerPort { - container_port: INTERNAL_ATTESTATION_KEY_REGISTER_PORT, + container_port: ATTESTATION_KEY_REGISTER_PORT, ..Default::default() }]), args: Some(vec![ "--port".to_string(), - INTERNAL_ATTESTATION_KEY_REGISTER_PORT.to_string(), + ATTESTATION_KEY_REGISTER_PORT.to_string(), ]), ..Default::default() }], @@ -98,13 +97,12 @@ pub async fn create_attestation_key_register_service( owner_reference: OwnerReference, attestation_key_register_port: Option, ) -> Result<()> { - let name = "attestation-key-register"; let app_label = "attestation-key-register"; let labels = BTreeMap::from([("app".to_string(), app_label.to_string())]); let service = Service { metadata: ObjectMeta { - name: Some(name.to_string()), + name: Some(ATTESTATION_KEY_REGISTER_SERVICE.to_string()), labels: Some(labels.clone()), owner_references: Some(vec![owner_reference]), ..Default::default() diff --git a/operator/src/register_server.rs b/operator/src/register_server.rs index b4ff6c34..c807e731 100644 --- a/operator/src/register_server.rs +++ b/operator/src/register_server.rs @@ -28,9 +28,8 @@ use std::{collections::BTreeMap, sync::Arc}; use crate::trustee; use operator::*; -use trusted_cluster_operator_lib::Machine; +use trusted_cluster_operator_lib::{Machine, endpoints::*}; -const INTERNAL_REGISTER_SERVER_PORT: i32 = 8000; /// Finalizer name to discard decryption keys when a machine is deleted const MACHINE_FINALIZER: &str = "finalizer.machine.trusted-execution-clusters.io"; @@ -39,13 +38,12 @@ pub async fn create_register_server_deployment( owner_reference: OwnerReference, image: &str, ) -> Result<()> { - let name = "register-server"; let app_label = "register-server"; let labels = BTreeMap::from([("app".to_string(), app_label.to_string())]); let deployment = Deployment { metadata: ObjectMeta { - name: Some(name.to_string()), + name: Some(REGISTER_SERVER_DEPLOYMENT.to_string()), owner_references: Some(vec![owner_reference]), ..Default::default() }, @@ -63,16 +61,13 @@ pub async fn create_register_server_deployment( spec: Some(PodSpec { service_account_name: Some("trusted-cluster-operator".to_string()), containers: vec![Container { - name: name.to_string(), + name: REGISTER_SERVER_DEPLOYMENT.to_string(), image: Some(image.to_string()), ports: Some(vec![ContainerPort { - container_port: INTERNAL_REGISTER_SERVER_PORT, + container_port: REGISTER_SERVER_PORT, ..Default::default() }]), - args: Some(vec![ - "--port".to_string(), - INTERNAL_REGISTER_SERVER_PORT.to_string(), - ]), + args: Some(vec!["--port".to_string(), REGISTER_SERVER_PORT.to_string()]), ..Default::default() }], ..Default::default() @@ -93,13 +88,12 @@ pub async fn create_register_server_service( owner_reference: OwnerReference, register_server_port: Option, ) -> Result<()> { - let name = "register-server"; let app_label = "register-server"; let labels = BTreeMap::from([("app".to_string(), app_label.to_string())]); let service = Service { metadata: ObjectMeta { - name: Some(name.to_string()), + name: Some(REGISTER_SERVER_SERVICE.to_string()), labels: Some(labels.clone()), owner_references: Some(vec![owner_reference]), ..Default::default() @@ -108,8 +102,8 @@ pub async fn create_register_server_service( selector: Some(labels), ports: Some(vec![ServicePort { name: Some("http".to_string()), - port: register_server_port.unwrap_or(INTERNAL_REGISTER_SERVER_PORT), - target_port: Some(IntOrString::Int(INTERNAL_REGISTER_SERVER_PORT)), + port: register_server_port.unwrap_or(REGISTER_SERVER_PORT), + target_port: Some(IntOrString::Int(REGISTER_SERVER_PORT)), protocol: Some("TCP".to_string()), ..Default::default() }]), diff --git a/operator/src/trustee.rs b/operator/src/trustee.rs index 74ad65d2..abdb9dbf 100644 --- a/operator/src/trustee.rs +++ b/operator/src/trustee.rs @@ -27,6 +27,8 @@ use operator::{RvContextData, create_or_info_if_exists}; use serde::{Serialize, Serializer}; use serde_json::{Value::String as JsonString, json}; use std::collections::BTreeMap; + +use trusted_cluster_operator_lib::endpoints::*; use trusted_cluster_operator_lib::reference_values::*; const TRUSTEE_DATA_DIR: &str = "/opt/trustee"; @@ -36,8 +38,6 @@ pub(crate) const REFERENCE_VALUES_FILE: &str = "reference-values.json"; pub(crate) const TRUSTEE_DATA_MAP: &str = "trustee-data"; const ATT_POLICY_MAP: &str = "attestation-policy"; -const DEPLOYMENT_NAME: &str = "trustee-deployment"; -const INTERNAL_KBS_PORT: i32 = 8080; const TRUSTED_AK_KEYS_VOLUME: &str = "trusted-ak-keys"; const TRUSTED_AK_KEYS_DIR: &str = "/etc/tpm/trusted_ak_keys"; @@ -140,24 +140,24 @@ fn generate_secret_volume(id: &str) -> (Volume, VolumeMount) { pub async fn mount_secret(client: Client, id: &str) -> Result<()> { let result = do_mount_secret(client, id, true).await; - info!("Mounted secret {id} to {DEPLOYMENT_NAME}"); + info!("Mounted secret {id} to {TRUSTEE_DEPLOYMENT}"); result } pub async fn unmount_secret(client: Client, id: &str) -> Result<()> { let result = do_mount_secret(client, id, false).await; - info!("Unmounted secret {id} from {DEPLOYMENT_NAME}"); + info!("Unmounted secret {id} from {TRUSTEE_DEPLOYMENT}"); result } pub async fn do_mount_secret(client: Client, id: &str, add: bool) -> Result<()> { let deployments: Api = Api::default_namespaced(client); - let mut deployment = deployments.get(DEPLOYMENT_NAME).await?; - let err = format!("Deployment {DEPLOYMENT_NAME} existed, but had no spec"); + let mut deployment = deployments.get(TRUSTEE_DEPLOYMENT).await?; + let err = format!("Deployment {TRUSTEE_DEPLOYMENT} existed, but had no spec"); let depl_spec = deployment.spec.as_mut().context(err)?; - let err = format!("Deployment {DEPLOYMENT_NAME} existed, but had no pod spec"); + let err = format!("Deployment {TRUSTEE_DEPLOYMENT} existed, but had no pod spec"); let pod_spec = depl_spec.template.spec.as_mut().context(err)?; - let err = format!("Deployment {DEPLOYMENT_NAME} existed, but had no containers"); + let err = format!("Deployment {TRUSTEE_DEPLOYMENT} existed, but had no containers"); let container = pod_spec.containers.get_mut(0).context(err)?; let vol_mounts = container.volume_mounts.get_or_insert_default(); @@ -183,7 +183,7 @@ pub async fn do_mount_secret(client: Client, id: &str, add: bool) -> Result<()> } deployments - .replace(DEPLOYMENT_NAME, &Default::default(), &deployment) + .replace(TRUSTEE_DEPLOYMENT, &Default::default(), &deployment) .await?; Ok(()) } @@ -212,10 +212,10 @@ pub async fn update_attestation_keys(client: Client) -> Result<()> { .collect(); let deployments: Api = Api::default_namespaced(client); - let deployment = deployments.get(DEPLOYMENT_NAME).await?; - let err = format!("Deployment {DEPLOYMENT_NAME} existed, but had no spec"); + let deployment = deployments.get(TRUSTEE_DEPLOYMENT).await?; + let err = format!("Deployment {TRUSTEE_DEPLOYMENT} existed, but had no spec"); let depl_spec = deployment.spec.as_ref().context(err)?; - let err = format!("Deployment {DEPLOYMENT_NAME} existed, but had no pod spec"); + let err = format!("Deployment {TRUSTEE_DEPLOYMENT} existed, but had no pod spec"); let pod_spec = depl_spec.template.spec.as_ref().context(err)?; // Get existing volumes and volumeMounts, filtering out the attestation key volume @@ -230,7 +230,7 @@ pub async fn update_attestation_keys(client: Client) -> Result<()> { }) .unwrap_or_default(); - let err = format!("Deployment {DEPLOYMENT_NAME} existed, but had no containers"); + let err = format!("Deployment {TRUSTEE_DEPLOYMENT} existed, but had no containers"); let container = pod_spec.containers.first().context(err)?; let mut vol_mounts: Vec = container .volume_mounts @@ -244,7 +244,9 @@ pub async fn update_attestation_keys(client: Client) -> Result<()> { .unwrap_or_default(); if ak_secrets.is_empty() { - info!("No AttestationKey secrets found, removing projected volume from {DEPLOYMENT_NAME}"); + info!( + "No AttestationKey secrets found, removing projected volume from {TRUSTEE_DEPLOYMENT}" + ); } else { // Build the projected volume with all AttestationKey secrets let projections: Vec = ak_secrets @@ -291,7 +293,7 @@ pub async fn update_attestation_keys(client: Client) -> Result<()> { "apiVersion": "apps/v1", "kind": "Deployment", "metadata": { - "name": DEPLOYMENT_NAME + "name": TRUSTEE_DEPLOYMENT }, "spec": { "template": { @@ -308,12 +310,12 @@ pub async fn update_attestation_keys(client: Client) -> Result<()> { deployments .patch( - DEPLOYMENT_NAME, + TRUSTEE_DEPLOYMENT, &PatchParams::apply("trusted-cluster-operator").force(), &Patch::Apply(&patch), ) .await?; - info!("Successfully patched {DEPLOYMENT_NAME} with attestation key volumes"); + info!("Successfully patched {TRUSTEE_DEPLOYMENT} with attestation key volumes"); } else { info!("No changes to attestation key volumes, skipping deployment update"); } @@ -394,12 +396,11 @@ pub async fn generate_kbs_service( owner_reference: OwnerReference, kbs_port: Option, ) -> Result<()> { - let svc_name = "kbs-service"; let selector = Some(BTreeMap::from([("app".to_string(), "kbs".to_string())])); let service = Service { metadata: ObjectMeta { - name: Some(svc_name.to_string()), + name: Some(TRUSTEE_SERVICE.to_string()), owner_references: Some(vec![owner_reference.clone()]), ..Default::default() }, @@ -407,8 +408,8 @@ pub async fn generate_kbs_service( selector: selector.clone(), ports: Some(vec![ServicePort { name: Some("kbs-port".to_string()), - port: kbs_port.unwrap_or(INTERNAL_KBS_PORT), - target_port: Some(IntOrString::Int(INTERNAL_KBS_PORT)), + port: kbs_port.unwrap_or(TRUSTEE_PORT), + target_port: Some(IntOrString::Int(TRUSTEE_PORT)), ..Default::default() }]), ..Default::default() @@ -474,7 +475,7 @@ fn generate_kbs_pod_spec(image: &str) -> PodSpec { image: Some(image.to_string()), name: "kbs".to_string(), ports: Some(vec![ContainerPort { - container_port: INTERNAL_KBS_PORT, + container_port: TRUSTEE_PORT, ..Default::default() }]), volume_mounts: Some( @@ -514,7 +515,7 @@ pub async fn generate_kbs_deployment( // Inspired by trustee-operator let deployment = Deployment { metadata: ObjectMeta { - name: Some(DEPLOYMENT_NAME.to_string()), + name: Some(TRUSTEE_DEPLOYMENT.to_string()), owner_references: Some(vec![owner_reference]), ..Default::default() }, diff --git a/register-server/src/main.rs b/register-server/src/main.rs index e7102bff..0fd9436b 100644 --- a/register-server/src/main.rs +++ b/register-server/src/main.rs @@ -18,6 +18,7 @@ use std::net::SocketAddr; use uuid::Uuid; use warp::{http::StatusCode, reply, Filter}; +use trusted_cluster_operator_lib::endpoints::REGISTER_SERVER_RESOURCE; use trusted_cluster_operator_lib::{Machine, MachineSpec, TrustedExecutionCluster}; #[derive(Parser)] @@ -168,7 +169,7 @@ async fn main() { let args = Args::parse(); - let register_route = warp::path("ignition-clevis-pin-trustee") + let register_route = warp::path(REGISTER_SERVER_RESOURCE) .and(warp::get()) .and(warp::addr::remote()) .and_then(register_handler); diff --git a/scripts/pre-pull-images.sh b/scripts/pre-pull-images.sh index 59b83db3..daa48b36 100755 --- a/scripts/pre-pull-images.sh +++ b/scripts/pre-pull-images.sh @@ -13,7 +13,7 @@ IMAGES=( "quay.io/kubevirt/virt-operator:${KV_VERSION}" "$TRUSTEE_IMAGE" "$APPROVED_IMAGE" - "quay.io/trusted-execution-clusters/fedora-coreos-kubevirt:2026-14-01" + "$TEST_IMAGE" ) for IMAGE in "${IMAGES[@]}"; do diff --git a/test_utils/src/lib.rs b/test_utils/src/lib.rs index d4826114..0231f802 100644 --- a/test_utils/src/lib.rs +++ b/test_utils/src/lib.rs @@ -1,19 +1,22 @@ // SPDX-FileCopyrightText: Alice Frosi +// SPDX-FileCopyrightText: Jakob Naucke // // SPDX-License-Identifier: MIT -use anyhow::anyhow; +use anyhow::{Result, anyhow}; use fs_extra::dir; use k8s_openapi::api::apps::v1::Deployment; use k8s_openapi::api::core::v1::{ConfigMap, Namespace}; use kube::api::DeleteParams; use kube::{Api, Client}; -use std::collections::BTreeMap; -use std::path::Path; -use std::sync::Once; -use std::time::Duration; +use std::path::{Path, PathBuf}; +use std::{collections::BTreeMap, env, sync::Once, time::Duration}; use tokio::process::Command; +use trusted_cluster_operator_lib::endpoints::*; +use trusted_cluster_operator_lib::openshift_ingresses::Ingress; +use trusted_cluster_operator_lib::routes::Route; + pub mod timer; pub use timer::Poller; pub mod mock_client; @@ -57,7 +60,7 @@ macro_rules! test_warn { } macro_rules! kube_apply { - ($file:expr, $test_name:expr, $log:literal $(, kustomize = $kustomize:literal)? $(, fssa = $fssa:literal)?) => { + ($file:expr, $test_name:expr, $log:expr $(, kustomize = $kustomize:literal)? $(, fssa = $fssa:literal)?) => { test_info!($test_name, $log); #[allow(unused_mut)] let mut opt = "-f"; @@ -79,12 +82,39 @@ macro_rules! kube_apply { .await?; if !apply_output.status.success() { let stderr = String::from_utf8_lossy(&apply_output.stderr); - return Err(anyhow::anyhow!("{} failed: {}", $log, stderr)); + return Err(anyhow!("{} failed: {}", $log, stderr)); + } + } +} + +fn get_env(name: &str) -> Result { + env::var(name).map_err(|e| anyhow!("Environment variable {name} is required: {e}")) +} + +pub async fn get_cluster_url( + client: Client, + namespace: &str, + service: &str, + port: i32, +) -> Result { + let check = |v: String| v != "openshift"; + if env::var(PLATFORM_ENV).map(check).unwrap_or(true) { + return Ok(format!("{namespace}.svc.cluster.local:{port}")); + } + let routes: Api = Api::namespaced(client.clone(), namespace); + match routes.get(service).await { + Ok(route) => Ok(route.spec.host.unwrap()), + Err(_) => { + // Fallback when route does not exist yet + let ingresses: Api = Api::all(client); + let ingress = ingresses.get("cluster").await?; + let domain = ingress.spec.domain.unwrap(); + Ok(format!("{service}-{namespace}.{domain}")) } } } -pub fn ensure_command(name: &str) -> anyhow::Result<()> { +pub fn ensure_command(name: &str) -> Result<()> { let result = which::which(name).map(|_| ()); result.map_err(|_| anyhow!("Command {name} not found. Please install {name} first.")) } @@ -99,7 +129,7 @@ pub struct TestContext { } impl TestContext { - pub async fn new(test_name: &str) -> anyhow::Result { + pub async fn new(test_name: &str) -> Result { INIT.call_once(|| { let _ = env_logger::builder().is_test(true).try_init(); }); @@ -152,7 +182,7 @@ impl TestContext { Ok(()) } - async fn create_namespace(&self) -> anyhow::Result<()> { + async fn create_namespace(&self) -> Result<()> { test_info!( &self.test_name, "Creating test namespace: {}", @@ -174,7 +204,7 @@ impl TestContext { Ok(()) } - async fn cleanup_namespace(&self) -> anyhow::Result<()> { + async fn cleanup_namespace(&self) -> Result<()> { let namespace_api: Api = Api::all(self.client.clone()); let dp = DeleteParams::default(); @@ -191,23 +221,19 @@ impl TestContext { Ok(()) } - fn create_temp_manifests_dir(&self) -> anyhow::Result { - let temp_dir = std::env::temp_dir(); + fn create_temp_manifests_dir(&self) -> Result { + let temp_dir = env::temp_dir(); let manifests_dir = temp_dir.join(format!("manifests-{}", uuid::Uuid::new_v4())); std::fs::create_dir_all(&manifests_dir)?; - let dir_str = manifests_dir - .to_str() - .ok_or_else(|| anyhow::anyhow!("Invalid temp directory path"))? - .to_string(); + let dir_str = manifests_dir.to_str().unwrap(); test_info!( &self.test_name, - "Created temp manifests directory: {}", - dir_str + "Created temp manifests directory: {dir_str}", ); - Ok(dir_str) + Ok(dir_str.to_string()) } - fn cleanup_manifests_dir(&self) -> anyhow::Result<()> { + fn cleanup_manifests_dir(&self) -> Result<()> { if Path::new(&self.manifests_dir).exists() { std::fs::remove_dir_all(&self.manifests_dir)?; test_info!( @@ -224,7 +250,7 @@ impl TestContext { deployments_api: &Api, deployment_name: &str, timeout_secs: u64, - ) -> anyhow::Result<()> { + ) -> Result<()> { test_info!( &self.test_name, "Waiting for deployment {} to be ready", @@ -255,7 +281,7 @@ impl TestContext { } } - Err(anyhow::anyhow!( + Err(anyhow!( "{name} deployment does not have 1 available replica yet" )) } @@ -263,15 +289,8 @@ impl TestContext { .await } - async fn apply_operator_manifests(&self) -> anyhow::Result<()> { - test_info!( - &self.test_name, - "Generating manifests in {}", - self.manifests_dir - ); - + async fn generate_manifests(&self, workspace_root: &PathBuf) -> Result<(PathBuf, PathBuf)> { let ns = self.test_namespace.clone(); - let workspace_root = std::env::current_dir()?.join(".."); let controller_gen_path = workspace_root.join("bin/controller-gen-v0.19.0"); test_info!( @@ -287,80 +306,67 @@ impl TestContext { let rbac_temp_dir = Path::new(&self.manifests_dir).join("rbac"); std::fs::create_dir_all(&crd_temp_dir)?; - let crd_temp_dir_str = crd_temp_dir - .to_str() - .ok_or_else(|| anyhow::anyhow!("Invalid CRD temp directory path"))?; - let rbac_temp_dir_str = rbac_temp_dir - .to_str() - .ok_or_else(|| anyhow::anyhow!("Invalid RBAC temp directory path"))?; - - let crd_gen_output = Command::new(&controller_gen_path) - .args([ - "rbac:roleName=trusted-cluster-operator-role", - "crd", - "webhook", - "paths=./...", - &format!("output:crd:artifacts:config={crd_temp_dir_str}"), - &format!("output:rbac:artifacts:config={rbac_temp_dir_str}"), - ]) - .current_dir(&workspace_root) - .output() - .await?; + let crd_temp_dir_str = crd_temp_dir.to_str().unwrap(); + let rbac_temp_dir_str = rbac_temp_dir.to_str().unwrap(); + + let role_name = "rbac:roleName=trusted-cluster-operator-role"; + let mut args = vec![&role_name, "crd", "webhook", "paths=./..."]; + let crd_artifacts = format!("output:crd:artifacts:config={crd_temp_dir_str}"); + let rbac_artifacts = format!("output:rbac:artifacts:config={rbac_temp_dir_str}"); + args.extend_from_slice(&[&crd_artifacts, &rbac_artifacts]); + let mut crd_gen_cmd = Command::new(&controller_gen_path); + let crd_gen = crd_gen_cmd.args(args).current_dir(workspace_root).output(); + let crd_gen_output = crd_gen.await?; if !crd_gen_output.status.success() { let stderr = String::from_utf8_lossy(&crd_gen_output.stderr); - return Err(anyhow::anyhow!( - "Failed to generate CRDs and RBAC: {stderr}" - )); + return Err(anyhow!("Failed to generate CRDs and RBAC: {stderr}")); } test_info!(&self.test_name, "CRDs and RBAC generated successfully"); let trusted_cluster_gen_path = workspace_root.join("trusted-cluster-gen"); if !trusted_cluster_gen_path.exists() { - return Err(anyhow::anyhow!( + return Err(anyhow!( "trusted-cluster-gen not found at {}. Run 'make trusted-cluster-gen' first.", trusted_cluster_gen_path.display() )); } - let repo = std::env::var("REGISTRY").unwrap_or_else(|_| "localhost:5000".to_string()); - let tag = std::env::var("TAG").unwrap_or_else(|_| "latest".to_string()); - let manifest_gen_output = Command::new(&trusted_cluster_gen_path) - .args([ - "-namespace", - &ns, - "-output-dir", - &self.manifests_dir, - "-image", - &format!("{repo}/trusted-cluster-operator:{tag}"), - "-pcrs-compute-image", - &format!("{repo}/compute-pcrs:{tag}"), - "-trustee-image", - "quay.io/trusted-execution-clusters/key-broker-service:20260106", - "-register-server-image", - &format!("{repo}/registration-server:{tag}"), - "-attestation-key-register-image", - &format!("{repo}/attestation-key-register:{tag}"), - "-approved-image", - "quay.io/trusted-execution-clusters/fedora-coreos@sha256:79a0657399e6c67c7c95b8a09193d18e5675b5aa3cfb4d75ea5c8d4d53b2af74" - ]) - .output() - .await?; + let repo = env::var("REGISTRY").unwrap_or_else(|_| "localhost:5000".to_string()); + let tag = env::var("TAG").unwrap_or_else(|_| "latest".to_string()); + let trustee_image = get_env("TRUSTEE_IMAGE")?; + let approved_image = get_env("APPROVED_IMAGE")?; + + let mut args = vec!["-namespace", &ns, "-output-dir", &self.manifests_dir]; + let operator_img = format!("{repo}/trusted-cluster-operator:{tag}"); + let compute_pcrs_img = format!("{repo}/compute-pcrs:{tag}"); + let reg_srv_img = format!("{repo}/registration-server:{tag}"); + let att_reg_img = format!("{repo}/attestation-key-register:{tag}"); + args.extend(&["-image", &operator_img]); + args.extend(&["-pcrs-compute-image", &compute_pcrs_img]); + args.extend(&["-trustee-image", &trustee_image]); + args.extend(&["-register-server-image", ®_srv_img]); + args.extend(&["-attestation-key-register-image", &att_reg_img]); + args.extend(&["-approved-image", &approved_image]); + let manifest_gen = Command::new(&trusted_cluster_gen_path).args(args).output(); + let manifest_gen_output = manifest_gen.await?; if !manifest_gen_output.status.success() { let stderr = String::from_utf8_lossy(&manifest_gen_output.stderr); - return Err(anyhow::anyhow!("Failed to generate manifests: {stderr}")); + return Err(anyhow!("Failed to generate manifests: {stderr}")); } + Ok((crd_temp_dir, rbac_temp_dir)) + } + async fn apply_operator_manifests(&self) -> Result<()> { + let manifests_dir = &self.manifests_dir; + test_info!(&self.test_name, "Generating manifests in {manifests_dir}"); + let workspace_root = env::current_dir()?.join(".."); + let (crd_temp_dir, rbac_temp_dir) = self.generate_manifests(&workspace_root).await?; test_info!(&self.test_name, "Manifests generated successfully"); - let crd_check_output = Command::new("kubectl") - .args([ - "get", - "crd", - "trustedexecutionclusters.trusted-execution-clusters.io", - ]) - .output() - .await?; + let tec = "trustedexecutionclusters.trusted-execution-clusters.io"; + let args = ["get", "crd", tec]; + let crd_check_output = Command::new("kubectl").args(args).output().await?; if crd_check_output.status.success() { test_info!( @@ -369,7 +375,7 @@ impl TestContext { ); } else { kube_apply!( - crd_temp_dir_str, + crd_temp_dir.to_str().unwrap(), &self.test_name, "Applying CRDs", fssa = true @@ -378,6 +384,7 @@ impl TestContext { test_info!(&self.test_name, "Preparing RBAC manifests"); + let ns = self.test_namespace.clone(); let sa_src = workspace_root.join("config/rbac/service_account.yaml"); let sa_content = std::fs::read_to_string(&sa_src)? .replace("namespace: system", &format!("namespace: {}", ns)); @@ -392,15 +399,11 @@ impl TestContext { std::fs::write(&role_path, role_content)?; let rb_src = workspace_root.join("config/rbac/role_binding.yaml"); + let rb = "name: manager-rolebinding"; + let role = "name: trusted-cluster-operator-role"; let rb_content = std::fs::read_to_string(&rb_src)? - .replace( - "name: manager-rolebinding", - &format!("name: {}-manager-rolebinding", ns), - ) - .replace( - "name: trusted-cluster-operator-role", - &format!("name: {}-trusted-cluster-operator-role", ns), - ) + .replace(rb, &format!("name: {}-manager-rolebinding", ns)) + .replace(role, &format!("name: {}-trusted-cluster-operator-role", ns)) .replace("namespace: system", &format!("namespace: {}", ns)); let rb_dst = rbac_temp_dir.join("role_binding.yaml"); std::fs::write(&rb_dst, rb_content)?; @@ -418,7 +421,7 @@ impl TestContext { std::fs::write(&le_rb_dst, le_rb_content)?; test_info!(&self.test_name, "Preparing RBAC kustomization"); - let platform = std::env::var("PLATFORM").unwrap_or_else(|_| "kind".to_string()); + let platform = env::var(PLATFORM_ENV).unwrap_or_else(|_| "kind".to_string()); let kustomization_src = workspace_root.join("config/rbac/kustomization.yaml.in"); let kustomization_content = std::fs::read_to_string(&kustomization_src)? .replace("namespace: NAMESPACE", &format!("namespace: {}", ns)) @@ -440,7 +443,7 @@ impl TestContext { std::fs::write(&scc_openshift_rb_dst, scc_openshift_rb_content)?; kube_apply!( - rbac_temp_dir_str, + rbac_temp_dir.to_str().unwrap(), &self.test_name, "Applying RBAC", kustomize = true @@ -448,9 +451,7 @@ impl TestContext { let manifests_path = Path::new(&self.manifests_dir); let operator_manifest_path = manifests_path.join("operator.yaml"); - let operator_manifest_str = operator_manifest_path - .to_str() - .ok_or_else(|| anyhow::anyhow!("Invalid operator manifest path"))?; + let operator_manifest_str = operator_manifest_path.to_str().unwrap(); kube_apply!( operator_manifest_str, &self.test_name, @@ -461,9 +462,14 @@ impl TestContext { &self.test_name, "Updating CR manifest with publicTrusteeAddr" ); - let cluster_url = - std::env::var("CLUSTER_URL").unwrap_or_else(|_| "svc.cluster.local".to_string()); - let trustee_addr = format!("kbs-service.{}.{}:8080", ns, cluster_url); + self.apply_operator_manifest(manifests_path, &platform) + .await + } + + async fn apply_operator_manifest(&self, manifests_path: &Path, platform: &str) -> Result<()> { + let ns = self.test_namespace.clone(); + let trustee_addr = + get_cluster_url(self.client.clone(), &ns, TRUSTEE_SERVICE, TRUSTEE_PORT).await?; let cr_manifest_path = manifests_path.join("trusted_execution_cluster_cr.yaml"); let cr_content = std::fs::read_to_string(&cr_manifest_path)?; @@ -483,19 +489,14 @@ impl TestContext { test_info!( &self.test_name, - "Updated CR manifest with publicTrusteeAddr: {}", - trustee_addr + "Updated CR manifest with publicTrusteeAddr: {trustee_addr}", ); - let cr_manifest_str = cr_manifest_path - .to_str() - .ok_or_else(|| anyhow::anyhow!("Invalid CR manifest path"))?; + let cr_manifest_str = cr_manifest_path.to_str().unwrap(); kube_apply!(cr_manifest_str, &self.test_name, "Applying CR manifest"); let approved_image_path = manifests_path.join("approved_image_cr.yaml"); - let approved_image_str = approved_image_path - .to_str() - .ok_or_else(|| anyhow::anyhow!("Invalid ApprovedImage manifest path"))?; + let approved_image_str = approved_image_path.to_str().unwrap(); kube_apply!( approved_image_str, &self.test_name, @@ -510,6 +511,20 @@ impl TestContext { .await?; self.wait_for_deployment_ready(&deployments_api, "trustee-deployment", 180) .await?; + self.wait_for_deployment_ready(&deployments_api, "attestation-key-register", 120) + .await?; + + if platform == "openshift" { + ensure_command("oc")?; + for svc in ["kbs-service", "attestation-key-register", "register-server"] { + let args = ["expose", "service", svc, "-n", &ns]; + let output = Command::new("oc").args(args).output().await?; + if !output.status.success() { + let stderr = String::from_utf8_lossy(&output.stderr); + return Err(anyhow!("oc command failed: {stderr}")); + } + } + } test_info!( &self.test_name, @@ -517,28 +532,25 @@ impl TestContext { ); let configmap_api: Api = Api::namespaced(self.client.clone(), &ns); + let err = format!("image-pcrs ConfigMap in the namespace {ns} not found"); let poller = Poller::new() .with_timeout(Duration::from_secs(60)) .with_interval(Duration::from_secs(5)) - .with_error_message(format!( - "image-pcrs ConfigMap in the namespace {} not found", - ns - )); + .with_error_message(err); let test_name_owned = self.test_name.clone(); - poller - .poll_async(move || { - let api = configmap_api.clone(); - let tn = test_name_owned.clone(); - async move { - let result = api.get("image-pcrs").await; - if result.is_ok() { - test_info!(&tn, "image-pcrs ConfigMap created"); - } - result + let check_fn = move || { + let api = configmap_api.clone(); + let tn = test_name_owned.clone(); + async move { + let result = api.get("image-pcrs").await; + if result.is_ok() { + test_info!(&tn, "image-pcrs ConfigMap created"); } - }) - .await?; + result + } + }; + poller.poll_async(check_fn).await?; Ok(()) } @@ -573,13 +585,15 @@ macro_rules! setup { () => {{ $crate::TestContext::new(TEST_NAME) }}; } -async fn setup_test_client() -> anyhow::Result { +async fn setup_test_client() -> Result { let client = Client::try_default().await?; Ok(client) } fn test_namespace_name() -> String { - format!("test-{}", &uuid::Uuid::new_v4().to_string()[..8]) + let namespace_prefix = env::var("TEST_NAMESPACE_PREFIX").unwrap_or_default(); + let uuid = &uuid::Uuid::new_v4().to_string()[..8]; + format!("{namespace_prefix}test-{uuid}") } pub async fn wait_for_resource_deleted( @@ -587,7 +601,7 @@ pub async fn wait_for_resource_deleted( resource_name: &str, timeout_secs: u64, interval_secs: u64, -) -> anyhow::Result<()> +) -> Result<()> where K: kube::Resource + Clone + std::fmt::Debug, K: k8s_openapi::serde::de::DeserializeOwned, diff --git a/test_utils/src/timer.rs b/test_utils/src/timer.rs index a1f70ec5..5686258e 100644 --- a/test_utils/src/timer.rs +++ b/test_utils/src/timer.rs @@ -51,12 +51,12 @@ impl Poller { loop { match check_fn().await { Ok(result) => return Ok(result), - Err(_) => { + Err(e) => { if start_time.elapsed() >= self.timeout { let error_msg = self.error_message.as_ref().cloned().unwrap_or_else(|| { format!("Polling timed out after {:?}", self.timeout) }); - return Err(anyhow::anyhow!(error_msg)); + return Err(anyhow::anyhow!("{error_msg}, last error was: {e:?}")); } tokio::time::sleep(self.interval).await; } diff --git a/test_utils/src/virt/kubevirt.rs b/test_utils/src/virt/kubevirt.rs index 25d8fcfd..9bd674bd 100644 --- a/test_utils/src/virt/kubevirt.rs +++ b/test_utils/src/virt/kubevirt.rs @@ -19,7 +19,7 @@ pub struct KubevirtBackend(pub VmConfig); #[async_trait::async_trait] impl VmBackend for KubevirtBackend { async fn create_vm(&self) -> Result<()> { - let ignition_json = generate_ignition(&self.0, true); + let ignition_json = generate_ignition(&self.0, true).await?; let vm = VirtualMachine { metadata: ObjectMeta { name: Some(self.0.vm_name.clone()), diff --git a/test_utils/src/virt/mod.rs b/test_utils/src/virt/mod.rs index a085d7a3..9d129108 100644 --- a/test_utils/src/virt/mod.rs +++ b/test_utils/src/virt/mod.rs @@ -11,9 +11,12 @@ use k8s_openapi::api::core::v1::Secret; use kube::{Api, Client}; use std::{env, path::PathBuf, time::Duration}; use tokio::process::Command; -use trusted_cluster_operator_lib::Machine; + +use endpoints::*; +use trusted_cluster_operator_lib::*; use super::Poller; +use crate::{get_cluster_url, get_env}; /// Environment variable name for selecting the VM provider pub const VIRT_PROVIDER_ENV: &str = "VIRT_PROVIDER"; @@ -71,17 +74,19 @@ pub fn generate_ssh_key_pair() -> Result<(String, PathBuf)> { Ok((public_key_str, key_path)) } -pub fn generate_ignition(config: &VmConfig, with_ak: bool) -> serde_json::Value { +pub async fn generate_ignition(config: &VmConfig, with_ak: bool) -> Result { use ignition_config::v3_5::*; - let register_server_url = format!( - "http://register-server.{}.svc.cluster.local:8000/ignition-clevis-pin-trustee", - config.namespace - ); + let client = config.client.clone(); + let ns = &config.namespace; + let register_server_url = + get_cluster_url(client, ns, REGISTER_SERVER_SERVICE, REGISTER_SERVER_PORT).await?; let ignition = Ignition { version: "3.6.0-experimental".to_string(), config: Some(IgnitionConfig { merge: Some(vec![Resource { - source: Some(register_server_url), + source: Some(format!( + "http://{register_server_url}/{REGISTER_SERVER_RESOURCE}" + )), ..Default::default() }]), ..Default::default() @@ -121,24 +126,29 @@ pub fn generate_ignition(config: &VmConfig, with_ak: bool) -> serde_json::Value let mut ignition_json = serde_json::to_value(&ignition_config).unwrap(); if with_ak { - ignition_json = patch_ak(&config.namespace, ignition_json); + ignition_json = patch_ak(config.client.clone(), ns, ignition_json).await?; } - ignition_json + Ok(ignition_json) } -fn patch_ak(namespace: &str, mut ignition: serde_json::Value) -> serde_json::Value { - let attestation_url = - format!("http://attestation-key-register.{namespace}.svc.cluster.local:8001/register-ak",); +async fn patch_ak( + client: Client, + namespace: &str, + mut ignition: serde_json::Value, +) -> Result { + let svc = ATTESTATION_KEY_REGISTER_SERVICE; + let port = ATTESTATION_KEY_REGISTER_PORT; + let attestation_url = get_cluster_url(client, namespace, svc, port).await?; let ign_json = serde_json::json!({ "attestation_key": { "registration": { - "url": attestation_url + "url": format!("http://{attestation_url}/{ATTESTATION_KEY_REGISTER_RESOURCE}"), } } }); let obj = ignition.as_object_mut().unwrap(); obj.insert("attestation".to_string(), ign_json); - ignition + Ok(ignition) } pub async fn ssh_exec(command: &str) -> Result { @@ -191,13 +201,14 @@ pub fn create_backend( ) -> Result> { let provider = get_virt_provider()?; let (public_key, key_path) = generate_ssh_key_pair()?; + let image = get_env("TEST_IMAGE")?; let config = VmConfig { client, namespace: namespace.to_string(), vm_name: vm_name.to_string(), ssh_public_key: public_key, ssh_private_key: key_path, - image: "quay.io/trusted-execution-clusters/fedora-coreos-kubevirt:2026-14-01".to_string(), + image, }; match provider { VirtProvider::Kubevirt => Ok(Box::new(kubevirt::KubevirtBackend(config))), From d93e5c98577ac4ceb783777d6f80d48d76ccc60e Mon Sep 17 00:00:00 2001 From: Jakob Naucke Date: Fri, 23 Jan 2026 10:23:31 +0100 Subject: [PATCH 6/6] [reword,readme] tests: Add Azure backend Signed-off-by: Jakob Naucke Co-authored-by: Uri Lublin Assisted-by: Gemini, Claude --- Cargo.lock | 794 +++++++++++++++++++++++++++++++++-- test_utils/Cargo.toml | 3 + test_utils/src/virt/azure.rs | 344 +++++++++++++++ test_utils/src/virt/mod.rs | 6 +- tests/attestation.rs | 6 +- 5 files changed, 1112 insertions(+), 41 deletions(-) create mode 100644 test_utils/src/virt/azure.rs diff --git a/Cargo.lock b/Cargo.lock index 256245aa..27d528d9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -135,6 +135,29 @@ dependencies = [ "pin-project-lite", ] +[[package]] +name = "async-compression" +version = "0.4.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d10e4f991a553474232bc0a31799f6d24b034a84c0971d80d2e2f78b2e576e40" +dependencies = [ + "compression-codecs", + "compression-core", + "pin-project-lite", + "tokio", +] + +[[package]] +name = "async-lock" +version = "3.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "290f7f2596bd5b78a9fec8088ccd89180d7f9f55b94b0576823bbbdc72ee8311" +dependencies = [ + "event-listener", + "event-listener-strategy", + "pin-project-lite", +] + [[package]] name = "async-stream" version = "0.3.6" @@ -252,6 +275,78 @@ dependencies = [ "thiserror 1.0.69", ] +[[package]] +name = "aws-lc-rs" +version = "1.15.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e84ce723ab67259cfeb9877c6a639ee9eb7a27b28123abd71db7f0d5d0cc9d86" +dependencies = [ + "aws-lc-sys", + "zeroize", +] + +[[package]] +name = "aws-lc-sys" +version = "0.36.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "43a442ece363113bd4bd4c8b18977a7798dd4d3c3383f34fb61936960e8f4ad8" +dependencies = [ + "cc", + "cmake", + "dunce", + "fs_extra", +] + +[[package]] +name = "azure_core" +version = "0.31.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dfe45c6bd7ce3a592327ee4e35b5bd16681714c4443c8a9884abb5731cc4d833" +dependencies = [ + "async-lock", + "async-trait", + "azure_core_macros", + "bytes", + "futures", + "pin-project", + "rustc_version", + "serde", + "serde_json", + "tracing", + "typespec", + "typespec_client_core", +] + +[[package]] +name = "azure_core_macros" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "190d6e0622d17e2a28239b55d2829d98b348269adcd4ab86a21d3304aa3500cb" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.110", + "tracing", +] + +[[package]] +name = "azure_identity" +version = "0.31.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c7c0c8cb8886f2bdabb3501476fa53f87fa9efea9d457ff991c5ce80052c4774" +dependencies = [ + "async-lock", + "async-trait", + "azure_core", + "futures", + "pin-project", + "serde", + "serde_json", + "time", + "tracing", + "url", +] + [[package]] name = "backon" version = "1.6.0" @@ -358,20 +453,34 @@ checksum = "d71b6127be86fdcfddb610f7182ac57211d4b18a3e9c82eb2d17662f2227ad6a" [[package]] name = "cc" -version = "1.2.45" +version = "1.2.54" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "35900b6c8d709fb1d854671ae27aeaa9eec2f8b01b364e1619a40da3e6fe2afe" +checksum = "6354c81bbfd62d9cfa9cb3c773c2b7b2a3a482d569de977fd0e961f6e7c00583" dependencies = [ "find-msvc-tools", + "jobserver", + "libc", "shlex", ] +[[package]] +name = "cesu8" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d43a04d8753f35258c91f8ec639f792891f748a1edbd759cf1dcea3382ad83c" + [[package]] name = "cfg-if" version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" +[[package]] +name = "cfg_aliases" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724" + [[package]] name = "chrono" version = "0.4.42" @@ -444,12 +553,48 @@ dependencies = [ "serde", ] +[[package]] +name = "cmake" +version = "0.1.57" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75443c44cd6b379beb8c5b45d85d0773baf31cce901fe7bb252f4eff3008ef7d" +dependencies = [ + "cc", +] + [[package]] name = "colorchoice" version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b05b61dc5112cbb17e4b6cd61790d9845d13888356391624cbe7e41efeac1e75" +[[package]] +name = "combine" +version = "4.6.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba5a308b75df32fe02788e748662718f03fde005016435c444eea572398219fd" +dependencies = [ + "bytes", + "memchr", +] + +[[package]] +name = "compression-codecs" +version = "0.4.36" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "00828ba6fd27b45a448e57dbfe84f1029d4c9f26b368157e9a448a5f49a2ec2a" +dependencies = [ + "compression-core", + "flate2", + "memchr", +] + +[[package]] +name = "compression-core" +version = "0.4.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75984efb6ed102a0d42db99afb6c1948f0380d1d91808d5529916e6c08b49d8d" + [[package]] name = "compute-pcrs" version = "0.1.0" @@ -525,6 +670,16 @@ dependencies = [ "libc", ] +[[package]] +name = "core-foundation" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2a6cd9ae233e7f62ba4e9353e81a88df7fc8a5987b8d445b4d90c879bd156f6" +dependencies = [ + "core-foundation-sys", + "libc", +] + [[package]] name = "core-foundation-sys" version = "0.8.7" @@ -562,7 +717,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0dc92fb57ca44df6db8059111ab3af99a63d5d0f8375d9972e319a379c6bab76" dependencies = [ "generic-array", - "rand_core", + "rand_core 0.6.4", "subtle", "zeroize", ] @@ -699,6 +854,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ececcb659e7ba858fb4f10388c250a7252eb0a27373f1a72b8748afdd248e587" dependencies = [ "powerfmt", + "serde_core", ] [[package]] @@ -775,6 +931,12 @@ dependencies = [ "syn 2.0.110", ] +[[package]] +name = "dunce" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92773504d58c093f6de2459af4af33faa518c13451eb8f2b5698ed3d36e7c813" + [[package]] name = "dyn-clone" version = "1.0.20" @@ -826,7 +988,7 @@ dependencies = [ "generic-array", "group", "pkcs8", - "rand_core", + "rand_core 0.6.4", "sec1", "subtle", "zeroize", @@ -939,15 +1101,15 @@ version = "0.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c0b50bfb653653f9ca9095b427bed08ab8d75a137839d9ad64eb11810d5b6393" dependencies = [ - "rand_core", + "rand_core 0.6.4", "subtle", ] [[package]] name = "find-msvc-tools" -version = "0.1.4" +version = "0.1.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "52051878f80a721bb68ebfbc930e07b65ba72f2da88968ea5c06fd6ca3d3a127" +checksum = "8591b0bcc8a98a64310a2fae1bb3e9b8564dd10e381e6e28010fde8e8e8568db" [[package]] name = "flate2" @@ -1009,6 +1171,7 @@ checksum = "65bc07b1a8bc7c85c5f2e110c476c7389b4554ba72af57d8445ea63a576b0876" dependencies = [ "futures-channel", "futures-core", + "futures-executor", "futures-io", "futures-sink", "futures-task", @@ -1031,6 +1194,17 @@ version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "05f29059c0c2090612e8d742178b0580d2dc940c837851ad723096f87af6663e" +[[package]] +name = "futures-executor" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e28d1d997f585e54aebc3f97d39e72338912123a67330d723fdbb564d646c9f" +dependencies = [ + "futures-core", + "futures-task", + "futures-util", +] + [[package]] name = "futures-io" version = "0.3.31" @@ -1096,8 +1270,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "335ff9f135e4384c8150d6f27c6daed433577f86b4750418338c01a1a2528592" dependencies = [ "cfg-if", + "js-sys", "libc", "wasi", + "wasm-bindgen", ] [[package]] @@ -1107,9 +1283,11 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "899def5c37c4fd7b2664648c28120ecec138e4d395b459e5ca34f9cce2dd77fd" dependencies = [ "cfg-if", + "js-sys", "libc", "r-efi", "wasip2", + "wasm-bindgen", ] [[package]] @@ -1175,7 +1353,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f0f9ef7462f7c099f518d754361858f86d8a07af53ba9af0fe635bbccb151a63" dependencies = [ "ff", - "rand_core", + "rand_core 0.6.4", "subtle", ] @@ -1198,6 +1376,25 @@ dependencies = [ "tracing", ] +[[package]] +name = "h2" +version = "0.4.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2f44da3a8150a6703ed5d34e164b875fd14c2cdab9af1252a9a1020bde2bdc54" +dependencies = [ + "atomic-waker", + "bytes", + "fnv", + "futures-core", + "futures-sink", + "http 1.4.0", + "indexmap 2.12.0", + "slab", + "tokio", + "tokio-util", + "tracing", +] + [[package]] name = "hashbrown" version = "0.12.3" @@ -1394,7 +1591,7 @@ dependencies = [ "futures-channel", "futures-core", "futures-util", - "h2", + "h2 0.3.27", "http 0.2.12", "http-body 0.4.6", "httparse", @@ -1418,6 +1615,7 @@ dependencies = [ "bytes", "futures-channel", "futures-core", + "h2 0.4.13", "http 1.4.0", "http-body 1.0.1", "httparse", @@ -1457,9 +1655,25 @@ dependencies = [ "futures-util", "http 0.2.12", "hyper 0.14.32", - "rustls", + "rustls 0.21.12", "tokio", - "tokio-rustls", + "tokio-rustls 0.24.1", +] + +[[package]] +name = "hyper-rustls" +version = "0.27.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3c93eb611681b207e1fe55d5a71ecf91572ec8a6705cdb6857f7d8d5242cf58" +dependencies = [ + "http 1.4.0", + "hyper 1.7.0", + "hyper-util", + "rustls 0.23.36", + "rustls-pki-types", + "tokio", + "tokio-rustls 0.26.4", + "tower-service", ] [[package]] @@ -1510,9 +1724,11 @@ dependencies = [ "percent-encoding", "pin-project-lite", "socket2 0.6.1", + "system-configuration 0.6.1", "tokio", "tower-service", "tracing", + "windows-registry", ] [[package]] @@ -1778,6 +1994,38 @@ dependencies = [ "syn 2.0.110", ] +[[package]] +name = "jni" +version = "0.21.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a87aa2bb7d2af34197c04845522473242e1aa17c12f4935d5856491a7fb8c97" +dependencies = [ + "cesu8", + "cfg-if", + "combine", + "jni-sys", + "log", + "thiserror 1.0.69", + "walkdir", + "windows-sys 0.45.0", +] + +[[package]] +name = "jni-sys" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8eaf4bc02d17cbdd7ff4c7438cafcdf7fb9a4613313ad11b4f8fefe7d3fa0130" + +[[package]] +name = "jobserver" +version = "0.1.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9afb3de4395d6b3e67a780b6de64b51c978ecf11cb9a462c66be7d4ca9039d33" +dependencies = [ + "getrandom 0.3.4", + "libc", +] + [[package]] name = "js-sys" version = "0.3.82" @@ -2072,6 +2320,12 @@ version = "0.4.29" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5e5032e24019045c762d3c0f28f5b6b8bbf38563a65908389bf7978758920897" +[[package]] +name = "lru-slab" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "112b39cec0b298b6c1999fee3e31427f74f676e4cb9879ed1a121b43661a4154" + [[package]] name = "memchr" version = "2.7.6" @@ -2183,10 +2437,10 @@ dependencies = [ "libc", "log", "openssl", - "openssl-probe", + "openssl-probe 0.1.6", "openssl-sys", "schannel", - "security-framework", + "security-framework 2.11.1", "security-framework-sys", "tempfile", ] @@ -2212,7 +2466,7 @@ dependencies = [ "num-integer", "num-iter", "num-traits", - "rand", + "rand 0.8.5", "smallvec", "zeroize", ] @@ -2371,6 +2625,12 @@ version = "0.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d05e27ee213611ffe7d6348b942e8f942b37114c00cc03cec254295a4a17852e" +[[package]] +name = "openssl-probe" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7c87def4c32ab89d880effc9e097653c8da5d6ef28e6b539d313baaacfbafcbe" + [[package]] name = "openssl-sys" version = "0.9.111" @@ -2460,7 +2720,7 @@ dependencies = [ "ecdsa", "elliptic-curve", "primeorder", - "rand_core", + "rand_core 0.6.4", "sha2", ] @@ -2723,6 +2983,62 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "quinn" +version = "0.11.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9e20a958963c291dc322d98411f541009df2ced7b5a4f2bd52337638cfccf20" +dependencies = [ + "bytes", + "cfg_aliases", + "pin-project-lite", + "quinn-proto", + "quinn-udp", + "rustc-hash", + "rustls 0.23.36", + "socket2 0.6.1", + "thiserror 2.0.17", + "tokio", + "tracing", + "web-time", +] + +[[package]] +name = "quinn-proto" +version = "0.11.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1906b49b0c3bc04b5fe5d86a77925ae6524a19b816ae38ce1e426255f1d8a31" +dependencies = [ + "aws-lc-rs", + "bytes", + "getrandom 0.3.4", + "lru-slab", + "rand 0.9.2", + "ring", + "rustc-hash", + "rustls 0.23.36", + "rustls-pki-types", + "slab", + "thiserror 2.0.17", + "tinyvec", + "tracing", + "web-time", +] + +[[package]] +name = "quinn-udp" +version = "0.5.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "addec6a0dcad8a8d96a771f815f0eaf55f9d1805756410b39f5fa81332574cbd" +dependencies = [ + "cfg_aliases", + "libc", + "once_cell", + "socket2 0.6.1", + "tracing", + "windows-sys 0.60.2", +] + [[package]] name = "quote" version = "1.0.42" @@ -2745,8 +3061,18 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404" dependencies = [ "libc", - "rand_chacha", - "rand_core", + "rand_chacha 0.3.1", + "rand_core 0.6.4", +] + +[[package]] +name = "rand" +version = "0.9.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6db2770f06117d490610c7488547d543617b21bfa07796d7a12f6f1bd53850d1" +dependencies = [ + "rand_chacha 0.9.0", + "rand_core 0.9.5", ] [[package]] @@ -2756,7 +3082,17 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" dependencies = [ "ppv-lite86", - "rand_core", + "rand_core 0.6.4", +] + +[[package]] +name = "rand_chacha" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3022b5f1df60f26e1ffddd6c66e8aa15de382ae63b3a0c1bfc0e4d3e3f325cb" +dependencies = [ + "ppv-lite86", + "rand_core 0.9.5", ] [[package]] @@ -2768,6 +3104,15 @@ dependencies = [ "getrandom 0.2.16", ] +[[package]] +name = "rand_core" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76afc826de14238e6e8c374ddcc1fa19e374fd8dd986b0d2af0d02377261d83c" +dependencies = [ + "getrandom 0.3.4", +] + [[package]] name = "redox_syscall" version = "0.5.18" @@ -2859,11 +3204,11 @@ dependencies = [ "encoding_rs", "futures-core", "futures-util", - "h2", + "h2 0.3.27", "http 0.2.12", "http-body 0.4.6", "hyper 0.14.32", - "hyper-rustls", + "hyper-rustls 0.24.2", "ipnet", "js-sys", "log", @@ -2871,15 +3216,15 @@ dependencies = [ "once_cell", "percent-encoding", "pin-project-lite", - "rustls", + "rustls 0.21.12", "rustls-pemfile", "serde", "serde_json", "serde_urlencoded", "sync_wrapper 0.1.2", - "system-configuration", + "system-configuration 0.5.1", "tokio", - "tokio-rustls", + "tokio-rustls 0.24.1", "tower-service", "url", "wasm-bindgen", @@ -2895,6 +3240,7 @@ version = "0.12.24" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9d0946410b9f7b082a427e4ef5c8ff541a88b357bc6c637c40db3a68ac70a36f" dependencies = [ + "async-compression", "base64 0.22.1", "bytes", "futures-core", @@ -2928,6 +3274,46 @@ dependencies = [ "web-sys", ] +[[package]] +name = "reqwest" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "04e9018c9d814e5f30cc16a0f03271aeab3571e609612d9fe78c1aa8d11c2f62" +dependencies = [ + "base64 0.22.1", + "bytes", + "encoding_rs", + "futures-core", + "h2 0.4.13", + "http 1.4.0", + "http-body 1.0.1", + "http-body-util", + "hyper 1.7.0", + "hyper-rustls 0.27.7", + "hyper-util", + "js-sys", + "log", + "mime", + "percent-encoding", + "pin-project-lite", + "quinn", + "rustls 0.23.36", + "rustls-pki-types", + "rustls-platform-verifier", + "serde", + "serde_json", + "sync_wrapper 1.0.2", + "tokio", + "tokio-rustls 0.26.4", + "tower", + "tower-http", + "tower-service", + "url", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", +] + [[package]] name = "rfc6979" version = "0.4.0" @@ -2965,7 +3351,7 @@ dependencies = [ "num-traits", "pkcs1", "pkcs8", - "rand_core", + "rand_core 0.6.4", "sha2", "signature", "spki", @@ -2979,6 +3365,21 @@ version = "0.1.26" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "56f7d92ca342cea22a06f2121d944b4fd82af56988c270852495420f961d4ace" +[[package]] +name = "rustc-hash" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "357703d41365b4b27c590e3ed91eabb1b663f07c4c084095e60cbed4362dff0d" + +[[package]] +name = "rustc_version" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cfcb3a22ef46e85b45de6ee7e79d063319ebb6594faafcf1c225ea92ab6e9b92" +dependencies = [ + "semver", +] + [[package]] name = "rustix" version = "1.1.2" @@ -3000,10 +3401,36 @@ checksum = "3f56a14d1f48b391359b22f731fd4bd7e43c97f3c50eee276f3aa09c94784d3e" dependencies = [ "log", "ring", - "rustls-webpki", + "rustls-webpki 0.101.7", "sct", ] +[[package]] +name = "rustls" +version = "0.23.36" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c665f33d38cea657d9614f766881e4d510e0eda4239891eea56b4cadcf01801b" +dependencies = [ + "aws-lc-rs", + "once_cell", + "rustls-pki-types", + "rustls-webpki 0.103.9", + "subtle", + "zeroize", +] + +[[package]] +name = "rustls-native-certs" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "612460d5f7bea540c490b2b6395d8e34a953e52b491accd6c86c8164c5932a63" +dependencies = [ + "openssl-probe 0.2.1", + "rustls-pki-types", + "schannel", + "security-framework 3.5.1", +] + [[package]] name = "rustls-pemfile" version = "1.0.4" @@ -3019,9 +3446,37 @@ version = "1.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "94182ad936a0c91c324cd46c6511b9510ed16af436d7b5bab34beab0afd55f7a" dependencies = [ + "web-time", "zeroize", ] +[[package]] +name = "rustls-platform-verifier" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d99feebc72bae7ab76ba994bb5e121b8d83d910ca40b36e0921f53becc41784" +dependencies = [ + "core-foundation 0.10.1", + "core-foundation-sys", + "jni", + "log", + "once_cell", + "rustls 0.23.36", + "rustls-native-certs", + "rustls-platform-verifier-android", + "rustls-webpki 0.103.9", + "security-framework 3.5.1", + "security-framework-sys", + "webpki-root-certs", + "windows-sys 0.61.2", +] + +[[package]] +name = "rustls-platform-verifier-android" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f87165f0995f63a9fbeea62b64d10b4d9d8e78ec6d7d51fb2125fda7bb36788f" + [[package]] name = "rustls-webpki" version = "0.101.7" @@ -3032,6 +3487,18 @@ dependencies = [ "untrusted", ] +[[package]] +name = "rustls-webpki" +version = "0.103.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d7df23109aa6c1567d1c575b9952556388da57401e4ace1d15f79eedad0d8f53" +dependencies = [ + "aws-lc-rs", + "ring", + "rustls-pki-types", + "untrusted", +] + [[package]] name = "rustversion" version = "1.0.22" @@ -3044,6 +3511,15 @@ version = "1.0.20" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "28d3b2b1366ec20994f1fd18c3c594f05c5dd4bc44d8bb0c1c632c8d6829481f" +[[package]] +name = "same-file" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502" +dependencies = [ + "winapi-util", +] + [[package]] name = "schannel" version = "0.1.28" @@ -3130,7 +3606,20 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "897b2245f0b511c87893af39b033e5ca9cce68824c4d7e7630b5a1d339658d02" dependencies = [ "bitflags 2.10.0", - "core-foundation", + "core-foundation 0.9.4", + "core-foundation-sys", + "libc", + "security-framework-sys", +] + +[[package]] +name = "security-framework" +version = "3.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b3297343eaf830f66ede390ea39da1d462b6b0c1b000f420d0a83f898bbbe6ef" +dependencies = [ + "bitflags 2.10.0", + "core-foundation 0.10.1", "core-foundation-sys", "libc", "security-framework-sys", @@ -3295,7 +3784,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "77549399552de45a898a580c1b41d445bf730df867cc44e6c0233bbc4b8329de" dependencies = [ "digest", - "rand_core", + "rand_core 0.6.4", ] [[package]] @@ -3389,7 +3878,7 @@ dependencies = [ "p256", "p384", "p521", - "rand_core", + "rand_core 0.6.4", "rsa", "sec1", "sha2", @@ -3519,8 +4008,19 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ba3a3adc5c275d719af8cb4272ea1c4a6d668a777f37e115f6d11ddbc1c8e0e7" dependencies = [ "bitflags 1.3.2", - "core-foundation", - "system-configuration-sys", + "core-foundation 0.9.4", + "system-configuration-sys 0.5.0", +] + +[[package]] +name = "system-configuration" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c879d448e9d986b661742763247d3693ed13609438cf3d006f51f5368a5ba6b" +dependencies = [ + "bitflags 2.10.0", + "core-foundation 0.9.4", + "system-configuration-sys 0.6.0", ] [[package]] @@ -3533,6 +4033,16 @@ dependencies = [ "libc", ] +[[package]] +name = "system-configuration-sys" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e1d1b10ced5ca923a1fcb8d03e96b8d3268065d724548c0211415ff6ac6bac4" +dependencies = [ + "core-foundation-sys", + "libc", +] + [[package]] name = "tempfile" version = "3.23.0" @@ -3614,10 +4124,13 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "91e7d9e3bb61134e77bde20dd4825b97c010155709965fedf0f49bb138e52a9d" dependencies = [ "deranged", + "itoa", + "js-sys", "num-conv", "powerfmt", "serde", "time-core", + "time-macros", ] [[package]] @@ -3626,6 +4139,16 @@ version = "0.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "40868e7c1d2f0b8d73e4a8c7f0ff63af4f6d19be117e90bd73eb1d62cf831c6b" +[[package]] +name = "time-macros" +version = "0.2.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "30cfb0125f12d9c277f35663a0a33f8c30190f4e4574868a330595412d34ebf3" +dependencies = [ + "num-conv", + "time-core", +] + [[package]] name = "tinystr" version = "0.8.2" @@ -3694,7 +4217,17 @@ version = "0.24.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c28327cf380ac148141087fbfb9de9d7bd4e84ab5d2c28fbc911d753de8a7081" dependencies = [ - "rustls", + "rustls 0.21.12", + "tokio", +] + +[[package]] +name = "tokio-rustls" +version = "0.26.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1729aa945f29d91ba541258c8df89027d5792d85a8841fb65e8bf0f4ede4ef61" +dependencies = [ + "rustls 0.23.36", "tokio", ] @@ -3746,9 +4279,9 @@ dependencies = [ [[package]] name = "tower-http" -version = "0.6.6" +version = "0.6.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "adc82fd73de2a9722ac5da747f12383d2bfdb93591ee6c58486e0097890f05f2" +checksum = "d4e6559d53cc268e5031cd8429d05415bc4cb4aefc4aa5d6cc35fbf5b924a1f8" dependencies = [ "base64 0.22.1", "bitflags 2.10.0", @@ -3827,6 +4360,8 @@ dependencies = [ "anyhow", "async-trait", "auto_impl", + "azure_core", + "azure_identity", "clevis-pin-trustee-lib", "compute-pcrs-lib", "env_logger", @@ -3836,7 +4371,8 @@ dependencies = [ "k8s-openapi", "kube", "log", - "rand_core", + "rand_core 0.6.4", + "reqwest 0.13.1", "serde", "serde_json", "serde_yaml", @@ -3882,7 +4418,7 @@ dependencies = [ "http 1.4.0", "httparse", "log", - "rand", + "rand 0.8.5", "sha1", "thiserror 1.0.69", "url", @@ -3895,6 +4431,57 @@ version = "1.19.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "562d481066bde0658276a35467c4af00bdc6ee726305698a55b86e61d7ad82bb" +[[package]] +name = "typespec" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4dd1eb4a538c1ab3d5c05437129bc16891296146b23c9b0bb3f5df99f5b3a18d" +dependencies = [ + "base64 0.22.1", + "bytes", + "futures", + "serde", + "serde_json", + "url", +] + +[[package]] +name = "typespec_client_core" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e632235c99ae896a3c451d1ead00cea11a2219aeda1b35a74027fe99ea3f3b72" +dependencies = [ + "async-trait", + "base64 0.22.1", + "dyn-clone", + "futures", + "getrandom 0.3.4", + "pin-project", + "rand 0.9.2", + "reqwest 0.12.24", + "serde", + "serde_json", + "time", + "tokio", + "tracing", + "typespec", + "typespec_macros", + "url", + "uuid", +] + +[[package]] +name = "typespec_macros" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7048df3b053daa72e8ea91894ebcb2f0511ba52737379834524d82074a94a458" +dependencies = [ + "proc-macro2", + "quote", + "rustc_version", + "syn 2.0.110", +] + [[package]] name = "ucd-trie" version = "0.1.7" @@ -4006,6 +4593,16 @@ version = "0.9.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" +[[package]] +name = "walkdir" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29790946404f91d9c5d06f9874efddea1dc06c5efe94541a7d6863108e3a5e4b" +dependencies = [ + "same-file", + "winapi-util", +] + [[package]] name = "want" version = "0.3.1" @@ -4140,6 +4737,25 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "web-time" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a6580f308b1fad9207618087a65c04e7a10bc77e02c8e84e9b00dd4b12fa0bb" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "webpki-root-certs" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "36a29fc0408b113f68cf32637857ab740edfafdf460c326cd2afaa2d84cc05dc" +dependencies = [ + "rustls-pki-types", +] + [[package]] name = "webpki-roots" version = "0.25.4" @@ -4173,6 +4789,15 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" +[[package]] +name = "winapi-util" +version = "0.1.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" +dependencies = [ + "windows-sys 0.61.2", +] + [[package]] name = "winapi-x86_64-pc-windows-gnu" version = "0.4.0" @@ -4188,8 +4813,8 @@ dependencies = [ "windows-implement", "windows-interface", "windows-link 0.2.1", - "windows-result", - "windows-strings", + "windows-result 0.4.1", + "windows-strings 0.5.1", ] [[package]] @@ -4226,6 +4851,26 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" +[[package]] +name = "windows-registry" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b8a9ed28765efc97bbc954883f4e6796c33a06546ebafacbabee9696967499e" +dependencies = [ + "windows-link 0.1.3", + "windows-result 0.3.4", + "windows-strings 0.4.2", +] + +[[package]] +name = "windows-result" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "56f42bd332cc6c8eac5af113fc0c1fd6a8fd2aa08a0119358686e5160d0586c6" +dependencies = [ + "windows-link 0.1.3", +] + [[package]] name = "windows-result" version = "0.4.1" @@ -4235,6 +4880,15 @@ dependencies = [ "windows-link 0.2.1", ] +[[package]] +name = "windows-strings" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "56e6c93f3a0c3b36176cb1327a4958a0353d5d166c2a35cb268ace15e91d3b57" +dependencies = [ + "windows-link 0.1.3", +] + [[package]] name = "windows-strings" version = "0.5.1" @@ -4244,6 +4898,15 @@ dependencies = [ "windows-link 0.2.1", ] +[[package]] +name = "windows-sys" +version = "0.45.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75283be5efb2831d37ea142365f009c02ec203cd29a3ebecbc093d52315b66d0" +dependencies = [ + "windows-targets 0.42.2", +] + [[package]] name = "windows-sys" version = "0.48.0" @@ -4289,6 +4952,21 @@ dependencies = [ "windows-link 0.2.1", ] +[[package]] +name = "windows-targets" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e5180c00cd44c9b1c88adb3693291f1cd93605ded80c250a75d472756b4d071" +dependencies = [ + "windows_aarch64_gnullvm 0.42.2", + "windows_aarch64_msvc 0.42.2", + "windows_i686_gnu 0.42.2", + "windows_i686_msvc 0.42.2", + "windows_x86_64_gnu 0.42.2", + "windows_x86_64_gnullvm 0.42.2", + "windows_x86_64_msvc 0.42.2", +] + [[package]] name = "windows-targets" version = "0.48.5" @@ -4337,6 +5015,12 @@ dependencies = [ "windows_x86_64_msvc 0.53.1", ] +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "597a5118570b68bc08d8d59125332c54f1ba9d9adeedeef5b99b02ba2b0698f8" + [[package]] name = "windows_aarch64_gnullvm" version = "0.48.5" @@ -4355,6 +5039,12 @@ version = "0.53.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a9d8416fa8b42f5c947f8482c43e7d89e73a173cead56d044f6a56104a6d1b53" +[[package]] +name = "windows_aarch64_msvc" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e08e8864a60f06ef0d0ff4ba04124db8b0fb3be5776a5cd47641e942e58c4d43" + [[package]] name = "windows_aarch64_msvc" version = "0.48.5" @@ -4373,6 +5063,12 @@ version = "0.53.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b9d782e804c2f632e395708e99a94275910eb9100b2114651e04744e9b125006" +[[package]] +name = "windows_i686_gnu" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c61d927d8da41da96a81f029489353e68739737d3beca43145c8afec9a31a84f" + [[package]] name = "windows_i686_gnu" version = "0.48.5" @@ -4403,6 +5099,12 @@ version = "0.53.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fa7359d10048f68ab8b09fa71c3daccfb0e9b559aed648a8f95469c27057180c" +[[package]] +name = "windows_i686_msvc" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "44d840b6ec649f480a41c8d80f9c65108b92d89345dd94027bfe06ac444d1060" + [[package]] name = "windows_i686_msvc" version = "0.48.5" @@ -4421,6 +5123,12 @@ version = "0.53.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1e7ac75179f18232fe9c285163565a57ef8d3c89254a30685b57d83a38d326c2" +[[package]] +name = "windows_x86_64_gnu" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8de912b8b8feb55c064867cf047dda097f92d51efad5b491dfb98f6bbb70cb36" + [[package]] name = "windows_x86_64_gnu" version = "0.48.5" @@ -4439,6 +5147,12 @@ version = "0.53.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9c3842cdd74a865a8066ab39c8a7a473c0778a3f29370b5fd6b4b9aa7df4a499" +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "26d41b46a36d453748aedef1486d5c7a85db22e56aff34643984ea85514e94a3" + [[package]] name = "windows_x86_64_gnullvm" version = "0.48.5" @@ -4457,6 +5171,12 @@ version = "0.53.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0ffa179e2d07eee8ad8f57493436566c7cc30ac536a3379fdf008f47f6bb7ae1" +[[package]] +name = "windows_x86_64_msvc" +version = "0.42.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9aec5da331524158c6d1a4ac0ab1541149c0b9505fde06423b02f5ef0106b9f0" + [[package]] name = "windows_x86_64_msvc" version = "0.48.5" diff --git a/test_utils/Cargo.toml b/test_utils/Cargo.toml index a3661779..1c2ddce8 100644 --- a/test_utils/Cargo.toml +++ b/test_utils/Cargo.toml @@ -15,6 +15,8 @@ virtualization = [] anyhow.workspace = true async-trait = "0.1" auto_impl = "1" +azure_core = "0.31.0" +azure_identity = "0.31.0" clevis-pin-trustee-lib.workspace = true compute-pcrs-lib.workspace = true env_logger.workspace = true @@ -24,6 +26,7 @@ k8s-openapi.workspace = true kube = { workspace = true } log.workspace = true rand_core = "0.6" +reqwest = { version = "0.13.1", features = ["json"] } serde.workspace = true serde_json.workspace = true serde_yaml.workspace = true diff --git a/test_utils/src/virt/azure.rs b/test_utils/src/virt/azure.rs new file mode 100644 index 00000000..87d37185 --- /dev/null +++ b/test_utils/src/virt/azure.rs @@ -0,0 +1,344 @@ +use anyhow::{Context, Result, anyhow}; +use azure_core::{base64, credentials::TokenCredential}; +use azure_identity::DeveloperToolsCredential; +use k8s_openapi::chrono::{self, Utc}; +use reqwest::{Client, header}; +use serde_json::{Value, json}; +use std::{env, sync::Arc, time}; + +use super::{VmBackend, VmConfig, generate_ignition, ssh_exec}; +use crate::{Poller, get_env}; + +const AZURE_MGMT: &str = "https://management.azure.com"; +const AZURE_SCOPE: &str = ".default"; +const NET_API_VERSION: &str = "api-version=2025-05-01"; +const VM_API_VERSION: &str = "api-version=2025-04-01"; +// Old. Consider Start/Stop VMs v2 +const SCHEDULES_API_VERSION: &str = "api-version=2018-09-15"; +const KEEP_ALIVE_MINUTES: i64 = 60; +const NET_PATH: &str = "providers/Microsoft.Network"; +const VMS_PATH: &str = "providers/Microsoft.Compute/virtualMachines"; + +pub struct AzureBackend { + config: VmConfig, + client: Client, + rg_path: String, + location: String, + cred: Arc, +} + +impl AzureBackend { + pub fn new(config: VmConfig) -> Result { + let subscription_id = get_env("AZURE_SUBSCRIPTION_ID")?; + let cred = DeveloperToolsCredential::new(None)?; + let resource_group = config.namespace.clone(); + Ok(Self { + config, + client: Client::new(), + rg_path: format!("subscriptions/{subscription_id}/resourceGroups/{resource_group}"), + location: env::var("AZURE_LOCATION").unwrap_or("eastus".to_string()), + cred, + }) + } + + async fn get_token(&self) -> Result { + let scope = format!("{AZURE_MGMT}/{AZURE_SCOPE}"); + let token_response = self.cred.get_token(&[&scope], None).await?; + Ok(token_response.token.secret().to_string()) + } + + async fn put_resource(&self, url: &str, body: &Value) -> Result { + let token = self.get_token().await?; + let req = self.client.put(url); + let headers = req + .header(header::AUTHORIZATION, format!("Bearer {token}")) + .header(header::CONTENT_TYPE, "application/json"); + let response = headers.json(body).send().await?; + if !response.status().is_success() { + let status = response.status(); + let error_body = response.text().await?; + return Err(anyhow!("PUT {url} failed: {status} - {error_body}")); + } + let result = response.json().await?; + Ok(result) + } + + async fn delete_resource(&self, url: &str) -> Result<()> { + let token = self.get_token().await?; + let req = self.client.delete(url); + let headers = req.header(header::AUTHORIZATION, format!("Bearer {token}")); + let response = headers.send().await?; + // 200, 202 (accepted), 204 (no content) are all success for DELETE + if !response.status().is_success() && response.status().as_u16() != 202 { + let status = response.status(); + let error_body = response.text().await?; + return Err(anyhow::anyhow!( + "DELETE {url} failed: {status} - {error_body}" + )); + } + Ok(()) + } + + async fn get_resource(&self, url: &str) -> Result { + let token = self.get_token().await?; + let req = self.client.get(url); + let headers = req.header(header::AUTHORIZATION, format!("Bearer {token}")); + let response = headers.send().await?; + if !response.status().is_success() { + let status = response.status(); + let error_body = response.text().await?; + return Err(anyhow!("GET {url} failed: {status} - {error_body}")); + } + let result = response.json().await?; + Ok(result) + } +} + +#[async_trait::async_trait] +impl VmBackend for AzureBackend { + async fn create_vm(&self) -> Result<()> { + // TODO consider CLI + let rg_path = &self.rg_path; + let mgmt_base = format!("{AZURE_MGMT}/{rg_path}"); + let rg_url = format!("{mgmt_base}?{VM_API_VERSION}"); + let rg_body = json!({"location": self.location}); + // TODO probably handle already_exists for parallel test + self.put_resource(&rg_url, &rg_body).await?; + + let vm_name = &self.config.vm_name; + let vnet_name = format!("{vm_name}-vnet"); + let vnet_url = + format!("{mgmt_base}/{NET_PATH}/virtualNetworks/{vnet_name}?{NET_API_VERSION}",); + // If Microsoft returns to making these structures available + // in a Rust SDK (was discontinued in version 0.22), use them. + let vnet_body = json!({ + "location": self.location, + "properties": { + "addressSpace": { + "addressPrefixes": ["10.0.0.0/16"] + }, + "subnets": [{ + "name": "default", + "properties": { + "addressPrefix": "10.0.0.0/24" + } + }] + } + }); + self.put_resource(&vnet_url, &vnet_body).await?; + + let ip_url = + format!("{mgmt_base}/{NET_PATH}/publicIPAddresses/{vm_name}-ip?{NET_API_VERSION}",); + let ip_body = json!({ + "location": self.location, + "sku": { + "name": "Standard" + }, + "properties": { + "publicIPAllocationMethod": "Static" + } + }); + let ip_result = self.put_resource(&ip_url, &ip_body).await?; + + let nsg_url = + format!("{mgmt_base}/{NET_PATH}/networkSecurityGroups/{vm_name}-nsg?{NET_API_VERSION}"); + let nsg_body = json!({ + "location": self.location, + "properties": { + "securityRules": [{ + "name": "AllowSSH", + "properties": { + "protocol": "Tcp", + "sourceAddressPrefix": "*", + "sourcePortRange": "*", + "destinationAddressPrefix": "*", + "destinationPortRange": "22", + "access": "Allow", + "direction": "Inbound", + "priority": 1000, + "description": "Allow SSH" + } + }] + } + }); + let nsg_result = self.put_resource(&nsg_url, &nsg_body).await?; + + let nic_url = + format!("{mgmt_base}/{NET_PATH}/networkInterfaces/{vm_name}-nic?{NET_API_VERSION}"); + let nic_body = json!({ + "location": self.location, + "properties": { + "networkSecurityGroup": { + "id": nsg_result["id"].as_str().unwrap(), + }, + "ipConfigurations": [{ + "name": "ipconfig1", + "properties": { + "subnet": { + "id": format!("{rg_path}/{NET_PATH}/virtualNetworks/{vnet_name}/subnets/default"), + }, + "publicIPAddress": { + "id": ip_result["id"].as_str().unwrap(), + } + } + }] + } + }); + let nic_result = self.put_resource(&nic_url, &nic_body).await?; + + let image_ref_json = if self.config.image.starts_with('/') { + json!({ "id": self.config.image }) + } else { + let parts: Vec<&str> = self.config.image.split(':').collect(); + if parts.len() < 4 { + let err = "Invalid Image URN. Expected 'Publisher:Offer:Sku:Version'"; + return Err(anyhow!(err)); + } + json!({ + "publisher": parts[0], + "offer": parts[1], + "sku": parts[2], + "version": parts[3] + }) + }; + + let admin_username = "core"; + let vm_path = format!("{rg_path}/{VMS_PATH}/{vm_name}"); + let vm_url = format!("{AZURE_MGMT}/{vm_path}?{VM_API_VERSION}"); + let ign = generate_ignition(&self.config, false).await?; + let vm_body = json!({ + "location": self.location, + "properties": { + "hardwareProfile": { + "vmSize": "Standard_DC2as_v5" + }, + "storageProfile": { + "imageReference": image_ref_json, + "osDisk": { + "createOption": "FromImage", + "deleteOption": "Delete", + "managedDisk": { + "storageAccountType": "StandardSSD_LRS", + "securityProfile": { + "securityEncryptionType": "VMGuestStateOnly" + } + } + } + }, + "osProfile": { + "computerName": vm_name, + "adminUsername": admin_username, + "linuxConfiguration": { + // TODO this didn't work earlier, so it might also be unnecessary + "disablePasswordAuthentication": true, + "ssh": { + "publicKeys": [ + { + "path": format!("/home/{}/.ssh/authorized_keys", admin_username), + "keyData": self.config.ssh_public_key + } + ] + } + }, + "customData": base64::encode(ign.to_string()), + }, + "networkProfile": { + "networkInterfaces": [ + { + "id": nic_result["id"].as_str().unwrap(), + "properties": { + "primary": true + } + } + ] + }, + "securityProfile": { + "securityType": "ConfidentialVM", + "uefiSettings": { + "secureBootEnabled": true, + "vTpmEnabled": true + } + } + } + }); + self.put_resource(&vm_url, &vm_body).await?; + + // Schedule VM shutdown at KEEP_ALIVE_MINUTES in the future to control costs if cleanup fails + let shutdown_time = Utc::now() + chrono::Duration::minutes(KEEP_ALIVE_MINUTES); + let shutdown_url = format!( + "{mgmt_base}/providers/Microsoft.DevTestLab/schedules/shutdown-computevm-{vm_name}?{SCHEDULES_API_VERSION}" + ); + let shutdown_body = json!({ + "location": self.location, + "properties": { + "status": "Enabled", + "taskType": "ComputeVmShutdownTask", + "dailyRecurrence": { + "time": shutdown_time.format("%H%M").to_string(), + }, + "targetResourceId": vm_path, + "timezoneId": "UTC", + } + }); + let warn = format!("=== WARNING === +Request to auto-shutdown the VM at {vm_path} has failed. Log in manually to verify the VM was removed correctly. +=== END OF WARNING ==="); + self.put_resource(&shutdown_url, &shutdown_body) + .await + .context(warn)?; + Ok(()) + } + + async fn wait_for_running(&self, timeout_secs: u64) -> Result<()> { + let poller = Poller::new() + .with_timeout(time::Duration::from_secs(timeout_secs)) + .with_interval(time::Duration::from_secs(5)) + .with_error_message(format!( + "virtualMachine {} did not reach PowerState/running status after {timeout_secs} seconds", + self.config.vm_name + )); + + let check_fn = || async move { + let vm_name = &self.config.vm_name; + let rg_path = &self.rg_path; + let url = format!( + "{AZURE_MGMT}/{rg_path}/{VMS_PATH}/{vm_name}/instanceView?{VM_API_VERSION}" + ); + let vm = self.get_resource(&url).await?; + let statuses = vm["statuses"].as_array().unwrap(); + let check = |s: &&Value| s["code"] == "PowerState/running"; + let err = anyhow!("virtualMachine {vm_name} is not in running PowerState yet"); + statuses.iter().find(check).map(|_| ()).ok_or(err) + }; + poller.poll_async(check_fn).await + } + + async fn ssh_exec(&self, command: &str) -> Result { + let ip_url = format!( + "{AZURE_MGMT}/{}/{NET_PATH}/publicIPAddresses/{}-ip?{NET_API_VERSION}", + self.rg_path, self.config.vm_name + ); + let response = self.get_resource(&ip_url).await?; + let public_ip = response["properties"]["ipAddress"].as_str().unwrap(); + + let full_cmd = format!( + "ssh -i {} -o IdentitiesOnly=yes -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null core@{public_ip} '{command}'", + self.config.ssh_private_key.display() + ); + ssh_exec(&full_cmd).await + } + + async fn get_root_key(&self) -> Result>> { + Ok(None) + } + + async fn cleanup(&self) -> Result<()> { + self.config.cleanup(); + let rg_path = &self.rg_path; + let url = format!("{AZURE_MGMT}/{rg_path}?{VM_API_VERSION}"); + let warn = format!("=== WARNING === +Request to cleanup the Azure resource group at {rg_path} failed. Log in manually to verify the resource group was removed correctly. +=== END OF WARNING ==="); + self.delete_resource(&url).await.context(warn) + } +} diff --git a/test_utils/src/virt/mod.rs b/test_utils/src/virt/mod.rs index 9d129108..65e4f767 100644 --- a/test_utils/src/virt/mod.rs +++ b/test_utils/src/virt/mod.rs @@ -3,6 +3,7 @@ // // SPDX-License-Identifier: MIT +pub mod azure; pub mod kubevirt; use anyhow::{Context, Result, anyhow}; @@ -179,14 +180,16 @@ pub async fn get_root_key(config: &VmConfig, ip: &str) -> Result> { pub enum VirtProvider { #[default] Kubevirt, + Azure, } fn get_virt_provider() -> Result { match env::var(VIRT_PROVIDER_ENV) { Ok(val) => match val.to_lowercase().as_str() { "kubevirt" => Ok(VirtProvider::Kubevirt), + "azure" => Ok(VirtProvider::Azure), v => Err(anyhow!( - "Unknown {VIRT_PROVIDER_ENV} '{v}'. Supported providers: kubevirt" + "Unknown {VIRT_PROVIDER_ENV} '{v}'. Supported providers: kubevirt, azure" )), }, Err(env::VarError::NotPresent) => Ok(VirtProvider::default()), @@ -212,6 +215,7 @@ pub fn create_backend( }; match provider { VirtProvider::Kubevirt => Ok(Box::new(kubevirt::KubevirtBackend(config))), + VirtProvider::Azure => Ok(Box::new(azure::AzureBackend::new(config)?)), } } diff --git a/tests/attestation.rs b/tests/attestation.rs index 1145d856..a682f4f0 100644 --- a/tests/attestation.rs +++ b/tests/attestation.rs @@ -44,7 +44,7 @@ impl SingleAttestationContext { backend.create_vm().await?; test_ctx.info(format!("Waiting for VM {} to reach Running state", vm_name)); - backend.wait_for_running(300).await?; + backend.wait_for_running(600).await?; test_ctx.info(format!("VM {} is Running", vm_name)); test_ctx.info(format!("Waiting for SSH access to VM {}", vm_name)); @@ -101,8 +101,8 @@ async fn test_parallel_vm_attestation() -> anyhow::Result<()> { // Wait for both VMs to reach Running state in parallel test_ctx.info("Waiting for both VMs to reach Running state"); let (vm1_running, vm2_running) = tokio::join!( - backend1.wait_for_running(300), - backend2.wait_for_running(300) + backend1.wait_for_running(600), + backend2.wait_for_running(600) ); vm1_running?;