From 52c0a3924d8a9484d1523b0ce5fd9b5f11cc44b6 Mon Sep 17 00:00:00 2001 From: Tyler Fanelli Date: Wed, 21 Jan 2026 22:26:56 -0500 Subject: [PATCH 01/17] nitro: Allow u64 rootfs buffer sizes Signed-off-by: Tyler Fanelli --- init/nitro/args_reader.c | 18 +++++++++--------- init/nitro/include/args_reader.h | 2 +- src/nitro/src/enclave/args_writer.rs | 8 ++++---- 3 files changed, 14 insertions(+), 14 deletions(-) diff --git a/init/nitro/args_reader.c b/init/nitro/args_reader.c index 44bac1fea..f079c4b70 100644 --- a/init/nitro/args_reader.c +++ b/init/nitro/args_reader.c @@ -30,22 +30,22 @@ enum { /* * Before reading data from the vsock, the vsock sends a 4-byte "header", * representing the size (in bytes) of the object that will be written over the - * stream. Read this size and store it within a uint32_t variable. + * stream. Read this size and store it within a uint64_t variable. */ -static int args_reader_len_read(int sock_fd, uint32_t *size) +static int args_reader_len_read(int sock_fd, uint64_t *size) { - uint8_t bytes[sizeof(uint32_t)]; + uint8_t bytes[sizeof(uint64_t)]; ssize_t ret; // Read the bytes (representing the size) from the vsock. - ret = read(sock_fd, bytes, sizeof(uint32_t)); - if (ret < sizeof(uint32_t)) { + ret = read(sock_fd, bytes, sizeof(uint64_t)); + if (ret < sizeof(uint64_t)) { perror("vsock byte buffer length read"); return -errno; } // Store the size within the "size" argument. - memcpy(size, bytes, sizeof(uint32_t)); + memcpy(size, bytes, sizeof(uint64_t)); return 0; } @@ -66,9 +66,9 @@ static void char_list_free(char **buf) /* * Read and store an object from the vsock stream. */ -static int args_reader_rcv(int sock_fd, void **buf_ptr, uint32_t *size) +static int args_reader_rcv(int sock_fd, void **buf_ptr, uint64_t *size) { - uint32_t len, idx; + uint64_t len, idx; ssize_t read_len; uint8_t *buf; int ret; @@ -122,7 +122,7 @@ static int args_reader_rcv(int sock_fd, void **buf_ptr, uint32_t *size) */ static int args_reader_char_list_build(int sock_fd, char ***buf_ptr) { - uint32_t size; + uint64_t size; char **buf; int ret, i; diff --git a/init/nitro/include/args_reader.h b/init/nitro/include/args_reader.h index e037e7827..e9b71c8c8 100644 --- a/init/nitro/include/args_reader.h +++ b/init/nitro/include/args_reader.h @@ -8,7 +8,7 @@ struct enclave_args { void *rootfs_archive; - uint32_t rootfs_archive_size; + uint64_t rootfs_archive_size; char *exec_path; char **exec_argv; char **exec_envp; diff --git a/src/nitro/src/enclave/args_writer.rs b/src/nitro/src/enclave/args_writer.rs index 40533d059..285d4ca57 100644 --- a/src/nitro/src/enclave/args_writer.rs +++ b/src/nitro/src/enclave/args_writer.rs @@ -147,21 +147,21 @@ impl EnclaveArg<'_> { match self { Self::RootFilesystem(buf) => { - let len: u32 = buf.len().try_into().unwrap(); + let len: u64 = buf.len().try_into().unwrap(); vsock.write_all(&len.to_ne_bytes()).unwrap(); vsock.write_all(buf).unwrap(); } Self::ExecArgv(vec) | Self::ExecEnvp(vec) => { - let len: u32 = vec.len().try_into().unwrap(); + let len: u64 = vec.len().try_into().unwrap(); vsock.write_all(&len.to_ne_bytes()).unwrap(); for string in vec { let bytes = Vec::from(CString::from_str(string).unwrap().as_bytes_with_nul()); - let len: u32 = bytes.len().try_into().unwrap(); + let len: u64 = bytes.len().try_into().unwrap(); vsock.write_all(&len.to_ne_bytes()).unwrap(); @@ -170,7 +170,7 @@ impl EnclaveArg<'_> { } Self::ExecPath(buf) => { let bytes = Vec::from(CString::from_str(buf).unwrap().as_bytes_with_nul()); - let len: u32 = bytes.len().try_into().unwrap(); + let len: u64 = bytes.len().try_into().unwrap(); vsock.write_all(&len.to_ne_bytes()).unwrap(); From b05fae16e437df65c737779e122dbc2d0c56e5dd Mon Sep 17 00:00:00 2001 From: Tyler Fanelli Date: Wed, 21 Jan 2026 00:05:53 -0500 Subject: [PATCH 02/17] nitro: Overhaul error return implementation Modularize each error within its respected module. This allows for clearer messages to determine the component that the error originated. Signed-off-by: Tyler Fanelli --- src/nitro/src/enclave/args_writer.rs | 130 ++++++++++----- src/nitro/src/enclave/device/devices/net.rs | 10 +- .../enclave/device/devices/signal_handler.rs | 2 +- src/nitro/src/enclave/device/mod.rs | 42 +++-- src/nitro/src/enclave/mod.rs | 108 ++++++------ src/nitro/src/error.rs | 155 +++++++++--------- 6 files changed, 251 insertions(+), 196 deletions(-) diff --git a/src/nitro/src/enclave/args_writer.rs b/src/nitro/src/enclave/args_writer.rs index 285d4ca57..42f1a7b07 100644 --- a/src/nitro/src/enclave/args_writer.rs +++ b/src/nitro/src/enclave/args_writer.rs @@ -1,15 +1,14 @@ // SPDX-License-Identifier: Apache-2.0 -use crate::{ - enclave::{device::DeviceProxyList, VsockPortOffset}, - error::NitroError, -}; +use crate::enclave::{device::DeviceProxyList, VsockPortOffset}; use libc::c_int; use nitro_enclaves::launch::PollTimeout; use nix::poll::{poll, PollFd, PollFlags, PollTimeout as NixPollTimeout}; use std::{ - ffi::CString, - io::{Read, Write}, + ffi::{self, CString}, + fmt, + io::{self, Read, Write}, + num::TryFromIntError, os::fd::AsFd, str::FromStr, }; @@ -17,7 +16,7 @@ use vsock::{VsockAddr, VsockListener, VsockStream, VMADDR_CID_ANY}; const ENCLAVE_VSOCK_LAUNCH_ARGS_READY: u8 = 0xb7; -type Result = std::result::Result; +type Result = std::result::Result; #[derive(Debug, Default)] pub struct EnclaveArgsWriter<'a> { @@ -66,26 +65,24 @@ impl<'a> EnclaveArgsWriter<'a> { VMADDR_CID_ANY, cid + (VsockPortOffset::ArgsReader as u32), )) - .unwrap(); + .map_err(Error::VsockBind)?; + self.poll(&listener, timeout)?; - let mut stream = listener.accept().unwrap(); + let mut stream = listener.accept().map_err(Error::VsockAccept)?; if stream.1.cid() != cid { - return Err(NitroError::HeartbeatCidMismatch); + return Err(Error::VsockCidMismatch); } let mut buf = [0u8]; - let bytes = stream.0.read(&mut buf).map_err(NitroError::HeartbeatRead)?; + let bytes = stream.0.read(&mut buf).map_err(Error::VsockRead)?; if bytes != 1 || buf[0] != ENCLAVE_VSOCK_LAUNCH_ARGS_READY { - return Err(NitroError::EnclaveHeartbeatNotDetected); + return Err(Error::ReadySignalNotDetected); } - stream - .0 - .write_all(&buf) - .map_err(NitroError::HeartbeatWrite)?; + stream.0.write_all(&buf).map_err(Error::VsockWrite)?; for arg in &self.args { arg.write(&mut stream.0)?; @@ -106,8 +103,8 @@ impl<'a> EnclaveArgsWriter<'a> { ); match result { - Ok(0) => Err(NitroError::PollNoSelectedEvents), - Ok(x) if x > 1 => Err(NitroError::PollMoreThanOneSelectedEvent), + Ok(0) => Err(Error::PollNoEvents), + Ok(x) if x > 1 => Err(Error::PollMoreThanOneSelectedEvent), _ => Ok(()), } } @@ -143,38 +140,60 @@ impl EnclaveArg<'_> { fn write(&self, vsock: &mut VsockStream) -> Result<()> { let id: [u8; 1] = [self.into()]; - vsock.write_all(&id).unwrap(); + vsock.write_all(&id).map_err(Error::VsockWrite)?; match self { Self::RootFilesystem(buf) => { - let len: u64 = buf.len().try_into().unwrap(); + let len: u64 = buf.len().try_into().map_err(Error::VsockBufferLenConvert)?; - vsock.write_all(&len.to_ne_bytes()).unwrap(); + vsock + .write_all(&len.to_ne_bytes()) + .map_err(Error::VsockWrite)?; - vsock.write_all(buf).unwrap(); + vsock.write_all(buf).map_err(Error::VsockWrite)?; } Self::ExecArgv(vec) | Self::ExecEnvp(vec) => { - let len: u64 = vec.len().try_into().unwrap(); + let len: u64 = vec.len().try_into().map_err(Error::VsockBufferLenConvert)?; - vsock.write_all(&len.to_ne_bytes()).unwrap(); + vsock + .write_all(&len.to_ne_bytes()) + .map_err(Error::VsockWrite)?; for string in vec { - let bytes = Vec::from(CString::from_str(string).unwrap().as_bytes_with_nul()); - - let len: u64 = bytes.len().try_into().unwrap(); - - vsock.write_all(&len.to_ne_bytes()).unwrap(); - - vsock.write_all(&bytes).unwrap(); + let bytes = Vec::from( + CString::from_str(string) + .map_err(Error::CStringConvert)? + .as_bytes_with_nul(), + ); + + let len: u64 = bytes + .len() + .try_into() + .map_err(Error::VsockBufferLenConvert)?; + + vsock + .write_all(&len.to_ne_bytes()) + .map_err(Error::VsockWrite)?; + + vsock.write_all(&bytes).map_err(Error::VsockWrite)?; } } Self::ExecPath(buf) => { - let bytes = Vec::from(CString::from_str(buf).unwrap().as_bytes_with_nul()); - let len: u64 = bytes.len().try_into().unwrap(); - - vsock.write_all(&len.to_ne_bytes()).unwrap(); - - vsock.write_all(&bytes).unwrap(); + let bytes = Vec::from( + CString::from_str(buf) + .map_err(Error::CStringConvert)? + .as_bytes_with_nul(), + ); + let len: u64 = bytes + .len() + .try_into() + .map_err(Error::VsockBufferLenConvert)?; + + vsock + .write_all(&len.to_ne_bytes()) + .map_err(Error::VsockWrite)?; + + vsock.write_all(&bytes).map_err(Error::VsockWrite)?; } _ => (), } @@ -182,3 +201,40 @@ impl EnclaveArg<'_> { Ok(()) } } + +#[derive(Debug)] +pub enum Error { + CStringConvert(ffi::NulError), + PollNoEvents, + PollMoreThanOneSelectedEvent, + ReadySignalNotDetected, + VsockAccept(io::Error), + VsockBind(io::Error), + VsockBufferLenConvert(TryFromIntError), + VsockCidMismatch, + VsockRead(io::Error), + VsockWrite(io::Error), +} + +impl fmt::Display for Error { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + let msg = match self { + Self::CStringConvert(e) => format!("unable to convert string to CString: {e}"), + Self::PollNoEvents => "no events on vsock detected".to_string(), + Self::PollMoreThanOneSelectedEvent => { + "more than one event on vsock detected".to_string() + } + Self::ReadySignalNotDetected => "ready signal not detected".to_string(), + Self::VsockAccept(e) => format!("unable to accept vsock connection: {e}"), + Self::VsockBind(e) => format!("unable to bind to vsock: {e}"), + Self::VsockBufferLenConvert(e) => { + format!("unable to convert vsock buffer size to u64: {e}") + } + Self::VsockCidMismatch => "CID mismatch on vsock".to_string(), + Self::VsockRead(e) => format!("unable to read from vsock: {e}"), + Self::VsockWrite(e) => format!("unable to write to vsock: {e}"), + }; + + write!(f, "{}", msg) + } +} diff --git a/src/nitro/src/enclave/device/devices/net.rs b/src/nitro/src/enclave/device/devices/net.rs index cef3ac512..2a3af09ad 100644 --- a/src/nitro/src/enclave/device/devices/net.rs +++ b/src/nitro/src/enclave/device/devices/net.rs @@ -67,7 +67,7 @@ impl DeviceProxy for NetProxy { .write_all(&vsock_buf[..size]) .map_err(Error::UnixWrite)?; } else { - tx.send(()).unwrap(); + let _ = tx.send(()); break; } } @@ -75,10 +75,10 @@ impl DeviceProxy for NetProxy { Ok(()) }); - let mut unix_stream_clone_read = unix_stream.try_clone().unwrap(); + let mut unix_stream_clone_read = unix_stream.try_clone().map_err(Error::UnixClone)?; unix_stream_clone_read .set_read_timeout(Some(Duration::from_millis(250))) - .unwrap(); + .map_err(Error::UnixReadTimeoutSet)?; // Unix let unix_thread: JoinHandle> = thread::spawn(move || { let mut unix_buf = [0u8; 1500]; @@ -102,12 +102,12 @@ impl DeviceProxy for NetProxy { if e == RecvTimeoutError::Timeout { continue; } else { - panic!(); + return Err(Error::ShutdownSignalReceive(e))?; } } } } - Err(_) => panic!(), + Err(e) => return Err(Error::UnixRead(e)), } } diff --git a/src/nitro/src/enclave/device/devices/signal_handler.rs b/src/nitro/src/enclave/device/devices/signal_handler.rs index cf89d6ee8..421051fc2 100644 --- a/src/nitro/src/enclave/device/devices/signal_handler.rs +++ b/src/nitro/src/enclave/device/devices/signal_handler.rs @@ -31,7 +31,7 @@ impl DeviceProxy for SignalHandler { fn _start(&mut self, vsock_port: u32) -> Result<()> { let term = Arc::new(AtomicBool::new(false)); - signal_hook::flag::register(SIGTERM, Arc::clone(&term)).unwrap(); + signal_hook::flag::register(SIGTERM, Arc::clone(&term)).map_err(Error::SignalRegister)?; let vsock_listener = VsockListener::bind(&VsockAddr::new(VMADDR_CID_ANY, vsock_port)) .map_err(Error::VsockBind)?; diff --git a/src/nitro/src/enclave/device/mod.rs b/src/nitro/src/enclave/device/mod.rs index bdcfff4ae..a9e37870c 100644 --- a/src/nitro/src/enclave/device/mod.rs +++ b/src/nitro/src/enclave/device/mod.rs @@ -7,6 +7,7 @@ pub use devices::*; use crate::enclave::{args_writer::EnclaveArg, VsockPortOffset}; use std::{ fmt, io, + sync::mpsc, thread::{self, JoinHandle}, }; @@ -53,7 +54,11 @@ pub enum Error { FileOpen(io::Error), FileWrite(io::Error), InvalidNetInterface, + ShutdownSignalReceive(mpsc::RecvTimeoutError), + SignalRegister(io::Error), UnixClone(io::Error), + UnixRead(io::Error), + UnixReadTimeoutSet(io::Error), UnixWrite(io::Error), VsockAccept(io::Error), VsockBind(io::Error), @@ -66,29 +71,30 @@ pub enum Error { impl fmt::Display for Error { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { let msg = match self { - Self::FileOpen(cause) => format!("unable to open file: {:?}", cause), - Self::FileWrite(cause) => { - format!("unable to write buffer to output file: {:?}", cause) + Self::FileOpen(e) => format!("unable to open file: {e}"), + Self::FileWrite(e) => format!("unable to write buffer to output file: {e}"), + Self::ShutdownSignalReceive(e) => { + format!("error while receiving read proxy shutdown signal: {e}") + } + Self::SignalRegister(e) => { + format!("unable to register signal in signal handler proxy: {e}") } Self::InvalidNetInterface => { - "invalid network proxy interface, must supply unix stream file descriptor" + "invalid network proxy interface, must supply UNIX stream file descriptor" .to_string() } - Self::UnixClone(cause) => format!("unable to clone unix stream: {:?}", cause), - Self::UnixWrite(cause) => format!("unable to write to unix stream: {:?}", cause), - Self::VsockAccept(cause) => format!( - "unable to accept connection from enclave vsock: {:?}", - cause - ), - Self::VsockBind(cause) => { - format!("unable to bind to enclave vsock: {:?}", cause) - } - Self::VsockConnect(cause) => format!("uanble to connect to enclave vsock: {:?}", cause), - Self::VsockClone(cause) => format!("unable to clone enclave vsock: {:?}", cause), - Self::VsockRead(cause) => { - format!("unable to read from enclave vsock: {:?}", cause) + Self::UnixClone(e) => format!("unable to clone unix stream: {e}"), + Self::UnixRead(e) => format!("unable to read from unix stream: {e}"), + Self::UnixReadTimeoutSet(e) => { + format!("unable to set read timeout for unix stream: {e}") } - Self::VsockWrite(cause) => format!("unable to write to enclave vsock: {:?}", cause), + Self::UnixWrite(e) => format!("unable to write to unix stream: {e}"), + Self::VsockAccept(e) => format!("unable to accept connection from vsock: {e}"), + Self::VsockBind(e) => format!("unable to bind to vsock: {e}"), + Self::VsockConnect(e) => format!("unable to connect to vsock: {e}"), + Self::VsockClone(e) => format!("unable to clone vsock: {e}"), + Self::VsockRead(e) => format!("unable to read from vsock: {e}"), + Self::VsockWrite(e) => format!("unable to write to vsock: {e}"), }; write!(f, "{}", msg) diff --git a/src/nitro/src/enclave/mod.rs b/src/nitro/src/enclave/mod.rs index a43f00db9..12de63523 100644 --- a/src/nitro/src/enclave/mod.rs +++ b/src/nitro/src/enclave/mod.rs @@ -1,8 +1,11 @@ // SPDX-License-Identifier: Apache-2.0 -pub mod args_writer; -pub mod device; +pub(crate) mod args_writer; +pub(crate) mod device; +use super::error::{ + return_code::Error as ReturnCodeListenerError, start::Error as StartError, Error, +}; use args_writer::EnclaveArgsWriter; use device::{ net::NetProxy, output::OutputProxy, signal_handler::SignalHandler, DeviceProxy, DeviceProxyList, @@ -18,12 +21,8 @@ use std::{ io::{self, Read}, path::PathBuf, }; -use vsock::{VsockAddr, VsockListener, VMADDR_CID_ANY}; - -use super::error::NitroError; use tar::HeaderMode; - -type Result = std::result::Result; +use vsock::{VsockAddr, VsockListener, VMADDR_CID_ANY}; const ROOTFS_DIR_DENYLIST: [&str; 6] = [ "proc", // /proc. @@ -58,10 +57,10 @@ pub struct NitroEnclave { impl NitroEnclave { /// Run the enclave. - pub fn run(mut self) -> Result<()> { - let rootfs_archive = self.rootfs_archive()?; + pub fn run(mut self) -> Result<(), Error> { + let rootfs_archive = self.rootfs_archive().map_err(Error::RootFsArchive)?; - let devices = self.devices()?; + let devices = self.devices().map_err(Error::Device)?; let writer = EnclaveArgsWriter::new( &rootfs_archive, @@ -71,15 +70,16 @@ impl NitroEnclave { &devices, ); - let (cid, timeout) = self.start()?; + let (cid, timeout) = self.start().map_err(Error::Start)?; - writer.write_args(cid, timeout)?; + writer.write_args(cid, timeout).map_err(Error::ArgsWrite)?; let retcode_listener = VsockListener::bind(&VsockAddr::new( VMADDR_CID_ANY, cid + (VsockPortOffset::ReturnCode as u32), )) - .unwrap(); + .map_err(ReturnCodeListenerError::VsockBind) + .map_err(Error::ReturnCodeListener)?; devices.start(cid); @@ -89,30 +89,35 @@ impl NitroEnclave { * code from the enclave. */ if !self.debug { - let ret = self.shutdown_ret(retcode_listener)?; + let ret = self + .shutdown_ret(retcode_listener) + .map_err(Error::ReturnCodeListener)?; if ret != 0 { - return Err(NitroError::AppReturn(ret)); + return Err(Error::AppReturn(ret)); } } Ok(()) } - fn start(&mut self) -> Result<(u32, PollTimeout)> { - let eif = eif()?; + fn start(&mut self) -> Result<(u32, PollTimeout), StartError> { + let path = env::var("KRUN_NITRO_EIF_PATH") + .unwrap_or("/usr/share/krun-nitro/krun-nitro.eif".to_string()); + + let eif = fs::read(path).map_err(StartError::EifRead)?; let timeout = PollTimeout::try_from((eif.as_slice(), self.mem_size_mib << 20)) - .map_err(NitroError::PollTimeoutCalculate)?; + .map_err(StartError::PollTimeoutCalculate)?; - let device = Device::open().map_err(NitroError::DeviceOpen)?; + let device = Device::open().map_err(StartError::DeviceOpen)?; - let mut launcher = Launcher::new(&device).map_err(NitroError::VmCreate)?; + let mut launcher = Launcher::new(&device).map_err(StartError::VmCreate)?; let mem = MemoryInfo::new(ImageType::Eif(&eif), self.mem_size_mib); - launcher.set_memory(mem).map_err(NitroError::VmMemorySet)?; + launcher.set_memory(mem).map_err(StartError::VmMemorySet)?; for _ in 0..self.vcpus { - launcher.add_vcpu(None).map_err(NitroError::VcpuAdd)?; + launcher.add_vcpu(None).map_err(StartError::VcpuAdd)?; } let mut start_flags = StartFlags::empty(); @@ -123,16 +128,16 @@ impl NitroEnclave { let cid = launcher .start(start_flags, None) - .map_err(NitroError::VmStart)?; + .map_err(StartError::VmStart)?; - Ok((cid.try_into().unwrap(), timeout)) // Safe to unwrap. + // Safe to unwrap. + Ok((cid.try_into().unwrap(), timeout)) } - fn devices(&self) -> Result { + fn devices(&self) -> Result { let mut proxies: Vec> = vec![]; - let output = - OutputProxy::new(&self.output_path, self.debug).map_err(NitroError::DeviceError)?; + let output = OutputProxy::new(&self.output_path, self.debug)?; proxies.push(Box::new(output)); if let Some(net) = self.net.clone() { @@ -144,7 +149,7 @@ impl NitroEnclave { Ok(DeviceProxyList(proxies)) } - fn rootfs_archive(&self) -> Result> { + fn rootfs_archive(&self) -> Result, io::Error> { let mut builder = tar::Builder::new(Vec::new()); builder.mode(HeaderMode::Deterministic); @@ -156,54 +161,47 @@ impl NitroEnclave { .file_name() .unwrap_or(OsStr::new("/")) .to_str() - .ok_or(NitroError::RootFsArchive(io::Error::other( - "unable to convert rootfs directory name to str", + .ok_or(io::Error::other(format!( + "unable to convert rootfs directory name (\"{:?}\") to str", + pathbuf_copy )))?; - for entry in fs::read_dir(pathbuf).map_err(NitroError::RootFsArchive)? { - let entry = entry.map_err(NitroError::RootFsArchive)?; - let filetype = entry.file_type().map_err(NitroError::RootFsArchive)?; - let filename = entry.file_name().into_string().map_err(|_| { - NitroError::RootFsArchive(io::Error::other( - "unable to convert file name to String object", + for entry in fs::read_dir(pathbuf)? { + let entry = entry?; + let filetype = entry.file_type()?; + let filename = entry.file_name().into_string().map_err(|e| { + io::Error::other(format!( + "unable to convert file name {:?} to String object", + e )) })?; if !ROOTFS_DIR_DENYLIST.contains(&filename.as_str()) && filename != rootfs_dirname { if filetype.is_dir() { - builder - .append_dir_all(format!("rootfs/{}", filename), entry.path()) - .map_err(NitroError::RootFsArchive)?; + builder.append_dir_all(format!("rootfs/{}", filename), entry.path())? } else if filetype.is_file() { - builder - .append_path_with_name(entry.path(), format!("rootfs/{}", filename)) - .map_err(NitroError::RootFsArchive)?; + builder.append_path_with_name(entry.path(), format!("rootfs/{}", filename))? } } } - builder.into_inner().map_err(NitroError::RootFsArchive) + builder.into_inner() } - fn shutdown_ret(&self, vsock_listener: VsockListener) -> Result { - let (mut vsock_stream, _vsock_addr) = vsock_listener.accept().unwrap(); + fn shutdown_ret(&self, vsock_listener: VsockListener) -> Result { + let (mut vsock_stream, _vsock_addr) = vsock_listener + .accept() + .map_err(ReturnCodeListenerError::VsockAccept)?; let mut buf = [0u8; 4]; - let _ = vsock_stream.read(&mut buf).unwrap(); + let _ = vsock_stream + .read(&mut buf) + .map_err(ReturnCodeListenerError::VsockRead)?; Ok(i32::from_ne_bytes(buf)) } } -fn eif() -> Result> { - let path = env::var("KRUN_NITRO_EIF_PATH") - .unwrap_or("/usr/share/krun-nitro/krun-nitro.eif".to_string()); - - let bytes = fs::read(path).map_err(NitroError::EifRead)?; - - Ok(bytes) -} - #[repr(u32)] pub enum VsockPortOffset { ArgsReader = 1, diff --git a/src/nitro/src/error.rs b/src/nitro/src/error.rs index 2eed35aa6..27a3249ae 100644 --- a/src/nitro/src/error.rs +++ b/src/nitro/src/error.rs @@ -1,98 +1,93 @@ // SPDX-License-Identifier: Apache-2.0 -use super::enclave::device; -use nitro_enclaves::launch::LaunchError; -use std::{ffi, fmt, io}; +use super::enclave::{args_writer, device}; +use std::{fmt, io}; #[derive(Debug)] -pub enum NitroError { +pub enum Error { AppReturn(i32), - DeviceOpen(io::Error), - VmCreate(LaunchError), - VmMemorySet(LaunchError), - VcpuAdd(LaunchError), - HeartbeatAccept(io::Error), - HeartbeatBind(io::Error), - HeartbeatRead(io::Error), - HeartbeatWrite(io::Error), - VmStart(LaunchError), - PollTimeoutCalculate(LaunchError), - PollNoSelectedEvents, - PollMoreThanOneSelectedEvent, - EnclaveHeartbeatNotDetected, + ArgsWrite(args_writer::Error), + Device(device::Error), + ReturnCodeListener(return_code::Error), RootFsArchive(io::Error), - HeartbeatCidMismatch, - VsockCreate, - VsockSetTimeout, - VsockConnect, - IpcWrite(io::Error), - VsockBytesLenWrite(io::Error), - VsockBytesWrite(io::Error), - VsockBytesTooLarge, - CStringConversion(ffi::NulError), - EifRead(io::Error), - EifTarExtract(io::Error), - DeviceError(device::Error), + Start(start::Error), } -impl fmt::Display for NitroError { +impl fmt::Display for Error { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { let msg = match self { - NitroError::AppReturn(ret) => format!("app returned non-zero return code: {ret}"), - NitroError::DeviceOpen(e) => format!("unable to open nitro enclaves device: {e}"), - NitroError::VmCreate(e) => format!("unable to create enclave VM: {e}"), - NitroError::VmMemorySet(e) => format!("unable to set enclave memory regions: {e}"), - NitroError::VcpuAdd(e) => format!("unable to add vCPU to enclave: {e}"), - NitroError::HeartbeatAccept(e) => { - format!("unable to accept enclave heartbeat vsock: {e}") + Self::AppReturn(ret) => format!("app returned non-zero return code: {ret}"), + Self::ArgsWrite(e) => format!("enclave VM argument writer error: {e}"), + Self::Device(e) => format!("device proxy error: {e}"), + Self::ReturnCodeListener(e) => { + format!("error with enclave VM return code listener: {e}") } - NitroError::HeartbeatBind(e) => { - format!("unable to bind to enclave heartbeat vsock: {e}") - } - NitroError::HeartbeatRead(e) => format!("unable to read enclave heartbeat vsock: {e}"), - NitroError::HeartbeatWrite(e) => { - format!("unable to write to enclave heartbeat vsock: {e}") - } - NitroError::VmStart(e) => format!("unable to start enclave: {e}"), - NitroError::PollTimeoutCalculate(e) => { - format!("unable to calculate vsock poll timeout: {e}") - } - NitroError::PollNoSelectedEvents => { - "no selected poll fds for heartbeat vsock found".to_string() - } - NitroError::PollMoreThanOneSelectedEvent => { - "more than one selected pollfd for heartbeat vsock found".to_string() - } - NitroError::EnclaveHeartbeatNotDetected => { - "enclave heartbeat message not detected".to_string() - } - NitroError::HeartbeatCidMismatch => "enclave heartbeat vsock CID mismatch".to_string(), - NitroError::VsockCreate => "unable to create enclave vsock".to_string(), - NitroError::VsockSetTimeout => { - "unable to set poll timeout for enclave vsock".to_string() - } - NitroError::VsockConnect => "unable to connect to enclave vsock".to_string(), - NitroError::RootFsArchive(e) => { + Self::RootFsArchive(e) => { format!("unable to archive rootfs: {e}") } - NitroError::IpcWrite(e) => { - format!("unable to write enclave vsock data to UNIX IPC socket: {e}") - } - NitroError::VsockBytesLenWrite(e) => { - format!("unable to write rootfs archive length to enclave: {e}") - } - NitroError::VsockBytesWrite(e) => { - format!("unable to write rootfs archive to enclave: {e}") - } - NitroError::VsockBytesTooLarge => { - "vsock write byte buffer size is larger than 64 bytes".to_string() - } - NitroError::CStringConversion(e) => format!("unable to convert String to CString: {e}"), - NitroError::EifRead(e) => format!("unable to read cached EIF file: {e}"), - NitroError::EifTarExtract(e) => format!("unable to extract EIF from tar archive: {e}"), - NitroError::DeviceError(e) => format!("device proxy error: {:?}", e), + Self::Start(e) => format!("error launching enclave VM: {e}"), }; write!(f, "{}", msg) } } + +pub mod start { + use super::*; + use nitro_enclaves::launch::LaunchError; + + #[derive(Debug)] + pub enum Error { + DeviceOpen(io::Error), + EifRead(io::Error), + PollTimeoutCalculate(LaunchError), + VcpuAdd(LaunchError), + VmCreate(LaunchError), + VmMemorySet(LaunchError), + VmStart(LaunchError), + } + + impl fmt::Display for Error { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + let msg = match self { + Self::DeviceOpen(e) => format!("unable to open nitro enclaves device: {e}"), + Self::EifRead(e) => format!("unable to read cached EIF file: {e}"), + Self::PollTimeoutCalculate(e) => { + format!("unable to calculate vsock poll timeout for enclave VM: {e}") + } + Self::VcpuAdd(e) => format!("unable to add vCPU to enclave VM: {e}"), + Self::VmCreate(e) => format!("unable to create enclave VM: {e}"), + Self::VmMemorySet(e) => { + format!("unable to set enclave VM memory regions: {e}") + } + Self::VmStart(e) => format!("unable to start enclave VM: {e}"), + }; + + write!(f, "{}", msg) + } + } +} + +pub mod return_code { + use super::*; + + #[derive(Debug)] + #[allow(clippy::enum_variant_names)] + pub enum Error { + VsockAccept(io::Error), + VsockBind(io::Error), + VsockRead(io::Error), + } + + impl fmt::Display for Error { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + let msg = match self { + Self::VsockAccept(e) => format!("unable to accept vsock connection: {e}"), + Self::VsockBind(e) => format!("unable to bind to vsock: {e}"), + Self::VsockRead(e) => format!("unable to read from vsock: {e}"), + }; + + write!(f, "{}", msg) + } + } +} From a1c05cae09238e3c260399d13a08949084e84a78 Mon Sep 17 00:00:00 2001 From: Tyler Fanelli Date: Thu, 22 Jan 2026 00:13:00 -0500 Subject: [PATCH 03/17] nitro: Block signals while creating enclave VM Creating the enclave VM can be thought of as a critical section that should not be interrupted. Disable all signals while starting the enclave VM. Signed-off-by: Tyler Fanelli --- src/nitro/src/enclave/mod.rs | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/src/nitro/src/enclave/mod.rs b/src/nitro/src/enclave/mod.rs index 12de63523..77d5a9719 100644 --- a/src/nitro/src/enclave/mod.rs +++ b/src/nitro/src/enclave/mod.rs @@ -70,6 +70,9 @@ impl NitroEnclave { &devices, ); + // Disable signals to launch enclave VM. + self.signals(false); + let (cid, timeout) = self.start().map_err(Error::Start)?; writer.write_args(cid, timeout).map_err(Error::ArgsWrite)?; @@ -81,6 +84,9 @@ impl NitroEnclave { .map_err(ReturnCodeListenerError::VsockBind) .map_err(Error::ReturnCodeListener)?; + // Enable signals now that enclave VM is started. + self.signals(true); + devices.start(cid); /* @@ -200,6 +206,21 @@ impl NitroEnclave { Ok(i32::from_ne_bytes(buf)) } + + // Enable or disable all signals. + fn signals(&self, enable: bool) { + let sig = if enable { + libc::SIG_UNBLOCK + } else { + libc::SIG_BLOCK + }; + + let mut set: libc::sigset_t = unsafe { std::mem::zeroed() }; + unsafe { + libc::sigfillset(&mut set); + libc::pthread_sigmask(sig, &set, std::ptr::null_mut()); + } + } } #[repr(u32)] From d3429028e6b85be2dddd7d2155761685160ebd5c Mon Sep 17 00:00:00 2001 From: Tyler Fanelli Date: Thu, 22 Jan 2026 00:35:51 -0500 Subject: [PATCH 04/17] nitro: Add error logging for proxy threads If an error occurs in device proxy threads, ensure they are logged. Signed-off-by: Tyler Fanelli --- src/nitro/src/enclave/device/devices/signal_handler.rs | 9 +++++++-- src/nitro/src/enclave/device/mod.rs | 4 ++-- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/src/nitro/src/enclave/device/devices/signal_handler.rs b/src/nitro/src/enclave/device/devices/signal_handler.rs index 421051fc2..ff1d32873 100644 --- a/src/nitro/src/enclave/device/devices/signal_handler.rs +++ b/src/nitro/src/enclave/device/devices/signal_handler.rs @@ -72,8 +72,13 @@ impl DeviceProxy for SignalHandler { Ok(()) }); - let _ = signal_handler.join().unwrap(); - let _ = shutdown_listener.join().unwrap(); + if let Ok(Err(e)) = signal_handler.join() { + log::error!("error in signal handler proxy: {e}"); + } + + if let Ok(Err(e)) = shutdown_listener.join() { + log::error!("error in signal handler device proxy shutdown listener: {e}"); + } Ok(()) } diff --git a/src/nitro/src/enclave/device/mod.rs b/src/nitro/src/enclave/device/mod.rs index a9e37870c..6f66c8727 100644 --- a/src/nitro/src/enclave/device/mod.rs +++ b/src/nitro/src/enclave/device/mod.rs @@ -41,8 +41,8 @@ impl DeviceProxyList { } for handle in handles.into_iter() { - let res = handle.join().unwrap(); - if let Err(err) = res { + let res = handle.join(); + if let Ok(Err(err)) = res { log::error!("error running enclave device proxy: {:?}", err); } } From 09120f1419c5d6caa0942c618366226e1fb1ac19 Mon Sep 17 00:00:00 2001 From: Tyler Fanelli Date: Thu, 22 Jan 2026 02:30:33 -0500 Subject: [PATCH 05/17] nitro: Extract shared behavior in device proxies There exists shared behavior within the device proxies such as dedicated threads for reading from and (optionally) writing to a vsock, signalling when the proxy has shut down, and more. Extract this shared behavior from the individual device proxy implementations themselves and implement them within the proxy list's method for running each proxy. Signed-off-by: Tyler Fanelli --- src/libkrun/src/lib.rs | 17 +-- src/nitro/src/enclave/device/devices/net.rs | 141 ++++++------------ .../src/enclave/device/devices/output.rs | 53 +++---- .../enclave/device/devices/signal_handler.rs | 93 +++++------- src/nitro/src/enclave/device/mod.rs | 78 ++++++++-- src/nitro/src/enclave/mod.rs | 10 +- src/nitro/src/lib.rs | 3 - 7 files changed, 188 insertions(+), 207 deletions(-) diff --git a/src/libkrun/src/lib.rs b/src/libkrun/src/lib.rs index e41ff2ccc..e497c0157 100644 --- a/src/libkrun/src/lib.rs +++ b/src/libkrun/src/lib.rs @@ -372,7 +372,7 @@ impl TryFrom for NitroEnclave { return Err(-libc::EINVAL); }; - let net = { + let net_unixfd = { let mut list = ctx.vmr.net.list; let len = list.len(); match len { @@ -381,13 +381,12 @@ impl TryFrom for NitroEnclave { let device = list.pop_front().unwrap(); let device = device.lock().unwrap(); - match nitro::NetProxy::try_from(&*device) { - Ok(net_proxy) => Some(net_proxy), - Err(e) => { - error!("unable to configure network device: {:?}", e); - return Err(-libc::EINVAL); - } - } + let fd = match device.cfg_backend { + VirtioNetBackend::UnixstreamFd(fd) => RawFd::from(fd), + _ => return Err(libc::EINVAL), + }; + + Some(fd) } _ => { error!( @@ -412,7 +411,7 @@ impl TryFrom for NitroEnclave { exec_path, exec_args, exec_env, - net, + net_unixfd, output_path, debug: *debug, }) diff --git a/src/nitro/src/enclave/device/devices/net.rs b/src/nitro/src/enclave/device/devices/net.rs index 2a3af09ad..54d6415d0 100644 --- a/src/nitro/src/enclave/device/devices/net.rs +++ b/src/nitro/src/enclave/device/devices/net.rs @@ -5,129 +5,84 @@ use crate::enclave::{ device::{DeviceProxy, Error, Result}, VsockPortOffset, }; -use devices::virtio::{net::device::VirtioNetBackend, Net}; use std::{ io::{ErrorKind, Read, Write}, os::{ fd::{FromRawFd, OwnedFd, RawFd}, unix::net::UnixStream, }, - sync::mpsc::{self, RecvTimeoutError}, - thread::{self, JoinHandle}, time::Duration, }; -use vsock::{VsockAddr, VsockListener, VMADDR_CID_ANY}; +use vsock::{VsockAddr, VsockListener, VsockStream, VMADDR_CID_ANY}; -#[derive(Clone)] pub struct NetProxy { - fd: RawFd, + buf: [u8; 1500], + unix: UnixStream, } -impl TryFrom<&Net> for NetProxy { +impl TryFrom for NetProxy { type Error = Error; - fn try_from(net: &Net) -> Result { - let fd = match net.cfg_backend { - VirtioNetBackend::UnixstreamFd(fd) => RawFd::from(fd), - _ => return Err(Error::InvalidNetInterface), - }; + fn try_from(fd: RawFd) -> Result { + let buf = [0u8; 1500]; - Ok(Self { fd }) + let unix = unsafe { UnixStream::from(OwnedFd::from_raw_fd(fd)) }; + unix.set_read_timeout(Some(Duration::from_millis(250))) + .map_err(Error::UnixReadTimeoutSet)?; + + Ok(Self { buf, unix }) } } impl DeviceProxy for NetProxy { - fn vsock_port_offset(&self) -> VsockPortOffset { - VsockPortOffset::Net - } + fn clone(&self) -> Result>> { + let unix = self.unix.try_clone().map_err(Error::UnixClone)?; - #[allow(unreachable_code)] - fn _start(&mut self, vsock_port: u32) -> Result<()> { - let vsock_listener = VsockListener::bind(&VsockAddr::new(VMADDR_CID_ANY, vsock_port)) - .map_err(Error::VsockBind)?; - - let mut vsock_stream = vsock_listener.accept().map_err(Error::VsockAccept)?; - - let mut vsock_stream_clone = vsock_stream.0.try_clone().map_err(Error::VsockClone)?; + Ok(Some(Box::new(Self { + buf: self.buf, + unix, + }))) + } + fn enclave_arg(&self) -> Option> { + Some(EnclaveArg::NetworkProxy) + } - let unix_stream = unsafe { UnixStream::from(OwnedFd::from_raw_fd(self.fd)) }; - let mut unix_stream_clone_write = unix_stream.try_clone().map_err(Error::UnixClone)?; + fn port_offset(&self) -> VsockPortOffset { + VsockPortOffset::Net + } - let (tx, rx) = mpsc::channel::<()>(); + fn rcv(&mut self, vsock: &mut VsockStream) -> Result { + let size = vsock.read(&mut self.buf).map_err(Error::VsockRead)?; + if size > 0 { + self.unix + .write_all(&self.buf[..size]) + .map_err(Error::UnixWrite)?; + } - // vsock - let vsock_thread: JoinHandle> = thread::spawn(move || { - let mut vsock_buf = [0u8; 1500]; - loop { - let size = vsock_stream_clone - .read(&mut vsock_buf) - .map_err(Error::VsockRead)?; + Ok(size) + } + fn send(&mut self, vsock: &mut VsockStream) -> Result { + match self.unix.read(&mut self.buf) { + Ok(size) => { if size > 0 { - unix_stream_clone_write - .write_all(&vsock_buf[..size]) - .map_err(Error::UnixWrite)?; - } else { - let _ = tx.send(()); - break; + let _ = vsock.write_all(&self.buf[..size]); } - } - Ok(()) - }); - - let mut unix_stream_clone_read = unix_stream.try_clone().map_err(Error::UnixClone)?; - unix_stream_clone_read - .set_read_timeout(Some(Duration::from_millis(250))) - .map_err(Error::UnixReadTimeoutSet)?; - // Unix - let unix_thread: JoinHandle> = thread::spawn(move || { - let mut unix_buf = [0u8; 1500]; - loop { - match unix_stream_clone_read.read(&mut unix_buf) { - Ok(size) => { - if size > 0 { - if vsock_stream.0.write_all(&unix_buf[..size]).is_err() { - continue; - } - } else { - break; - } - } - Err(ref e) - if e.kind() == ErrorKind::TimedOut || e.kind() == ErrorKind::WouldBlock => - { - match rx.recv_timeout(Duration::from_micros(500)) { - Ok(_) => break, - Err(e) => { - if e == RecvTimeoutError::Timeout { - continue; - } else { - return Err(Error::ShutdownSignalReceive(e))?; - } - } - } - } - Err(e) => return Err(Error::UnixRead(e)), - } + Ok(size) } - - Ok(()) - }); - - if let Ok(Err(err)) = vsock_thread.join() { - log::error!("error with network vsock stream listener thread: {:?}", err); - return Err(err); + Err(ref e) if e.kind() == ErrorKind::TimedOut || e.kind() == ErrorKind::WouldBlock => { + Ok(0) + } + Err(e) => Err(Error::UnixRead(e)), } + } - if let Ok(Err(err)) = unix_thread.join() { - log::error!("error with network UNIX stream listener thread: {:?}", err); - return Err(err); - } + fn vsock(&self, port: u32) -> Result { + let listener = + VsockListener::bind(&VsockAddr::new(VMADDR_CID_ANY, port)).map_err(Error::VsockBind)?; - Ok(()) - } + let (vsock, _) = listener.accept().map_err(Error::VsockAccept)?; - fn enclave_arg(&self) -> Option> { - Some(EnclaveArg::NetworkProxy) + Ok(vsock) } } diff --git a/src/nitro/src/enclave/device/devices/output.rs b/src/nitro/src/enclave/device/devices/output.rs index 7b92ae726..aa8aec774 100644 --- a/src/nitro/src/enclave/device/devices/output.rs +++ b/src/nitro/src/enclave/device/devices/output.rs @@ -18,6 +18,7 @@ type Result = std::result::Result; pub struct OutputProxy { file: File, debug: bool, + buf: [u8; 1500], } impl OutputProxy { @@ -28,54 +29,54 @@ impl OutputProxy { .open(path) .map_err(Error::FileOpen)?; - Ok(Self { file, debug }) + let buf = [0u8; 1500]; + + Ok(Self { file, debug, buf }) } } impl DeviceProxy for OutputProxy { + fn clone(&self) -> Result>> { + Ok(None) + } fn enclave_arg(&self) -> Option> { match self.debug { true => Some(EnclaveArg::Debug), false => None, } } - - fn vsock_port_offset(&self) -> VsockPortOffset { + fn port_offset(&self) -> VsockPortOffset { match self.debug { true => VsockPortOffset::Console, false => VsockPortOffset::AppOutput, } } + fn rcv(&mut self, vsock: &mut VsockStream) -> Result { + let size = vsock.read(&mut self.buf).map_err(Error::VsockRead)?; + if size > 0 { + self.file + .write_all(&self.buf[..size]) + .map_err(Error::FileWrite)?; + } - fn _start(&mut self, vsock_port: u32) -> Result<()> { - let mut vsock_stream = if self.debug { - VsockStream::connect(&VsockAddr::new(VMADDR_CID_HYPERVISOR, vsock_port)) + Ok(size) + } + fn send(&mut self, _vsock: &mut VsockStream) -> Result { + Ok(0) + } + fn vsock(&self, port: u32) -> Result { + let vsock = if self.debug { + VsockStream::connect(&VsockAddr::new(VMADDR_CID_HYPERVISOR, port)) .map_err(Error::VsockConnect)? } else { - let vsock_listener = VsockListener::bind(&VsockAddr::new(VMADDR_CID_ANY, vsock_port)) + let listener = VsockListener::bind(&VsockAddr::new(VMADDR_CID_ANY, port)) .map_err(Error::VsockBind)?; - let (vsock_stream, _vsock_addr) = - vsock_listener.accept().map_err(Error::VsockAccept)?; + let (vsock, _) = listener.accept().map_err(Error::VsockAccept)?; - vsock_stream + vsock }; - let mut vsock_buf = [0u8; 1500]; - loop { - let size = vsock_stream - .read(&mut vsock_buf) - .map_err(Error::VsockRead)?; - - if size > 0 { - self.file - .write_all(&vsock_buf[..size]) - .map_err(Error::FileWrite)?; - } else { - break; - } - } - - Ok(()) + Ok(vsock) } } diff --git a/src/nitro/src/enclave/device/devices/signal_handler.rs b/src/nitro/src/enclave/device/devices/signal_handler.rs index ff1d32873..6a356defa 100644 --- a/src/nitro/src/enclave/device/devices/signal_handler.rs +++ b/src/nitro/src/enclave/device/devices/signal_handler.rs @@ -9,77 +9,58 @@ use std::{ io::{Read, Write}, sync::{ atomic::{AtomicBool, Ordering}, - mpsc::{self, RecvTimeoutError}, Arc, }, - thread::{self, JoinHandle}, - time::Duration, }; -use vsock::{VsockAddr, VsockListener, VMADDR_CID_ANY}; +use vsock::{VsockAddr, VsockListener, VsockStream, VMADDR_CID_ANY}; -#[derive(Default)] -pub struct SignalHandler; +#[derive(Clone)] +pub struct SignalHandler { + sig: Arc, + buf: [u8; 1], +} + +impl SignalHandler { + pub fn new() -> Result { + let sig = Arc::new(AtomicBool::new(false)); + signal_hook::flag::register(SIGTERM, Arc::clone(&sig)).map_err(Error::SignalRegister)?; + + let buf = [0u8; 1]; + + Ok(Self { sig, buf }) + } +} impl DeviceProxy for SignalHandler { + fn clone(&self) -> Result>> { + Ok(Some(Box::new(Clone::clone(self)))) + } fn enclave_arg(&self) -> Option> { None } - - fn vsock_port_offset(&self) -> VsockPortOffset { + fn port_offset(&self) -> VsockPortOffset { VsockPortOffset::SignalHandler } + fn rcv(&mut self, vsock: &mut VsockStream) -> Result { + vsock.read(&mut self.buf).map_err(Error::VsockRead) + } - fn _start(&mut self, vsock_port: u32) -> Result<()> { - let term = Arc::new(AtomicBool::new(false)); - signal_hook::flag::register(SIGTERM, Arc::clone(&term)).map_err(Error::SignalRegister)?; - - let vsock_listener = VsockListener::bind(&VsockAddr::new(VMADDR_CID_ANY, vsock_port)) - .map_err(Error::VsockBind)?; - - let (mut vsock_stream, _vsock_addr) = - vsock_listener.accept().map_err(Error::VsockAccept)?; - - let (tx, rx) = mpsc::channel::<()>(); - let mut vsock_stream_clone = vsock_stream.try_clone().map_err(Error::VsockClone)?; - - let signal_handler: JoinHandle> = thread::spawn(move || { - while !term.load(Ordering::Relaxed) { - match rx.recv_timeout(Duration::from_micros(500)) { - Ok(_) => return Ok(()), - Err(e) => { - if e == RecvTimeoutError::Timeout { - continue; - } - } - } - } - - let sig = libc::SIGTERM; - vsock_stream - .write(&sig.to_ne_bytes()) - .map_err(Error::VsockWrite)?; - - Ok(()) - }); - - let shutdown_listener: JoinHandle> = thread::spawn(move || { - let mut vsock_buf = [0u8; 1]; - let _ = vsock_stream_clone - .read(&mut vsock_buf) - .map_err(Error::VsockRead)?; - let _ = tx.send(()); + fn send(&mut self, vsock: &mut VsockStream) -> Result { + if !self.sig.load(Ordering::Relaxed) { + return Ok(0); + } - Ok(()) - }); + let sig = libc::SIGTERM; + vsock.write(&sig.to_ne_bytes()).map_err(Error::VsockWrite)?; - if let Ok(Err(e)) = signal_handler.join() { - log::error!("error in signal handler proxy: {e}"); - } + Ok(0) + } + fn vsock(&self, port: u32) -> Result { + let listener = + VsockListener::bind(&VsockAddr::new(VMADDR_CID_ANY, port)).map_err(Error::VsockBind)?; - if let Ok(Err(e)) = shutdown_listener.join() { - log::error!("error in signal handler device proxy shutdown listener: {e}"); - } + let (vsock, _) = listener.accept().map_err(Error::VsockAccept)?; - Ok(()) + Ok(vsock) } } diff --git a/src/nitro/src/enclave/device/mod.rs b/src/nitro/src/enclave/device/mod.rs index 6f66c8727..f920c7e70 100644 --- a/src/nitro/src/enclave/device/mod.rs +++ b/src/nitro/src/enclave/device/mod.rs @@ -7,32 +7,81 @@ pub use devices::*; use crate::enclave::{args_writer::EnclaveArg, VsockPortOffset}; use std::{ fmt, io, - sync::mpsc, + sync::mpsc::{self, RecvTimeoutError}, thread::{self, JoinHandle}, + time::Duration, }; +use vsock::*; type Result = std::result::Result; -pub trait DeviceProxy { +pub trait DeviceProxy: Send { + fn clone(&self) -> Result>>; fn enclave_arg(&self) -> Option>; - fn vsock_port_offset(&self) -> VsockPortOffset; - fn start(&mut self, cid: u32) -> Result<()> { - let port = cid + (self.vsock_port_offset() as u32); - - self._start(port) - } - fn _start(&mut self, vsock_port: u32) -> Result<()>; + fn rcv(&mut self, vsock: &mut VsockStream) -> Result; + fn send(&mut self, vsock: &mut VsockStream) -> Result; + fn port_offset(&self) -> VsockPortOffset; + fn vsock(&self, port: u32) -> Result; } pub struct DeviceProxyList(pub Vec>); impl DeviceProxyList { - pub fn start(self, cid: u32) { + pub fn start(self, cid: u32) -> Result<()> { let mut handles: Vec>> = Vec::new(); for mut device in self.0 { + let mut vsock_rcv = device.vsock(cid + (device.port_offset() as u32))?; + let handle: JoinHandle> = thread::spawn(move || { - device.start(cid)?; + let clone = device.clone()?; + let mut vsock_send = vsock_rcv.try_clone().map_err(Error::VsockClone)?; + + let (tx, rx) = mpsc::channel::<()>(); + + let rcv: JoinHandle> = thread::spawn(move || loop { + match device.rcv(&mut vsock_rcv) { + Ok(0) => { + let _ = tx.send(()); + return Ok(()); + } + Ok(_) => continue, + Err(e) => { + let _ = tx.send(()); + return Err(e); + } + } + }); + + let send: JoinHandle> = thread::spawn(move || { + if let Some(mut sender) = clone { + loop { + let size = sender.send(&mut vsock_send)?; + if size == 0 { + match rx.recv_timeout(Duration::from_micros(500)) { + Ok(_) => break, + Err(e) => { + if e == RecvTimeoutError::Timeout { + continue; + } else { + return Err(Error::ShutdownSignalReceive(e))?; + } + } + } + } + } + } + + Ok(()) + }); + + if let Ok(Err(e)) = rcv.join() { + log::error!("error in device proxy receive thread: {e}"); + } + + if let Ok(Err(e)) = send.join() { + log::error!("error in device proxy send thread: {e}"); + } Ok(()) }); @@ -46,6 +95,8 @@ impl DeviceProxyList { log::error!("error running enclave device proxy: {:?}", err); } } + + Ok(()) } } @@ -53,7 +104,6 @@ impl DeviceProxyList { pub enum Error { FileOpen(io::Error), FileWrite(io::Error), - InvalidNetInterface, ShutdownSignalReceive(mpsc::RecvTimeoutError), SignalRegister(io::Error), UnixClone(io::Error), @@ -79,10 +129,6 @@ impl fmt::Display for Error { Self::SignalRegister(e) => { format!("unable to register signal in signal handler proxy: {e}") } - Self::InvalidNetInterface => { - "invalid network proxy interface, must supply UNIX stream file descriptor" - .to_string() - } Self::UnixClone(e) => format!("unable to clone unix stream: {e}"), Self::UnixRead(e) => format!("unable to read from unix stream: {e}"), Self::UnixReadTimeoutSet(e) => { diff --git a/src/nitro/src/enclave/mod.rs b/src/nitro/src/enclave/mod.rs index 77d5a9719..0775d5ff3 100644 --- a/src/nitro/src/enclave/mod.rs +++ b/src/nitro/src/enclave/mod.rs @@ -19,6 +19,7 @@ use std::{ ffi::OsStr, fs, io::{self, Read}, + os::fd::RawFd, path::PathBuf, }; use tar::HeaderMode; @@ -48,7 +49,7 @@ pub struct NitroEnclave { /// Execution environment. pub exec_env: String, /// Network proxy. - pub net: Option, + pub net_unixfd: Option, /// Path to redirect enclave output to. pub output_path: PathBuf, // Output kernel and initramfs debug logs from enclave. @@ -87,7 +88,7 @@ impl NitroEnclave { // Enable signals now that enclave VM is started. self.signals(true); - devices.start(cid); + devices.start(cid).map_err(Error::Device)?; /* * In debug mode, the console device doesn't shut down until the enclave @@ -146,11 +147,12 @@ impl NitroEnclave { let output = OutputProxy::new(&self.output_path, self.debug)?; proxies.push(Box::new(output)); - if let Some(net) = self.net.clone() { + if let Some(fd) = self.net_unixfd { + let net = NetProxy::try_from(fd)?; proxies.push(Box::new(net)); } - proxies.push(Box::new(SignalHandler)); + proxies.push(Box::new(SignalHandler::new()?)); Ok(DeviceProxyList(proxies)) } diff --git a/src/nitro/src/lib.rs b/src/nitro/src/lib.rs index 16c3ac562..2b9e282be 100644 --- a/src/nitro/src/lib.rs +++ b/src/nitro/src/lib.rs @@ -5,6 +5,3 @@ pub mod enclave; #[cfg(feature = "nitro")] mod error; - -#[cfg(feature = "nitro")] -pub use enclave::device::net::NetProxy; From 82e509ca3be04e37707342a775ff993d15e9d02d Mon Sep 17 00:00:00 2001 From: Tyler Fanelli Date: Fri, 23 Jan 2026 23:08:57 -0500 Subject: [PATCH 06/17] nitro: Remove vsock port offset proxy method Refer to the port offset directly when creating the proxy's vsock. Signed-off-by: Tyler Fanelli --- src/nitro/src/enclave/device/devices/net.rs | 9 +++------ src/nitro/src/enclave/device/devices/output.rs | 16 +++++++++------- .../src/enclave/device/devices/signal_handler.rs | 7 +++---- src/nitro/src/enclave/device/mod.rs | 7 +++---- 4 files changed, 18 insertions(+), 21 deletions(-) diff --git a/src/nitro/src/enclave/device/devices/net.rs b/src/nitro/src/enclave/device/devices/net.rs index 54d6415d0..a052c7172 100644 --- a/src/nitro/src/enclave/device/devices/net.rs +++ b/src/nitro/src/enclave/device/devices/net.rs @@ -46,11 +46,6 @@ impl DeviceProxy for NetProxy { fn enclave_arg(&self) -> Option> { Some(EnclaveArg::NetworkProxy) } - - fn port_offset(&self) -> VsockPortOffset { - VsockPortOffset::Net - } - fn rcv(&mut self, vsock: &mut VsockStream) -> Result { let size = vsock.read(&mut self.buf).map_err(Error::VsockRead)?; if size > 0 { @@ -77,7 +72,9 @@ impl DeviceProxy for NetProxy { } } - fn vsock(&self, port: u32) -> Result { + fn vsock(&self, cid: u32) -> Result { + let port = cid + (VsockPortOffset::Net as u32); + let listener = VsockListener::bind(&VsockAddr::new(VMADDR_CID_ANY, port)).map_err(Error::VsockBind)?; diff --git a/src/nitro/src/enclave/device/devices/output.rs b/src/nitro/src/enclave/device/devices/output.rs index aa8aec774..bbe5c864f 100644 --- a/src/nitro/src/enclave/device/devices/output.rs +++ b/src/nitro/src/enclave/device/devices/output.rs @@ -45,12 +45,6 @@ impl DeviceProxy for OutputProxy { false => None, } } - fn port_offset(&self) -> VsockPortOffset { - match self.debug { - true => VsockPortOffset::Console, - false => VsockPortOffset::AppOutput, - } - } fn rcv(&mut self, vsock: &mut VsockStream) -> Result { let size = vsock.read(&mut self.buf).map_err(Error::VsockRead)?; if size > 0 { @@ -64,7 +58,15 @@ impl DeviceProxy for OutputProxy { fn send(&mut self, _vsock: &mut VsockStream) -> Result { Ok(0) } - fn vsock(&self, port: u32) -> Result { + fn vsock(&self, cid: u32) -> Result { + let port = { + let offset = match self.debug { + true => VsockPortOffset::Console, + false => VsockPortOffset::AppOutput, + }; + + cid + (offset as u32) + }; let vsock = if self.debug { VsockStream::connect(&VsockAddr::new(VMADDR_CID_HYPERVISOR, port)) .map_err(Error::VsockConnect)? diff --git a/src/nitro/src/enclave/device/devices/signal_handler.rs b/src/nitro/src/enclave/device/devices/signal_handler.rs index 6a356defa..17c023add 100644 --- a/src/nitro/src/enclave/device/devices/signal_handler.rs +++ b/src/nitro/src/enclave/device/devices/signal_handler.rs @@ -38,9 +38,6 @@ impl DeviceProxy for SignalHandler { fn enclave_arg(&self) -> Option> { None } - fn port_offset(&self) -> VsockPortOffset { - VsockPortOffset::SignalHandler - } fn rcv(&mut self, vsock: &mut VsockStream) -> Result { vsock.read(&mut self.buf).map_err(Error::VsockRead) } @@ -55,7 +52,9 @@ impl DeviceProxy for SignalHandler { Ok(0) } - fn vsock(&self, port: u32) -> Result { + fn vsock(&self, cid: u32) -> Result { + let port = cid + (VsockPortOffset::SignalHandler as u32); + let listener = VsockListener::bind(&VsockAddr::new(VMADDR_CID_ANY, port)).map_err(Error::VsockBind)?; diff --git a/src/nitro/src/enclave/device/mod.rs b/src/nitro/src/enclave/device/mod.rs index f920c7e70..c6b27aa31 100644 --- a/src/nitro/src/enclave/device/mod.rs +++ b/src/nitro/src/enclave/device/mod.rs @@ -4,7 +4,7 @@ mod devices; pub use devices::*; -use crate::enclave::{args_writer::EnclaveArg, VsockPortOffset}; +use crate::enclave::args_writer::EnclaveArg; use std::{ fmt, io, sync::mpsc::{self, RecvTimeoutError}, @@ -20,8 +20,7 @@ pub trait DeviceProxy: Send { fn enclave_arg(&self) -> Option>; fn rcv(&mut self, vsock: &mut VsockStream) -> Result; fn send(&mut self, vsock: &mut VsockStream) -> Result; - fn port_offset(&self) -> VsockPortOffset; - fn vsock(&self, port: u32) -> Result; + fn vsock(&self, cid: u32) -> Result; } pub struct DeviceProxyList(pub Vec>); @@ -31,7 +30,7 @@ impl DeviceProxyList { let mut handles: Vec>> = Vec::new(); for mut device in self.0 { - let mut vsock_rcv = device.vsock(cid + (device.port_offset() as u32))?; + let mut vsock_rcv = device.vsock(cid)?; let handle: JoinHandle> = thread::spawn(move || { let clone = device.clone()?; From 190ca8477fbf715f287bf007aef8d84712eab921 Mon Sep 17 00:00:00 2001 From: Tyler Fanelli Date: Fri, 23 Jan 2026 23:12:00 -0500 Subject: [PATCH 07/17] nitro: Rename enclave_arg method to arg Signed-off-by: Tyler Fanelli --- src/nitro/src/enclave/args_writer.rs | 2 +- src/nitro/src/enclave/device/devices/net.rs | 6 +++--- src/nitro/src/enclave/device/devices/output.rs | 8 ++++---- src/nitro/src/enclave/device/devices/signal_handler.rs | 6 +++--- src/nitro/src/enclave/device/mod.rs | 2 +- 5 files changed, 12 insertions(+), 12 deletions(-) diff --git a/src/nitro/src/enclave/args_writer.rs b/src/nitro/src/enclave/args_writer.rs index 42f1a7b07..b3d77ed69 100644 --- a/src/nitro/src/enclave/args_writer.rs +++ b/src/nitro/src/enclave/args_writer.rs @@ -53,7 +53,7 @@ impl<'a> EnclaveArgsWriter<'a> { ]); for device in &devices.0 { - if let Some(arg) = device.enclave_arg() { + if let Some(arg) = device.arg() { args.push(arg); } } diff --git a/src/nitro/src/enclave/device/devices/net.rs b/src/nitro/src/enclave/device/devices/net.rs index a052c7172..bd9b02f79 100644 --- a/src/nitro/src/enclave/device/devices/net.rs +++ b/src/nitro/src/enclave/device/devices/net.rs @@ -35,6 +35,9 @@ impl TryFrom for NetProxy { } impl DeviceProxy for NetProxy { + fn arg(&self) -> Option> { + Some(EnclaveArg::NetworkProxy) + } fn clone(&self) -> Result>> { let unix = self.unix.try_clone().map_err(Error::UnixClone)?; @@ -43,9 +46,6 @@ impl DeviceProxy for NetProxy { unix, }))) } - fn enclave_arg(&self) -> Option> { - Some(EnclaveArg::NetworkProxy) - } fn rcv(&mut self, vsock: &mut VsockStream) -> Result { let size = vsock.read(&mut self.buf).map_err(Error::VsockRead)?; if size > 0 { diff --git a/src/nitro/src/enclave/device/devices/output.rs b/src/nitro/src/enclave/device/devices/output.rs index bbe5c864f..e919ae086 100644 --- a/src/nitro/src/enclave/device/devices/output.rs +++ b/src/nitro/src/enclave/device/devices/output.rs @@ -36,15 +36,15 @@ impl OutputProxy { } impl DeviceProxy for OutputProxy { - fn clone(&self) -> Result>> { - Ok(None) - } - fn enclave_arg(&self) -> Option> { + fn arg(&self) -> Option> { match self.debug { true => Some(EnclaveArg::Debug), false => None, } } + fn clone(&self) -> Result>> { + Ok(None) + } fn rcv(&mut self, vsock: &mut VsockStream) -> Result { let size = vsock.read(&mut self.buf).map_err(Error::VsockRead)?; if size > 0 { diff --git a/src/nitro/src/enclave/device/devices/signal_handler.rs b/src/nitro/src/enclave/device/devices/signal_handler.rs index 17c023add..3c4eec385 100644 --- a/src/nitro/src/enclave/device/devices/signal_handler.rs +++ b/src/nitro/src/enclave/device/devices/signal_handler.rs @@ -32,12 +32,12 @@ impl SignalHandler { } impl DeviceProxy for SignalHandler { + fn arg(&self) -> Option> { + None + } fn clone(&self) -> Result>> { Ok(Some(Box::new(Clone::clone(self)))) } - fn enclave_arg(&self) -> Option> { - None - } fn rcv(&mut self, vsock: &mut VsockStream) -> Result { vsock.read(&mut self.buf).map_err(Error::VsockRead) } diff --git a/src/nitro/src/enclave/device/mod.rs b/src/nitro/src/enclave/device/mod.rs index c6b27aa31..92a1e9fbc 100644 --- a/src/nitro/src/enclave/device/mod.rs +++ b/src/nitro/src/enclave/device/mod.rs @@ -16,8 +16,8 @@ use vsock::*; type Result = std::result::Result; pub trait DeviceProxy: Send { + fn arg(&self) -> Option>; fn clone(&self) -> Result>>; - fn enclave_arg(&self) -> Option>; fn rcv(&mut self, vsock: &mut VsockStream) -> Result; fn send(&mut self, vsock: &mut VsockStream) -> Result; fn vsock(&self, cid: u32) -> Result; From 718d60707104f55eafbf37c7d75445c576d3f5b9 Mon Sep 17 00:00:00 2001 From: Tyler Fanelli Date: Fri, 23 Jan 2026 23:20:02 -0500 Subject: [PATCH 08/17] nitro: Rename device start method to run The proxy threads do not return until they are finished running. Reflect this by naming the method `run` instead of `start` which may indicate returning after starting the proxies. Signed-off-by: Tyler Fanelli --- src/nitro/src/enclave/device/mod.rs | 2 +- src/nitro/src/enclave/mod.rs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/nitro/src/enclave/device/mod.rs b/src/nitro/src/enclave/device/mod.rs index 92a1e9fbc..f4c8d5552 100644 --- a/src/nitro/src/enclave/device/mod.rs +++ b/src/nitro/src/enclave/device/mod.rs @@ -26,7 +26,7 @@ pub trait DeviceProxy: Send { pub struct DeviceProxyList(pub Vec>); impl DeviceProxyList { - pub fn start(self, cid: u32) -> Result<()> { + pub fn run(self, cid: u32) -> Result<()> { let mut handles: Vec>> = Vec::new(); for mut device in self.0 { diff --git a/src/nitro/src/enclave/mod.rs b/src/nitro/src/enclave/mod.rs index 0775d5ff3..13490a0d2 100644 --- a/src/nitro/src/enclave/mod.rs +++ b/src/nitro/src/enclave/mod.rs @@ -88,7 +88,7 @@ impl NitroEnclave { // Enable signals now that enclave VM is started. self.signals(true); - devices.start(cid).map_err(Error::Device)?; + devices.run(cid).map_err(Error::Device)?; /* * In debug mode, the console device doesn't shut down until the enclave From a51fbd1de3e2620e8ea4e23987d09bb2279e0043 Mon Sep 17 00:00:00 2001 From: Tyler Fanelli Date: Fri, 23 Jan 2026 23:27:12 -0500 Subject: [PATCH 09/17] nitro: Rename device, devices modules Rather than direct device emulation, the proxies act as intermediaries between the guest and host device to provide device services. `device` may lead some to believe that the module refers to device emulations. Rename it to `proxy` to better reflect its/their goals. Signed-off-by: Tyler Fanelli --- src/nitro/src/enclave/args_writer.rs | 8 ++++---- src/nitro/src/enclave/mod.rs | 16 ++++++++-------- src/nitro/src/enclave/{device => proxy}/mod.rs | 12 ++++++------ .../{device/devices => proxy/proxies}/mod.rs | 0 .../{device/devices => proxy/proxies}/net.rs | 2 +- .../{device/devices => proxy/proxies}/output.rs | 2 +- .../devices => proxy/proxies}/signal_handler.rs | 2 +- src/nitro/src/error.rs | 6 +++--- 8 files changed, 24 insertions(+), 24 deletions(-) rename src/nitro/src/enclave/{device => proxy}/mod.rs (95%) rename src/nitro/src/enclave/{device/devices => proxy/proxies}/mod.rs (100%) rename src/nitro/src/enclave/{device/devices => proxy/proxies}/net.rs (98%) rename src/nitro/src/enclave/{device/devices => proxy/proxies}/output.rs (98%) rename src/nitro/src/enclave/{device/devices => proxy/proxies}/signal_handler.rs (97%) diff --git a/src/nitro/src/enclave/args_writer.rs b/src/nitro/src/enclave/args_writer.rs index b3d77ed69..f86f9b517 100644 --- a/src/nitro/src/enclave/args_writer.rs +++ b/src/nitro/src/enclave/args_writer.rs @@ -1,6 +1,6 @@ // SPDX-License-Identifier: Apache-2.0 -use crate::enclave::{device::DeviceProxyList, VsockPortOffset}; +use crate::enclave::{proxy::DeviceProxyList, VsockPortOffset}; use libc::c_int; use nitro_enclaves::launch::PollTimeout; use nix::poll::{poll, PollFd, PollFlags, PollTimeout as NixPollTimeout}; @@ -29,7 +29,7 @@ impl<'a> EnclaveArgsWriter<'a> { exec_path: &str, argv_str: &str, envp_str: &str, - devices: &'a DeviceProxyList, + proxies: &'a DeviceProxyList, ) -> Self { let mut args: Vec> = Vec::new(); @@ -52,8 +52,8 @@ impl<'a> EnclaveArgsWriter<'a> { EnclaveArg::ExecEnvp(envp), ]); - for device in &devices.0 { - if let Some(arg) = device.arg() { + for proxy in &proxies.0 { + if let Some(arg) = proxy.arg() { args.push(arg); } } diff --git a/src/nitro/src/enclave/mod.rs b/src/nitro/src/enclave/mod.rs index 13490a0d2..15898f47e 100644 --- a/src/nitro/src/enclave/mod.rs +++ b/src/nitro/src/enclave/mod.rs @@ -1,19 +1,19 @@ // SPDX-License-Identifier: Apache-2.0 pub(crate) mod args_writer; -pub(crate) mod device; +pub(crate) mod proxy; use super::error::{ return_code::Error as ReturnCodeListenerError, start::Error as StartError, Error, }; use args_writer::EnclaveArgsWriter; -use device::{ - net::NetProxy, output::OutputProxy, signal_handler::SignalHandler, DeviceProxy, DeviceProxyList, -}; use nitro_enclaves::{ launch::{ImageType, Launcher, MemoryInfo, PollTimeout, StartFlags}, Device, }; +use proxy::{ + net::NetProxy, output::OutputProxy, signal_handler::SignalHandler, DeviceProxy, DeviceProxyList, +}; use std::{ env, ffi::OsStr, @@ -61,14 +61,14 @@ impl NitroEnclave { pub fn run(mut self) -> Result<(), Error> { let rootfs_archive = self.rootfs_archive().map_err(Error::RootFsArchive)?; - let devices = self.devices().map_err(Error::Device)?; + let proxies = self.proxies().map_err(Error::DeviceProxy)?; let writer = EnclaveArgsWriter::new( &rootfs_archive, &self.exec_path, &self.exec_args, &self.exec_env, - &devices, + &proxies, ); // Disable signals to launch enclave VM. @@ -88,7 +88,7 @@ impl NitroEnclave { // Enable signals now that enclave VM is started. self.signals(true); - devices.run(cid).map_err(Error::Device)?; + proxies.run(cid).map_err(Error::DeviceProxy)?; /* * In debug mode, the console device doesn't shut down until the enclave @@ -141,7 +141,7 @@ impl NitroEnclave { Ok((cid.try_into().unwrap(), timeout)) } - fn devices(&self) -> Result { + fn proxies(&self) -> Result { let mut proxies: Vec> = vec![]; let output = OutputProxy::new(&self.output_path, self.debug)?; diff --git a/src/nitro/src/enclave/device/mod.rs b/src/nitro/src/enclave/proxy/mod.rs similarity index 95% rename from src/nitro/src/enclave/device/mod.rs rename to src/nitro/src/enclave/proxy/mod.rs index f4c8d5552..426aceed6 100644 --- a/src/nitro/src/enclave/device/mod.rs +++ b/src/nitro/src/enclave/proxy/mod.rs @@ -1,8 +1,8 @@ // SPDX-License-Identifier: Apache-2.0 -mod devices; +mod proxies; -pub use devices::*; +pub use proxies::*; use crate::enclave::args_writer::EnclaveArg; use std::{ @@ -29,17 +29,17 @@ impl DeviceProxyList { pub fn run(self, cid: u32) -> Result<()> { let mut handles: Vec>> = Vec::new(); - for mut device in self.0 { - let mut vsock_rcv = device.vsock(cid)?; + for mut proxy in self.0 { + let mut vsock_rcv = proxy.vsock(cid)?; let handle: JoinHandle> = thread::spawn(move || { - let clone = device.clone()?; + let clone = proxy.clone()?; let mut vsock_send = vsock_rcv.try_clone().map_err(Error::VsockClone)?; let (tx, rx) = mpsc::channel::<()>(); let rcv: JoinHandle> = thread::spawn(move || loop { - match device.rcv(&mut vsock_rcv) { + match proxy.rcv(&mut vsock_rcv) { Ok(0) => { let _ = tx.send(()); return Ok(()); diff --git a/src/nitro/src/enclave/device/devices/mod.rs b/src/nitro/src/enclave/proxy/proxies/mod.rs similarity index 100% rename from src/nitro/src/enclave/device/devices/mod.rs rename to src/nitro/src/enclave/proxy/proxies/mod.rs diff --git a/src/nitro/src/enclave/device/devices/net.rs b/src/nitro/src/enclave/proxy/proxies/net.rs similarity index 98% rename from src/nitro/src/enclave/device/devices/net.rs rename to src/nitro/src/enclave/proxy/proxies/net.rs index bd9b02f79..cf52e4032 100644 --- a/src/nitro/src/enclave/device/devices/net.rs +++ b/src/nitro/src/enclave/proxy/proxies/net.rs @@ -2,7 +2,7 @@ use crate::enclave::{ args_writer::EnclaveArg, - device::{DeviceProxy, Error, Result}, + proxy::{DeviceProxy, Error, Result}, VsockPortOffset, }; use std::{ diff --git a/src/nitro/src/enclave/device/devices/output.rs b/src/nitro/src/enclave/proxy/proxies/output.rs similarity index 98% rename from src/nitro/src/enclave/device/devices/output.rs rename to src/nitro/src/enclave/proxy/proxies/output.rs index e919ae086..ec21fd008 100644 --- a/src/nitro/src/enclave/device/devices/output.rs +++ b/src/nitro/src/enclave/proxy/proxies/output.rs @@ -2,7 +2,7 @@ use crate::enclave::{ args_writer::EnclaveArg, - device::{DeviceProxy, Error}, + proxy::{DeviceProxy, Error}, VsockPortOffset, }; use std::{ diff --git a/src/nitro/src/enclave/device/devices/signal_handler.rs b/src/nitro/src/enclave/proxy/proxies/signal_handler.rs similarity index 97% rename from src/nitro/src/enclave/device/devices/signal_handler.rs rename to src/nitro/src/enclave/proxy/proxies/signal_handler.rs index 3c4eec385..fac020edc 100644 --- a/src/nitro/src/enclave/device/devices/signal_handler.rs +++ b/src/nitro/src/enclave/proxy/proxies/signal_handler.rs @@ -1,7 +1,7 @@ // SPDX-License-Identifier: Apache-2.0 use crate::enclave::{ - device::{EnclaveArg, Error, Result}, + proxy::{EnclaveArg, Error, Result}, DeviceProxy, VsockPortOffset, }; use signal_hook::consts::SIGTERM; diff --git a/src/nitro/src/error.rs b/src/nitro/src/error.rs index 27a3249ae..fe995a884 100644 --- a/src/nitro/src/error.rs +++ b/src/nitro/src/error.rs @@ -1,13 +1,13 @@ // SPDX-License-Identifier: Apache-2.0 -use super::enclave::{args_writer, device}; +use super::enclave::{args_writer, proxy}; use std::{fmt, io}; #[derive(Debug)] pub enum Error { AppReturn(i32), ArgsWrite(args_writer::Error), - Device(device::Error), + DeviceProxy(proxy::Error), ReturnCodeListener(return_code::Error), RootFsArchive(io::Error), Start(start::Error), @@ -18,7 +18,7 @@ impl fmt::Display for Error { let msg = match self { Self::AppReturn(ret) => format!("app returned non-zero return code: {ret}"), Self::ArgsWrite(e) => format!("enclave VM argument writer error: {e}"), - Self::Device(e) => format!("device proxy error: {e}"), + Self::DeviceProxy(e) => format!("device proxy error: {e}"), Self::ReturnCodeListener(e) => { format!("error with enclave VM return code listener: {e}") } From 2159a0abd2e061d00e52b95dc24c9ec48eace4ff Mon Sep 17 00:00:00 2001 From: Tyler Fanelli Date: Sat, 24 Jan 2026 00:19:18 -0500 Subject: [PATCH 10/17] nitro: Document code with comments Signed-off-by: Tyler Fanelli --- src/nitro/src/enclave/args_writer.rs | 50 +++++++++++++++ src/nitro/src/enclave/mod.rs | 63 ++++++++++++++----- src/nitro/src/enclave/proxy/mod.rs | 61 +++++++++++++++++- src/nitro/src/enclave/proxy/proxies/net.rs | 15 ++++- src/nitro/src/enclave/proxy/proxies/output.rs | 20 ++++++ .../enclave/proxy/proxies/signal_handler.rs | 15 +++++ src/nitro/src/error.rs | 19 ++++++ 7 files changed, 224 insertions(+), 19 deletions(-) diff --git a/src/nitro/src/enclave/args_writer.rs b/src/nitro/src/enclave/args_writer.rs index f86f9b517..25c8d7ea6 100644 --- a/src/nitro/src/enclave/args_writer.rs +++ b/src/nitro/src/enclave/args_writer.rs @@ -14,16 +14,23 @@ use std::{ }; use vsock::{VsockAddr, VsockListener, VsockStream, VMADDR_CID_ANY}; +// A known byte that libkrun-nitro and the enclave initramfs will exchange to confirm that startup +// was successful and the initramfs is ready to begin reading enclave arguments. const ENCLAVE_VSOCK_LAUNCH_ARGS_READY: u8 = 0xb7; type Result = std::result::Result; +/// The service responsible for writing the configuration (rootfs, execution environment, and +// optional device proxies) to the enclave. #[derive(Debug, Default)] pub struct EnclaveArgsWriter<'a> { + // List of enclave arguments. pub args: Vec>, } impl<'a> EnclaveArgsWriter<'a> { + /// Create a new arguments writer. An enclave's rootfs and execution path are required + /// arguments. Some device proxies are required, but others are optional. pub fn new( rootfs_archive: &'a [u8], exec_path: &str, @@ -33,18 +40,21 @@ impl<'a> EnclaveArgsWriter<'a> { ) -> Self { let mut args: Vec> = Vec::new(); + // Split the argv string into a vector. let argv: Vec = argv_str .replace("\"", "") .split(' ') .map(|s| s.to_string()) .collect(); + // Split the envp string into a vector. let envp: Vec = envp_str .replace("\"", "") .split(' ') .map(|s| s.to_string()) .collect(); + // Create the initial argument list from the required arguments. args.append(&mut vec![ EnclaveArg::RootFilesystem(rootfs_archive), EnclaveArg::ExecPath(exec_path.to_string()), @@ -52,6 +62,8 @@ impl<'a> EnclaveArgsWriter<'a> { EnclaveArg::ExecEnvp(envp), ]); + // Add an enclave argument for each device proxy that includes one. Any optional device + // proxy has an enclave argument. for proxy in &proxies.0 { if let Some(arg) = proxy.arg() { args.push(arg); @@ -60,7 +72,10 @@ impl<'a> EnclaveArgsWriter<'a> { Self { args } } + + /// Write the arguments to the enclave. pub fn write_args(&self, cid: u32, timeout: PollTimeout) -> Result<()> { + // Establish a vsock connection to the enclave's initramfs. let listener = VsockListener::bind(&VsockAddr::new( VMADDR_CID_ANY, cid + (VsockPortOffset::ArgsReader as u32), @@ -75,6 +90,7 @@ impl<'a> EnclaveArgsWriter<'a> { return Err(Error::VsockCidMismatch); } + // Exchange the ready signal to ensure the initramfs is ready to receive arguments. let mut buf = [0u8]; let bytes = stream.0.read(&mut buf).map_err(Error::VsockRead)?; @@ -84,6 +100,7 @@ impl<'a> EnclaveArgsWriter<'a> { stream.0.write_all(&buf).map_err(Error::VsockWrite)?; + // Write each argument. for arg in &self.args { arg.write(&mut stream.0)?; } @@ -95,6 +112,7 @@ impl<'a> EnclaveArgsWriter<'a> { Ok(()) } + /// The enclave's initramfs may take some time to connect over vsock. Poll for the connection. fn poll(&self, listener: &VsockListener, timeout: PollTimeout) -> Result<()> { let mut poll_fds = [PollFd::new(listener.as_fd(), PollFlags::POLLIN)]; let result = poll( @@ -110,17 +128,27 @@ impl<'a> EnclaveArgsWriter<'a> { } } +/// An enclave argument. #[derive(Debug)] pub enum EnclaveArg<'a> { + // Enclave rootfs. RootFilesystem(&'a [u8]), + // Enclave execution environment (path, argv, envp). ExecPath(String), ExecArgv(Vec), ExecEnvp(Vec), + // Network proxy. NetworkProxy, + // Debug logs. Debug, + + // Placeholder argument where libkrun notifies the initramfs that all arguments have been + // written and it can now close the vsock connection. Finished, } +/// Each argument has a unique code/ID for the initramfs to understand how to read its parameters. +/// This code is represented as a one-byte value. impl From<&EnclaveArg<'_>> for u8 { fn from(arg: &EnclaveArg) -> u8 { match arg { @@ -137,12 +165,16 @@ impl From<&EnclaveArg<'_>> for u8 { } impl EnclaveArg<'_> { + /// Write an argument to the enclave. fn write(&self, vsock: &mut VsockStream) -> Result<()> { let id: [u8; 1] = [self.into()]; + // Write the argument's ID for the enclave to understand how to read the argument's + // parameters. vsock.write_all(&id).map_err(Error::VsockWrite)?; match self { + // rootfs argument writes the rootfs tar archive. Self::RootFilesystem(buf) => { let len: u64 = buf.len().try_into().map_err(Error::VsockBufferLenConvert)?; @@ -152,13 +184,17 @@ impl EnclaveArg<'_> { vsock.write_all(buf).map_err(Error::VsockWrite)?; } + // Execution argv and envp arguments write their respective contents as string arrays. Self::ExecArgv(vec) | Self::ExecEnvp(vec) => { + // Write the amount of strings the enclave will read. let len: u64 = vec.len().try_into().map_err(Error::VsockBufferLenConvert)?; vsock .write_all(&len.to_ne_bytes()) .map_err(Error::VsockWrite)?; + // For each string, write the length (i.e. the number of bytes the enclave should + // read) and the string itself. for string in vec { let bytes = Vec::from( CString::from_str(string) @@ -178,6 +214,7 @@ impl EnclaveArg<'_> { vsock.write_all(&bytes).map_err(Error::VsockWrite)?; } } + // Execution path argument writes the path as a string. Self::ExecPath(buf) => { let bytes = Vec::from( CString::from_str(buf) @@ -195,6 +232,8 @@ impl EnclaveArg<'_> { vsock.write_all(&bytes).map_err(Error::VsockWrite)?; } + + // Other arguments write solely their ID. The enclave will initialize them. _ => (), } @@ -202,17 +241,28 @@ impl EnclaveArg<'_> { } } +/// Error in the process of writing the enclave's arguments. #[derive(Debug)] pub enum Error { + // Convert a string to a CString. CStringConvert(ffi::NulError), + // No events detected on vsock. PollNoEvents, + // More than one event found on vsock. PollMoreThanOneSelectedEvent, + // Ready signal not detected. ReadySignalNotDetected, + // Accepting the vsock connection. VsockAccept(io::Error), + // Binding to the vsock. VsockBind(io::Error), + // Converting a byte buffer's length to a u64. VsockBufferLenConvert(TryFromIntError), + // CID mismatch with communicating enclave. VsockCidMismatch, + // Reading from the vsock. VsockRead(io::Error), + // Writing to the vsock. VsockWrite(io::Error), } diff --git a/src/nitro/src/enclave/mod.rs b/src/nitro/src/enclave/mod.rs index 15898f47e..9884075e4 100644 --- a/src/nitro/src/enclave/mod.rs +++ b/src/nitro/src/enclave/mod.rs @@ -25,13 +25,15 @@ use std::{ use tar::HeaderMode; use vsock::{VsockAddr, VsockListener, VMADDR_CID_ANY}; +/// Directories within the configured rootfs that will be ignored when writing to the enclave. The +/// enclave is responsible for initializing these directories within the guest operating system. const ROOTFS_DIR_DENYLIST: [&str; 6] = [ - "proc", // /proc. - "run", // /run. - "tmp", // /tmp. - "dev", // /dev. - "sys", // /sys. - "usr/share/krun-nitro", + "proc", // /proc. + "run", // /run. + "tmp", // /tmp. + "dev", // /dev. + "sys", // /sys. + "usr/share/krun-nitro", // Cached EIF file (and possibly other metadata). ]; /// Nitro Enclave data. @@ -57,10 +59,11 @@ pub struct NitroEnclave { } impl NitroEnclave { - /// Run the enclave. + /// Run an application within a nitro enclave. pub fn run(mut self) -> Result<(), Error> { + // Collect all launch parameters (rootfs, execution arguments, device proxies) and establish + // an enclave argument writer to write this data to the nitro enclave when started. let rootfs_archive = self.rootfs_archive().map_err(Error::RootFsArchive)?; - let proxies = self.proxies().map_err(Error::DeviceProxy)?; let writer = EnclaveArgsWriter::new( @@ -74,10 +77,12 @@ impl NitroEnclave { // Disable signals to launch enclave VM. self.signals(false); + // Launch the enclave and write the configured launch parameters to the initramfs. let (cid, timeout) = self.start().map_err(Error::Start)?; writer.write_args(cid, timeout).map_err(Error::ArgsWrite)?; + // Establish the vsock listener for the application's return code upon termination. let retcode_listener = VsockListener::bind(&VsockAddr::new( VMADDR_CID_ANY, cid + (VsockPortOffset::ReturnCode as u32), @@ -88,17 +93,19 @@ impl NitroEnclave { // Enable signals now that enclave VM is started. self.signals(true); + // Run the device proxies. Each proxy is run within its own thread that can only be + // terminated by the enclave (by closing the vsock connection). proxies.run(cid).map_err(Error::DeviceProxy)?; - /* - * In debug mode, the console device doesn't shut down until the enclave - * itself exits. Thus, libkrun will be unable to retrieve the shutdown - * code from the enclave. - */ + // In debug mode, the console device doesn't shut down until the enclave itself exits. Thus, + // libkrun will be unable to retrieve the shutdown code from the enclave. if !self.debug { + // Retrieve the application return code from the enclave. let ret = self .shutdown_ret(retcode_listener) .map_err(Error::ReturnCodeListener)?; + + // A non-zero return code indicates an error. Wrap this code within an Error object. if ret != 0 { return Err(Error::AppReturn(ret)); } @@ -107,15 +114,22 @@ impl NitroEnclave { Ok(()) } + /// Start a nitro enclave. fn start(&mut self) -> Result<(u32, PollTimeout), StartError> { - let path = env::var("KRUN_NITRO_EIF_PATH") - .unwrap_or("/usr/share/krun-nitro/krun-nitro.eif".to_string()); + // Read the cached EIF file required to run the enclave. + let eif = { + let path = env::var("KRUN_NITRO_EIF_PATH") + .unwrap_or("/usr/share/krun-nitro/krun-nitro.eif".to_string()); - let eif = fs::read(path).map_err(StartError::EifRead)?; + fs::read(path).map_err(StartError::EifRead) + }?; + // Calculate the poll timeout (based on the size of the EIF file and amount of RAM allocated + // to the enclave) for the enclave to indicate that has successfully started. let timeout = PollTimeout::try_from((eif.as_slice(), self.mem_size_mib << 20)) .map_err(StartError::PollTimeoutCalculate)?; + // Launch an enclave VM with the configured number of vCPUs and amount of RAM. let device = Device::open().map_err(StartError::DeviceOpen)?; let mut launcher = Launcher::new(&device).map_err(StartError::VmCreate)?; @@ -127,12 +141,14 @@ impl NitroEnclave { launcher.add_vcpu(None).map_err(StartError::VcpuAdd)?; } + // Indicate to the enclave to start in debug mode if configured. let mut start_flags = StartFlags::empty(); if self.debug { start_flags |= StartFlags::DEBUG; } + // Start the enclave. let cid = launcher .start(start_flags, None) .map_err(StartError::VmStart)?; @@ -141,9 +157,11 @@ impl NitroEnclave { Ok((cid.try_into().unwrap(), timeout)) } + /// Initialize and collect all device proxies used for the enclave. fn proxies(&self) -> Result { let mut proxies: Vec> = vec![]; + // All enclaves will include a proxy for debug/application output. let output = OutputProxy::new(&self.output_path, self.debug)?; proxies.push(Box::new(output)); @@ -152,11 +170,15 @@ impl NitroEnclave { proxies.push(Box::new(net)); } + // All enclaves will include a proxy for signal handling (e.g. forwarding SIGTERM signals to + // application running within the enclave). proxies.push(Box::new(SignalHandler::new()?)); Ok(DeviceProxyList(proxies)) } + /// Produce a tarball of the enclave's rootfs (to be written to and extracted by the enclave + // initramfs). fn rootfs_archive(&self) -> Result, io::Error> { let mut builder = tar::Builder::new(Vec::new()); @@ -174,6 +196,8 @@ impl NitroEnclave { pathbuf_copy )))?; + // Traverse each directory and file within the root directory tree. If a directory is not + // found within the denylist, add it to the archive. for entry in fs::read_dir(pathbuf)? { let entry = entry?; let filetype = entry.file_type()?; @@ -196,6 +220,8 @@ impl NitroEnclave { builder.into_inner() } + // Receive a 4-byte (representing an i32) return code from the enclave via vsock. This + // represents the return code of the application that ran within the enclave. fn shutdown_ret(&self, vsock_listener: VsockListener) -> Result { let (mut vsock_stream, _vsock_addr) = vsock_listener .accept() @@ -225,6 +251,11 @@ impl NitroEnclave { } } +/// Each service provided to an enclave is done so via vsock. Each service has a designated port +/// offset (relative to the enclave VM's CID) to connect to for service. The port number for each of +/// an enclave's services can be calculated as: +/// +/// vsock port = (Enclave VM CID + vsock port offset) #[repr(u32)] pub enum VsockPortOffset { ArgsReader = 1, diff --git a/src/nitro/src/enclave/proxy/mod.rs b/src/nitro/src/enclave/proxy/mod.rs index 426aceed6..9cb3b68fd 100644 --- a/src/nitro/src/enclave/proxy/mod.rs +++ b/src/nitro/src/enclave/proxy/mod.rs @@ -15,36 +15,60 @@ use vsock::*; type Result = std::result::Result; +/// Device proxy trait to describe shared behavior between all proxies. pub trait DeviceProxy: Send { + /// Enclave argument of the proxy. fn arg(&self) -> Option>; + /// Clone a proxy's contents. fn clone(&self) -> Result>>; + /// Receive data from the proxy's vsock. Perhaps perform some other functions. fn rcv(&mut self, vsock: &mut VsockStream) -> Result; + /// Write data to the enclave's vsock. Perhaps perform some other functions. fn send(&mut self, vsock: &mut VsockStream) -> Result; + /// Establish the proxy's respective vsock connection. fn vsock(&self, cid: u32) -> Result; } +/// List of all configured device proxies. pub struct DeviceProxyList(pub Vec>); impl DeviceProxyList { + /// Run each proxy's send and receive processes within their own dedicated threads. pub fn run(self, cid: u32) -> Result<()> { + // This function will not return until all device proxies' dedicated threads have returned. + // Under normal conditions, this will only happen when the enclave completes execution and + // gracefully closes all proxy vsock connections. Store each thread's JoinHandle in a list + // to keep track of completed proxy threads. let mut handles: Vec>> = Vec::new(); for mut proxy in self.0 { + // Get a proxy's vsock connection for the its receiver thread. let mut vsock_rcv = proxy.vsock(cid)?; let handle: JoinHandle> = thread::spawn(move || { + // Clone the proxy and vsock connection data for the proxy's sender thread. let clone = proxy.clone()?; let mut vsock_send = vsock_rcv.try_clone().map_err(Error::VsockClone)?; + // Establish a message passing channel for the receiver thread to notify the send + // thread that the enclave has closed the connection. let (tx, rx) = mpsc::channel::<()>(); + // Receiver thread. Receive data from the vsock and perform some proxy-dependent + // action with the data. let rcv: JoinHandle> = thread::spawn(move || loop { + // Proxy rcv method returns the number of bytes read from the vsock. match proxy.rcv(&mut vsock_rcv) { + // Zero bytes read indicates the enclave has closed the vsock connection. + // Notify the sender thread that the vsock was closed. Ok(0) => { let _ = tx.send(()); return Ok(()); } + // Bytes were read, continue the receive process. Ok(_) => continue, + // An error occured, exit the receiver thread and notify the sender thread to + // also exit. Err(e) => { let _ = tx.send(()); return Err(e); @@ -52,17 +76,32 @@ impl DeviceProxyList { } }); + // Sender thread. Perform some proxy-dependent action and (if applicable) write data + // to the vsock. let send: JoinHandle> = thread::spawn(move || { + // Some proxies (like output/debug) do not send data to the enclave. If there is + // nothing to be done, exit the thread. if let Some(mut sender) = clone { loop { + // Proxy send method returns the number of bytes written to the vsock. let size = sender.send(&mut vsock_send)?; + + // No data was written to the vsock. This may indicate that the timeout + // has occurred without data being retrieved from the device's other + // party. This may indicate that the proxy is complete. Check for this + // by reading if a message was sent by the receiver thread. if size == 0 { match rx.recv_timeout(Duration::from_micros(500)) { + // Message was sent indicating the enclave has closed the + // connection, exit from this thread. Ok(_) => break, Err(e) => { + // The receiver thread has not sent a shutdown signal. + // Continue execution. if e == RecvTimeoutError::Timeout { continue; } else { + // Error in fetching message from receiver thread. return Err(Error::ShutdownSignalReceive(e))?; } } @@ -75,19 +114,22 @@ impl DeviceProxyList { }); if let Ok(Err(e)) = rcv.join() { - log::error!("error in device proxy receive thread: {e}"); + log::error!("error in device proxy receiver thread: {e}"); } if let Ok(Err(e)) = send.join() { - log::error!("error in device proxy send thread: {e}"); + log::error!("error in device proxy sender thread: {e}"); } Ok(()) }); + // Add the proxy's control thread JoinHandle into the list. handles.push(handle); } + // Traverse over each device proxy thread and ensure it closes and exits correctly. Do not + // return until all do. for handle in handles.into_iter() { let res = handle.join(); if let Ok(Err(err)) = res { @@ -99,21 +141,36 @@ impl DeviceProxyList { } } +/// Error while running a device proxy. #[derive(Debug)] pub enum Error { + // Opening a file (for proxies also communicating with files/sockets). FileOpen(io::Error), + // Writing to a file. FileWrite(io::Error), + // Receiving a shutdown signal from a proxy's receiver thread. ShutdownSignalReceive(mpsc::RecvTimeoutError), + // Registering a signal for the signal handler. SignalRegister(io::Error), + // Cloning a unix socket. UnixClone(io::Error), + // Reading from a unix socket. UnixRead(io::Error), + // Setting the read timeout for a unix socket. UnixReadTimeoutSet(io::Error), + // Writing to a unix socket. UnixWrite(io::Error), + // Accepting the vsock connection. VsockAccept(io::Error), + // Binding to the vsock. VsockBind(io::Error), + // Cloning the vsock. VsockClone(io::Error), + // Connecting to the vsock. VsockConnect(io::Error), + // Reading from the vsock. VsockRead(io::Error), + // Writing to the vsock. VsockWrite(io::Error), } diff --git a/src/nitro/src/enclave/proxy/proxies/net.rs b/src/nitro/src/enclave/proxy/proxies/net.rs index cf52e4032..27740f417 100644 --- a/src/nitro/src/enclave/proxy/proxies/net.rs +++ b/src/nitro/src/enclave/proxy/proxies/net.rs @@ -15,9 +15,13 @@ use std::{ }; use vsock::{VsockAddr, VsockListener, VsockStream, VMADDR_CID_ANY}; +/// Network proxy. Forwards data to/from a UNIX socket and vsock within an enclave to provide +/// network access. pub struct NetProxy { - buf: [u8; 1500], + // Unix socket connected to service providing network access. unix: UnixStream, + // Buffer to send/receive data to/from vsock. + buf: [u8; 1500], } impl TryFrom for NetProxy { @@ -35,9 +39,12 @@ impl TryFrom for NetProxy { } impl DeviceProxy for NetProxy { + /// Enclave argument of the proxy. fn arg(&self) -> Option> { Some(EnclaveArg::NetworkProxy) } + + /// Clone a proxy's contents (notably, its connected unix socket). fn clone(&self) -> Result>> { let unix = self.unix.try_clone().map_err(Error::UnixClone)?; @@ -46,6 +53,8 @@ impl DeviceProxy for NetProxy { unix, }))) } + + /// Receive data from the proxy's vsock. Forward the data to the connected unix socket. fn rcv(&mut self, vsock: &mut VsockStream) -> Result { let size = vsock.read(&mut self.buf).map_err(Error::VsockRead)?; if size > 0 { @@ -56,6 +65,8 @@ impl DeviceProxy for NetProxy { Ok(size) } + + /// Receive data from the connected unix socket. Forward the data to the proxy's vsock. fn send(&mut self, vsock: &mut VsockStream) -> Result { match self.unix.read(&mut self.buf) { Ok(size) => { @@ -65,6 +76,7 @@ impl DeviceProxy for NetProxy { Ok(size) } + // No data read from unix socket before timeout. Err(ref e) if e.kind() == ErrorKind::TimedOut || e.kind() == ErrorKind::WouldBlock => { Ok(0) } @@ -72,6 +84,7 @@ impl DeviceProxy for NetProxy { } } + /// Establish the proxy's vsock connection. fn vsock(&self, cid: u32) -> Result { let port = cid + (VsockPortOffset::Net as u32); diff --git a/src/nitro/src/enclave/proxy/proxies/output.rs b/src/nitro/src/enclave/proxy/proxies/output.rs index ec21fd008..913ebbd3b 100644 --- a/src/nitro/src/enclave/proxy/proxies/output.rs +++ b/src/nitro/src/enclave/proxy/proxies/output.rs @@ -15,13 +15,19 @@ use vsock::{VsockAddr, VsockListener, VsockStream, VMADDR_CID_ANY, VMADDR_CID_HY type Result = std::result::Result; +/// Output proxy. May output application process logs or (in debug mode) kernel+initramfs logs as +// well. pub struct OutputProxy { + // The file to write enclave output to. file: File, + // Indicator of debug mode. debug: bool, + // Buffer to receive data from the vsock. buf: [u8; 1500], } impl OutputProxy { + /// Open the file in which to forward enclave output to. pub fn new(path: &PathBuf, debug: bool) -> Result { let file = OpenOptions::new() .read(false) @@ -36,15 +42,22 @@ impl OutputProxy { } impl DeviceProxy for OutputProxy { + /// Enclave argument of the proxy. fn arg(&self) -> Option> { + // The enclave only needs to be made aware that it is to be run in debug mode. match self.debug { true => Some(EnclaveArg::Debug), false => None, } } + + /// The output proxy doesn't send any data to the enclave, so there is no need for cloning it + /// for a sender thread. fn clone(&self) -> Result>> { Ok(None) } + + /// Receive data from the proxy's vsock. Forward the data to the output file. fn rcv(&mut self, vsock: &mut VsockStream) -> Result { let size = vsock.read(&mut self.buf).map_err(Error::VsockRead)?; if size > 0 { @@ -55,10 +68,15 @@ impl DeviceProxy for OutputProxy { Ok(size) } + + /// The output proxy does not send data to the enclave. fn send(&mut self, _vsock: &mut VsockStream) -> Result { Ok(0) } + + /// Establish the proxy's vsock connection. fn vsock(&self, cid: u32) -> Result { + // If debug mode is enabled, connect to the enclave's console for kernel+initramfs logs. let port = { let offset = match self.debug { true => VsockPortOffset::Console, @@ -67,6 +85,8 @@ impl DeviceProxy for OutputProxy { cid + (offset as u32) }; + + // If debug mode is enabled, the enclave already binds to the console vsock. let vsock = if self.debug { VsockStream::connect(&VsockAddr::new(VMADDR_CID_HYPERVISOR, port)) .map_err(Error::VsockConnect)? diff --git a/src/nitro/src/enclave/proxy/proxies/signal_handler.rs b/src/nitro/src/enclave/proxy/proxies/signal_handler.rs index fac020edc..859adf571 100644 --- a/src/nitro/src/enclave/proxy/proxies/signal_handler.rs +++ b/src/nitro/src/enclave/proxy/proxies/signal_handler.rs @@ -14,13 +14,18 @@ use std::{ }; use vsock::{VsockAddr, VsockListener, VsockStream, VMADDR_CID_ANY}; +/// Signal handler proxy. Forwards signals from the host to the enclave. Currently, only SIGTERM is +/// supported. #[derive(Clone)] pub struct SignalHandler { + // Signal hook to determine when a SIGTERM is caught. sig: Arc, + // Buffer to forward the SIGTERM to the enclave. buf: [u8; 1], } impl SignalHandler { + // Create a new signal handler proxy with the SIGTERM hook set to false (not caught yet). pub fn new() -> Result { let sig = Arc::new(AtomicBool::new(false)); signal_hook::flag::register(SIGTERM, Arc::clone(&sig)).map_err(Error::SignalRegister)?; @@ -32,16 +37,24 @@ impl SignalHandler { } impl DeviceProxy for SignalHandler { + /// Enclave argument of the proxy. fn arg(&self) -> Option> { None } + + /// Clone a proxy's contents. The cloned signal handler is not used, only the vsock connection. fn clone(&self) -> Result>> { Ok(Some(Box::new(Clone::clone(self)))) } + + /// Receive data from the proxy's vsock. This should never read any actual data, but be a + /// placeholder to indicate that the enclave has closed the vsock connection. fn rcv(&mut self, vsock: &mut VsockStream) -> Result { vsock.read(&mut self.buf).map_err(Error::VsockRead) } + /// Check if a SIGTERM was caught. If so, write the signal to the enclave indicating it should + // gracefully shut down. fn send(&mut self, vsock: &mut VsockStream) -> Result { if !self.sig.load(Ordering::Relaxed) { return Ok(0); @@ -52,6 +65,8 @@ impl DeviceProxy for SignalHandler { Ok(0) } + + /// Establish the proxy's vsock connection. fn vsock(&self, cid: u32) -> Result { let port = cid + (VsockPortOffset::SignalHandler as u32); diff --git a/src/nitro/src/error.rs b/src/nitro/src/error.rs index fe995a884..93fcb6ccf 100644 --- a/src/nitro/src/error.rs +++ b/src/nitro/src/error.rs @@ -3,13 +3,20 @@ use super::enclave::{args_writer, proxy}; use std::{fmt, io}; +/// Error in the running of a nitro enclave. #[derive(Debug)] pub enum Error { + // Application running within the enclave returned a non-zero return code. AppReturn(i32), + // Argument writing process. ArgsWrite(args_writer::Error), + // Error in device proxy execution. DeviceProxy(proxy::Error), + // Error in listener for application return code. ReturnCodeListener(return_code::Error), + // Error in rootfs tar archiving. RootFsArchive(io::Error), + // Error in launching the enclave. Start(start::Error), } @@ -36,14 +43,22 @@ pub mod start { use super::*; use nitro_enclaves::launch::LaunchError; + /// Error in launching the enclave. #[derive(Debug)] pub enum Error { + // Opening the /dev/nitro_enclaves device. DeviceOpen(io::Error), + // Reading the cached EIF. EifRead(io::Error), + // Calculating the poll timeout. PollTimeoutCalculate(LaunchError), + // Adding a vCPU to an enclave VM. VcpuAdd(LaunchError), + // Creating the enclave VM. VmCreate(LaunchError), + // Setting the enclave VM's memory. VmMemorySet(LaunchError), + // Starting the enclave VM. VmStart(LaunchError), } @@ -71,11 +86,15 @@ pub mod start { pub mod return_code { use super::*; + /// Error in listener for application return code. #[derive(Debug)] #[allow(clippy::enum_variant_names)] pub enum Error { + // Accepting the vsock connection. VsockAccept(io::Error), + // Binding to the vsock. VsockBind(io::Error), + // Reading from the vsock. VsockRead(io::Error), } From 0fdfe149efd0dbe6f59fdc9123ebc9f39e5f0ba2 Mon Sep 17 00:00:00 2001 From: Tyler Fanelli Date: Mon, 26 Jan 2026 20:41:59 -0500 Subject: [PATCH 11/17] init/nitro: Close vsock device before error check Signed-off-by: Tyler Fanelli --- init/nitro/main.c | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/init/nitro/main.c b/init/nitro/main.c index 531c4929b..a23e4614d 100644 --- a/init/nitro/main.c +++ b/init/nitro/main.c @@ -285,14 +285,13 @@ unsigned int cid_fetch(void) } ret = ioctl(fd, IOCTL_VM_SOCKETS_GET_LOCAL_CID, &cid); + close(fd); + if (ret < 0) { - close(fd); perror("unable to fetch VM CID:"); return 0; } - close(fd); - return cid; } From e8a86fd5e39686afb65f853bc7aa9eb42c5d4309 Mon Sep 17 00:00:00 2001 From: Tyler Fanelli Date: Mon, 26 Jan 2026 20:51:33 -0500 Subject: [PATCH 12/17] nitro: Write close signal to app return code vsock Originally, the enclave's main process would sleep to give libkrun a chance to read the return code before the enclave shut down. Rather than sleep, write a 4-byte "close signal" to the enclave to notify it that libkrun has read the return code and the enclave can now exit. Signed-off-by: Tyler Fanelli --- init/nitro/main.c | 10 +++++++--- src/nitro/src/enclave/mod.rs | 7 ++++++- src/nitro/src/error.rs | 3 +++ 3 files changed, 16 insertions(+), 4 deletions(-) diff --git a/init/nitro/main.c b/init/nitro/main.c index a23e4614d..2443c725b 100644 --- a/init/nitro/main.c +++ b/init/nitro/main.c @@ -340,6 +340,13 @@ static int app_ret_write(int code, unsigned int cid) return -errno; } + ret = read(sock_fd, (void *)&code, sizeof(int)); + if (ret < sizeof(int)) { + perror("unable to read close signal from application return vsock"); + close(sock_fd); + return -errno; + } + close(sock_fd); return 0; @@ -551,9 +558,6 @@ int main(int argc, char *argv[]) goto out; ret = app_ret_write(ret_code, cid); - - // Allow the host to read the return code before exiting the enclave. - sleep(1); } out: diff --git a/src/nitro/src/enclave/mod.rs b/src/nitro/src/enclave/mod.rs index 9884075e4..ed8d2284b 100644 --- a/src/nitro/src/enclave/mod.rs +++ b/src/nitro/src/enclave/mod.rs @@ -18,7 +18,7 @@ use std::{ env, ffi::OsStr, fs, - io::{self, Read}, + io::{self, Read, Write}, os::fd::RawFd, path::PathBuf, }; @@ -232,6 +232,11 @@ impl NitroEnclave { .read(&mut buf) .map_err(ReturnCodeListenerError::VsockRead)?; + let close_signal: u32 = 0; + vsock_stream + .write_all(&close_signal.to_ne_bytes()) + .map_err(ReturnCodeListenerError::VsockWrite)?; + Ok(i32::from_ne_bytes(buf)) } diff --git a/src/nitro/src/error.rs b/src/nitro/src/error.rs index 93fcb6ccf..0218ec6f9 100644 --- a/src/nitro/src/error.rs +++ b/src/nitro/src/error.rs @@ -96,6 +96,8 @@ pub mod return_code { VsockBind(io::Error), // Reading from the vsock. VsockRead(io::Error), + // Writing to the vsock. + VsockWrite(io::Error), } impl fmt::Display for Error { @@ -104,6 +106,7 @@ pub mod return_code { Self::VsockAccept(e) => format!("unable to accept vsock connection: {e}"), Self::VsockBind(e) => format!("unable to bind to vsock: {e}"), Self::VsockRead(e) => format!("unable to read from vsock: {e}"), + Self::VsockWrite(e) => format!("unable to write to vsock: {e}"), }; write!(f, "{}", msg) From bbd6698b68cd144b9ca138be1f420f90c5fb15e9 Mon Sep 17 00:00:00 2001 From: Tyler Fanelli Date: Mon, 26 Jan 2026 22:18:35 -0500 Subject: [PATCH 13/17] init/nitro: Reorganize and document module Signed-off-by: Tyler Fanelli --- init/nitro/args_reader.c | 13 ++- init/nitro/device/app_stdio_output.c | 10 ++ init/nitro/device/device.c | 14 +++ init/nitro/device/include/device.h | 8 ++ init/nitro/device/net_tap_afvsock.c | 31 ++++- init/nitro/device/signal.c | 12 ++ init/nitro/fs.c | 42 +++++++ init/nitro/include/args_reader.h | 17 +-- init/nitro/include/fs.h | 1 + init/nitro/main.c | 163 ++++++++++++++++----------- 10 files changed, 233 insertions(+), 78 deletions(-) diff --git a/init/nitro/args_reader.c b/init/nitro/args_reader.c index f079c4b70..032c0ec70 100644 --- a/init/nitro/args_reader.c +++ b/init/nitro/args_reader.c @@ -16,6 +16,9 @@ #define ENCLAVE_VSOCK_LAUNCH_ARGS_READY 0xb7 +/* + * Enclave argument IDs. + */ enum { ENCLAVE_ARG_ID_ROOTFS, ENCLAVE_ARG_ID_EXEC_PATH, @@ -162,9 +165,9 @@ static int args_reader_char_list_build(int sock_fd, char ***buf_ptr) */ static int args_reader_signal(unsigned int vsock_port) { - uint8_t buf[1]; struct sockaddr_vm saddr; int ret, sock_fd; + uint8_t buf[1]; buf[0] = ENCLAVE_VSOCK_LAUNCH_ARGS_READY; errno = -EINVAL; @@ -218,6 +221,9 @@ static int args_reader_signal(unsigned int vsock_port) return -errno; } +/* + * Read each enclave argument from the host. + */ static int __args_reader_read(int sock_fd, struct enclave_args *args) { uint8_t id; @@ -261,11 +267,16 @@ static int __args_reader_read(int sock_fd, struct enclave_args *args) return 0; } + // Error occurred. Return error code. if (ret < 0) return ret; } } +/* + * Establish communication with the host's argument writer and read the enclave + * configuration (via the arguments) from it. + */ int args_reader_read(struct enclave_args *args, unsigned int vsock_port) { int ret, sock_fd; diff --git a/init/nitro/device/app_stdio_output.c b/init/nitro/device/app_stdio_output.c index f6696cbe3..30a9dffef 100644 --- a/init/nitro/device/app_stdio_output.c +++ b/init/nitro/device/app_stdio_output.c @@ -11,6 +11,10 @@ static int APP_STDIO_OUTPUT_VSOCK_FD = -1; +/* + * Redirect std{err, out} output to a vsock connected to the host. Allows the + * host to read application output. + */ int app_stdio_output(unsigned int vsock_port) { int streams[2] = {STDOUT_FILENO, STDERR_FILENO}; @@ -18,6 +22,7 @@ int app_stdio_output(unsigned int vsock_port) struct timeval timeval; int ret, sock_fd, i; + // Open a vsock and connect to the host. sock_fd = socket(AF_VSOCK, SOCK_STREAM, 0); if (sock_fd < 0) { perror("unable to create guest socket"); @@ -47,6 +52,7 @@ int app_stdio_output(unsigned int vsock_port) return -errno; } + // Refer the std{err, out} file descriptors to the connected vsock. for (i = 0; i < 2; i++) { ret = dup2(sock_fd, streams[i]); if (ret < 0) { @@ -57,11 +63,15 @@ int app_stdio_output(unsigned int vsock_port) } } + // Store the vsock's file descriptor for eventual closing. APP_STDIO_OUTPUT_VSOCK_FD = sock_fd; return 0; } +/* + * Dereference and close the application output vsock. + */ void app_stdio_close(void) { close(STDOUT_FILENO); diff --git a/init/nitro/device/device.c b/init/nitro/device/device.c index 89abda167..53e8551d7 100644 --- a/init/nitro/device/device.c +++ b/init/nitro/device/device.c @@ -8,12 +8,20 @@ #include "include/device.h" +/* + * Upon receiving SIGUSR1 from a device proxy process, set the proxy ready + * variable to indicate the proxy is finished initializing and the main process + * can continue. + */ void device_proxy_sig_handler(int sig) { if (sig == SIGUSR1) DEVICE_PROXY_READY = 1; } +/* + * Initialize a specific device proxy. + */ int device_init(enum krun_nitro_device dev, int vsock_port, int shutdown_fd) { int ret; @@ -22,6 +30,12 @@ int device_init(enum krun_nitro_device dev, int vsock_port, int shutdown_fd) DEVICE_PROXY_READY = 0; switch (dev) { + /* + * Some proxies will fork to produce separate processes. These processes + * will send a signal to the main process to indicate when they have + * finished initialization. When applicable, the main process must wait for + * this signal before continuing execution. + */ case KRUN_NE_DEV_SIGNAL_HANDLER: ret = sig_handler_init(vsock_port, shutdown_fd); while (!DEVICE_PROXY_READY) diff --git a/init/nitro/device/include/device.h b/init/nitro/device/include/device.h index 0d91d561a..4f81d2054 100644 --- a/init/nitro/device/include/device.h +++ b/init/nitro/device/include/device.h @@ -5,8 +5,16 @@ #include +/* + * Variable for device proxies to indicate to the main process that they have + * finished initialization. + */ static volatile sig_atomic_t DEVICE_PROXY_READY = 0; +/* + * Device proxy signal handler. Used by device proxy processes to notify the + * main process that they have finished initialization. + */ void device_proxy_sig_handler(int); enum krun_nitro_device { diff --git a/init/nitro/device/net_tap_afvsock.c b/init/nitro/device/net_tap_afvsock.c index 5444473ef..c0ce0a5e9 100644 --- a/init/nitro/device/net_tap_afvsock.c +++ b/init/nitro/device/net_tap_afvsock.c @@ -31,6 +31,10 @@ #define TUN_DEV_MAJOR 10 #define TUN_DEV_MINOR 200 +/* + * Forward ethernet packets to/from the host vsock providing network access and + * the guest TAP device routing application network traffic. + */ static int tap_vsock_forward(int tun_fd, int vsock_fd, int shutdown_fd, char *tap_name) { @@ -38,9 +42,13 @@ static int tap_vsock_forward(int tun_fd, int vsock_fd, int shutdown_fd, unsigned char *buf; bool event_found; struct ifreq ifr; - ssize_t nread; int ret, sock_fd; + ssize_t nread; + /* + * Fetch the TAP device's Maximum Transfer Unit (MTU) and allocate a buffer + * in that size to transfer ethernet frames to/from the host. + */ sock_fd = socket(AF_INET, SOCK_DGRAM, 0); if (sock_fd < 0) { perror("creating INET socket to get TAP MTU"); @@ -54,7 +62,7 @@ static int tap_vsock_forward(int tun_fd, int vsock_fd, int shutdown_fd, if (ret < 0) { close(sock_fd); perror("fetch MTU of TAP device"); - exit(ret); + exit(-errno); } close(sock_fd); @@ -74,10 +82,12 @@ static int tap_vsock_forward(int tun_fd, int vsock_fd, int shutdown_fd, pfds[2].fd = shutdown_fd; pfds[2].events = POLLIN; + // Signal to the parent process that initialization is complete. kill(getppid(), SIGUSR1); while (poll(pfds, 3, -1) > 0) { event_found = false; + // Event on vsock. Read the frame and write it to the TAP device. if (pfds[0].revents & POLLIN) { unsigned int sz; nread = read(vsock_fd, &sz, 4); @@ -92,6 +102,7 @@ static int tap_vsock_forward(int tun_fd, int vsock_fd, int shutdown_fd, event_found = true; } + // Event on the TAP device. Read the frame and write it to the vsock. if (pfds[1].revents & POLLIN) { nread = read(tun_fd, buf, ifr.ifr_mtu); if (nread > 0) { @@ -120,6 +131,9 @@ static int tap_vsock_forward(int tun_fd, int vsock_fd, int shutdown_fd, exit(0); } +/* + * Initialize the enclave TAP device to route all network traffic to the host. + */ static int tun_init(void) { struct stat statbuf; @@ -165,6 +179,9 @@ static int tun_init(void) return 0; } +/* + * Assign IP data to route enclave network traffic to the TAP device. + */ static int tap_assign_ipaddr(char *name) { struct sockaddr_in *addr; @@ -279,6 +296,9 @@ static int tap_assign_ipaddr(char *name) return 0; } +/* + * Allocate a TAP device for enclave network traffic. + */ static int tap_alloc(char *name) { struct ifreq ifr; @@ -303,6 +323,7 @@ static int tap_alloc(char *name) strcpy(name, ifr.ifr_name); + // Assign the IP data to the TAP device. ret = tap_assign_ipaddr(name); if (ret < 0) return ret; @@ -310,6 +331,9 @@ static int tap_alloc(char *name) return fd; } +/* + * Initialize a TAP device to route network traffic to/from. + */ int tap_afvsock_init(unsigned int vsock_port, int shutdown_fd) { int ret, tun_fd, vsock_fd; @@ -335,6 +359,7 @@ int tap_afvsock_init(unsigned int vsock_port, int shutdown_fd) perror("network proxy process"); exit(EXIT_FAILURE); case 0: + // Initialize the vsock used for network proxying. vsock_fd = socket(AF_VSOCK, SOCK_STREAM, 0); if (vsock_fd < 0) { perror("network vsock creation"); @@ -350,7 +375,6 @@ int tap_afvsock_init(unsigned int vsock_port, int shutdown_fd) return -errno; } - // Initialize the vsock used for network proxying. memset(&saddr, 0, sizeof(struct sockaddr_vm)); saddr.svm_family = AF_VSOCK; saddr.svm_cid = VMADDR_CID_HOST; @@ -363,6 +387,7 @@ int tap_afvsock_init(unsigned int vsock_port, int shutdown_fd) exit(EXIT_FAILURE); } + // Forward network traffic between the host and TAP device. ret = tap_vsock_forward(tun_fd, vsock_fd, shutdown_fd, tap_name); if (ret < 0) exit(EXIT_FAILURE); diff --git a/init/nitro/device/signal.c b/init/nitro/device/signal.c index 751f952b8..734176125 100644 --- a/init/nitro/device/signal.c +++ b/init/nitro/device/signal.c @@ -15,6 +15,9 @@ #include "include/device.h" +/* + * Forward signals from the host to the parent process. + */ static int sig_handler_start(int vsock_fd, int shutdown_fd) { struct pollfd pfds[2]; @@ -27,9 +30,11 @@ static int sig_handler_start(int vsock_fd, int shutdown_fd) pfds[1].fd = shutdown_fd; pfds[1].events = POLLIN; + // Signal to the parent process that initialization is complete. kill(getppid(), SIGUSR1); while (poll(pfds, 2, -1) > 0) { + // Event on vsock. Read the signal and forward it to the parent process. if (pfds[0].revents & POLLIN) { len = read(vsock_fd, (void *)&sig, sizeof(int)); if (len != sizeof(int)) { @@ -39,6 +44,7 @@ static int sig_handler_start(int vsock_fd, int shutdown_fd) kill(getppid(), sig); } + // Event on shutdown FD. Close the vsock and exit. if (pfds[1].revents & POLLIN) break; } @@ -48,6 +54,10 @@ static int sig_handler_start(int vsock_fd, int shutdown_fd) exit(0); } +/* + * Initialize a signal handling proxy to forward signals from the host to the + * parent process. + */ int sig_handler_init(unsigned int vsock_port, int shutdown_fd) { struct sockaddr_vm saddr; @@ -61,6 +71,7 @@ int sig_handler_init(unsigned int vsock_port, int shutdown_fd) perror("signal handler proxy process"); return -errno; case 0: + // Initialize the vsock used for signal forwarding. vsock_fd = socket(AF_VSOCK, SOCK_STREAM, 0); if (vsock_fd < 0) { perror("signal handler vsock creation"); @@ -90,6 +101,7 @@ int sig_handler_init(unsigned int vsock_port, int shutdown_fd) return -errno; } + // Forward signals from the host to the parent process. ret = sig_handler_start(vsock_fd, shutdown_fd); if (ret < 0) { close(vsock_fd); diff --git a/init/nitro/fs.c b/init/nitro/fs.c index 65284a327..9b9fba93a 100644 --- a/init/nitro/fs.c +++ b/init/nitro/fs.c @@ -14,6 +14,48 @@ #define SYS_FS_CGROUP_PATH "/sys/fs/cgroup/" #define CGROUP_SUB_PATH_SIZE (sizeof(SYS_FS_CGROUP_PATH) - 1 + 64) +/* + * Initialize /dev/console and redirect std{err, in, out} to it for early debug + * output. + */ +int console_init() +{ + const char *path = "/dev/console"; + FILE *file; + int ret; + + ret = mount("dev", "/dev", "devtmpfs", MS_NOSUID | MS_NOEXEC, NULL); + if (ret < 0 && errno != EBUSY) { + perror("mount /dev"); + return -errno; + } + + // Redirect stdin, stdout, and stderr to /dev/console. + file = freopen(path, "r", stdin); + if (file == NULL) { + perror("freopen stdin"); + return -errno; + } + + file = freopen(path, "w", stdout); + if (file == NULL) { + perror("freopen stdout"); + goto err; + } + + file = freopen(path, "w", stderr); + if (file == NULL) { + perror("freopen stderr"); + goto err; + } + + return 0; + +err: + fclose(file); + return -errno; +} + /* * Initialize the cgroups. */ diff --git a/init/nitro/include/args_reader.h b/init/nitro/include/args_reader.h index e9b71c8c8..d73895d1d 100644 --- a/init/nitro/include/args_reader.h +++ b/init/nitro/include/args_reader.h @@ -6,14 +6,17 @@ #include #include +/* + * Enclave configuration arguments written from the host. + */ struct enclave_args { - void *rootfs_archive; - uint64_t rootfs_archive_size; - char *exec_path; - char **exec_argv; - char **exec_envp; - bool network_proxy; - bool debug; + void *rootfs_archive; // rootfs tar archive. + uint64_t rootfs_archive_size; // Size of rootfs tar archive. + char *exec_path; // Path of execution binary. + char **exec_argv; // Execution argument vector. + char **exec_envp; // Execution environment pointer. + bool network_proxy; // Indicate if networking is configured. + bool debug; // Indicate if running in debug mode. }; int args_reader_read(struct enclave_args *, unsigned int); diff --git a/init/nitro/include/fs.h b/init/nitro/include/fs.h index b07509985..359cea8ba 100644 --- a/init/nitro/include/fs.h +++ b/init/nitro/include/fs.h @@ -3,6 +3,7 @@ #ifndef _FS_INIT_H #define _FS_INIT_H +int console_init(); int filesystem_init(); int cgroups_init(); diff --git a/init/nitro/main.c b/init/nitro/main.c index 2443c725b..2b5c53403 100644 --- a/init/nitro/main.c +++ b/init/nitro/main.c @@ -42,55 +42,14 @@ enum { }; /* - * Initialize /dev/console and redirect std{err, in, out} to it for early debug - * output. + * Load the NSM kernel module. */ -int console_init() -{ - const char *path = "/dev/console"; - FILE *file; - int ret; - - ret = mount("dev", "/dev", "devtmpfs", MS_NOSUID | MS_NOEXEC, NULL); - if (ret < 0 && errno != EBUSY) { - perror("mount /dev"); - return -errno; - } - - // Redirect stdin, stdout, and stderr to /dev/console. - file = freopen(path, "r", stdin); - if (file == NULL) { - perror("freopen stdin"); - return -errno; - } - - file = freopen(path, "w", stdout); - if (file == NULL) { - perror("freopen stdout"); - goto err; - } - - file = freopen(path, "w", stderr); - if (file == NULL) { - perror("freopen stderr"); - goto err; - } - - return 0; - -err: - fclose(file); - return -errno; -} - -/* - * Initialize/load the NSM kernel module. - */ -int nsm_init() +static int nsm_load(void) { const char *file_name = "nsm.ko"; int fd, ret; + // Open and load the kernel module. fd = open(file_name, O_RDONLY | O_CLOEXEC); if (fd < 0 && errno == ENOENT) return 0; @@ -99,7 +58,6 @@ int nsm_init() return -errno; } - // Load the NSM module. ret = finit_module(fd, "", 0); if (ret < 0) { close(fd); @@ -127,7 +85,7 @@ int nsm_init() /* * Mount the extracted rootfs and switch the root directory to it. */ -static int rootfs_mount() +static int rootfs_mount(void) { int ret; @@ -172,7 +130,7 @@ static int rootfs_mount() /* * Launch the application specified with argv and envp. */ -pid_t launch(char **argv, char **envp) +static pid_t launch(char **argv, char **envp) { int ret; @@ -200,11 +158,9 @@ pid_t launch(char **argv, char **envp) } /* - * Measure the enclave rootfs and execution variables (path, argv, envp) with - * the NSM PCRs. + * Measure the enclave execution environment (path, argv, envp) in NSM PCR 17. * - * NSM PCR 16 contains the measurement of the root filesystem. - * NSM PCR 17 contains the measurement of the execution variables (path, argv, + * NSM PCR 17 contains the measurement of the execution environment (path, argv, * envp). */ static int nsm_pcrs_exec_path_extend(int nsm_fd, char *path, char **argv, @@ -217,14 +173,14 @@ static int nsm_pcrs_exec_path_extend(int nsm_fd, char *path, char **argv, pcr_data_size = 256; - // Measure the execution path with NSM PCR 17. + // Measure the execution path. exec_ptr = path; ret = nsm_extend_pcr(nsm_fd, NSM_PCR_EXEC_DATA, (uint8_t *)exec_ptr, strlen(exec_ptr), (void *)pcr_data, &pcr_data_size); if (ret != ERROR_CODE_SUCCESS) goto out; - // Measure each execution argument with NSM PCR 17. + // Measure each execution argument. for (i = 0; (exec_ptr = argv[i]) != NULL; ++i) { ret = nsm_extend_pcr(nsm_fd, NSM_PCR_EXEC_DATA, (uint8_t *)exec_ptr, @@ -233,7 +189,7 @@ static int nsm_pcrs_exec_path_extend(int nsm_fd, char *path, char **argv, goto out; } - // Measure each environment variable with NSM PCR 17. + // Measure each environment variable. for (i = 0; (exec_ptr = envp[i]) != NULL; ++i) { ret = nsm_extend_pcr(nsm_fd, NSM_PCR_EXEC_DATA, (uint8_t *)exec_ptr, @@ -249,7 +205,7 @@ static int nsm_pcrs_exec_path_extend(int nsm_fd, char *path, char **argv, } /* - * Lock PCRs measured by initramfs and close the NSM handle. + * Lock PCRs measured by init process and close the NSM handle. */ static int nsm_exit(int nsm_fd) { @@ -258,7 +214,7 @@ static int nsm_exit(int nsm_fd) /* * Lock PCRs 16 and 17 so they cannot be extended further. This is to ensure * there can no further data measured other than the rootfs and execution - * variables. + * environment. */ ret = nsm_lock_pcrs(nsm_fd, NSM_PCR_EXEC_DATA); if (ret != ERROR_CODE_SUCCESS) @@ -268,12 +224,15 @@ static int nsm_exit(int nsm_fd) nsm_lib_exit(nsm_fd); ret = 0; - out: return -ret; } -unsigned int cid_fetch(void) +/* + * Fetch the enclave VM's CID in order to calculate vsock port offsets for host + * communication. + */ +static unsigned int cid_fetch(void) { unsigned int cid; int ret, fd; @@ -295,12 +254,15 @@ unsigned int cid_fetch(void) return cid; } +/* + * Forward the application return code to the host. + */ static int app_ret_write(int code, unsigned int cid) { - int ret, sock_fd; unsigned int vsock_port; struct sockaddr_vm addr; struct timeval timeval; + int ret, sock_fd; sock_fd = socket(AF_VSOCK, SOCK_STREAM, 0); if (sock_fd < 0) { @@ -318,6 +280,10 @@ static int app_ret_write(int code, unsigned int cid) memset(&timeval, 0, sizeof(struct timeval)); timeval.tv_sec = 5; + /* + * The host needs to join all device proxy threads before reading the return + * code. Allow some time for the host to connect to the return code vsock. + */ ret = setsockopt(sock_fd, AF_VSOCK, SO_VM_SOCKETS_CONNECT_TIMEOUT, (void *)&timeval, sizeof(struct timeval)); if (ret < 0) { @@ -333,6 +299,7 @@ static int app_ret_write(int code, unsigned int cid) return -errno; } + // Write the return code. ret = write(sock_fd, (void *)&code, sizeof(int)); if (ret < sizeof(int)) { perror("unable to write application return code"); @@ -340,6 +307,11 @@ static int app_ret_write(int code, unsigned int cid) return -errno; } + /* + * Read a return code (value is irrelevant) from the host. This is to ensure + * that the host was able to read the return code from the vsock before the + * enclave exits. + */ ret = read(sock_fd, (void *)&code, sizeof(int)); if (ret < sizeof(int)) { perror("unable to read close signal from application return vsock"); @@ -352,7 +324,10 @@ static int app_ret_write(int code, unsigned int cid) return 0; } -static int devices_init(int cid, struct enclave_args *args, int shutdown_fd) +/* + * Initialize each configured device proxy for the enclave. + */ +static int proxies_init(int cid, struct enclave_args *args, int shutdown_fd) { struct sigaction sa; int ret; @@ -363,17 +338,28 @@ static int devices_init(int cid, struct enclave_args *args, int shutdown_fd) sigaddset(&sa.sa_mask, SIGUSR1); sigprocmask(SIG_UNBLOCK, &sa.sa_mask, NULL); + /* + * Each proxy will send a SIGUSR1 message to indicate when it has started. + * Enable this signal so the main process can wait and be notified when each + * proxy has initialized itself. + */ ret = sigaction(SIGUSR1, &sa, NULL); if (ret < 0) { perror("sigaction enable SIGUSR1 for device proxies"); return -errno; } + /* + * If not running in debug mode, initialize the application output proxy. + * In debug mode, the enclave uses the console (which is already connected) + * for output. + */ if (!args->debug) { ret = device_init(KRUN_NE_DEV_APP_OUTPUT_STDIO, cid + VSOCK_PORT_OFFSET_OUTPUT, shutdown_fd); } + // Initialize the network proxy if configured. if (args->network_proxy) { ret = device_init(KRUN_NE_DEV_NET_TAP_AF_VSOCK, cid + VSOCK_PORT_OFFSET_NET, shutdown_fd); @@ -381,19 +367,29 @@ static int devices_init(int cid, struct enclave_args *args, int shutdown_fd) return ret; } + /* + * The signal proxy is always initialized to allow the host to send signals + * to the enclave. + */ ret = device_init(KRUN_NE_DEV_SIGNAL_HANDLER, cid + VSOCK_PORT_OFFSET_SIGNAL_HANDLER, shutdown_fd); - if (ret < 0) - return ret; return ret; } -static int devices_exit(struct enclave_args *args, int shutdown_fd) +/* + * Close and exit each device proxy. + */ +static int proxies_exit(struct enclave_args *args, int shutdown_fd) { uint64_t sfd_val; int ret; + /* + * The shutdown value is irrelevant, it acts as a signal to all device proxy + * threads that the enclave is exiting. Upon receiving this signal, each + * device proxy will close their respective vsock and exit. + */ sfd_val = 1; ret = write(shutdown_fd, &sfd_val, sizeof(uint64_t)); if (ret < 0) { @@ -401,19 +397,28 @@ static int devices_exit(struct enclave_args *args, int shutdown_fd) ret = -errno; } + // If not in debug mode, close the application output vsock. if (!args->debug) app_stdio_close(); return ret; } +// The PID of the application process. static pid_t KRUN_NITRO_APP_PID = -1; +// Indicates if a SIGTERM signal was caught by the enclave signal handler. static bool KRUN_NITRO_SIGTERM_CAUGHT = false; +/* + * Forward a signal from the signal handler to the application process. + * Currently, only SIGTERM is supported. + */ void shutdown_sig_handler(int sig) { if ((sig == SIGTERM) && (KRUN_NITRO_APP_PID > 0)) { + // Send the signal to the application process. kill(KRUN_NITRO_APP_PID, sig); + // Indicate that the SIGTERM signal was caught. KRUN_NITRO_SIGTERM_CAUGHT = true; } } @@ -453,10 +458,11 @@ int main(int argc, char *argv[]) goto out; // Initialize the NSM kernel module. - ret = nsm_init(); + ret = nsm_load(); if (ret < 0) goto out; + // Read the enclave arguments from the host. ret = args_reader_read(&args, cid + VSOCK_PORT_OFFSET_ARGS_READER); if (ret < 0) goto out; @@ -469,7 +475,7 @@ int main(int argc, char *argv[]) goto out; } - // Measure the rootfs and execution variables in the NSM PCRs. + // Measure the rootfs and execution environment in the NSM PCRs. ret = nsm_pcrs_exec_path_extend(nsm_fd, args.exec_path, args.exec_argv, args.exec_envp); if (ret < 0) @@ -508,6 +514,10 @@ int main(int argc, char *argv[]) if (ret < 0) goto out; + /* + * Create a shutdown eventfd that can be written to in order to notify each + * device proxy to close and exit at some point. + */ shutdown_fd = eventfd(0, 0); if (shutdown_fd < 0) { perror("creating shutdown FD"); @@ -515,7 +525,8 @@ int main(int argc, char *argv[]) goto out; } - ret = devices_init(cid, &args, shutdown_fd); + // Initialize each configured device proxy. + ret = proxies_init(cid, &args, shutdown_fd); if (ret < 0) goto out; @@ -537,8 +548,16 @@ int main(int argc, char *argv[]) ret = launch(args.exec_argv, args.exec_envp); break; default: + /* + * Store the application process' PID in the event of a signal needing + * to be forwarded to it. + */ KRUN_NITRO_APP_PID = pid; + /* + * Initialize the shutdown handler for signals to be forwarded to the + * application process. + */ memset(&sa, 0, sizeof(struct sigaction)); sa.sa_handler = shutdown_sig_handler; @@ -548,15 +567,25 @@ int main(int argc, char *argv[]) return -errno; } + // Wait for the application process to exit. waitpid(pid, &ret_code, 0); + /* + * If the process was ended by a signal, the return code may represent a + * value that under normal circumstances would indicate an error. + * Therefore, if the application ended from a signal, zero-out the + * return code (indicating that the application process exited + * gracefully). + */ if (KRUN_NITRO_SIGTERM_CAUGHT) ret_code = 0; - ret = devices_exit(&args, shutdown_fd); + // Close and exit each device proxy. + ret = proxies_exit(&args, shutdown_fd); if (ret < 0) goto out; + // Write the return code to the host. ret = app_ret_write(ret_code, cid); } From dbf30939265fa11f247b95398f5d1d69c40ccfb9 Mon Sep 17 00:00:00 2001 From: Tyler Fanelli Date: Tue, 27 Jan 2026 19:38:21 -0500 Subject: [PATCH 14/17] nitro: Create constants for EIF path strings Signed-off-by: Tyler Fanelli --- src/nitro/src/enclave/mod.rs | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/nitro/src/enclave/mod.rs b/src/nitro/src/enclave/mod.rs index ed8d2284b..d2be74d31 100644 --- a/src/nitro/src/enclave/mod.rs +++ b/src/nitro/src/enclave/mod.rs @@ -25,6 +25,9 @@ use std::{ use tar::HeaderMode; use vsock::{VsockAddr, VsockListener, VMADDR_CID_ANY}; +const KRUN_NITRO_EIF_PATH_ENV_VAR: &str = "KRUN_NITRO_EIF_PATH"; +const KRUN_NITRO_EIF_PATH_DEFAULT: &str = "/usr/share/krun-nitro/krun-nitro.eif"; + /// Directories within the configured rootfs that will be ignored when writing to the enclave. The /// enclave is responsible for initializing these directories within the guest operating system. const ROOTFS_DIR_DENYLIST: [&str; 6] = [ @@ -118,8 +121,8 @@ impl NitroEnclave { fn start(&mut self) -> Result<(u32, PollTimeout), StartError> { // Read the cached EIF file required to run the enclave. let eif = { - let path = env::var("KRUN_NITRO_EIF_PATH") - .unwrap_or("/usr/share/krun-nitro/krun-nitro.eif".to_string()); + let path = env::var(KRUN_NITRO_EIF_PATH_ENV_VAR) + .unwrap_or(KRUN_NITRO_EIF_PATH_DEFAULT.to_string()); fs::read(path).map_err(StartError::EifRead) }?; From 1e8ecf6d85d767af4c22c25b22e01f37c7f153c6 Mon Sep 17 00:00:00 2001 From: Tyler Fanelli Date: Tue, 27 Jan 2026 19:42:48 -0500 Subject: [PATCH 15/17] nitro: Remove aliasing of error types Signed-off-by: Tyler Fanelli --- src/nitro/src/enclave/mod.rs | 32 ++++++++++++++++---------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/src/nitro/src/enclave/mod.rs b/src/nitro/src/enclave/mod.rs index d2be74d31..9dce60ffa 100644 --- a/src/nitro/src/enclave/mod.rs +++ b/src/nitro/src/enclave/mod.rs @@ -3,9 +3,7 @@ pub(crate) mod args_writer; pub(crate) mod proxy; -use super::error::{ - return_code::Error as ReturnCodeListenerError, start::Error as StartError, Error, -}; +use super::error::{return_code, start, Error}; use args_writer::EnclaveArgsWriter; use nitro_enclaves::{ launch::{ImageType, Launcher, MemoryInfo, PollTimeout, StartFlags}, @@ -90,7 +88,7 @@ impl NitroEnclave { VMADDR_CID_ANY, cid + (VsockPortOffset::ReturnCode as u32), )) - .map_err(ReturnCodeListenerError::VsockBind) + .map_err(return_code::Error::VsockBind) .map_err(Error::ReturnCodeListener)?; // Enable signals now that enclave VM is started. @@ -118,30 +116,32 @@ impl NitroEnclave { } /// Start a nitro enclave. - fn start(&mut self) -> Result<(u32, PollTimeout), StartError> { + fn start(&mut self) -> Result<(u32, PollTimeout), start::Error> { // Read the cached EIF file required to run the enclave. let eif = { let path = env::var(KRUN_NITRO_EIF_PATH_ENV_VAR) .unwrap_or(KRUN_NITRO_EIF_PATH_DEFAULT.to_string()); - fs::read(path).map_err(StartError::EifRead) + fs::read(path).map_err(start::Error::EifRead) }?; // Calculate the poll timeout (based on the size of the EIF file and amount of RAM allocated // to the enclave) for the enclave to indicate that has successfully started. let timeout = PollTimeout::try_from((eif.as_slice(), self.mem_size_mib << 20)) - .map_err(StartError::PollTimeoutCalculate)?; + .map_err(start::Error::PollTimeoutCalculate)?; // Launch an enclave VM with the configured number of vCPUs and amount of RAM. - let device = Device::open().map_err(StartError::DeviceOpen)?; + let device = Device::open().map_err(start::Error::DeviceOpen)?; - let mut launcher = Launcher::new(&device).map_err(StartError::VmCreate)?; + let mut launcher = Launcher::new(&device).map_err(start::Error::VmCreate)?; let mem = MemoryInfo::new(ImageType::Eif(&eif), self.mem_size_mib); - launcher.set_memory(mem).map_err(StartError::VmMemorySet)?; + launcher + .set_memory(mem) + .map_err(start::Error::VmMemorySet)?; for _ in 0..self.vcpus { - launcher.add_vcpu(None).map_err(StartError::VcpuAdd)?; + launcher.add_vcpu(None).map_err(start::Error::VcpuAdd)?; } // Indicate to the enclave to start in debug mode if configured. @@ -154,7 +154,7 @@ impl NitroEnclave { // Start the enclave. let cid = launcher .start(start_flags, None) - .map_err(StartError::VmStart)?; + .map_err(start::Error::VmStart)?; // Safe to unwrap. Ok((cid.try_into().unwrap(), timeout)) @@ -225,20 +225,20 @@ impl NitroEnclave { // Receive a 4-byte (representing an i32) return code from the enclave via vsock. This // represents the return code of the application that ran within the enclave. - fn shutdown_ret(&self, vsock_listener: VsockListener) -> Result { + fn shutdown_ret(&self, vsock_listener: VsockListener) -> Result { let (mut vsock_stream, _vsock_addr) = vsock_listener .accept() - .map_err(ReturnCodeListenerError::VsockAccept)?; + .map_err(return_code::Error::VsockAccept)?; let mut buf = [0u8; 4]; let _ = vsock_stream .read(&mut buf) - .map_err(ReturnCodeListenerError::VsockRead)?; + .map_err(return_code::Error::VsockRead)?; let close_signal: u32 = 0; vsock_stream .write_all(&close_signal.to_ne_bytes()) - .map_err(ReturnCodeListenerError::VsockWrite)?; + .map_err(return_code::Error::VsockWrite)?; Ok(i32::from_ne_bytes(buf)) } From 14cb8cd12b466273cb793cd97362757fbccaebbd Mon Sep 17 00:00:00 2001 From: Tyler Fanelli Date: Tue, 27 Jan 2026 23:37:48 -0500 Subject: [PATCH 16/17] nitro: Calculate vsock buffers on proxy basis The size of the vsock buffers should be determined on a per-proxy basis. For example, the network proxy buffer must be determined by the enclave TAP device's MTU. Allocate these buffers accordingly. Signed-off-by: Tyler Fanelli --- init/nitro/device/net_tap_afvsock.c | 7 ++++++ src/nitro/src/enclave/proxy/mod.rs | 8 +++++- src/nitro/src/enclave/proxy/proxies/net.rs | 25 +++++++++++++++---- src/nitro/src/enclave/proxy/proxies/output.rs | 14 +++++++---- .../enclave/proxy/proxies/signal_handler.rs | 2 +- 5 files changed, 44 insertions(+), 12 deletions(-) diff --git a/init/nitro/device/net_tap_afvsock.c b/init/nitro/device/net_tap_afvsock.c index c0ce0a5e9..dd8b65f5e 100644 --- a/init/nitro/device/net_tap_afvsock.c +++ b/init/nitro/device/net_tap_afvsock.c @@ -73,6 +73,13 @@ static int tap_vsock_forward(int tun_fd, int vsock_fd, int shutdown_fd, exit(-1); } + // Forward the MTU to the host for it to allocate a corresponding buffer. + ret = write(vsock_fd, (void *)&ifr.ifr_mtu, sizeof(int)); + if (ret < sizeof(int)) { + perror("write TAP device MTU to host"); + exit(-errno); + } + pfds[0].fd = vsock_fd; pfds[0].events = POLLIN; diff --git a/src/nitro/src/enclave/proxy/mod.rs b/src/nitro/src/enclave/proxy/mod.rs index 9cb3b68fd..eace1d2d7 100644 --- a/src/nitro/src/enclave/proxy/mod.rs +++ b/src/nitro/src/enclave/proxy/mod.rs @@ -7,6 +7,7 @@ pub use proxies::*; use crate::enclave::args_writer::EnclaveArg; use std::{ fmt, io, + num::TryFromIntError, sync::mpsc::{self, RecvTimeoutError}, thread::{self, JoinHandle}, time::Duration, @@ -26,7 +27,7 @@ pub trait DeviceProxy: Send { /// Write data to the enclave's vsock. Perhaps perform some other functions. fn send(&mut self, vsock: &mut VsockStream) -> Result; /// Establish the proxy's respective vsock connection. - fn vsock(&self, cid: u32) -> Result; + fn vsock(&mut self, cid: u32) -> Result; } /// List of all configured device proxies. @@ -164,6 +165,8 @@ pub enum Error { VsockAccept(io::Error), // Binding to the vsock. VsockBind(io::Error), + // Converting a byte buffer's length to a u64. + VsockBufferLenConvert(TryFromIntError), // Cloning the vsock. VsockClone(io::Error), // Connecting to the vsock. @@ -193,6 +196,9 @@ impl fmt::Display for Error { Self::UnixWrite(e) => format!("unable to write to unix stream: {e}"), Self::VsockAccept(e) => format!("unable to accept connection from vsock: {e}"), Self::VsockBind(e) => format!("unable to bind to vsock: {e}"), + Self::VsockBufferLenConvert(e) => { + format!("unable to convert vsock buffer size to u32: {e}") + } Self::VsockConnect(e) => format!("unable to connect to vsock: {e}"), Self::VsockClone(e) => format!("unable to clone vsock: {e}"), Self::VsockRead(e) => format!("unable to read from vsock: {e}"), diff --git a/src/nitro/src/enclave/proxy/proxies/net.rs b/src/nitro/src/enclave/proxy/proxies/net.rs index 27740f417..ef98570ea 100644 --- a/src/nitro/src/enclave/proxy/proxies/net.rs +++ b/src/nitro/src/enclave/proxy/proxies/net.rs @@ -7,6 +7,7 @@ use crate::enclave::{ }; use std::{ io::{ErrorKind, Read, Write}, + mem::size_of, os::{ fd::{FromRawFd, OwnedFd, RawFd}, unix::net::UnixStream, @@ -21,14 +22,14 @@ pub struct NetProxy { // Unix socket connected to service providing network access. unix: UnixStream, // Buffer to send/receive data to/from vsock. - buf: [u8; 1500], + buf: Vec, } impl TryFrom for NetProxy { type Error = Error; fn try_from(fd: RawFd) -> Result { - let buf = [0u8; 1500]; + let buf = Vec::new(); let unix = unsafe { UnixStream::from(OwnedFd::from_raw_fd(fd)) }; unix.set_read_timeout(Some(Duration::from_millis(250))) @@ -49,7 +50,7 @@ impl DeviceProxy for NetProxy { let unix = self.unix.try_clone().map_err(Error::UnixClone)?; Ok(Some(Box::new(Self { - buf: self.buf, + buf: self.buf.clone(), unix, }))) } @@ -85,13 +86,27 @@ impl DeviceProxy for NetProxy { } /// Establish the proxy's vsock connection. - fn vsock(&self, cid: u32) -> Result { + fn vsock(&mut self, cid: u32) -> Result { let port = cid + (VsockPortOffset::Net as u32); let listener = VsockListener::bind(&VsockAddr::new(VMADDR_CID_ANY, port)).map_err(Error::VsockBind)?; - let (vsock, _) = listener.accept().map_err(Error::VsockAccept)?; + let (mut vsock, _) = listener.accept().map_err(Error::VsockAccept)?; + + /* + * Upon initial connection, read the MTU size from the enclave and allocate the buffer + * accordingly. + */ + let size = { + let mut size_buf = [0u8; size_of::()]; + let _ = vsock.read(&mut size_buf).map_err(Error::VsockRead)?; + + u32::from_ne_bytes(size_buf) + }; + + self.buf + .resize(size.try_into().map_err(Error::VsockBufferLenConvert)?, 0); Ok(vsock) } diff --git a/src/nitro/src/enclave/proxy/proxies/output.rs b/src/nitro/src/enclave/proxy/proxies/output.rs index 913ebbd3b..047016232 100644 --- a/src/nitro/src/enclave/proxy/proxies/output.rs +++ b/src/nitro/src/enclave/proxy/proxies/output.rs @@ -15,6 +15,8 @@ use vsock::{VsockAddr, VsockListener, VsockStream, VMADDR_CID_ANY, VMADDR_CID_HY type Result = std::result::Result; +const OUTPUT_BUFFER_SIZE: usize = 1500; + /// Output proxy. May output application process logs or (in debug mode) kernel+initramfs logs as // well. pub struct OutputProxy { @@ -23,7 +25,7 @@ pub struct OutputProxy { // Indicator of debug mode. debug: bool, // Buffer to receive data from the vsock. - buf: [u8; 1500], + buf: Vec, } impl OutputProxy { @@ -35,9 +37,11 @@ impl OutputProxy { .open(path) .map_err(Error::FileOpen)?; - let buf = [0u8; 1500]; - - Ok(Self { file, debug, buf }) + Ok(Self { + file, + debug, + buf: vec![0u8; OUTPUT_BUFFER_SIZE], + }) } } @@ -75,7 +79,7 @@ impl DeviceProxy for OutputProxy { } /// Establish the proxy's vsock connection. - fn vsock(&self, cid: u32) -> Result { + fn vsock(&mut self, cid: u32) -> Result { // If debug mode is enabled, connect to the enclave's console for kernel+initramfs logs. let port = { let offset = match self.debug { diff --git a/src/nitro/src/enclave/proxy/proxies/signal_handler.rs b/src/nitro/src/enclave/proxy/proxies/signal_handler.rs index 859adf571..e43649ea8 100644 --- a/src/nitro/src/enclave/proxy/proxies/signal_handler.rs +++ b/src/nitro/src/enclave/proxy/proxies/signal_handler.rs @@ -67,7 +67,7 @@ impl DeviceProxy for SignalHandler { } /// Establish the proxy's vsock connection. - fn vsock(&self, cid: u32) -> Result { + fn vsock(&mut self, cid: u32) -> Result { let port = cid + (VsockPortOffset::SignalHandler as u32); let listener = From 84fb54e852d9a93e431edbb2ce3f4756aa89b90f Mon Sep 17 00:00:00 2001 From: Tyler Fanelli Date: Wed, 28 Jan 2026 01:20:28 -0500 Subject: [PATCH 17/17] nitro: Return 0 bytes written on signal proxy err With this, the signal proxy can return zero bytes written and wait for a shutdown signal from the receiver thread. Signed-off-by: Tyler Fanelli --- .../src/enclave/proxy/proxies/signal_handler.rs | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/src/nitro/src/enclave/proxy/proxies/signal_handler.rs b/src/nitro/src/enclave/proxy/proxies/signal_handler.rs index e43649ea8..7b4ab7b02 100644 --- a/src/nitro/src/enclave/proxy/proxies/signal_handler.rs +++ b/src/nitro/src/enclave/proxy/proxies/signal_handler.rs @@ -6,7 +6,7 @@ use crate::enclave::{ }; use signal_hook::consts::SIGTERM; use std::{ - io::{Read, Write}, + io::{ErrorKind, Read, Write}, sync::{ atomic::{AtomicBool, Ordering}, Arc, @@ -61,9 +61,15 @@ impl DeviceProxy for SignalHandler { } let sig = libc::SIGTERM; - vsock.write(&sig.to_ne_bytes()).map_err(Error::VsockWrite)?; - - Ok(0) + match vsock.write(&sig.to_ne_bytes()) { + Ok(size) => Ok(size), + /* + * If connection was already closed by enclave, return zero bytes written in order to + * listen for receiver shutdown signal. + */ + Err(e) if e.kind() == ErrorKind::BrokenPipe => Ok(0), + Err(e) => Err(Error::VsockWrite(e)), + } } /// Establish the proxy's vsock connection.