nitro: Reorganize and document modules

init/nitro/args_reader.c

@@ -16,6 +16,9 @@
 
 #define ENCLAVE_VSOCK_LAUNCH_ARGS_READY 0xb7
 
+/*
+ * Enclave argument IDs.
+ */
 enum {
     ENCLAVE_ARG_ID_ROOTFS,
     ENCLAVE_ARG_ID_EXEC_PATH,
@@ -30,22 +33,22 @@ enum {
 /*
  * Before reading data from the vsock, the vsock sends a 4-byte "header",
  * representing the size (in bytes) of the object that will be written over the
- * stream. Read this size and store it within a uint32_t variable.
+ * stream. Read this size and store it within a uint64_t variable.
  */
-static int args_reader_len_read(int sock_fd, uint32_t *size)
+static int args_reader_len_read(int sock_fd, uint64_t *size)
 {
-    uint8_t bytes[sizeof(uint32_t)];
+    uint8_t bytes[sizeof(uint64_t)];
     ssize_t ret;
 
     // Read the bytes (representing the size) from the vsock.
-    ret = read(sock_fd, bytes, sizeof(uint32_t));
-    if (ret < sizeof(uint32_t)) {
+    ret = read(sock_fd, bytes, sizeof(uint64_t));
+    if (ret < sizeof(uint64_t)) {
         perror("vsock byte buffer length read");
         return -errno;
     }
 
     // Store the size within the "size" argument.
-    memcpy(size, bytes, sizeof(uint32_t));
+    memcpy(size, bytes, sizeof(uint64_t));
 
     return 0;
 }
@@ -66,9 +69,9 @@ static void char_list_free(char **buf)
 /*
  * Read and store an object from the vsock stream.
  */
-static int args_reader_rcv(int sock_fd, void **buf_ptr, uint32_t *size)
+static int args_reader_rcv(int sock_fd, void **buf_ptr, uint64_t *size)
 {
-    uint32_t len, idx;
+    uint64_t len, idx;
     ssize_t read_len;
     uint8_t *buf;
     int ret;
@@ -122,7 +125,7 @@ static int args_reader_rcv(int sock_fd, void **buf_ptr, uint32_t *size)
  */
 static int args_reader_char_list_build(int sock_fd, char ***buf_ptr)
 {
-    uint32_t size;
+    uint64_t size;
     char **buf;
     int ret, i;
 
@@ -162,9 +165,9 @@ static int args_reader_char_list_build(int sock_fd, char ***buf_ptr)
  */
 static int args_reader_signal(unsigned int vsock_port)
 {
-    uint8_t buf[1];
     struct sockaddr_vm saddr;
     int ret, sock_fd;
+    uint8_t buf[1];
 
     buf[0] = ENCLAVE_VSOCK_LAUNCH_ARGS_READY;
     errno = -EINVAL;
@@ -218,6 +221,9 @@ static int args_reader_signal(unsigned int vsock_port)
     return -errno;
 }
 
+/*
+ * Read each enclave argument from the host.
+ */
 static int __args_reader_read(int sock_fd, struct enclave_args *args)
 {
     uint8_t id;
@@ -261,11 +267,16 @@ static int __args_reader_read(int sock_fd, struct enclave_args *args)
             return 0;
         }
 
+        // Error occurred. Return error code.
         if (ret < 0)
             return ret;
     }
 }
 
+/*
+ * Establish communication with the host's argument writer and read the enclave
+ * configuration (via the arguments) from it.
+ */
 int args_reader_read(struct enclave_args *args, unsigned int vsock_port)
 {
     int ret, sock_fd;

init/nitro/device/app_stdio_output.c

@@ -11,13 +11,18 @@
 
 static int APP_STDIO_OUTPUT_VSOCK_FD = -1;
 
+/*
+ * Redirect std{err, out} output to a vsock connected to the host. Allows the
+ * host to read application output.
+ */
 int app_stdio_output(unsigned int vsock_port)
 {
     int streams[2] = {STDOUT_FILENO, STDERR_FILENO};
     struct sockaddr_vm addr;
     struct timeval timeval;
     int ret, sock_fd, i;
 
+    // Open a vsock and connect to the host.
     sock_fd = socket(AF_VSOCK, SOCK_STREAM, 0);
     if (sock_fd < 0) {
         perror("unable to create guest socket");
@@ -47,6 +52,7 @@ int app_stdio_output(unsigned int vsock_port)
         return -errno;
     }
 
+    // Refer the std{err, out} file descriptors to the connected vsock.
     for (i = 0; i < 2; i++) {
         ret = dup2(sock_fd, streams[i]);
         if (ret < 0) {
@@ -57,11 +63,15 @@ int app_stdio_output(unsigned int vsock_port)
         }
     }
 
+    // Store the vsock's file descriptor for eventual closing.
     APP_STDIO_OUTPUT_VSOCK_FD = sock_fd;
 
     return 0;
 }
 
+/*
+ * Dereference and close the application output vsock.
+ */
 void app_stdio_close(void)
 {
     close(STDOUT_FILENO);

init/nitro/device/device.c

@@ -8,12 +8,20 @@
 
 #include "include/device.h"
 
+/*
+ * Upon receiving SIGUSR1 from a device proxy process, set the proxy ready
+ * variable to indicate the proxy is finished initializing and the main process
+ * can continue.
+ */
 void device_proxy_sig_handler(int sig)
 {
     if (sig == SIGUSR1)
         DEVICE_PROXY_READY = 1;
 }
 
+/*
+ * Initialize a specific device proxy.
+ */
 int device_init(enum krun_nitro_device dev, int vsock_port, int shutdown_fd)
 {
     int ret;
@@ -22,6 +30,12 @@ int device_init(enum krun_nitro_device dev, int vsock_port, int shutdown_fd)
     DEVICE_PROXY_READY = 0;
 
     switch (dev) {
+    /*
+     * Some proxies will fork to produce separate processes. These processes
+     * will send a signal to the main process to indicate when they have
+     * finished initialization. When applicable, the main process must wait for
+     * this signal before continuing execution.
+     */
     case KRUN_NE_DEV_SIGNAL_HANDLER:
         ret = sig_handler_init(vsock_port, shutdown_fd);
         while (!DEVICE_PROXY_READY)

init/nitro/device/include/device.h

@@ -5,8 +5,16 @@
 
 #include <signal.h>
 
+/*
+ * Variable for device proxies to indicate to the main process that they have
+ * finished initialization.
+ */
 static volatile sig_atomic_t DEVICE_PROXY_READY = 0;
 
+/*
+ * Device proxy signal handler. Used by device proxy processes to notify the
+ * main process that they have finished initialization.
+ */
 void device_proxy_sig_handler(int);
 
 enum krun_nitro_device {

init/nitro/device/net_tap_afvsock.c

@@ -31,16 +31,24 @@
 #define TUN_DEV_MAJOR 10
 #define TUN_DEV_MINOR 200
 
+/*
+ * Forward ethernet packets to/from the host vsock providing network access and
+ * the guest TAP device routing application network traffic.
+ */
 static int tap_vsock_forward(int tun_fd, int vsock_fd, int shutdown_fd,
                              char *tap_name)
 {
     struct pollfd pfds[3];
     unsigned char *buf;
     bool event_found;
     struct ifreq ifr;
-    ssize_t nread;
     int ret, sock_fd;
+    ssize_t nread;
 
+    /*
+     * Fetch the TAP device's Maximum Transfer Unit (MTU) and allocate a buffer
+     * in that size to transfer ethernet frames to/from the host.
+     */
     sock_fd = socket(AF_INET, SOCK_DGRAM, 0);
     if (sock_fd < 0) {
         perror("creating INET socket to get TAP MTU");
@@ -54,7 +62,7 @@ static int tap_vsock_forward(int tun_fd, int vsock_fd, int shutdown_fd,
     if (ret < 0) {
         close(sock_fd);
         perror("fetch MTU of TAP device");
-        exit(ret);
+        exit(-errno);
     }
 
     close(sock_fd);
@@ -65,6 +73,13 @@ static int tap_vsock_forward(int tun_fd, int vsock_fd, int shutdown_fd,
         exit(-1);
     }
 
+    // Forward the MTU to the host for it to allocate a corresponding buffer.
+    ret = write(vsock_fd, (void *)&ifr.ifr_mtu, sizeof(int));
+    if (ret < sizeof(int)) {
+        perror("write TAP device MTU to host");
+        exit(-errno);
+    }
+
     pfds[0].fd = vsock_fd;
     pfds[0].events = POLLIN;
 
@@ -74,10 +89,12 @@ static int tap_vsock_forward(int tun_fd, int vsock_fd, int shutdown_fd,
     pfds[2].fd = shutdown_fd;
     pfds[2].events = POLLIN;
 
+    // Signal to the parent process that initialization is complete.
     kill(getppid(), SIGUSR1);
 
     while (poll(pfds, 3, -1) > 0) {
         event_found = false;
+        // Event on vsock. Read the frame and write it to the TAP device.
         if (pfds[0].revents & POLLIN) {
             unsigned int sz;
             nread = read(vsock_fd, &sz, 4);
@@ -92,6 +109,7 @@ static int tap_vsock_forward(int tun_fd, int vsock_fd, int shutdown_fd,
             event_found = true;
         }
 
+        // Event on the TAP device. Read the frame and write it to the vsock.
         if (pfds[1].revents & POLLIN) {
             nread = read(tun_fd, buf, ifr.ifr_mtu);
             if (nread > 0) {
@@ -120,6 +138,9 @@ static int tap_vsock_forward(int tun_fd, int vsock_fd, int shutdown_fd,
     exit(0);
 }
 
+/*
+ * Initialize the enclave TAP device to route all network traffic to the host.
+ */
 static int tun_init(void)
 {
     struct stat statbuf;
@@ -165,6 +186,9 @@ static int tun_init(void)
     return 0;
 }
 
+/*
+ * Assign IP data to route enclave network traffic to the TAP device.
+ */
 static int tap_assign_ipaddr(char *name)
 {
     struct sockaddr_in *addr;
@@ -279,6 +303,9 @@ static int tap_assign_ipaddr(char *name)
     return 0;
 }
 
+/*
+ * Allocate a TAP device for enclave network traffic.
+ */
 static int tap_alloc(char *name)
 {
     struct ifreq ifr;
@@ -303,13 +330,17 @@ static int tap_alloc(char *name)
 
     strcpy(name, ifr.ifr_name);
 
+    // Assign the IP data to the TAP device.
     ret = tap_assign_ipaddr(name);
     if (ret < 0)
         return ret;
 
     return fd;
 }
 
+/*
+ * Initialize a TAP device to route network traffic to/from.
+ */
 int tap_afvsock_init(unsigned int vsock_port, int shutdown_fd)
 {
     int ret, tun_fd, vsock_fd;
@@ -335,6 +366,7 @@ int tap_afvsock_init(unsigned int vsock_port, int shutdown_fd)
         perror("network proxy process");
         exit(EXIT_FAILURE);
     case 0:
+        // Initialize the vsock used for network proxying.
         vsock_fd = socket(AF_VSOCK, SOCK_STREAM, 0);
         if (vsock_fd < 0) {
             perror("network vsock creation");
@@ -350,7 +382,6 @@ int tap_afvsock_init(unsigned int vsock_port, int shutdown_fd)
             return -errno;
         }
 
-        // Initialize the vsock used for network proxying.
         memset(&saddr, 0, sizeof(struct sockaddr_vm));
         saddr.svm_family = AF_VSOCK;
         saddr.svm_cid = VMADDR_CID_HOST;
@@ -363,6 +394,7 @@ int tap_afvsock_init(unsigned int vsock_port, int shutdown_fd)
             exit(EXIT_FAILURE);
         }
 
+        // Forward network traffic between the host and TAP device.
         ret = tap_vsock_forward(tun_fd, vsock_fd, shutdown_fd, tap_name);
         if (ret < 0)
             exit(EXIT_FAILURE);

init/nitro/device/signal.c

@@ -15,6 +15,9 @@
 
 #include "include/device.h"
 
+/*
+ * Forward signals from the host to the parent process.
+ */
 static int sig_handler_start(int vsock_fd, int shutdown_fd)
 {
     struct pollfd pfds[2];
@@ -27,9 +30,11 @@ static int sig_handler_start(int vsock_fd, int shutdown_fd)
     pfds[1].fd = shutdown_fd;
     pfds[1].events = POLLIN;
 
+    // Signal to the parent process that initialization is complete.
     kill(getppid(), SIGUSR1);
 
     while (poll(pfds, 2, -1) > 0) {
+        // Event on vsock. Read the signal and forward it to the parent process.
         if (pfds[0].revents & POLLIN) {
             len = read(vsock_fd, (void *)&sig, sizeof(int));
             if (len != sizeof(int)) {
@@ -39,6 +44,7 @@ static int sig_handler_start(int vsock_fd, int shutdown_fd)
             kill(getppid(), sig);
         }
 
+        // Event on shutdown FD. Close the vsock and exit.
         if (pfds[1].revents & POLLIN)
             break;
     }
@@ -48,6 +54,10 @@ static int sig_handler_start(int vsock_fd, int shutdown_fd)
     exit(0);
 }
 
+/*
+ * Initialize a signal handling proxy to forward signals from the host to the
+ * parent process.
+ */
 int sig_handler_init(unsigned int vsock_port, int shutdown_fd)
 {
     struct sockaddr_vm saddr;
@@ -61,6 +71,7 @@ int sig_handler_init(unsigned int vsock_port, int shutdown_fd)
         perror("signal handler proxy process");
         return -errno;
     case 0:
+        // Initialize the vsock used for signal forwarding.
         vsock_fd = socket(AF_VSOCK, SOCK_STREAM, 0);
         if (vsock_fd < 0) {
             perror("signal handler vsock creation");
@@ -90,6 +101,7 @@ int sig_handler_init(unsigned int vsock_port, int shutdown_fd)
             return -errno;
         }
 
+        // Forward signals from the host to the parent process.
         ret = sig_handler_start(vsock_fd, shutdown_fd);
         if (ret < 0) {
             close(vsock_fd);