Towards Trusted Contextualization of Confidential Virtual Machines

Master's Thesis

Florian Lubitz

2025-02-17

Contextualization in Cloud Computing

• Rapid scaling
• Many individual VMs
• Based on a base image
• Same image, different context

Confidential Computing in the Cloud

Security Shortcomings with Contextualization

• Contextualization with Base Image
• Contextualization changes Image
• Third Party involvement in Software Stack
• No Record of Contextualization

Enabling Post Boot Verification

Goals revisited

Components

Related Work

Academic

• Johnson et al. :
  Parma: Confidential containers via attested execution policies
• Pecholt and Wessel :
  Cocotpm: Trusted platform modules for virtual machines in confidential computing environments
• Wilke and Scopelliti:
  Snpguard: Remote attestation of sev-snp vms using open source tools

Commercial

• cloud-init & Ansible
• Integrity Measurement Architecture
• Tetragon, Kubearmor, AppArmor
• Confidential Containers

Conclusion

• Secure, Immutable, Integrity protected System
• Post Boot Attestation of Contextualized System
• Post Boot Attestation of Dynamic Workloads
• Policy Enforcer for Contextualization and Hardening
• Integrated into Kubernetes
• Minimal overhead
• Extendable for more features and other systems

1. Get Inodes of STIEVM dirs
2. Load eBPF program into kernel
3. Populate eBPF maps with Inodes
4. Attach eBPF program to LSM hooks
5. Pin eBPF program to filesystem

Logs all actions

SEC("lsm/file_open")
int BPF_PROG(file_guard, struct file* file) {
  current_pid = get_current_pid();
  for (int i = 0; i < MAX_DEPTH; i++) {
    inode = file.inode
    if (isInodeInMap(inode)) {
      if (current_pid != stievm_pid) {
        return -EACCES;
      } else{
        return 0
      }
    }

    if (! hasParent(file)){
      break;
    }   
  }
}

Sending Secrets to HTTP Server

POST / HTTP/1.1
Host: 172.27.16.46:3000
Accept: */*
Accept-Encoding: gzip, deflate
Connection: keep-alive
User-Agent: python-httpx/0.27.2
Authorization: florian;QjsyxWGSPtFpIGf8cUJbzLodt411WeqVauYFmZYebXvUEgdEXnDOrvWwHFKWam9847Er88oLp+pa59VjI5dafg0+rZUjiWCtFrhYXwtXyiZj1smdEYZNjTOl2pM1dQoE
Content-Type: application/json
Content-Length: 82

{"luks_devices": {"device_keys": [{"id": "my-device-1", "key": "SomeSecretKey"}]}}

Basic Configuration

pub fn basic_configuration(config: &PolicyConfig) -> Vec<Box<dyn configurator::Configurator>> {
  let mut configurators: Vec<Box<dyn configurator::Configurator>> = vec![
    Box::new(NetworkConfigurator::new()),
    Box::new(AttestationConfigurator::new()),
    Box::new(SshConfigurator::new()),
  ];
  for configurator in &mut configurators {
    configurator.configure(config);
  }
  stievm_log!("stievm", "contextualizer", "Finished basic configuration");
  configurators
  }

/report/attest

POST /report/attest HTTP/1.1
Host: 172.27.16.46:3000
Accept: */*
Accept-Encoding: gzip, deflate
Connection: keep-alive
User-Agent: python-httpx/0.27.2
Authorization: florian;a/HykOzqBVawZQCoDsqKU2828It96tChUxhc3GitvVbr882OWWvPkMml1d7AZ/vkeJNENbb6xzBv+k/n517yIU0UNjOPhXHRvMdovt7CX0KayKgmu1sEfQBIyywmBjEC
Content-Type: application/json
Content-Length: 254

{
  "nonce": "32c9254a-be71-4232-8053-1e4956e114be",
  "evidence": [
    {
      "type": "fs_hash",
      "path": "/home/florian/projects/ttccvm/cvm",
      "hash": "Crz7McEuZ8IZXL2ViXcEVGCHj3Ox1LV5EsmuPhlebn8="
    },
    {
      "type": "log"
    }
  ]
}

/report/attest

{
  "nonce": "32c9254a-be71-4232-8053-1e4956e114be",
  "evidence": [ 
    {
      "hash": "Crz7McEuZ8IZXL2ViXcEVGCHj3Ox1LV5EsmuPhlebn8=",
      "value": [
        {
          "path": "",
          "hash": "Crz7McEuZ8IZXL2ViXcEVGCHj3Ox1LV5EsmuPhlebn8="
        },
        {
          "path": "create-cvm-clean.sh",
          "hash": "dgphKtWQamZHeGlmyGNlvbSOroCaH2YP5Gadat6fcow="
        },
      ]
    },
    {
      "hash": "pfM6IlgDIG2sFdKq978hQHsWJ9e23Rfd8N1qVDqQ59s=",
      "value": "
      [INFO ] - [stievm][main][stievm] Starting stievm
      [INFO ] - [stievm][policy_parser][stievm::policy_parser::parser] Re
      [INFO ] - [stievm][stievm_guards][stievm::enforcer] stievm_guards A
      [INFO ] - [stievm][contextualizer][stievm::config_modules] Starting
      [INFO ] - [stievm][contextualizer][stievm::config_modules] Finished
      [INFO ] - [stievm][contextualizer][stievm::config_modules] Starting
      [INFO ] - [stievm][contextualizer][stievm::config_modules] Finished
      [INFO ] - [stievm][web_server][stievm::att_web_server::web_server] 
      "
    }
  ]
}

uRYmc+Whyuc5FySyFzCkRFpoZeJa
HrTpZR0dnLvwBd1woTvKZU6wvbVl
wISqErK1Qa6bvC2z848Bd23nAMmdAg==

Attestation Report (1184 bytes):

Report Data:                  
4f 53 cd a1 8c 2b aa 0c 03 54 bb 5f 9a 3e cb e5 
ed 12 ab 4d 8e 11 ba 87 3c 2f 11 16 12 02 b9 45 
88 42 f3 6b ca c0 fb 09 14 b1 b8 3d 41 fe 6b a9 
d4 ab cd f9 8c 49 16 72 b4 38 f7 5f c1 55 d6 f6

Workload Guard

pub fn install_workload_guards(
  docker_container_id: &str,
  workload_id: &str,
  policy: &WorkloadPolicy,
) {
  let docker_pid = get_docker_pid(docker_container_id)?;
  let container_pid = get_container_pid(docker_pid)?;

  check_bpf_lsm_enabled()?;

    // Get the pid namespace of the container
  let container_pid_ns = container_ns_inode(&docker_pid);

  log_info(...);

  let inodes_to_block = get_inodes(policy.path_guards);

  bpf.container_pid = container_pid;

  bpf.load();
  bpf.inode_map.pin();

  for block_pair in inodes_to_block {
    bpf.inode_map.add(
      &block_pair.path_inode,
      &block_pair.command_inode,
        )
  }

  bpf.attach();
  bpf.pin()

  log_info(...)
}

Workload Vault BPF

SEC("lsm/file_open")
int BPF_PROG(check_file_open, struct file* file, int ret) {

  if (ret != 0) {
    return 0;
  }

  if (is_in_container()) {

    // Get the inode of the current task
    struct task_struct* task;
    struct mm_struct* mm;
    struct file* exe_file;
    struct dentry* exe_dentry;
    struct inode* exe_inode;

    task = (struct task_struct*)bpf_get_current_task_btf();
    mm = task->mm;
    exe_file = mm->exe_file;
    exe_dentry = exe_file->f_path.dentry;
    exe_inode = exe_dentry->d_inode;
    u64 exe_inode_num = exe_inode->i_ino;

    // Check if the current process may access the specified inode
    struct dentry* dentry = file->f_path.dentry;
    // Traverse up the directory hierarchy to check for restrictions
    for (__u16 i = 0; i < MAX_DIR_DEPTH; i++) {  // Limit traversal depth to avoid verifier issues
      if (!dentry) {
        bpf_printk("No dentry found");
        break;
      }

      u64 inode = dentry->d_inode->i_ino;
      __u32 current_pid = bpf_get_current_pid_tgid() >> 32;
      u32 allowed_inode = is_inode_restricted(inode);
      // Check if this inode (file or directory) is restricted
      if (allowed_inode != 0) {
        bpf_printk("inode: %d tries to access restricted %d", exe_inode_num, inode);
        if (exe_inode_num != allowed_inode) {
          return -EACCES;  // Access denied for non-allowed PID
        } else {
          return 0;  // Access allowed for the allowed PID
        }
      }

      // Move up to the parent directory
      if (dentry->d_parent == dentry) {
        break;  // Reached the root directory
      }
      dentry = dentry->d_parent;
    }
  }

  return 0;  // Access allowed if no restricted inode found in path
}