diff --git a/lib/hyper/node/fire_vmm/boot_spec.ex b/lib/hyper/node/fire_vmm/boot_spec.ex index dc663854..1c85a1c7 100644 --- a/lib/hyper/node/fire_vmm/boot_spec.ex +++ b/lib/hyper/node/fire_vmm/boot_spec.ex @@ -1,11 +1,13 @@ defmodule Hyper.Node.FireVMM.BootSpec do @moduledoc """ - Resolves a `Hyper.Vm.source()` + instance `type` into a concrete, API-shaped - cold-boot spec the `:configuring` state issues: machine config (from the - instance type), a kernel boot source, and a root drive. + Resolves a `Hyper.Vm.source()` + instance `type` + `arch` into a concrete, + API-shaped cold-boot spec the `:configuring` state issues: machine config + (from the instance type), the guest CPU config, a kernel boot source, and a + root drive. """ - alias Hyper.Firecracker.Api.{BootSource, Drive, MachineConfiguration} + alias Hyper.Firecracker.Api.{BootSource, CpuConfig, Drive, MachineConfiguration} + alias Hyper.Node.FireVMM.GuestCpu alias Hyper.Vm.Instance # Default cmdline. No `console=`: serial printk is synchronous at 115200 baud @@ -18,20 +20,24 @@ defmodule Hyper.Node.FireVMM.BootSpec do @default_boot_args "reboot=k panic=1 pci=off init=/hyper-init" defmodule Cold do - @moduledoc "A resolved cold boot." + @moduledoc """ + A resolved cold boot. `cpu_config` is `nil` where the guest boots with + Firecracker's default CPU config. + """ @enforce_keys [:machine_config, :boot_source, :drives] - defstruct [:machine_config, :boot_source, drives: [], network_interfaces: []] + defstruct [:machine_config, :cpu_config, :boot_source, drives: [], network_interfaces: []] @type t :: %__MODULE__{ machine_config: MachineConfiguration.t(), + cpu_config: CpuConfig.t() | nil, boot_source: BootSource.t(), drives: [Drive.t()], network_interfaces: [Hyper.Firecracker.Api.NetworkInterface.t()] } end - @spec resolve(Hyper.Vm.source(), Instance.t()) :: Cold.t() - def resolve(source, type) when is_map(source) do + @spec resolve(Hyper.Vm.source(), Instance.t(), Instance.arch()) :: Cold.t() + def resolve(source, type, arch) when is_map(source) do # Networking is mandatory: every VM gets a NIC on the uniform inner-world # address plus the kernel `ip=`/`hyper.resolver=` cmdline. A node that # reached here has already passed the startup network preflight. @@ -45,6 +51,7 @@ defmodule Hyper.Node.FireVMM.BootSpec do %Cold{ machine_config: Instance.Spec.machine_config(Instance.spec(type)), + cpu_config: GuestCpu.config(arch), boot_source: %BootSource{ kernel_image_path: Map.fetch!(source, :kernel_image_path), boot_args: boot_args diff --git a/lib/hyper/node/fire_vmm/chroot_jail.ex b/lib/hyper/node/fire_vmm/chroot_jail.ex index cab7364c..3460a0ec 100644 --- a/lib/hyper/node/fire_vmm/chroot_jail.ex +++ b/lib/hyper/node/fire_vmm/chroot_jail.ex @@ -17,7 +17,7 @@ defmodule Hyper.Node.FireVMM.ChrootJail do alias Hyper.Node.FireVMM.Jailer alias Hyper.SuidHelper - # In-jail filenames + the rootfs drive id (matches BootSpec.resolve/2). + # In-jail filenames + the rootfs drive id (matches BootSpec.resolve/3). @kernel_name "vmlinux" @root_name "rootfs" @root_drive_id "rootfs" diff --git a/lib/hyper/node/fire_vmm/guest_cpu.ex b/lib/hyper/node/fire_vmm/guest_cpu.ex new file mode 100644 index 00000000..abc46f32 --- /dev/null +++ b/lib/hyper/node/fire_vmm/guest_cpu.ex @@ -0,0 +1,46 @@ +defmodule Hyper.Node.FireVMM.GuestCpu do + @moduledoc """ + The CPU features a guest boots with. + + Guests never see a hardware-virtualization extension. Firecracker passes VMX + and SVM through from the host unmasked, so on a host whose KVM offers nested + virt (the `kvm_intel`/`kvm_amd` default) every guest could run its own + hypervisor on top of the host's nested-paging code (CVE-2026-53359). KVM + refuses CR4.VMXE and EFER.SVME when the guest's CPUID lacks the bit, so the + mask holds whatever the host's `nested` parameter says. + """ + + alias Hyper.Firecracker.Api.{CpuConfig, CpuidLeafModifier, CpuidRegisterModifier} + + @vmx_leaf "0x1" + @vmx_ecx_bit 5 + @svm_leaf "0x80000001" + @svm_ecx_bit 2 + + @doc "The `/cpu-config` for an `arch` guest, or `nil` on aarch64 (CPUID modifiers are x86_64-only)." + @spec config(Sys.Arch.t()) :: CpuConfig.t() | nil + def config(:aarch64), do: nil + + def config(:x86_64) do + %CpuConfig{ + cpuid_modifiers: [ + clear_ecx_bit(@vmx_leaf, @vmx_ecx_bit), + clear_ecx_bit(@svm_leaf, @svm_ecx_bit) + ] + } + end + + # Firecracker's bitmap is `0b` + 32 chars, bit 31 first: `0` clears, `1` sets, + # `x` keeps the host's value. + @spec clear_ecx_bit(String.t(), 0..31) :: CpuidLeafModifier.t() + defp clear_ecx_bit(leaf, bit) do + bitmap = String.duplicate("x", 31 - bit) <> "0" <> String.duplicate("x", bit) + + %CpuidLeafModifier{ + leaf: leaf, + subleaf: "0x0", + flags: 0, + modifiers: [%CpuidRegisterModifier{register: "ecx", bitmap: "0b" <> bitmap}] + } + end +end diff --git a/lib/hyper/node/fire_vmm/net.ex b/lib/hyper/node/fire_vmm/net.ex index a323f0ab..f084efeb 100644 --- a/lib/hyper/node/fire_vmm/net.ex +++ b/lib/hyper/node/fire_vmm/net.ex @@ -23,7 +23,7 @@ defmodule Hyper.Node.FireVMM.Net do Kernel `hyper.resolver=` cmdline fragment carrying the DNS resolver IP the PID-1 guest agent writes to `/etc/resolv.conf` — kernel `ip=` autoconfig sets the guest's address/route but never DNS, so this closes that gap. Pure: - callers (`BootSpec.resolve/2`) read the configured resolver themselves. + callers (`BootSpec.resolve/3`) read the configured resolver themselves. """ @spec resolver_cmdline(String.t()) :: String.t() def resolver_cmdline(resolver), do: "hyper.resolver=#{resolver}" diff --git a/lib/hyper/node/fire_vmm/state.ex b/lib/hyper/node/fire_vmm/state.ex index 4e1b7bbe..c8b519f7 100644 --- a/lib/hyper/node/fire_vmm/state.ex +++ b/lib/hyper/node/fire_vmm/state.ex @@ -87,13 +87,19 @@ defmodule Hyper.Node.FireVMM.State do # device off the per-VM mutable layer, resolve the boot spec, set the readiness # deadline, and start probing the API. def init( - %Opts{vm_id: id, mutable: mutable, kernel: kernel, boot_args: boot_args, type: type} = - opts + %Opts{ + vm_id: id, + mutable: mutable, + kernel: kernel, + boot_args: boot_args, + type: type, + arch: arch + } = opts ) do case Hyper.Cluster.Routing.register_self({id, :state}) do :ok -> source = boot_source(id, kernel, Mutable.blk_path(mutable), boot_args) - spec = BootSpec.resolve(source, type) + spec = BootSpec.resolve(source, type, arch) deadline = System.monotonic_time(:millisecond) + Time.as_ms(@ready_timeout) data = %State{opts: opts, spec: spec, boot_deadline: deadline} @@ -224,7 +230,7 @@ defmodule Hyper.Node.FireVMM.State do use OpenTelemetryDecorator - alias Hyper.Firecracker.Api.{InstanceActionInfo, Operations, Vsock} + alias Hyper.Firecracker.Api.{CpuConfig, InstanceActionInfo, Operations, Vsock} alias Hyper.Node.FireVMM.{BootSpec, ChrootJail, Client, Jailer, Opts} alias Unit.Time @@ -271,18 +277,17 @@ defmodule Hyper.Node.FireVMM.State do {:keep_state_and_data, [{:reply, from, {:error, :not_running}}]} end - # Device PUT sequence: machine-config -> boot-source -> drives -> NICs -> vsock. - # InstanceStart is deferred until the vsock UDS grant succeeds. + # Device PUT sequence: machine-config -> cpu-config -> boot-source -> drives -> + # NICs -> vsock. InstanceStart is deferred until the vsock UDS grant succeeds. @spec configure_devices(Hyper.Vm.Id.t(), BootSpec.Cold.t()) :: :ok | {:error, term()} @decorate with_span("Hyper.Node.FireVMM.State.Configuring.configure_devices", include: [:id]) defp configure_devices(id, %BootSpec.Cold{} = cold) do via = Client.via(id) ops = - [ - fn opts -> Operations.put_machine_configuration(cold.machine_config, opts) end, - fn opts -> Operations.put_guest_boot_source(cold.boot_source, opts) end - ] ++ + [fn opts -> Operations.put_machine_configuration(cold.machine_config, opts) end] ++ + cpu_config_ops(cold.cpu_config) ++ + [fn opts -> Operations.put_guest_boot_source(cold.boot_source, opts) end] ++ Enum.map(cold.drives, fn drive -> fn opts -> Operations.put_guest_drive_by_id(drive.drive_id, drive, opts) end end) ++ @@ -306,6 +311,12 @@ defmodule Hyper.Node.FireVMM.State do end) end + @spec cpu_config_ops(CpuConfig.t() | nil) :: [(keyword() -> :ok | {:error, term()})] + defp cpu_config_ops(nil), do: [] + + defp cpu_config_ops(%CpuConfig{} = config), + do: [fn opts -> Operations.put_cpu_configuration(config, opts) end] + # Attempt to hand the vsock UDS to the node user. Firecracker may not have # created the socket yet immediately after PUT /vsock, so retry on # :socket_pending at @probe_interval, mirroring AwaitingApi's grant-api shape. diff --git a/lib/hyper/vm.ex b/lib/hyper/vm.ex index 219a9c3a..f49b9ac2 100644 --- a/lib/hyper/vm.ex +++ b/lib/hyper/vm.ex @@ -9,7 +9,7 @@ defmodule Hyper.Vm do What a VM boots from: explicit, already-jail-visible artifact paths for a cold boot (kernel + root drive). `boot_args` defaults to a standard serial-console cmdline when omitted. `vm_id` is required — networking is mandatory and - `BootSpec.resolve/2` derives the guest NIC (`ip=` cmdline + MAC) from it. + `BootSpec.resolve/3` derives the guest NIC (`ip=` cmdline + MAC) from it. """ @type source :: %{ required(:kernel_image_path) => Path.t(), diff --git a/test/e2e/guest_cpu_test.exs b/test/e2e/guest_cpu_test.exs new file mode 100644 index 00000000..948bc906 --- /dev/null +++ b/test/e2e/guest_cpu_test.exs @@ -0,0 +1,41 @@ +defmodule Hyper.E2e.GuestCpuTest do + @moduledoc """ + Live contract that a booted x86_64 guest sees no hardware-virtualization + extension: `vmx` and `svm` are absent from its `/proc/cpuinfo` flags + (`Hyper.Node.FireVMM.GuestCpu`). + + Only has teeth on a host whose KVM offers nested virt, which is the + `kvm_intel`/`kvm_amd` default; a `nested=0` host passes either way. CI's + `integration` runner keeps the default. + + Runs only under `--only integration` / `--include integration` on a host + provisioned per docs/cookbook/install.md. + """ + use ExUnit.Case, async: false + + import Hyper.E2e + + @moduletag :integration + @moduletag timeout: :timer.minutes(10) + + if Sys.Arch.current() != {:ok, :x86_64} do + @moduletag skip: "VMX and SVM are x86 CPUID features" + end + + @image System.get_env("HYPER_E2E_IMAGE", "public.ecr.aws/docker/library/alpine:3.19") + + test "a booted guest's CPU flags carry neither vmx nor svm" do + assert {:ok, img_id} = Hyper.Img.OciLoader.load(@image) + assert {:ok, vm} = Hyper.create_vm(%Hyper.Vm.Spec{img_id: img_id, type: :micro}) + on_exit(fn -> Hyper.Node.stop_image_vm(vm) end) + + assert {:ok, %{stdout: flags, exit_code: 0}} = + await_exec( + vm, + ["/bin/sh", "-c", "grep -m1 '^flags' /proc/cpuinfo"], + :timer.seconds(120) + ) + + refute flags =~ ~r/\b(vmx|svm)\b/, "guest exposes a virtualization extension:\n#{flags}" + end +end diff --git a/test/hyper/node/fire_vmm/boot_spec_test.exs b/test/hyper/node/fire_vmm/boot_spec_test.exs index b7aa8ea4..209bac3e 100644 --- a/test/hyper/node/fire_vmm/boot_spec_test.exs +++ b/test/hyper/node/fire_vmm/boot_spec_test.exs @@ -11,7 +11,7 @@ defmodule Hyper.Node.FireVMM.BootSpecTest do @source %{vm_id: "vabc", kernel_image_path: "/vmlinux", root_drive_path: "/rootfs"} test "default cmdline boots our agent as init with no serial console" do - cold = BootSpec.resolve(@source, :micro) + cold = BootSpec.resolve(@source, :micro, :x86_64) assert cold.boot_source.boot_args =~ "init=/hyper-init" # No console by default: serial printk blocks the boot vCPU. A debug boot # re-adds console=ttyS0 through the per-VM boot_args override. @@ -19,7 +19,7 @@ defmodule Hyper.Node.FireVMM.BootSpecTest do end test "every VM gets a NIC and the ip=/hyper.resolver= cmdline" do - cold = BootSpec.resolve(@source, :micro) + cold = BootSpec.resolve(@source, :micro, :x86_64) assert length(cold.network_interfaces) == 1 assert cold.boot_source.boot_args =~ "ip=" assert cold.boot_source.boot_args =~ "hyper.resolver=" diff --git a/test/hyper/node/fire_vmm/guest_cpu_properties_test.exs b/test/hyper/node/fire_vmm/guest_cpu_properties_test.exs new file mode 100644 index 00000000..a24336a9 --- /dev/null +++ b/test/hyper/node/fire_vmm/guest_cpu_properties_test.exs @@ -0,0 +1,62 @@ +defmodule Hyper.Node.FireVMM.GuestCpuPropertiesTest do + @moduledoc """ + Laws for `GuestCpu.config(:x86_64)`, checked by applying its modifiers to + arbitrary host CPUID values the way Firecracker documents the bitmap (`0b` + + 32 chars, bit 31 first; `0` clears, `1` sets, `x` keeps): + + * masked -- VMX (`CPUID.01H:ECX[5]`) and SVM (`CPUID.80000001H:ECX[2]`) + are clear in the guest for every host value; + * minimal -- every other bit keeps the host's value, and no modifier + targets a leaf, subleaf or register outside those two. + """ + use ExUnit.Case, async: true + use ExUnitProperties + + import Bitwise + + alias Hyper.Firecracker.Api.{CpuidLeafModifier, CpuidRegisterModifier} + alias Hyper.Node.FireVMM.GuestCpu + + @u32 0..0xFFFF_FFFF + + property "VMX and SVM are cleared and every other host bit survives" do + check all(leaf_1_ecx <- integer(@u32), ext_leaf_1_ecx <- integer(@u32)) do + host = %{{0x1, 0, :ecx} => leaf_1_ecx, {0x80000001, 0, :ecx} => ext_leaf_1_ecx} + + guest = apply_modifiers(host, GuestCpu.config(:x86_64).cpuid_modifiers) + + assert guest == %{ + {0x1, 0, :ecx} => leaf_1_ecx &&& bnot(1 <<< 5), + {0x80000001, 0, :ecx} => ext_leaf_1_ecx &&& bnot(1 <<< 2) + } + end + end + + # A modifier for a register the host table lacks raises, so a stray leaf, + # subleaf or register fails the property rather than slipping past it. + defp apply_modifiers(host, modifiers) do + for %CpuidLeafModifier{leaf: leaf, subleaf: subleaf, modifiers: regs} <- modifiers, + %CpuidRegisterModifier{register: register, bitmap: bitmap} <- regs, + reduce: host do + acc -> + key = {parse_index(leaf), parse_index(subleaf), String.to_existing_atom(register)} + Map.update!(acc, key, &apply_bitmap(bitmap, &1)) + end + end + + defp apply_bitmap("0b" <> bits, value) when byte_size(bits) == 32 do + bits + |> String.graphemes() + |> Enum.reverse() + |> Enum.with_index() + |> Enum.reduce(value, fn + {"x", _bit}, acc -> acc + {"0", bit}, acc -> acc &&& bnot(1 <<< bit) + {"1", bit}, acc -> acc ||| 1 <<< bit + end) + end + + defp parse_index("0x" <> hex), do: String.to_integer(hex, 16) + defp parse_index("0b" <> bin), do: String.to_integer(bin, 2) + defp parse_index(dec), do: String.to_integer(dec) +end