feat(networking): ipv6-based routing

.github/scripts/provision-kvm-host.sh

@@ -18,9 +18,6 @@
 #     the node)
 #   - DEBIAN_FRONTEND=noninteractive (install.md is written for an interactive
 #     operator session; CI has no tty to prompt on)
-#   - [network] is always written here (install.md presents it as an optional
-#     section) since the runner's default route interface is detected fresh
-#     each run and integration coverage wants egress networking exercised
 set -euo pipefail
 
 export DEBIAN_FRONTEND=noninteractive
@@ -34,20 +31,12 @@ sudo udevadm trigger --name-match=kvm
 
 sudo apt-get update
 sudo apt-get install -y \
-  coreutils e2fsprogs iproute2 libc-bin lvm2 nftables skopeo thin-provisioning-tools util-linux \
+  coreutils e2fsprogs libc-bin lvm2 skopeo thin-provisioning-tools util-linux \
   "linux-modules-extra-$(uname -r)"
 
 # -a is load-bearing: without it modprobe reads the 2nd+ names as module
 # PARAMETERS of the first, and the load fails.
-#
-# nf_tables + nf_nat back host-init's `hyper` nftables table and its
-# masquerade/SNAT/DNAT rules. A fresh GitHub-hosted runner does not have them
-# loaded, and the netfilter family does not always autoload on first `nft` use,
-# so `nft add table ip hyper` fails (empty stderr, non-zero exit) and the node
-# refuses to start. Load the base explicitly, as we do for the dm modules; the
-# nat expression sub-modules (nft_chain_nat, nft_masq) autoload once the base is
-# present and a rule is issued.
-sudo modprobe -av dm_snapshot dm_thin_pool loop nf_tables nf_nat
+sudo modprobe -av dm_snapshot dm_thin_pool loop
 targets="$(sudo dmsetup targets)"
 echo "dmsetup targets: ${targets}"
 grep -q thin-pool <<<"${targets}" || { echo "ERROR: thin-pool dm target missing" >&2; exit 1; }
@@ -59,33 +48,6 @@ command -v thin_dump >/dev/null || { echo "ERROR: thin_dump missing (thin-provis
 sudo install -o root -g root -m 0755 "$(readlink -f "$(command -v thin_dump)")" /usr/local/sbin/thin_dump
 [ ! -L /usr/local/sbin/thin_dump ] || { echo "ERROR: /usr/local/sbin/thin_dump is still a symlink" >&2; exit 1; }
 
-# host-init (run by the node at start) asserts ip_forward=1 rather than
-# setting it, so provisioning must turn it on and persist it across reboots.
-sudo sysctl -w net.ipv4.ip_forward=1
-echo 'net.ipv4.ip_forward=1' \
-  | sudo tee /etc/sysctl.d/99-hyper-ip-forward.conf >/dev/null
-
-# GitHub-hosted runners ship Docker, which sets the iptables `filter` FORWARD
-# policy to DROP and only admits its own bridge traffic. Hyper's guest egress is
-# *forwarded* (per-VM netns veth -> uplink), so Docker's drop silently eats
-# every packet the guest sends — the guest configures its NIC fine but nothing
-# ever returns. Admit the clone pool (172.31.0.0/16, the helper's default) both
-# directions via DOCKER-USER, which Docker evaluates before its own drops; the
-# `hyper` nftables forward chain still enforces guest isolation. Fall back to an
-# accepting FORWARD policy on a host without Docker's chain.
-clone_pool="172.31.0.0/16"
-if sudo iptables -L DOCKER-USER >/dev/null 2>&1; then
-  sudo iptables -I DOCKER-USER -s "${clone_pool}" -j ACCEPT
-  sudo iptables -I DOCKER-USER -d "${clone_pool}" -j ACCEPT
-else
-  sudo iptables -P FORWARD ACCEPT
-fi
-
-# The default-route interface is the uplink guests NAT egress through.
-uplink="$(ip route show default | awk '{print $5; exit}')"
-[ -n "${uplink}" ] || { echo "ERROR: could not detect default-route uplink interface" >&2; exit 1; }
-echo "detected uplink: ${uplink}"
-
 sudo mkdir -p /etc/hyper
 sudo tee /etc/hyper/config.toml >/dev/null <<EOF
 work_dir = "/srv/hyper"
@@ -104,11 +66,6 @@ cgroup = "hyper"
 # load is still decaying, and nothing else runs here worth protecting.
 [budget]
 cpu_max_load = 4.0
-
-# Enables per-VM egress NAT. The nft table itself is owned by host-init
-# (created idempotently at node start), not by this script.
-[network]
-uplink = "${uplink}"
 EOF
 sudo chown root:root /etc/hyper/config.toml
 sudo chmod 0644 /etc/hyper/config.toml

CHANGELOG.md

@@ -14,6 +14,15 @@ adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
   value is now rejected with `INVALID_ARGUMENT`; `ListVms` is now paginated
   (`page_size` / `page_token` / `next_page_token`).
 
+### Removed
+- **Breaking:** VM networking (per-VM netns + veth + TAP + nftables NAT egress,
+  and the guest `/etc/resolv.conf` from `hyper.resolver=`) has been removed
+  pending a ground-up rewrite. VMs boot with no NIC, a node no longer requires
+  or reads a `[network]` config table (an existing one is ignored), and the
+  `Hyper.SuidHelper.Network`, `Hyper.Node.FireVMM.Net` and `Hyper.Cfg.Network`
+  modules and the setuid helper's `network` subcommands remain as interfaces
+  but return/raise unimplemented.
+
 ## [0.1.0]
 
 First public release: a distributed orchestrator for Firecracker microVMs on the

README.md

@@ -86,6 +86,9 @@ integrating Hyper.
   call it from any language you already use.
   ⚠️ The gRPC API is unauthenticated and off by default — bind it to a trusted network or front it with an authenticating proxy. See [the gRPC guide](docs/grpc.md).
 
+> **VM networking** is being rewritten from the ground up and is currently
+> unavailable: VMs boot with no NIC and have no network egress.
+
 ## Docs
 
 Full docs on getting started, as well as useful diagrams are available on

docs/cookbook/config.md

@@ -110,29 +110,10 @@ uid_gid_range = [900000, 999999]
 
 ## VM Networking Configuration
 
-**Required.** Every VM gets NAT'd egress out through a physical host interface,
-and a node refuses to start without a `[network]` table (see the
-[install guide](install.md#vm-networking-required) for the host prerequisites:
-`iproute2`/`nftables` and `net.ipv4.ip_forward=1`). `config.toml`-only — the
-setuid helper reads the same `uplink` and `clone_pool` to build each VM's netns,
-veth pair, and NAT rules, so it cannot live in `config.exs`.
-
-### `config.toml`
-
-```toml
-[network]
-# **required**. The physical uplink interface guest egress is NAT'd out through.
-# On a single-uplink host, the default-route NIC is the usual choice:
-#   ip route show default | awk '{print $5; exit}'
-uplink = "eth0"
-
-# optional -- defaults shown. The IPv4 CIDR the per-VM /30s are carved from.
-clone_pool = "172.31.0.0/16"
-
-# optional -- defaults shown. DNS resolver handed to guests via the kernel
-# cmdline (the guest agent writes it to /etc/resolv.conf).
-resolver = "1.1.1.1"
-```
+VM networking is being rewritten from the ground up and is currently
+unavailable: VMs boot with no NIC, and a node no longer requires or reads a
+`[network]` table. An existing `[network]` table in `config.toml` is accepted
+but ignored (see the [install guide](install.md#vm-networking)).
 
 ## gRPC Configuration
 

docs/cookbook/install.md

@@ -193,71 +193,12 @@ cgroup = "hyper"
 For more details on configuring and tuning Hyper, we suggest you see the
 [configuration guide](config.md).
 
-### VM Networking (required)
+### VM networking
 
-VM networking is **mandatory**: a node refuses to start without a `[network]`
-table in `/etc/hyper/config.toml`, and every VM gets NAT'd egress out through a
-physical interface on the host. There is no opt-out — a node that fails this
-preflight will not boot (`{:error, :network_not_configured}`, a missing uplink,
-or IPv4 forwarding off). The `hyper` nft table itself is created (and owned) by
-`host-init`, which runs automatically at node start — you never create nft rules
-by hand. You must prepare the host first:
-
-Install `iproute2` and `nftables`:
-
-```sh
-sudo apt install -y iproute2 nftables   # dnf install -y iproute nftables
-```
-
-Enable IPv4 forwarding. `host-init` only *asserts* this is on — it refuses to
-proceed rather than setting it for you:
-
-```sh
-sudo sysctl -w net.ipv4.ip_forward=1
-```
-
-> #### Persistent Config {: .warning}
->
-> Loading this via `sysctl -w` is ephemeral and resets on reboot. Persist it:
->
-> ```sh
-> echo 'net.ipv4.ip_forward=1' \
->     | sudo tee /etc/sysctl.d/99-hyper-ip-forward.conf
-> ```
-
-> #### Conflicting FORWARD firewall {: .warning}
->
-> Guest egress is *forwarded* (per-VM netns veth → uplink). If the host already
-> runs a firewall that defaults the `filter` FORWARD chain to DROP — Docker does
-> this, and so do some hardened base images — those drops silently eat every
-> guest packet: the guest configures its NIC correctly but nothing ever returns.
-> Admit the clone pool through the firewall's user hook (Docker evaluates
-> `DOCKER-USER` before its own drops); the `hyper` nft forward chain still
-> enforces guest isolation:
->
-> ```sh
-> sudo iptables -I DOCKER-USER -s 172.31.0.0/16 -j ACCEPT
-> sudo iptables -I DOCKER-USER -d 172.31.0.0/16 -j ACCEPT
-> ```
->
-> On a host without Docker, ensure nothing else sets the FORWARD policy to DROP.
-
-Then set `uplink` to the physical NIC guests should NAT out through — the
-default-route interface is a reasonable choice on a single-uplink host:
-
-```sh
-ip route show default | awk '{print $5; exit}'
-```
-
-```toml
-[network]
-# **required**. The physical uplink interface guests NAT egress through.
-uplink = "eth0"
-
-# optional, defaults shown
-clone_pool = "172.31.0.0/16"
-resolver = "1.1.1.1"
-```
+VM networking is being rewritten from the ground up and is currently
+unavailable: VMs boot with no NIC, so guests have no egress. No `[network]`
+table, `nftables`, `iproute2` or IPv4 forwarding is needed on the host. An
+existing `[network]` table in `/etc/hyper/config.toml` is accepted but ignored.
 
 ### Cgroups
 

lib/hyper/cfg/network.ex

@@ -1,53 +1,30 @@
 defmodule Hyper.Cfg.Network do
   @moduledoc """
-  VM egress networking settings from the `[network]` table — `config.toml`-only
-  because the setuid helper reads the same `uplink` and `clone_pool` to build
-  each VM's netns, veth, and NAT rules.
+  Unimplemented placeholder pending the VM-networking rewrite.
 
-  Networking is **mandatory**: a node refuses to start unless `[network]` is
-  configured (see `Hyper.Node.FireVMM.Jailer.Checks.network_ready/0`). Every VM
-  gets a NIC; there is no opt-out. `configured?/0` exists only so the startup
-  preflight can fail fast with a clear message rather than crash later.
+  A `[network]` table in `/etc/hyper/config.toml` is tolerated but ignored
+  (`Hyper.Cfg.Toml` is a loose dotted-path lookup), and a node no longer
+  requires it to start. Every function raises; nothing in Hyper calls them.
   """
 
-  import Hyper.Cfg, only: [get_cfg: 1]
-
-  @default_clone_pool "172.31.0.0/16"
-  @default_resolver "1.1.1.1"
-
-  @doc """
-  Whether the required `[network] uplink` is present. Used only by the startup
-  preflight to fail fast; once a node is running, networking is guaranteed
-  configured, so runtime code calls `uplink/0` directly.
-  """
-  @spec configured?() :: boolean()
-  def configured?, do: not is_nil(get_cfg(toml: "network.uplink", default: nil))
-
-  @doc "Physical uplink interface guest egress is NAT'd out of. Raises if unset (a running node has already passed the preflight)."
-  @spec uplink() :: String.t()
-  def uplink do
-    case get_cfg(toml: "network.uplink", default: nil) do
-      nil ->
-        raise Hyper.Cfg.MissingError, "network.uplink is required — VM networking is mandatory"
-
-      v when is_binary(v) ->
-        v
-
-      other ->
-        raise ArgumentError, "network.uplink must be a string, got: #{inspect(other)}"
-    end
-  end
-
-  @doc "IPv4 CIDR the per-VM clone /30s are carved from. `[network] clone_pool`."
-  @spec clone_pool() :: String.t()
-  def clone_pool, do: get_cfg(toml: "network.clone_pool", default: @default_clone_pool)
-
-  @doc """
-  DNS resolver IP handed to the guest via the `hyper.resolver=` kernel cmdline
-  param, since the kernel's `ip=` autoconfig sets the guest's address/route but
-  never DNS. The PID-1 guest agent reads this back out of `/proc/cmdline` and
-  writes it to `/etc/resolv.conf`. `[network] resolver`.
-  """
-  @spec resolver() :: String.t()
-  def resolver, do: get_cfg(toml: "network.resolver", default: @default_resolver)
+  @doc "Unimplemented: whether VM networking is configured."
+  @spec configured?() :: no_return()
+  def configured?,
+    do:
+      raise("Hyper.Cfg.Network.configured?/0 is unimplemented: VM networking is being rewritten")
+
+  @doc "Unimplemented: the uplink interface guest egress leaves through."
+  @spec uplink() :: no_return()
+  def uplink,
+    do: raise("Hyper.Cfg.Network.uplink/0 is unimplemented: VM networking is being rewritten")
+
+  @doc "Unimplemented: the address pool per-VM links are carved from."
+  @spec clone_pool() :: no_return()
+  def clone_pool,
+    do: raise("Hyper.Cfg.Network.clone_pool/0 is unimplemented: VM networking is being rewritten")
+
+  @doc "Unimplemented: the DNS resolver handed to guests."
+  @spec resolver() :: no_return()
+  def resolver,
+    do: raise("Hyper.Cfg.Network.resolver/0 is unimplemented: VM networking is being rewritten")
 end

lib/hyper/node.ex

@@ -14,7 +14,7 @@ defmodule Hyper.Node do
 
     * `Hyper.Node.VMSupervisor` - a **local** `DynamicSupervisor` that starts one
       `Hyper.Node.FireVMM` per VM. Local on purpose: a firecracker VM is pinned to
-      this machine's kernel/rootfs/cgroup/tap devices and cannot migrate, so we
+      this machine's kernel/rootfs/cgroup and cannot migrate, so we
       deliberately avoid `Horde.DynamicSupervisor` (which would try to restart
       VMs on a surviving node - cold-booting a ghost).
 
@@ -310,25 +310,11 @@ defmodule Hyper.Node do
          :ok <- Hyper.Node.Layer.Repo.test_system(),
          :ok <- Hyper.SuidHelper.test_system(),
          {:ok, base} <- Hyper.SuidHelper.sys_test(),
-         :ok <- check_helper_base(base),
-         :ok <- init_network() do
+         :ok <- check_helper_base(base) do
       Hyper.Node.FireVMM.test_system()
     end
   end
 
-  # VM networking is mandatory: refuse to start unless `[network]` is
-  # configured, then idempotently reconcile the host-wide `hyper` nftables
-  # table. Runs after `Hyper.SuidHelper.test_system/0` so the helper is
-  # confirmed present first.
-  @spec init_network :: :ok | {:error, term()}
-  defp init_network do
-    if Hyper.Cfg.Network.configured?() do
-      Hyper.SuidHelper.Network.host_init()
-    else
-      {:error, :network_not_configured}
-    end
-  end
-
   @spec check_firecracker_bins ::
           :ok
           | {:error, {:firecracker_bin_missing | :jailer_bin_missing, Path.t()}}

lib/hyper/node/fire_vmm/boot_spec.ex

@@ -20,34 +20,22 @@ defmodule Hyper.Node.FireVMM.BootSpec do
   defmodule Cold do
     @moduledoc "A resolved cold boot."
     @enforce_keys [:machine_config, :boot_source, :drives]
-    defstruct [:machine_config, :boot_source, drives: [], network_interfaces: []]
+    defstruct [:machine_config, :boot_source, drives: []]
 
     @type t :: %__MODULE__{
             machine_config: MachineConfiguration.t(),
             boot_source: BootSource.t(),
-            drives: [Drive.t()],
-            network_interfaces: [Hyper.Firecracker.Api.NetworkInterface.t()]
+            drives: [Drive.t()]
           }
   end
 
   @spec resolve(Hyper.Vm.source(), Instance.t()) :: Cold.t()
   def resolve(source, type) when is_map(source) do
-    # Networking is mandatory: every VM gets a NIC on the uniform inner-world
-    # address plus the kernel `ip=`/`hyper.resolver=` cmdline. A node that
-    # reached here has already passed the startup network preflight.
-    vm_id = Map.fetch!(source, :vm_id)
-
-    boot_args =
-      Map.get(source, :boot_args, @default_boot_args) <>
-        " " <>
-        Hyper.Node.FireVMM.Net.ip_cmdline() <>
-        " " <> Hyper.Node.FireVMM.Net.resolver_cmdline(Hyper.Cfg.Network.resolver())
-
     %Cold{
       machine_config: Instance.Spec.machine_config(Instance.spec(type)),
       boot_source: %BootSource{
         kernel_image_path: Map.fetch!(source, :kernel_image_path),
-        boot_args: boot_args
+        boot_args: Map.get(source, :boot_args, @default_boot_args)
       },
       drives: [
         %Drive{
@@ -56,8 +44,7 @@ defmodule Hyper.Node.FireVMM.BootSpec do
           is_read_only: false,
           path_on_host: Map.fetch!(source, :root_drive_path)
         }
-      ],
-      network_interfaces: [Hyper.Node.FireVMM.Net.interface(vm_id)]
+      ]
     }
   end
 end