diff --git a/.sops.yaml b/.sops.yaml index af9d39b..31130ff 100644 --- a/.sops.yaml +++ b/.sops.yaml @@ -1,7 +1,12 @@ keys: - &root age1y0tj3kt67pfnj38t9c8g2ghry3a0mhcq8rrqv5xr4jekwepxaelqzu3dkf - &user age16v8w7q4wmn22hhakq2uzaus2508rhldm7lcwh0kukshzjzyhuqesqz44ze + - &sandbox age1mr42xh9llvrggrvuawr5253qk7gygf4gcg9clqdvjecw3d6tj59qsqxswd creation_rules: + - path_regex: microvm/secrets/sandbox\.yaml$ + key_groups: + - age: + - *sandbox - path_regex: secrets/[^/]+\.yaml$ key_groups: - age: diff --git a/executables/opencode.sh b/executables/opencode.sh new file mode 100644 index 0000000..8735693 --- /dev/null +++ b/executables/opencode.sh @@ -0,0 +1,12 @@ +#!/usr/bin/env bash +set -euo pipefail + +SSH_TARGET="sandbox@vsock%3" + +if [[ "$PWD" == /home/user* ]]; then + project="${PWD/\/home\/user/\/home\/sandbox}" +else + project="$(ssh "$SSH_TARGET" 'mktemp -d -p /tmp opencode-XXXXXX')" +fi + +exec ssh -t "$SSH_TARGET" opencode "$project" "$@" diff --git a/home/user.nix b/home/user.nix index d5fdafc..afe9204 100644 --- a/home/user.nix +++ b/home/user.nix @@ -367,6 +367,11 @@ # (writeShellScriptBin "agent" '' # machinectl shell agent@ ${waypipe}/bin/waypipe --socket /run/waypipe.sock server fish # '') + (writeShellApplication { + name = "opencode-sandbox"; + runtimeInputs = [ openssh ]; + text = builtins.readFile ../executables/opencode.sh; + }) (writeShellApplication { name = "tmux-sessionizer"; runtimeInputs = [ diff --git a/microvm/sandbox.nix b/microvm/sandbox.nix index 32b31be..cd530ed 100644 --- a/microvm/sandbox.nix +++ b/microvm/sandbox.nix @@ -1,104 +1,234 @@ { - config, - pkgs, - lib, + impermanence, + sops-nix, + home-manager, ... }: +let + impermanenceModule = impermanence.nixosModules.impermanence; + sopsModule = sops-nix.nixosModules.sops; + homeManagerModule = home-manager.nixosModules.home-manager; +in { microvm.host.useNotifySockets = true; microvm.vms."sandbox" = { - config = { - users.users.root = { - password = ""; - openssh.authorizedKeys.keys = [ - "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAILABd/iSJ4gn/ystDqNxLJTG0n0z5VIC9YXlmdUfOhHf desktop@icefox.sh" + config = + { + config, + pkgs, + lib, + ... + }: + { + imports = [ + impermanenceModule + sopsModule + homeManagerModule ]; - }; - users.users.sandbox = { - isNormalUser = true; - uid = 1001; - group = "users"; - extraGroups = [ - "video" - "render" - ]; - password = ""; - openssh.authorizedKeys.keys = [ - "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAILABd/iSJ4gn/ystDqNxLJTG0n0z5VIC9YXlmdUfOhHf desktop@icefox.sh" - ]; - }; - services.openssh.enable = true; - boot.kernelModules = [ - "drm" - "virtio_gpu" - ]; - boot.blacklistedKernelModules = lib.mkForce [ ]; - - environment.sessionVariables = { - WAYLAND_DISPLAY = "wayland-1"; - DISPLAY = ":0"; - QT_QPA_PLATFORM = "wayland"; - GDK_BACKEND = "wayland"; - XDG_SESSION_TYPE = "wayland"; - }; - - hardware.graphics.enable = true; - system.stateVersion = lib.trivial.release; - - environment.systemPackages = with pkgs; [ - xdg-utils - ungoogled-chromium - ghostty - tmux - waypipe - pciutils - ]; - - microvm = { - vsock = { - cid = 3; - ssh.enable = true; + sops.defaultSopsFile = ./secrets/sandbox.yaml; + sops.age.keyFile = "/.persist/secrets/age-keys.txt"; + sops.secrets."root-password-hash" = { + neededForUsers = true; }; - writableStoreOverlay = "/nix/.rw-store"; - # graphics.enable = true; - mem = 4096; - interfaces = [ - { - type = "user"; - id = "vm-1"; - mac = "02:00:00:01:01:01"; - } + sops.secrets."sandbox-password-hash" = { + neededForUsers = true; + }; + sops.secrets."deepseek-api-key" = { + owner = "sandbox"; + mode = "0400"; + }; + + fileSystems."/.persist".neededForBoot = true; + + users.users.root = { + hashedPasswordFile = config.sops.secrets."root-password-hash".path; + openssh.authorizedKeys.keys = [ + "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAILABd/iSJ4gn/ystDqNxLJTG0n0z5VIC9YXlmdUfOhHf desktop@icefox.sh" + ]; + }; + users.users.host = { + isNormalUser = true; + uid = 1000; + group = "users"; + }; + users.users.sandbox = { + isNormalUser = true; + uid = 1001; + group = "users"; + extraGroups = [ + "video" + "render" + ]; + hashedPasswordFile = config.sops.secrets."sandbox-password-hash".path; + openssh.authorizedKeys.keys = [ + "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAILABd/iSJ4gn/ystDqNxLJTG0n0z5VIC9YXlmdUfOhHf desktop@icefox.sh" + ]; + }; + services.openssh.enable = true; + + boot.kernelModules = [ + "drm" + "virtio_gpu" ]; - volumes = [ - { - mountPoint = "/var"; - image = "var.img"; - size = 256; - } - { - mountPoint = "/nix/.rw-store"; - image = "nix-store-overlay.img"; - size = 512; - } + boot.blacklistedKernelModules = lib.mkForce [ ]; + + environment.sessionVariables = { + WAYLAND_DISPLAY = "wayland-1"; + DISPLAY = ":0"; + QT_QPA_PLATFORM = "wayland"; + GDK_BACKEND = "wayland"; + XDG_SESSION_TYPE = "wayland"; + }; + + hardware.graphics.enable = true; + system.stateVersion = lib.trivial.release; + + environment.systemPackages = with pkgs; [ + xdg-utils + tmux + waypipe + opencode + fd + ripgrep + podman-compose ]; - shares = [ - { - proto = "virtiofs"; - tag = "projects"; - source = "/home/user/dev"; - mountPoint = "/home/user/dev"; - } - { - proto = "virtiofs"; - tag = "ro-store"; - source = "/nix/store"; - mountPoint = "/nix/.ro-store"; - readOnly = true; - } - ]; - hypervisor = "qemu"; + + environment.persistence."/.persist" = { + enable = true; + hideMounts = true; + users.sandbox = { + directories = [ + ".local/share/opencode" + ".claude" + ]; + files = [ + ".claude.json" + ]; + }; + + }; + + home-manager = { + useGlobalPkgs = true; + users.sandbox = { + home.stateVersion = "26.11"; + xdg.configFile."containers/containers.conf".text = '' + [engine] + compose_warning_logs=false + events_logger="file" + + [containers] + log_driver="k8s-file" + ''; + xdg.configFile."opencode/opencode.json".text = builtins.toJSON { + "$schema" = "https://opencode.ai/config.json"; + provider = { + deepseek = { + npm = "@ai-sdk/openai-compatible"; + name = "DeepSeek"; + options = { + baseURL = "https://api.deepseek.com"; + apiKey = "{file:${config.sops.secrets."deepseek-api-key".path}}"; + }; + models = { + "deepseek-v4-pro" = { + name = "DeepSeek-V4-Pro"; + limit = { + context = 1048576; + output = 262144; + }; + options = { + reasoningEffort = "high"; + thinking.type = "enabled"; + }; + }; + "deepseek-v4-flash" = { + name = "DeepSeek-V4-Flash"; + limit = { + context = 1048576; + output = 262144; + }; + options = { + reasoningEffort = "high"; + thinking.type = "enabled"; + }; + }; + }; + }; + }; + }; + }; + }; + + virtualisation.containers.enable = true; + virtualisation.podman = { + enable = true; + dockerCompat = true; + defaultNetwork.settings.dns_enabled = true; + }; + microvm = { + vsock = { + cid = 3; + ssh.enable = true; + }; + writableStoreOverlay = "/nix/.rw-store"; + # graphics.enable = true; + mem = 4096; + interfaces = [ + { + type = "user"; + id = "vm-1"; + mac = "02:00:00:01:01:01"; + } + ]; + volumes = [ + { + mountPoint = "/var"; + image = "var.img"; + size = 256; + } + { + mountPoint = "/nix/.rw-store"; + image = "nix-store-overlay.img"; + size = 512; + } + { + mountPoint = "/.persist"; + image = "persist.img"; + size = 1; + } + ]; + shares = [ + { + proto = "virtiofs"; + tag = "projects"; + source = "/home/user/dev"; + mountPoint = "/home/sandbox/dev"; + } + { + proto = "virtiofs"; + tag = "work"; + source = "/home/user/work"; + mountPoint = "/home/sandbox/work"; + } + { + proto = "virtiofs"; + tag = "screenshots"; + source = "/home/user/pictures"; + mountPoint = "/home/sandbox/pictures"; + readOnly = true; + } + { + proto = "virtiofs"; + tag = "ro-store"; + source = "/nix/store"; + mountPoint = "/nix/.ro-store"; + readOnly = true; + } + ]; + hypervisor = "qemu"; + }; }; - }; }; } diff --git a/microvm/secrets/sandbox.yaml b/microvm/secrets/sandbox.yaml new file mode 100644 index 0000000..0b93259 --- /dev/null +++ b/microvm/secrets/sandbox.yaml @@ -0,0 +1,18 @@ +root-password-hash: ENC[AES256_GCM,data:YTQn1UGwgrlCD381u+Z7xXMgUoigKXp+Na1/Aj0GExikEeIs9SkBIcAyQ2Dw0FimL0wgG4z+AxFhdAnzRHcbecvvb3tRrTX3VjFt+CLPWscZar1jjNesPyV1inXI2IFW06twHbAl4oQHFQ==,iv:Va0B4aFUlavJd3AGzDbAnmAnAX3fEow4ByIQzfPbheM=,tag:OIjXY7ngZow7Y3tRFZ8iiA==,type:str] +sandbox-password-hash: ENC[AES256_GCM,data:bxVKDAsk6ao0D+xIgAR2Sh1msoPJGEE8P0+O9YGZS/F+DZl33r/JQbNFYJzZgbHDqzLDbab0tnzyRwbfepT2eV/KWaroQDFcBCLJ07t4lTOtdJ3mR/dB1UftrCvvmoPubVrguDPvftQLAg==,iv:tDErYBYhfPrOKyH7W41SUAFgT54bA7QvUYmo35GWMIs=,tag:vNJVKrlCwCuZl4z8KFby2Q==,type:str] +deepseek-api-key: ENC[AES256_GCM,data:pzg/7L92Ig+m3nhZBiYFSfdLuJFmamWWmPQwMxmctRS2kVQ=,iv:yCu+kPXkOxhfpVLlOvUpB/BwygedRaJviPe2D0tCW78=,tag:YTM7hx4UdZEsB9qfXs5LXQ==,type:str] +sops: + age: + - enc: | + -----BEGIN AGE ENCRYPTED FILE----- + YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBzQXlvQWl3aWtSWjU5QWVN + K3JGSzg0RElZSkVxNWFmTDNMNzVHckFuZkM4CnJiVDFDTTZ3cDBrTG5RR2RTR3A0 + RmdDSVA1T1dDbUZTNWJQNzB1K0dhYnMKLS0tIDk3VVJBSUFLVjMrdzV5MTFzRXlF + N0ZZa3JnSXN2bHdLa1hoVFlmSUhXWWMK5JCznoaqS6svxcBl6UF9XqbfirZ9L2np + /CTlYyWRadRvRmxrNkYNdgGn4V+is9FG+Et0u8JHAMvI7x/+W99yGA== + -----END AGE ENCRYPTED FILE----- + recipient: age1mr42xh9llvrggrvuawr5253qk7gygf4gcg9clqdvjecw3d6tj59qsqxswd + lastmodified: "2026-07-18T18:23:56Z" + mac: ENC[AES256_GCM,data:d/vRaH5vFzTkb8WopwIDb9Ku7n0EXUR5PyE/fmYhDWcCGX71PcM23Y1DPX1SCY6ydnpxqtGxfRiJOZMfIAx+tCBKX5Szj2aPnKXpogtukZbWWT4pF5jYWnlXcA9R7b9CY0sSAXIetyKhAaH086PJ2i2tAx8TiqZNkUFyoR6g0+E=,iv:R+cOmhYJnVPMjaiL5J0zbFUR69sNR78Gz5ea3CV6X6Y=,tag:RlaTyampHdMtmzvHskM4Dw==,type:str] + unencrypted_suffix: _unencrypted + version: 3.13.1 diff --git a/tailscale.nix b/tailscale.nix new file mode 100644 index 0000000..af1693e --- /dev/null +++ b/tailscale.nix @@ -0,0 +1,273 @@ +{ + config, + lib, + pkgs, + ... +}: + +with lib; + +let + cfg = config.services.tailscale-netns; + + instanceOptions = + { name, ... }: + { + options = { + authKeyFile = mkOption { + type = types.nullOr types.path; + default = null; + description = "Path to file containing a Tailscale auth key."; + }; + + hostname = mkOption { + type = types.nullOr types.str; + default = null; + description = '' + Tailscale hostname for this instance. + Defaults to "-". + ''; + }; + + advertiseRoutes = mkOption { + type = types.listOf types.str; + default = [ ]; + description = "Subnet routes to advertise (e.g. [ \"10.0.0.0/24\" ])."; + }; + + dns = mkOption { + type = types.nullOr types.str; + default = null; + description = '' + Bootstrap DNS server for the namespace before Tailscale is connected. + Defaults to `1.1.1.1`. Only used for the initial coordination server + connection; once Tailscale is up it manages DNS inside the namespace. + ''; + }; + + advertiseExitNode = mkOption { + type = types.bool; + default = false; + description = "Whether to advertise this node as an exit node."; + }; + + acceptRoutes = mkOption { + type = types.bool; + default = false; + description = "Whether to accept routes advertised by other nodes."; + }; + + exitNode = mkOption { + type = types.nullOr types.str; + default = null; + description = "Tailscale IP or hostname of exit node to use."; + }; + + extraUpFlags = mkOption { + type = types.listOf types.str; + default = [ ]; + description = "Extra flags passed to `tailscale up`."; + }; + + extraDaemonFlags = mkOption { + type = types.listOf types.str; + default = [ ]; + description = "Extra flags passed to `tailscaled`."; + }; + }; + }; + + mkUpFlags = + instanceCfg: + let + hostFlag = optional ( + instanceCfg.hostname != null + ) "--hostname=${escapeShellArg instanceCfg.hostname}"; + routeFlags = map (r: "--advertise-routes=${escapeShellArg r}") instanceCfg.advertiseRoutes; + exitFlag = optional instanceCfg.advertiseExitNode "--advertise-exit-node"; + acceptFlag = optional instanceCfg.acceptRoutes "--accept-routes"; + exitNodeFlag = optional ( + instanceCfg.exitNode != null + ) "--exit-node=${escapeShellArg instanceCfg.exitNode}"; + in + hostFlag ++ routeFlags ++ exitFlag ++ acceptFlag ++ exitNodeFlag ++ instanceCfg.extraUpFlags; + + mkAutoconnectScript = + name: instanceCfg: + pkgs.writers.writeBash "tailscale-autoconnect-${name}" '' + set -euo pipefail + + TS="${pkgs.tailscale}/bin/tailscale" + SOCKET="/run/tailscale-${name}/tailscaled.sock" + TS_ARGS="--socket $SOCKET" + + get_state() { + $TS $TS_ARGS status --json --peers=false 2>/dev/null | ${pkgs.jq}/bin/jq -r '.BackendState // "NoState"' + } + + last_state="" + while state="$(get_state)"; do + if [[ "$state" != "$last_state" ]]; then + case "$state" in + NeedsLogin|NeedsMachineAuth|Stopped) + echo "[tailscale-netns:${name}] server needs authentication, sending auth key" + $TS $TS_ARGS up \ + --auth-key "$(cat ${instanceCfg.authKeyFile})" \ + ${escapeShellArgs (mkUpFlags instanceCfg)} + ;; + Running) + echo "[tailscale-netns:${name}] tailscale is running" + ${pkgs.systemd}/bin/systemd-notify --ready + exit 0 + ;; + NoState) + echo "[tailscale-netns:${name}] tailscaled not ready yet" + ;; + *) + echo "[tailscale-netns:${name}] waiting for Running (state=$state)" + ;; + esac + fi + last_state="$state" + sleep 1 + done + ''; + + mkNetnsSetup = + index: name: instanceCfg: + let + ns = "ts-${name}ns"; + hostIP = "10.201.${toString index}.1/24"; + nsIP = "10.201.${toString index}.2/24"; + vethHost = "veth-ts-${name}"; + vethNS = "veth-ts-${name}-ns"; + in + { + "netns@ts-${name}" = { + description = "Network namespace for Tailscale instance ${name}"; + before = [ "network.target" ]; + serviceConfig = { + Type = "oneshot"; + RemainAfterExit = true; + ExecStart = pkgs.writers.writeBash "ts-${name}-netns-up" '' + set -euo pipefail + ${pkgs.coreutils}/bin/mkdir -p /etc/netns/${ns} + + # Build resolv.conf for the netns. Skip loopback addresses + # (systemd-resolved stub at 127.0.0.53 is not reachable from the netns). + NS_DNS="${if instanceCfg.dns != null then instanceCfg.dns else "1.1.1.1"}" + { + ${pkgs.gawk}/bin/awk '$1 == "nameserver" && $2 !~ /^127\./ && $2 != "::1" {print}' /etc/resolv.conf + echo "nameserver $NS_DNS" + } > /etc/netns/${ns}/resolv.conf + + ${pkgs.iproute2}/bin/ip netns add ${ns} + ${pkgs.iproute2}/bin/ip link add ${vethHost} type veth peer name ${vethNS} + ${pkgs.iproute2}/bin/ip link set ${vethNS} netns ${ns} + + ${pkgs.iproute2}/bin/ip addr add ${hostIP} dev ${vethHost} + ${pkgs.iproute2}/bin/ip link set ${vethHost} up + + ${pkgs.iproute2}/bin/ip -n ${ns} addr add ${nsIP} dev ${vethNS} + ${pkgs.iproute2}/bin/ip -n ${ns} link set ${vethNS} up + ${pkgs.iproute2}/bin/ip -n ${ns} link set lo up + + ${pkgs.iptables}/bin/iptables -t nat -C POSTROUTING -s ${nsIP} -j MASQUERADE 2>/dev/null \ + || ${pkgs.iptables}/bin/iptables -t nat -A POSTROUTING -s ${nsIP} -j MASQUERADE + ''; + ExecStop = pkgs.writers.writeBash "ts-${name}-netns-down" '' + set -euo pipefail + ${pkgs.iptables}/bin/iptables -t nat -D POSTROUTING -s ${nsIP} -j MASQUERADE 2>/dev/null || true + ${pkgs.iproute2}/bin/ip link del ${vethHost} 2>/dev/null || true + ${pkgs.iproute2}/bin/ip netns del ${ns} 2>/dev/null || true + ''; + }; + }; + + "tailscaled-${name}" = { + description = "Tailscale daemon for instance ${name}"; + bindsTo = [ "netns@ts-${name}.service" ]; + after = [ "netns@ts-${name}.service" ]; + wantedBy = [ "multi-user.target" ]; + serviceConfig = { + ExecStart = pkgs.writers.writeBash "tailscaled-${name}-start" '' + exec ${pkgs.iproute2}/bin/ip netns exec ${ns} \ + ${pkgs.tailscale}/bin/tailscaled \ + --statedir=/var/lib/tailscale-${name} \ + --socket=/run/tailscale-${name}/tailscaled.sock \ + --tun=ts-${name} \ + --port=41641 \ + ${escapeShellArgs instanceCfg.extraDaemonFlags} + ''; + ExecStopPost = pkgs.writers.writeBash "tailscaled-${name}-stop" '' + ${pkgs.iproute2}/bin/ip netns exec ${ns} \ + ${pkgs.tailscale}/bin/tailscaled --cleanup \ + --statedir=/var/lib/tailscale-${name} \ + --socket=/run/tailscale-${name}/tailscaled.sock \ + 2>/dev/null || true + ''; + Restart = "on-failure"; + RuntimeDirectory = "tailscale-${name}"; + RuntimeDirectoryMode = "0755"; + StateDirectory = "tailscale-${name}"; + StateDirectoryMode = "0700"; + CacheDirectory = "tailscale-${name}"; + CacheDirectoryMode = "0750"; + + AmbientCapabilities = "CAP_NET_ADMIN CAP_NET_RAW"; + LimitNOFILE = "infinity"; + }; + }; + }; + +in +{ + options.services.tailscale-netns = { + enable = mkEnableOption "Tailscale instances in isolated network namespaces"; + + instances = mkOption { + type = types.attrsOf (types.submodule instanceOptions); + default = { }; + description = "Tailscale instances configured in separate network namespaces."; + example = literalExpression '' + { + work = { + authKeyFile = "/run/secrets/ts-work/authkey"; + acceptRoutes = true; + }; + } + ''; + }; + }; + + config = mkIf cfg.enable { + systemd.services = listToAttrs ( + flatten ( + imap1 ( + index: elem: + let + name = elem.name; + instanceCfg = elem.value; + units = mkNetnsSetup index name instanceCfg; + in + [ + units."netns@ts-${name}" + units."tailscaled-${name}" + ] + ++ optional (instanceCfg.authKeyFile != null) { + "tailscaled-autoconnect-${name}" = { + description = "Tailscale autoconnect for instance ${name}"; + after = [ "tailscaled-${name}.service" ]; + requires = [ "tailscaled-${name}.service" ]; + wantedBy = [ "multi-user.target" ]; + serviceConfig = { + Type = "notify"; + }; + script = "${mkAutoconnectScript name instanceCfg}"; + }; + } + ) (attrsToList cfg.instances) + ) + ); + }; +}