shuttle,nix: make nix work in the alpine microvm image · oscillatory.net/core@cd8e962

+1

flake.nix

··· 241 241 242 242 spindle-alpine-image = linuxPkgs.callPackage ./nix/pkgs/spindle-alpine-image.nix { 243 243 shuttle = (mkPackageSet linuxPkgs).shuttle-static; 244 + nix = linuxPkgs.nixStatic; 244 245 git = 245 246 (linuxPkgs.pkgsStatic.gitMinimal.override { 246 247 curl = linuxPkgs.pkgsStatic.curlMinimal;

+47 -1

nix/pkgs/spindle-alpine-image.nix

··· 5 5 squashfsTools, 6 6 shuttle, 7 7 git, 8 + nix, 8 9 }: let 9 10 version = "3.24.0"; 10 11 branch = "v3.24"; ··· 73 74 74 75 inittab = writeText "inittab" '' 75 76 ::sysinit:/sbin/spindle-setup 76 - ::respawn:/usr/bin/shuttle 77 + ::respawn:/usr/local/bin/nix-daemon 78 + ::respawn:env NIX_REMOTE=daemon /usr/bin/shuttle 77 79 ::ctrlaltdel:/sbin/reboot 78 80 ''; 79 81 80 82 profileScript = writeText "spindle-profile" '' 81 83 export SSL_CERT_FILE=/etc/ssl/certs/ca-certificates.crt 82 84 export GIT_SSL_CAINFO=/etc/ssl/certs/ca-certificates.crt 85 + export NIX_REMOTE=daemon 86 + ''; 87 + 88 + # mirror nix/microvm/base.nix and nix/modules/shuttle.nix 89 + nixConf = writeText "nix.conf" '' 90 + experimental-features = nix-command flakes 91 + trusted-users = root 92 + allowed-users = spindle-workflow 93 + post-build-hook = /usr/libexec/spindle-post-build-hook 94 + !include /run/spindle/nix.conf 95 + ''; 96 + 97 + postBuildHook = writeText "spindle-post-build-hook" '' 98 + #!/bin/sh 99 + set -f 100 + 101 + if [ -z "''${OUT_PATHS:-}" ]; then 102 + exit 0 103 + fi 104 + 105 + # OUT_PATHS is intentionally split into individual store paths 106 + exec /usr/bin/shuttle enqueue-built-paths $OUT_PATHS 83 107 ''; 84 108 85 109 imageSpecJSON = writeText "spindle-alpine-${system}.json" ( ··· 134 158 install -D -m 0755 ${setupScript} rootfs/sbin/spindle-setup 135 159 install -D -m 0644 ${inittab} rootfs/etc/inittab 136 160 install -D -m 0644 ${profileScript} rootfs/etc/profile.d/01-spindle.sh 161 + install -D -m 0644 ${nixConf} rootfs/etc/nix/nix.conf 162 + install -D -m 0755 ${postBuildHook} rootfs/usr/libexec/spindle-post-build-hook 137 163 138 164 # install git 139 165 mkdir -p rootfs/nix/store rootfs/usr/local/bin ··· 142 168 ln -s "$bin" "rootfs/usr/local/bin/$(basename "$bin")" 143 169 done 144 170 171 + # install nix 172 + install -D -m 0755 ${nix}/bin/nix rootfs/usr/local/bin/nix 173 + for bin in ${nix}/bin/*; do 174 + name=$(basename "$bin") 175 + if [ "$name" != "nix" ]; then 176 + ln -s nix "rootfs/usr/local/bin/$name" 177 + fi 178 + done 179 + 180 + # todo(dawn): ideally we should generecize this installation of dependencies probably ^^ 181 + 145 182 echo "spindle-microvm" > rootfs/etc/hostname 146 183 printf 'nameserver 10.0.3.3\nnameserver 1.1.1.1\n' > rootfs/etc/resolv.conf 147 184 printf '%s\n' \ ··· 153 190 echo "spindle-workflow:x:970:" >> rootfs/etc/group 154 191 echo "spindle-workflow:!::0:::::" >> rootfs/etc/shadow 155 192 mkdir -p rootfs/workspace 193 + 194 + # setup nix build users for the daemon 195 + members="" 196 + for i in $(seq 1 8); do 197 + echo "nixbld$i:x:$((30000 + i)):30000:nix build user $i:/var/empty:/sbin/nologin" >> rootfs/etc/passwd 198 + echo "nixbld$i:!::0:::::" >> rootfs/etc/shadow 199 + members="$members''${members:+,}nixbld$i" 200 + done 201 + echo "nixbld:x:30000:$members" >> rootfs/etc/group 156 202 157 203 mkdir -p "$out" 158 204 mksquashfs rootfs "$out/store-disk" -comp zstd -Xcompression-level 19 -noappend -no-xattrs -all-root -quiet

+2 -2

shuttle/src/cache/mod.rs

··· 1 1 use crate::command::{self, Spec}; 2 - use crate::nix_config::{NIX_EXECUTABLE, SPINDLE_RUN_DIR, clean_store_paths}; 2 + use crate::nix_config::{SPINDLE_RUN_DIR, clean_store_paths, nix_executable}; 3 3 use crate::protocol::{Message, v1}; 4 4 use anyhow::{Context, Result}; 5 5 use nix::unistd::{Group, chown}; ··· 376 376 let dest_url = add_query_param(&dest_url, "compression-level", "3"); 377 377 let dest_url = add_query_param(&dest_url, "parallel-compression", "true"); 378 378 379 - let spec = Spec::new(NIX_EXECUTABLE) 379 + let spec = Spec::new(nix_executable()) 380 380 .args(["copy", "--to", &dest_url]) 381 381 .args(paths.iter().cloned()) 382 382 .timeout(Duration::from_secs(10 * 60));

+65 -12

shuttle/src/nix_config.rs

··· 15 15 pub const SPINDLE_RUN_DIR: &str = "/run/spindle"; 16 16 pub const SPINDLE_NIX_CONFIG: &str = "/run/spindle/nix.conf"; 17 17 pub const SPINDLE_CACHE_CONFIG: &str = "/run/spindle/cache.json"; 18 - pub const NIX_EXECUTABLE: &str = "/run/current-system/sw/bin/nix"; 19 18 pub const NIX_BUILD_EXECUTABLE: &str = "/run/current-system/sw/bin/nix-build"; 20 19 pub const NIX_STORE_EXECUTABLE: &str = "/run/current-system/sw/bin/nix-store"; 21 20 pub const SYSTEMCTL_EXECUTABLE: &str = "/run/current-system/sw/bin/systemctl"; 21 + 22 + // nix lives in different places depending on the guest OS (NixOS system 23 + // profile vs. plain /usr/local on e.g. alpine) 24 + pub fn nix_executable() -> &'static str { 25 + static NIX: once_cell::sync::Lazy<&'static str> = once_cell::sync::Lazy::new(|| { 26 + let paths = [ 27 + "/run/current-system/sw/bin/nix", 28 + "/usr/local/bin/nix", 29 + "/usr/bin/nix", 30 + ]; 31 + for candidate in paths { 32 + if Path::new(candidate).exists() { 33 + return candidate; 34 + } 35 + } 36 + "/run/current-system/sw/bin/nix" 37 + }); 38 + &NIX 39 + } 22 40 23 41 #[derive(Clone, Debug, Default, Deserialize, Serialize)] 24 42 pub struct RuntimeCacheConfig { ··· 102 120 } 103 121 104 122 pub async fn nix_version() -> String { 105 - let spec = Spec::new(NIX_EXECUTABLE) 123 + let spec = Spec::new(nix_executable()) 106 124 .arg("--version") 107 125 .timeout(Duration::from_secs(1)); 108 126 ··· 142 160 // todo: ...and also here, the installed file? 143 161 } 144 162 163 + const NIX_DAEMON_SOCKET: &str = "/nix/var/nix/daemon-socket/socket"; 164 + 145 165 async fn restart_nix_daemon() { 146 - if !Path::new(SYSTEMCTL_EXECUTABLE).exists() { 147 - info!("no systemctl in guest, skipping nix-daemon restart"); 148 - return; 149 - } 150 - let spec = Spec::new(SYSTEMCTL_EXECUTABLE) 151 - .args(["try-restart", "nix-daemon.service"]) 152 - .timeout(Duration::from_secs(5)); 166 + let systemd = Path::new(SYSTEMCTL_EXECUTABLE).exists(); 167 + let spec = if systemd { 168 + Spec::new(SYSTEMCTL_EXECUTABLE) 169 + .args(["try-restart", "nix-daemon.service"]) 170 + .timeout(Duration::from_secs(5)) 171 + } else { 172 + // on non-systemd we can just kill the daemon and it should restart 173 + Spec::new("pkill") 174 + .args(["-f", "nix-daemon"]) 175 + .timeout(Duration::from_secs(5)) 176 + }; 153 177 154 178 match command::run_capture(spec).await { 155 - Ok(output) if output.success() => {} 179 + Ok(output) if output.success() => { 180 + if !systemd { 181 + // init has to respawn the daemon before any step needs it 182 + wait_for_nix_daemon_socket(Duration::from_secs(5)).await; 183 + } 184 + } 185 + // pkill exits 1 when nothing matched, ie. no daemon to restart 186 + Ok(output) if !systemd && output.exit.exit_code == 1 => { 187 + info!("no nix-daemon running, skipping restart") 188 + } 156 189 Ok(output) => warn!( 157 190 exit_code = output.exit.exit_code, 158 191 error = ?output.exit.error, 159 192 output = %output.combined_lossy(), 160 - "nix-daemon try-restart failed" 193 + "nix-daemon restart failed" 161 194 ), 162 - Err(error) => warn!(%error, "nix-daemon try-restart failed"), 195 + Err(error) => warn!(%error, "nix-daemon restart failed"), 196 + } 197 + } 198 + 199 + async fn wait_for_nix_daemon_socket(timeout: Duration) { 200 + let deadline = tokio::time::Instant::now() + timeout; 201 + loop { 202 + if tokio::net::UnixStream::connect(NIX_DAEMON_SOCKET) 203 + .await 204 + .is_ok() 205 + { 206 + return; 207 + } 208 + if tokio::time::Instant::now() >= deadline { 209 + warn!( 210 + socket = NIX_DAEMON_SOCKET, 211 + "nix-daemon did not come back after restart" 212 + ); 213 + return; 214 + } 215 + tokio::time::sleep(Duration::from_millis(100)).await; 163 216 } 164 217 } 165 218

+1 -1

shuttle/src/session.rs

··· 207 207 208 208 tokio::spawn(async move { 209 209 tokio::time::sleep(Duration::from_millis(100)).await; 210 - 210 + 211 211 // prefer a clean shutdown through the init system when one is around 212 212 // (systemd on NixOS, busybox/openrc elsewhere), then fall back to the 213 213 // raw reboot(2) syscall on minimal guests

+40

spindle/engines/microvm/test-spindle-microvm.sh

··· 469 469 echo "success: alpine guest booted, ran as workflow user, wrote workspace, cloned over https" 470 470 } 471 471 472 + test_alpine_nix() { 473 + local test_store_path 474 + test_store_path=$(nix-build -E 'with import <nixpkgs> {}; writeText "alpine-nix-test" "hello from cache to alpine"' --no-out-link) 475 + nix copy --to "$CACHE_UPLOAD_URL?secret-key=$CACHE_SECRET_KEY_PATH" "$test_store_path" 476 + 477 + local out 478 + out=$(run_vm --spec "$ALPINE_IMAGE_SPEC_JSON" --name "alpine-nix" --timeout "180s" --upload -- /bin/sh -lc ' 479 + set -eu 480 + export HOME=/workspace 481 + store_path=$1 482 + nix-store --realise "$store_path" >/dev/null 483 + echo "substituted=$(cat "$store_path")" 484 + built=$(nix-build -E "derivation { name = \"alpine-built\"; system = \"x86_64-linux\"; builder = \"/bin/sh\"; args = [ \"-c\" \"echo built-on-alpine > \$out\" ]; }" --no-out-link) 485 + echo "built=$built" 486 + ' sh "$test_store_path") || return 1 487 + 488 + if ! echo "$out" | strip_ansi | grep -q "substituted=hello from cache to alpine"; then 489 + echo "error: alpine guest did not substitute the test path from the cache" >&2 490 + echo "$out" | strip_ansi >&2 491 + return 1 492 + fi 493 + 494 + local built_path 495 + built_path=$(echo "$out" | strip_ansi | grep -o 'built=/nix/store/[a-z0-9]*-alpine-built' | cut -d= -f2) 496 + if [ -z "$built_path" ]; then 497 + echo "error: could not find built store path in alpine guest output" >&2 498 + echo "$out" | strip_ansi >&2 499 + return 1 500 + fi 501 + 502 + local hash 503 + hash=$(basename "$built_path" | cut -d'-' -f1) 504 + if ! curl -s -f "http://127.0.0.1:8501/${hash}.narinfo" > /dev/null; then 505 + echo "error: alpine-built store path was not uploaded to the binary cache" >&2 506 + return 1 507 + fi 508 + echo "success: alpine guest substituted from cache, built a derivation, and uploaded it" 509 + } 510 + 472 511 test_case "alpine" test_alpine 512 + test_case "alpine-nix" test_alpine_nix 473 513 test_case "realize" test_realize 474 514 test_case "build-upload" test_build_upload 475 515 test_case "networking" test_networking

Configure Feed

Configure Feed