Bench rig

# .mise.toml — task runner for this research notebook.
#
# Invoke with: `mise run <task>` or `mise run <task-a> <task-b>`.
# Reproducibility is the point — if a number is on the site, the mise
# task that produced it is in this file, and the underlying shell rig
# is in benchmarks/rigs/. The site reads both at build time.

[tools]
node = "22"
pnpm = "10"
python = "3.12"

[env]
CUBESANDBOX_COMMIT = "c439bb513f5124d4d9389451b31b8aeb87ab539c"
HONOR_HOST         = "honor"
HONOR_RIG_DIR      = "/tmp/bench-rigs"
# Shallow defaults for first-pass signal. Bump these (200/100/50) once
# we like the shape. Jail creates aren't cheap — cc=1 at ~1s each.
BENCH_ITERS_CC1    = "30"
BENCH_ITERS_CC10   = "30"
BENCH_ITERS_CC50   = "50"

# ──────────────────────────── site ─────────────────────────────

[tasks.dev]
description = "Run the Astro dev server"
run = "pnpm dev"

[tasks.build]
description = "Build the static site"
run = "pnpm build"

[tasks.test]
description = "Run component + schema tests"
run = "pnpm test"

[tasks.check]
description = "Run astro check"
run = "pnpm check"

[tasks.links]
description = "Check built site for broken links"
run = ["pnpm build", "pnpm links"]

# ──────────────────────────── research ──────────────────────────

[tasks."research:clone"]
description = "Clone CubeSandbox at the pinned commit into /tmp/cubesandbox-research"
run = '''
set -eu
mkdir -p /tmp/cubesandbox-research
cd /tmp/cubesandbox-research
[ -d CubeSandbox/.git ] || git clone https://github.com/TencentCloud/CubeSandbox.git
cd CubeSandbox
git fetch origin main
git checkout "$CUBESANDBOX_COMMIT"
echo "Pinned at $(git rev-parse HEAD)"
'''

# ──────────────────────────── benchmarks ────────────────────────

[tasks."bench:sync"]
description = "Sync benchmarks/rigs to $HONOR_HOST:$HONOR_RIG_DIR"
run = 'scp -r benchmarks/rigs "$HONOR_HOST:$HONOR_RIG_DIR"'

[tasks."bench:setup-pf"]
description = "Apply the benchmark pf ruleset on $HONOR_HOST with a dead-man switch that auto-disables pf after DMS_TIMEOUT seconds if we lose control."
depends = ["bench:sync"]
run = '''
set -eu
: "${DMS_TIMEOUT:=60}"
ssh "$HONOR_HOST" "sudo sh -c 'DMS_TIMEOUT=$DMS_TIMEOUT sh $HONOR_RIG_DIR/setup-pf.sh $HONOR_RIG_DIR/pf.bench.conf'"
# Independent verification from the dev machine — if this fails, the
# dead-man is running and pf will self-disable within DMS_TIMEOUT.
sleep 2
ssh "$HONOR_HOST" 'echo "SSH still alive: $(hostname)"'
'''

[tasks."bench:host-info"]
description = "Capture $HONOR_HOST's kernel/release/CPU/RAM into /tmp/honor-host.json"
run = '''
ssh "$HONOR_HOST" 'python3 - <<PY
import json, platform, subprocess
def s(k): return subprocess.run(["sysctl","-n",k], capture_output=True, text=True).stdout.strip()
print(json.dumps({
  "hostname": platform.node(), "kernel": platform.system(),
  "release": platform.release(), "cpuModel": s("hw.model"),
  "cpuCount": int(s("hw.ncpu") or 0),
  "memGB": round(int(s("hw.physmem") or 0) / (1024**3), 2),
}))
PY' > /tmp/honor-host.json
cat /tmp/honor-host.json
'''

# Internal helper: execute one config × (cc1/10/50 cold-start + idle RSS).
# Takes $CONFIG as env var. RSS concurrency defaults to 32 (jails); set
# $RSS_CC for configs where 32 is impractical (bhyve VMs at 512MB each
# would reserve 16GB — we use cc=8 there). Cold-start iteration counts
# per concurrency may also be overridden via $ITERS_CC1/10/50 for configs
# where a boot takes seconds, not milliseconds.
[tasks."bench:_run"]
description = "Internal: run one config's full sweep (cold-start cc1/10/50 + idle RSS)"
hide = true
run = '''
set -eu
: "${CONFIG:?must set CONFIG=jail-raw|jail-vnet-pf|jail-zfs-clone|bhyve-*}"
: "${RSS_CC:=32}"
: "${ITERS_CC1:=$BENCH_ITERS_CC1}"
: "${ITERS_CC10:=$BENCH_ITERS_CC10}"
: "${ITERS_CC50:=$BENCH_ITERS_CC50}"
: "${RIG_ENV:=}"
mkdir -p benchmarks/results
HOST_JSON=$(cat /tmp/honor-host.json)

for CC in 1 10 50; do
  case $CC in 1) ITERS=$ITERS_CC1;; 10) ITERS=$ITERS_CC10;; 50) ITERS=$ITERS_CC50;; esac
  echo "▸ $CONFIG cold-start @ cc=$CC (iters=$ITERS)"
  ssh "$HONOR_HOST" "cd $HONOR_RIG_DIR && sudo sh -c '$RIG_ENV sh $CONFIG.sh $CC $ITERS'" > "/tmp/$CONFIG-cc$CC.tsv"
  python3 benchmarks/rigs/summarize.py \
    --config "$CONFIG" --metric cold-start-ms --concurrency "$CC" \
    --script "benchmarks/rigs/$CONFIG.sh" --input "/tmp/$CONFIG-cc$CC.tsv" \
    --output "benchmarks/results/${CONFIG}_cold-start-ms_cc${CC}.json" \
    --host-info-json "$HOST_JSON"
done

echo "▸ $CONFIG idle RSS @ cc=$RSS_CC"
ssh "$HONOR_HOST" "cd $HONOR_RIG_DIR && sudo sh $CONFIG-rss.sh" > "/tmp/$CONFIG-rss.tsv"
python3 benchmarks/rigs/summarize.py \
  --config "$CONFIG" --metric rss-kb-idle-1s --concurrency "$RSS_CC" \
  --script "benchmarks/rigs/$CONFIG-rss.sh" --input "/tmp/$CONFIG-rss.tsv" \
  --output "benchmarks/results/${CONFIG}_rss-kb-idle-1s_cc${RSS_CC}.json" \
  --host-info-json "$HOST_JSON"
'''

[tasks."bench:jail-raw"]
description = "jail-raw: cold-start @ cc=1,10,50 + idle RSS @ cc=32"
depends = ["bench:sync", "bench:host-info"]
env = { CONFIG = "jail-raw" }
run = "mise run bench:_run"

[tasks."bench:jail-vnet-pf"]
description = "jail-vnet-pf (VNET + pf egress filter): cold-start + idle RSS"
depends = ["bench:sync", "bench:host-info"]
env = { CONFIG = "jail-vnet-pf" }
run = "mise run bench:_run"

[tasks."bench:jail-zfs-clone"]
description = "jail-zfs-clone (per-jail ZFS clone rootfs): cold-start + idle RSS"
depends = ["bench:sync", "bench:host-info"]
env = { CONFIG = "jail-zfs-clone" }
run = "mise run bench:_run"

[tasks."bench:all-jails"]
description = "All three jail configurations"
depends = ["bench:jail-raw", "bench:jail-vnet-pf", "bench:jail-zfs-clone"]

# bhyve tasks. Each config follows the same pattern as the jail tasks:
# bench:_run runs cold-start at cc=1/10/50 plus an idle-RSS sweep.
# bhyve VMs dominate the runtime budget — cc=50 cold-start can take
# minutes at full guest memory. See the rig scripts for iter counts.

[tasks."bench:bhyve-full"]
description = "bhyve-full (full FreeBSD 15 GENERIC guest per iter): cold-start + idle RSS"
depends = ["bench:sync", "bench:host-info"]
env = { CONFIG = "bhyve-full", RSS_CC = "8", ITERS_CC1 = "10", ITERS_CC10 = "10", ITERS_CC50 = "50", RIG_ENV = "TIMEOUT_SEC=180" }
run = "mise run bench:_run"

[tasks."bench:bhyve-minimal"]
description = "bhyve-minimal (MINIMAL-BHYVE kernel — STUB: guest rc.conf not yet trimmed)"
depends = ["bench:sync", "bench:host-info"]
env = { CONFIG = "bhyve-minimal", RSS_CC = "8", ITERS_CC1 = "10", ITERS_CC10 = "10", ITERS_CC50 = "50", RIG_ENV = "TIMEOUT_SEC=60" }
run = "mise run bench:_run"

[tasks."bench:bhyve-prewarm-pool"]
description = "bhyve-prewarm-pool (SIGSTOP/SIGCONT proxy for CH snapshot-restore): cold-start + idle RSS"
depends = ["bench:sync", "bench:host-info"]
env = { CONFIG = "bhyve-prewarm-pool", RSS_CC = "8", ITERS_CC1 = "30", ITERS_CC10 = "30", ITERS_CC50 = "50", RIG_ENV = "POOL_SIZE=50 BOOT_TIMEOUT=240" }
run = "mise run bench:_run"

[tasks."bench:bhyve-durable-pool-setup"]
description = "One-time pool-setup: boot N VMs, bhyvectl --suspend each to /vms/pool/ (requires SNAPSHOT kernel)"
depends = ["bench:sync"]
run = 'ssh "$HONOR_HOST" "sudo sh $HONOR_RIG_DIR/bhyve-durable-pool-setup.sh"'

[tasks."bench:bhyve-durable-pool"]
description = "bhyve-durable-pool: resume from on-disk bhyvectl --suspend checkpoint (SNAPSHOT kernel)"
depends = ["bench:sync", "bench:host-info"]
env = { CONFIG = "bhyve-durable-pool" }
run = "mise run bench:_run"

[tasks."bench:all"]
description = "Run every bench config that's implemented"
depends = ["bench:all-jails"]

[tasks."bench:clean-remote"]
description = "Remove /tmp/bench-rigs and bench tempfiles on $HONOR_HOST"
run = 'ssh "$HONOR_HOST" "rm -rf $HONOR_RIG_DIR /tmp/*.tsv /tmp/bhyve-*.log" || true'

# ─────────────────────────── e2b-compat ─────────────────────────

[tasks."e2b:build"]
description = "Build the e2b-compat binary (cargo build --release)"
run = "cargo build --release --manifest-path e2b-compat/Cargo.toml"

[tasks."e2b:check"]
description = "cargo check the e2b-compat crate"
run = "cargo check --manifest-path e2b-compat/Cargo.toml"

[tasks."e2b:serve"]
description = "Run e2b-compat locally (requires root or sudoers for zfs/jail/jls/ps/kill/jexec)"
run = '''
cargo run --release --manifest-path e2b-compat/Cargo.toml -- \
  --listen 127.0.0.1:3000 \
  --zfs-pool zroot/jails \
  --template-snapshot zroot/jails/_template@base \
  --jails-root /jails
'''

[tasks."e2b:sync-honor"]
description = "rsync e2b-compat sources to $HONOR_HOST:/tmp/e2b-compat-src/"
run = '''
ssh "$HONOR_HOST" 'mkdir -p /tmp/e2b-compat-src'
rsync -az --delete --exclude=target/ --exclude=Cargo.lock e2b-compat/ "$HONOR_HOST:/tmp/e2b-compat-src/"
'''

[tasks."e2b:build-honor"]
description = "Build the e2b-compat binary on $HONOR_HOST (FreeBSD-native)"
depends = ["e2b:sync-honor"]
run = 'ssh "$HONOR_HOST" "cd /tmp/e2b-compat-src && cargo build --release"'

[tasks."e2b:serve-honor"]
description = "Run e2b-compat on $HONOR_HOST as root, listen 127.0.0.1:3000"
depends = ["e2b:build-honor"]
run = 'ssh "$HONOR_HOST" "sudo /tmp/e2b-compat-src/target/release/e2b-compat --listen 127.0.0.1:3000 --zfs-pool zroot/jails --template-snapshot zroot/jails/_template@base --jails-root /jails"'

[tasks."e2b:smoke"]
description = "Hit the running e2b-compat with the example smoke-test script"
run = 'sh e2b-compat/examples/smoke-test.sh "${E2B_COMPAT_URL:-http://honor:3000}"'

# ──────────────────────────── demos ────────────────────────────

[tasks."demo:notebook"]
description = "Execute examples/notebook-demo.ipynb against the Coppice gateway"
run = '''
set -eu
HONOR="${HONOR_HOST:-honor}"

# Gateway binds to 127.0.0.1 on honor. Tunnel API + envd through SSH
# so the local SDK reaches them as localhost:3000 / localhost:49999
# — which is what E2B_DEBUG=true hardcodes anyway.
echo "opening SSH tunnel: $HONOR → localhost 3000 + 49999"
ssh -fN -L 3000:127.0.0.1:3000 -L 49999:127.0.0.1:49999 "$HONOR"
ssh_pid=$(pgrep -f "ssh -fN -L 3000:127.0.0.1:3000.*$HONOR" | head -1 || true)
cleanup() { [ -n "${ssh_pid:-}" ] && kill "$ssh_pid" 2>/dev/null || true; }
trap cleanup EXIT

export E2B_API_URL="http://localhost:3000"
export E2B_DEBUG="true"
export E2B_API_KEY="${E2B_API_KEY:-local}"
echo "gateway: $E2B_API_URL (tunneled from $HONOR)"

# uv run with inline deps — no pip, no venv to manage.
uv run --with jupyter --with nbclient --with e2b-code-interpreter \
  jupyter nbconvert \
    --to notebook \
    --execute examples/notebook-demo.ipynb \
    --output notebook-demo.executed.ipynb \
    --ExecutePreprocessor.timeout=120
echo
echo "executed: examples/notebook-demo.executed.ipynb"
echo "render to HTML:  mise run demo:notebook:html"
echo "open live:       mise run demo:notebook:view"
'''

[tasks."demo:notebook:view"]
description = "Open the notebook live in nbclassic, tunnel + deps wired up"
run = '''
set -eu
HONOR="${HONOR_HOST:-honor}"

# Same tunnel as demo:notebook — keeps localhost:3000/:49999 pointed at
# the gateway on honor while nbclassic is running.
ssh -fN -L 3000:127.0.0.1:3000 -L 49999:127.0.0.1:49999 "$HONOR"
ssh_pid=$(pgrep -f "ssh -fN -L 3000:127.0.0.1:3000.*$HONOR" | head -1 || true)
cleanup() { [ -n "${ssh_pid:-}" ] && kill "$ssh_pid" 2>/dev/null || true; }
trap cleanup EXIT

export E2B_API_URL="http://localhost:3000"
export E2B_DEBUG="true"
export E2B_API_KEY="${E2B_API_KEY:-local}"
echo "gateway: $E2B_API_URL (tunneled from $HONOR)"

# Run both the server and the in-notebook kernel under the same uv
# environment so `from e2b_code_interpreter import Sandbox` resolves.
# The source ipynb is the live version — open that, not the
# .executed one (nbconvert-rendered outputs confuse readers trying
# to re-run cells).
exec uv run \
  --with nbclassic \
  --with e2b-code-interpreter \
  --with matplotlib --with pandas --with numpy \
  jupyter nbclassic examples/notebook-demo.ipynb
'''

[tasks."demo:notebook:html"]
description = "Render the executed notebook to a static HTML page"
depends = ["demo:notebook"]
run = '''
uv run --with jupyter --with nbconvert \
  jupyter nbconvert \
    --to html \
    examples/notebook-demo.executed.ipynb \
    --output notebook-demo.html
echo "rendered: examples/notebook-demo.html"
'''

#!/bin/sh
# Shared helpers for all rigs. Sourced, not run directly.
set -eu

timestamp_ms() {
  # FreeBSD date supports %N only through gdate; use python for portability.
  python3 -c 'import time; print(int(time.time()*1000))'
}

host_info_json() {
  python3 - <<'PY'
import json, platform, subprocess
def sysctl(k):
    return subprocess.run(['sysctl','-n',k], capture_output=True, text=True).stdout.strip()
print(json.dumps({
    'hostname': platform.node(),
    'kernel': platform.system(),
    'release': platform.release(),
    'cpuModel': sysctl('hw.model'),
    'cpuCount': int(sysctl('hw.ncpu') or 0),
    'memGB': round(int(sysctl('hw.physmem') or 0) / (1024**3), 2),
}))
PY
}

run_concurrent() {
  # Usage: run_concurrent N CMD...
  # Runs CMD N times in parallel, prints per-iteration elapsed_ms TSV.
  # The loop index (0..N-1) is appended as an extra argument so each
  # worker can form a unique name even though $$ is shared by subshells.
  # _rc_n/_rc_j are intentionally prefixed to avoid clobbering the
  # caller's loop variable (sh functions share scope with their callers).
  _rc_n=$1; shift
  _rc_j=0
  while [ $_rc_j -lt $_rc_n ]; do
    ( s=$(timestamp_ms); "$@" "$_rc_j" >/dev/null 2>&1; e=$(timestamp_ms); printf "%d\t%d\n" "$_rc_j" "$((e - s))" ) &
    _rc_j=$((_rc_j + 1))
  done
  wait
}

#!/usr/bin/env python3
"""Wrap raw TSV samples into the BenchmarkRun JSON schema."""
import argparse, json, statistics, datetime, sys, subprocess

def host_info():
    r = subprocess.run(['python3', '-c', '''
import json, platform, subprocess
def s(k): return subprocess.run(["sysctl","-n",k], capture_output=True, text=True).stdout.strip()
print(json.dumps({
    "hostname": platform.node(),
    "kernel": platform.system(),
    "release": platform.release(),
    "cpuModel": s("hw.model") or "unknown",
    "cpuCount": int(s("hw.ncpu") or 0),
    "memGB": round(int(s("hw.physmem") or 0) / (1024**3), 2),
}))
'''], capture_output=True, text=True)
    return json.loads(r.stdout)

def summarize(samples):
    ss = sorted(samples)
    def pct(p): return ss[min(len(ss) - 1, int(len(ss) * p / 100))]
    return {
        'mean': statistics.mean(ss),
        'p50': pct(50), 'p95': pct(95), 'p99': pct(99),
        'min': min(ss), 'max': max(ss), 'n': len(ss),
    }

def main():
    ap = argparse.ArgumentParser()
    ap.add_argument('--config', required=True)
    ap.add_argument('--metric', required=True)
    ap.add_argument('--concurrency', type=int, required=True)
    ap.add_argument('--script', required=True)
    ap.add_argument('--input', required=True)
    ap.add_argument('--output', required=True)
    ap.add_argument('--host-info-json', help='Pre-collected host_info_json; if omitted, shells out to collect')
    args = ap.parse_args()

    samples = []
    with open(args.input) as f:
        for line in f:
            line = line.strip()
            if not line: continue
            parts = line.split('\t')
            samples.append(float(parts[-1]))

    host = json.loads(args.host_info_json) if args.host_info_json else host_info()

    out = {
        'config': args.config,
        'host': host,
        'metric': args.metric,
        'concurrency': args.concurrency,
        'samples': samples,
        'summary': summarize(samples),
        'scriptPath': args.script,
        'runAt': datetime.datetime.utcnow().isoformat() + 'Z',
    }
    with open(args.output, 'w') as f:
        json.dump(out, f, indent=2)

if __name__ == '__main__':
    main()

# pf.bench.conf — ruleset applied on honor for the jail-vnet-pf rig.
#
# IMPORTANT: designed to NEVER lock out the SSH control plane. The
# earlier version of this ruleset (`block out all` with no management
# pass rules) wedged honor at 2026-04-21T22:54 because the outbound
# half of the active SSH session got dropped. Do not reintroduce that
# shape. Both `pass in` and `pass out` for TCP/22 are explicit here,
# and `set skip on lo0` guarantees loopback is never filtered.
#
# Behavior: default-deny egress on the external NIC, except the SSH
# management traffic and ICMP. This matches the posture CubeVS
# enforces per-sandbox — block all except explicitly allowed.

# Loopback — never filter.
set skip on lo0

# SSH management plane, both directions. Keep-state required so once a
# session is established, reply traffic is authorized from the state
# entry rather than needing its own rule.
pass in  quick proto tcp to port 22 keep state
pass out quick proto tcp from port 22 keep state

# ICMP — lets ping-based health checks keep working.
pass quick proto icmp
pass quick proto icmp6

# Default egress: block everything else. This is the part we are
# actually measuring — the cost of per-packet pf filtering on jail
# egress traffic.
block out all

#!/bin/sh
# setup-pf.sh — apply pf.bench.conf with a dead-man's-switch that auto-
# reverts if SSH isn't confirmed alive within DMS_TIMEOUT seconds.
#
# Usage (as root, typically via `mise run bench:setup-pf`):
#     sh setup-pf.sh /path/to/pf.bench.conf
#
# Safety model:
#   1. Spawn a detached child via daemon(8) that sleeps DMS_TIMEOUT
#      seconds and then calls `pfctl -d`. Its PID is written to a
#      pidfile so we can cancel it.
#   2. Apply the ruleset with pfctl -f and enable pf.
#   3. If we reach the end of the script without the shell dying, cancel
#      the dead-man.
#
# If SSH goes away during (2) — e.g. because the rules were wrong — the
# dead-man fires, pf is disabled, and the host becomes reachable again.

set -eu

RULES=${1:?usage: setup-pf.sh /path/to/pf.bench.conf}
DMS_TIMEOUT=${DMS_TIMEOUT:-60}
DMS_PIDFILE=/tmp/bench-pf-deadman.pid

if [ ! -f "$RULES" ]; then
  echo "setup-pf: rules file not found: $RULES" >&2
  exit 2
fi

# Ensure pf kernel module is loaded.
kldload pf 2>/dev/null || true

# Fire the dead-man via daemon(8) so it survives our shell exiting.
# daemon -f detaches, -p writes the child's pid.
daemon -f -p "$DMS_PIDFILE" /bin/sh -c "sleep $DMS_TIMEOUT; pfctl -d; logger 'bench-safe: pf disabled by deadman after ${DMS_TIMEOUT}s'"
sleep 0.2  # let daemon write the pidfile

# Apply ruleset and enable pf. `pfctl -f` preserves the existing state
# table, so an active SSH session is kept alive across reload.
pfctl -f "$RULES"
pfctl -e 2>/dev/null || true
pfctl -s info | head -1

# Cancel the dead-man — we're alive and rules applied cleanly.
if [ -f "$DMS_PIDFILE" ]; then
  DMS_PID=$(cat "$DMS_PIDFILE")
  kill -TERM "$DMS_PID" 2>/dev/null || true
  rm -f "$DMS_PIDFILE"
fi

echo "setup-pf: bench ruleset active; deadman cancelled"