added to remote repo

backend/src/routes/backup.js

@@ -0,0 +1,78 @@
+import { Router } from 'express';
+import { config } from '../config.js';
+import { spawnCommand, runRestic } from '../executor.js';
+import { appendJobLog, createJob, finishJob, setJobRunning, setJobStep } from '../jobs.js';
+import { validateVmExists } from '../validators.js';
+
+export const backupRouter = Router();
+
+backupRouter.post('/:vmName', async (req, res, next) => {
+  try {
+    const vmName = req.params.vmName;
+    await validateVmExists(vmName);
+    const job = createJob('backup', vmName);
+    runBackupJob(job).catch(() => {});
+    res.status(202).json({ jobId: job.id, message: 'Backup job started.' });
+  } catch (error) {
+    next(error);
+  }
+});
+
+async function runBackupJob(job) {
+  const timestamp = new Date().toISOString().replace(/[-:.TZ]/g, '').slice(0, 14);
+  const snapshotName = `s3-backup-${timestamp}`;
+  const zvol = `${config.zfsPoolName}/virtual-machines/${job.vmName}.block`;
+  const snapshotDevice = `/dev/zvol/${zvol}@snapshot-${snapshotName}`;
+
+  try {
+    setJobRunning(job, 'Creating Incus snapshot');
+    await spawnCommand('incus', ['snapshot', 'create', job.vmName, snapshotName], { log: (line) => appendJobLog(job, line) });
+
+    setJobStep(job, 'Making ZFS snapshot device visible');
+    await spawnCommand('zfs', ['set', 'snapdev=visible', zvol], { log: (line) => appendJobLog(job, line) });
+
+    setJobStep(job, 'Waiting for snapshot device');
+    await new Promise((resolve) => setTimeout(resolve, 2000));
+
+    setJobStep(job, 'Streaming block device to Restic');
+    await spawnCommand('restic', ['backup', '--stdin', '--stdin-filename', `${job.vmName}.raw`, '--tag', job.vmName], {
+      env: { ...process.env, ...config.resticEnv },
+      input: await deviceReadStream(snapshotDevice),
+      log: (line) => appendJobLog(job, line),
+    });
+
+    setJobStep(job, 'Hiding ZFS snapshot device');
+    await spawnCommand('zfs', ['set', 'snapdev=hidden', zvol], { log: (line) => appendJobLog(job, line) });
+
+    setJobStep(job, 'Deleting temporary Incus snapshot');
+    await spawnCommand('incus', ['snapshot', 'delete', job.vmName, snapshotName], { log: (line) => appendJobLog(job, line) });
+
+    setJobStep(job, 'Applying Restic retention policy');
+    await runRestic(['forget', '--tag', job.vmName, '--keep-daily', '7', '--prune'], {
+      log: (line) => appendJobLog(job, line),
+    });
+
+    finishJob(job, 'success');
+  } catch (error) {
+    appendJobLog(job, error.message);
+    await cleanupBackup(job, zvol, snapshotName);
+    finishJob(job, 'failed', error);
+  }
+}
+
+async function deviceReadStream(path) {
+  const { createReadStream } = await import('node:fs');
+  return createReadStream(path);
+}
+
+async function cleanupBackup(job, zvol, snapshotName) {
+  setJobStep(job, 'Running cleanup');
+  await spawnCommand('zfs', ['set', 'snapdev=hidden', zvol], {
+    ignoreExitCode: true,
+    log: (line) => appendJobLog(job, line),
+  }).catch((error) => appendJobLog(job, error.message));
+  await spawnCommand('incus', ['snapshot', 'delete', job.vmName, snapshotName], {
+    ignoreExitCode: true,
+    log: (line) => appendJobLog(job, line),
+  }).catch((error) => appendJobLog(job, error.message));
+}

backend/src/routes/health.js

@@ -0,0 +1,41 @@
+import { Router } from 'express';
+import { missingEnvVars } from '../config.js';
+import { spawnCommand, runRestic } from '../executor.js';
+
+export const healthRouter = Router();
+
+healthRouter.get('/', async (_req, res) => {
+  const missing = missingEnvVars();
+  const checks = {
+    config: missing.length ? `missing: ${missing.join(', ')}` : 'ok',
+    incus: 'pending',
+    zfs: 'pending',
+    restic: 'pending',
+  };
+
+  await Promise.all([
+    checkCommand('incus', ['version']).then((value) => {
+      checks.incus = value;
+    }),
+    checkCommand('zfs', ['version']).then((value) => {
+      checks.zfs = value;
+    }),
+    runRestic(['snapshots', '--json'], { ignoreExitCode: true }).then((result) => {
+      checks.restic = result.exitCode === 0 ? 'ok' : result.stderr.trim() || 'failed';
+    }).catch((error) => {
+      checks.restic = error.message;
+    }),
+  ]);
+
+  const ok = Object.values(checks).every((value) => value === 'ok');
+  res.status(ok ? 200 : 503).json({ ok, checks });
+});
+
+async function checkCommand(command, args) {
+  try {
+    const result = await spawnCommand(command, args, { ignoreExitCode: true });
+    return result.exitCode === 0 ? 'ok' : result.stderr.trim() || 'failed';
+  } catch (error) {
+    return error.message;
+  }
+}

backend/src/routes/jobs.js

@@ -0,0 +1,17 @@
+import { Router } from 'express';
+import { getJob, listJobs } from '../jobs.js';
+
+export const jobsRouter = Router();
+
+jobsRouter.get('/', (_req, res) => {
+  res.json(listJobs());
+});
+
+jobsRouter.get('/:jobId', (req, res) => {
+  const job = getJob(req.params.jobId);
+  if (!job) {
+    res.status(404).json({ error: 'Job not found.' });
+    return;
+  }
+  res.json(job);
+});

backend/src/routes/restore.js

@@ -0,0 +1,77 @@
+import { Router } from 'express';
+import { config } from '../config.js';
+import { spawnCommand, streamResticDumpToDd } from '../executor.js';
+import { appendJobLog, createJob, finishJob, setJobRunning, setJobStep } from '../jobs.js';
+import { validateSnapshotForVm, validateVmExists } from '../validators.js';
+
+export const restoreRouter = Router();
+
+restoreRouter.post('/:vmName', async (req, res, next) => {
+  try {
+    const vmName = req.params.vmName;
+    const { snapshotId, confirmVmName } = req.body || {};
+    if (confirmVmName !== vmName) {
+      const error = new Error('Restore confirmation does not match VM name.');
+      error.status = 400;
+      throw error;
+    }
+
+    await validateVmExists(vmName);
+    const snapshot = await validateSnapshotForVm(vmName, snapshotId);
+    const job = createJob('restore', vmName);
+    runRestoreJob(job, snapshot.id).catch(() => {});
+    res.status(202).json({ jobId: job.id, message: 'Restore job started.' });
+  } catch (error) {
+    next(error);
+  }
+});
+
+async function runRestoreJob(job, snapshotId) {
+  const zvol = `${config.zfsPoolName}/virtual-machines/${job.vmName}.block`;
+  const device = `/dev/zvol/${zvol}`;
+  let volmodeDev = false;
+  let diskWriteOk = false;
+
+  try {
+    setJobRunning(job, 'Stopping VM');
+    await spawnCommand('incus', ['stop', job.vmName, '--force'], {
+      ignoreExitCode: true,
+      log: (line) => appendJobLog(job, line),
+    });
+
+    setJobStep(job, 'Setting ZFS volume to device mode');
+    await spawnCommand('zfs', ['set', 'volmode=dev', zvol], { log: (line) => appendJobLog(job, line) });
+    volmodeDev = true;
+
+    setJobStep(job, 'Settling device nodes');
+    await spawnCommand('udevadm', ['trigger'], { log: (line) => appendJobLog(job, line) });
+    await spawnCommand('udevadm', ['settle'], { log: (line) => appendJobLog(job, line) });
+    await new Promise((resolve) => setTimeout(resolve, 2000));
+
+    setJobStep(job, 'Writing Restic snapshot to block device');
+    await streamResticDumpToDd(snapshotId, `${job.vmName}.raw`, device, (line) => appendJobLog(job, line));
+    diskWriteOk = true;
+
+    setJobStep(job, 'Restoring ZFS volume mode');
+    await spawnCommand('zfs', ['set', 'volmode=none', zvol], { log: (line) => appendJobLog(job, line) });
+    volmodeDev = false;
+
+    setJobStep(job, 'Starting VM');
+    await spawnCommand('incus', ['start', job.vmName], { log: (line) => appendJobLog(job, line) });
+
+    finishJob(job, 'success');
+  } catch (error) {
+    appendJobLog(job, error.message);
+    if (volmodeDev) {
+      setJobStep(job, 'Restoring ZFS volume mode after failure');
+      await spawnCommand('zfs', ['set', 'volmode=none', zvol], {
+        ignoreExitCode: true,
+        log: (line) => appendJobLog(job, line),
+      }).catch((cleanupError) => appendJobLog(job, cleanupError.message));
+    }
+    if (!diskWriteOk) {
+      appendJobLog(job, 'VM was not restarted because disk restore did not complete successfully.');
+    }
+    finishJob(job, 'failed', error);
+  }
+}

backend/src/routes/settings.js

@@ -0,0 +1,23 @@
+import { Router } from 'express';
+import { readEnvSettings, writeEnvSettings } from '../config.js';
+
+export const settingsRouter = Router();
+
+settingsRouter.get('/', async (_req, res, next) => {
+  try {
+    res.json({ fields: await readEnvSettings() });
+  } catch (error) {
+    next(error);
+  }
+});
+
+settingsRouter.put('/', async (req, res, next) => {
+  try {
+    res.json({
+      fields: await writeEnvSettings(req.body?.values || {}),
+      message: 'Settings saved.',
+    });
+  } catch (error) {
+    next(error);
+  }
+});

backend/src/routes/snapshots.js

@@ -0,0 +1,12 @@
+import { Router } from 'express';
+import { listSnapshotsForVm } from '../validators.js';
+
+export const snapshotsRouter = Router();
+
+snapshotsRouter.get('/:vmName', async (req, res, next) => {
+  try {
+    res.json(await listSnapshotsForVm(req.params.vmName));
+  } catch (error) {
+    next(error);
+  }
+});

backend/src/routes/vms.js

@@ -0,0 +1,32 @@
+import { Router } from 'express';
+import { activeJobForVm, latestJobForVm } from '../jobs.js';
+import { listIncusVms } from '../validators.js';
+
+export const vmsRouter = Router();
+
+vmsRouter.get('/', async (_req, res, next) => {
+  try {
+    const vms = await listIncusVms();
+    res.json(vms.map((vm) => {
+      const activeJob = activeJobForVm(vm.name);
+      const latestJob = latestJobForVm(vm.name);
+      return {
+        name: vm.name,
+        status: vm.status,
+        activeJob: activeJob ? summarizeJob(activeJob) : null,
+        lastJobStatus: latestJob?.status || null,
+      };
+    }));
+  } catch (error) {
+    next(error);
+  }
+});
+
+function summarizeJob(job) {
+  return {
+    id: job.id,
+    type: job.type,
+    status: job.status,
+    currentStep: job.currentStep,
+  };
+}