incus-backup-ui/backend/src/routes/backup.js

import { Router } from 'express';
import { finished } from 'node:stream/promises';
import { setTimeout as delay } from 'node:timers/promises';
import { config } from '../config.js';
import { createProgressStream, spawnCommand, runRestic } from '../executor.js';
import { appendJobLog, createJob, finishJob, setJobProgress, setJobRunning, setJobStep } from '../jobs.js';
import { validateVmExists } from '../validators.js';

export const backupRouter = Router();

backupRouter.post('/:vmName', async (req, res, next) => {
  try {
    const vmName = req.params.vmName;
    const job = await startBackupForVm(vmName);
    res.status(202).json({ jobId: job.id, message: 'Backup job started.' });
  } catch (error) {
    next(error);
  }
});

export async function startBackupForVm(vmName) {
  await validateVmExists(vmName);
  const job = createJob('backup', vmName);
  runBackupJob(job).catch(() => {});
  return job;
}

export async function runBackupJob(job) {
  const timestamp = new Date().toISOString().replace(/[-:.TZ]/g, '').slice(0, 14);
  const snapshotName = `s3-backup-${timestamp}`;
  const zvol = `${config.zfsPoolName}/virtual-machines/${job.vmName}.block`;
  const snapshotDevice = `/dev/zvol/${zvol}@snapshot-${snapshotName}`;

  try {
    const totalBytes = await zfsVolumeSize(zvol);

    setJobRunning(job, 'Creating Incus snapshot');
    setJobProgress(job, { percent: 2 });
    await spawnCommand('incus', ['snapshot', 'create', job.vmName, snapshotName], { log: (line) => appendJobLog(job, line) });

    setJobStep(job, 'Making ZFS snapshot device visible');
    setJobProgress(job, { percent: 5 });
    await spawnCommand('zfs', ['set', 'snapdev=visible', zvol], { log: (line) => appendJobLog(job, line) });

    setJobStep(job, 'Waiting for snapshot device');
    setJobProgress(job, { percent: 8 });
    await new Promise((resolve) => setTimeout(resolve, 2000));

    setJobStep(job, 'Streaming block device to Restic');
    const snapshotStream = await deviceReadStream(snapshotDevice);
    const progressStream = createProgressStream(totalBytes, ({ currentBytes, totalBytes: bytesTotal, percent }) => {
      setJobProgress(job, {
        currentBytes,
        totalBytes: bytesTotal,
        percent: 10 + percent * 0.78,
        detail: `Streaming ${formatBytes(currentBytes)} of ${formatBytes(bytesTotal)}`,
      });
    });
    try {
      await spawnCommand('restic', ['backup', '--stdin', '--stdin-filename', `${job.vmName}.raw`, '--tag', job.vmName], {
        env: { ...process.env, ...config.resticEnv },
        input: snapshotStream.pipe(progressStream),
        log: (line) => appendJobLog(job, line),
      });
    } finally {
      if (!snapshotStream.destroyed) {
        snapshotStream.destroy();
      }
      await finished(snapshotStream, { cleanup: true }).catch(() => {});
    }

    setJobStep(job, 'Hiding ZFS snapshot device');
    setJobProgress(job, { percent: 90 });
    await spawnCommand('zfs', ['set', 'snapdev=hidden', zvol], { log: (line) => appendJobLog(job, line) });
    await settleUdev(job);

    setJobStep(job, 'Deleting temporary Incus snapshot');
    setJobProgress(job, { percent: 93 });
    await deleteIncusSnapshotWithRetry(job, snapshotName);

    setJobStep(job, 'Applying Restic retention policy');
    setJobProgress(job, { percent: 96 });
    await runRestic(['forget', '--tag', job.vmName, '--keep-daily', '7', '--prune'], {
      log: (line) => appendJobLog(job, line),
    });

    finishJob(job, 'success');
  } catch (error) {
    appendJobLog(job, error.message);
    await cleanupBackup(job, zvol, snapshotName);
    finishJob(job, 'failed', error);
  }
}

async function deviceReadStream(path) {
  const { createReadStream } = await import('node:fs');
  return createReadStream(path);
}

async function zfsVolumeSize(zvol) {
  const result = await spawnCommand('zfs', ['get', '-Hp', '-o', 'value', 'volsize', zvol]);
  const size = Number(result.stdout.trim());
  return Number.isFinite(size) && size > 0 ? size : null;
}

function formatBytes(value) {
  if (!value) return '-';
  const units = ['B', 'KiB', 'MiB', 'GiB', 'TiB'];
  let size = Number(value);
  let unitIndex = 0;
  while (size >= 1024 && unitIndex < units.length - 1) {
    size /= 1024;
    unitIndex += 1;
  }
  return `${size.toFixed(unitIndex === 0 ? 0 : 1)} ${units[unitIndex]}`;
}

async function cleanupBackup(job, zvol, snapshotName) {
  setJobStep(job, 'Running cleanup');
  await spawnCommand('zfs', ['set', 'snapdev=hidden', zvol], {
    ignoreExitCode: true,
    log: (line) => appendJobLog(job, line),
  }).catch((error) => appendJobLog(job, error.message));
  await settleUdev(job);
  await deleteIncusSnapshotWithRetry(job, snapshotName, { ignoreExitCode: true });
}

async function settleUdev(job) {
  await spawnCommand('udevadm', ['trigger'], {
    ignoreExitCode: true,
    log: (line) => appendJobLog(job, line),
  }).catch((error) => appendJobLog(job, error.message));
  await spawnCommand('udevadm', ['settle'], {
    ignoreExitCode: true,
    log: (line) => appendJobLog(job, line),
  }).catch((error) => appendJobLog(job, error.message));
}

async function deleteIncusSnapshotWithRetry(job, snapshotName, options = {}) {
  const maxAttempts = 5;
  for (let attempt = 1; attempt <= maxAttempts; attempt += 1) {
    try {
      return await spawnCommand('incus', ['snapshot', 'delete', job.vmName, snapshotName], {
        ignoreExitCode: options.ignoreExitCode,
        log: (line) => appendJobLog(job, line),
      });
    } catch (error) {
      const retryable = error.message.includes('dataset is busy');
      if (!retryable || attempt === maxAttempts) {
        if (options.ignoreExitCode) {
          appendJobLog(job, error.message);
          return null;
        }
        throw error;
      }
      appendJobLog(job, `Snapshot device still busy; retrying delete (${attempt}/${maxAttempts}).`);
      await delay(2000);
    }
  }
  return null;
}