import { Router } from 'express'; import { finished } from 'node:stream/promises'; import { setTimeout as delay } from 'node:timers/promises'; import { config } from '../config.js'; import { createProgressStream, spawnCommand, runRestic } from '../executor.js'; import { appendJobLog, createJob, finishJob, setJobProgress, setJobRunning, setJobStep } from '../jobs.js'; import { validateVmExists } from '../validators.js'; export const backupRouter = Router(); backupRouter.post('/:vmName', async (req, res, next) => { try { const vmName = req.params.vmName; const job = await startBackupForVm(vmName); res.status(202).json({ jobId: job.id, message: 'Backup job started.' }); } catch (error) { next(error); } }); export async function startBackupForVm(vmName) { await validateVmExists(vmName); const job = createJob('backup', vmName); runBackupJob(job).catch(() => {}); return job; } export async function runBackupJob(job) { const timestamp = new Date().toISOString().replace(/[-:.TZ]/g, '').slice(0, 14); const snapshotName = `s3-backup-${timestamp}`; const zvol = `${config.zfsPoolName}/virtual-machines/${job.vmName}.block`; const snapshotDevice = `/dev/zvol/${zvol}@snapshot-${snapshotName}`; try { const totalBytes = await zfsVolumeSize(zvol); setJobRunning(job, 'Creating Incus snapshot'); setJobProgress(job, { percent: 2 }); await spawnCommand('incus', ['snapshot', 'create', job.vmName, snapshotName], { log: (line) => appendJobLog(job, line) }); setJobStep(job, 'Making ZFS snapshot device visible'); setJobProgress(job, { percent: 5 }); await spawnCommand('zfs', ['set', 'snapdev=visible', zvol], { log: (line) => appendJobLog(job, line) }); setJobStep(job, 'Waiting for snapshot device'); setJobProgress(job, { percent: 8 }); await new Promise((resolve) => setTimeout(resolve, 2000)); setJobStep(job, 'Streaming block device to Restic'); const snapshotStream = await deviceReadStream(snapshotDevice); const progressStream = createProgressStream(totalBytes, ({ currentBytes, totalBytes: bytesTotal, percent }) => { setJobProgress(job, { currentBytes, totalBytes: bytesTotal, percent: 10 + percent * 0.78, detail: `Streaming ${formatBytes(currentBytes)} of ${formatBytes(bytesTotal)}`, }); }); try { await spawnCommand('restic', ['backup', '--stdin', '--stdin-filename', `${job.vmName}.raw`, '--tag', job.vmName], { env: { ...process.env, ...config.resticEnv }, input: snapshotStream.pipe(progressStream), log: (line) => appendJobLog(job, line), }); } finally { if (!snapshotStream.destroyed) { snapshotStream.destroy(); } await finished(snapshotStream, { cleanup: true }).catch(() => {}); } setJobStep(job, 'Hiding ZFS snapshot device'); setJobProgress(job, { percent: 90 }); await spawnCommand('zfs', ['set', 'snapdev=hidden', zvol], { log: (line) => appendJobLog(job, line) }); await settleUdev(job); setJobStep(job, 'Deleting temporary Incus snapshot'); setJobProgress(job, { percent: 93 }); await deleteIncusSnapshotWithRetry(job, snapshotName); setJobStep(job, 'Applying Restic retention policy'); setJobProgress(job, { percent: 96 }); await runRestic(['forget', '--tag', job.vmName, '--keep-daily', '7', '--prune'], { log: (line) => appendJobLog(job, line), }); finishJob(job, 'success'); } catch (error) { appendJobLog(job, error.message); await cleanupBackup(job, zvol, snapshotName); finishJob(job, 'failed', error); } } async function deviceReadStream(path) { const { createReadStream } = await import('node:fs'); return createReadStream(path); } async function zfsVolumeSize(zvol) { const result = await spawnCommand('zfs', ['get', '-Hp', '-o', 'value', 'volsize', zvol]); const size = Number(result.stdout.trim()); return Number.isFinite(size) && size > 0 ? size : null; } function formatBytes(value) { if (!value) return '-'; const units = ['B', 'KiB', 'MiB', 'GiB', 'TiB']; let size = Number(value); let unitIndex = 0; while (size >= 1024 && unitIndex < units.length - 1) { size /= 1024; unitIndex += 1; } return `${size.toFixed(unitIndex === 0 ? 0 : 1)} ${units[unitIndex]}`; } async function cleanupBackup(job, zvol, snapshotName) { setJobStep(job, 'Running cleanup'); await spawnCommand('zfs', ['set', 'snapdev=hidden', zvol], { ignoreExitCode: true, log: (line) => appendJobLog(job, line), }).catch((error) => appendJobLog(job, error.message)); await settleUdev(job); await deleteIncusSnapshotWithRetry(job, snapshotName, { ignoreExitCode: true }); } async function settleUdev(job) { await spawnCommand('udevadm', ['trigger'], { ignoreExitCode: true, log: (line) => appendJobLog(job, line), }).catch((error) => appendJobLog(job, error.message)); await spawnCommand('udevadm', ['settle'], { ignoreExitCode: true, log: (line) => appendJobLog(job, line), }).catch((error) => appendJobLog(job, error.message)); } async function deleteIncusSnapshotWithRetry(job, snapshotName, options = {}) { const maxAttempts = 5; for (let attempt = 1; attempt <= maxAttempts; attempt += 1) { try { return await spawnCommand('incus', ['snapshot', 'delete', job.vmName, snapshotName], { ignoreExitCode: options.ignoreExitCode, log: (line) => appendJobLog(job, line), }); } catch (error) { const retryable = error.message.includes('dataset is busy'); if (!retryable || attempt === maxAttempts) { if (options.ignoreExitCode) { appendJobLog(job, error.message); return null; } throw error; } appendJobLog(job, `Snapshot device still busy; retrying delete (${attempt}/${maxAttempts}).`); await delay(2000); } } return null; }