import { Router } from 'express'; import { spawn } from 'node:child_process'; import { mkdtemp, rm, writeFile } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import path from 'node:path'; import { finished } from 'node:stream/promises'; import { setTimeout as delay } from 'node:timers/promises'; import { config } from '../config.js'; import { createProgressStream, resticSnapshotFileSize, spawnCommand, runRestic } from '../executor.js'; import { appendJobLog, createJob, finishJob, setJobProgress, setJobRunning, setJobStep } from '../jobs.js'; import { validateVmExists } from '../validators.js'; export const backupRouter = Router(); backupRouter.post('/:vmName', async (req, res, next) => { try { const vmName = req.params.vmName; const job = await startBackupForVm(vmName); res.status(202).json({ jobId: job.id, message: 'Backup job started.' }); } catch (error) { next(error); } }); export async function startBackupForVm(vmName) { const instance = await validateVmExists(vmName); const job = createJob('backup', vmName); job.instanceType = instance.type; runBackupJob(job, instance).catch(() => {}); return job; } export async function runBackupJob(job, instance = null) { const currentInstance = instance || await validateVmExists(job.vmName); if (currentInstance.type === 'container') { await runContainerBackupJob(job); return; } const timestamp = new Date().toISOString().replace(/[-:.TZ]/g, '').slice(0, 14); const snapshotName = `s3-backup-${timestamp}`; const zvol = `${config.zfsPoolName}/virtual-machines/${job.vmName}.block`; const snapshotDevice = `/dev/zvol/${zvol}@snapshot-${snapshotName}`; try { const totalBytes = await zfsVolumeSize(zvol); setJobRunning(job, 'Creating Incus snapshot'); setJobProgress(job, { percent: 2 }); await spawnCommand('incus', ['snapshot', 'create', job.vmName, snapshotName], { log: (line) => appendJobLog(job, line) }); setJobStep(job, 'Making ZFS snapshot device visible'); setJobProgress(job, { percent: 5 }); await spawnCommand('zfs', ['set', 'snapdev=visible', zvol], { log: (line) => appendJobLog(job, line) }); setJobStep(job, 'Waiting for snapshot device'); setJobProgress(job, { percent: 8 }); await new Promise((resolve) => setTimeout(resolve, 2000)); setJobStep(job, 'Streaming block device to Restic'); const snapshotStream = await deviceReadStream(snapshotDevice); let streamedBytes = 0; const progressStream = createProgressStream(totalBytes, ({ currentBytes, totalBytes: bytesTotal, percent }) => { streamedBytes = currentBytes; setJobProgress(job, { currentBytes, totalBytes: bytesTotal, percent: 10 + percent * 0.78, detail: `Streaming ${formatBytes(currentBytes)} of ${formatBytes(bytesTotal)}`, }); }); try { const result = await spawnCommand('restic', ['backup', '--stdin', '--stdin-filename', `${job.vmName}.raw`, '--tag', job.vmName, '--tag', 'data', '--tag', 'virtual-machine'], { env: { ...process.env, ...config.resticEnv }, input: snapshotStream.pipe(progressStream), log: (line) => appendJobLog(job, line), }); const snapshotId = parseResticSnapshotId(result); await verifyResticFileSize(job, snapshotId, `${job.vmName}.raw`, streamedBytes || totalBytes); } finally { if (!snapshotStream.destroyed) { snapshotStream.destroy(); } await finished(snapshotStream, { cleanup: true }).catch(() => {}); } setJobStep(job, 'Hiding ZFS snapshot device'); setJobProgress(job, { percent: 90 }); await spawnCommand('zfs', ['set', 'snapdev=hidden', zvol], { log: (line) => appendJobLog(job, line) }); await settleUdev(job); setJobStep(job, 'Deleting temporary Incus snapshot'); setJobProgress(job, { percent: 93 }); await deleteIncusSnapshotWithRetry(job, snapshotName); setJobStep(job, 'Backing up Incus metadata'); setJobProgress(job, { percent: 95 }); await backupInstanceMetadata(job, 'virtual-machine'); setJobStep(job, 'Applying Restic retention policy'); setJobProgress(job, { percent: 97 }); await runRestic(retentionArgs(job.vmName), { log: (line) => appendJobLog(job, line), }); finishJob(job, 'success'); } catch (error) { appendJobLog(job, error.message); await cleanupBackup(job, zvol, snapshotName); finishJob(job, 'failed', error); } } async function runContainerBackupJob(job) { const timestamp = new Date().toISOString().replace(/[-:.TZ]/g, '').slice(0, 14); const snapshotName = `s3-backup-${timestamp}`; const dataset = `${config.zfsPoolName}/containers/${job.vmName}`; const snapshot = `${dataset}@snapshot-${snapshotName}`; try { const totalBytes = await zfsDatasetUsed(dataset); setJobRunning(job, 'Creating Incus snapshot'); setJobProgress(job, { percent: 2 }); await spawnCommand('incus', ['snapshot', 'create', job.vmName, snapshotName], { log: (line) => appendJobLog(job, line) }); setJobStep(job, 'Streaming ZFS snapshot to Restic'); const zfs = spawn('zfs', ['send', snapshot], { stdio: ['ignore', 'pipe', 'pipe'] }); const zfsClosed = waitForProcess(zfs, 'zfs send'); zfs.stderr.on('data', (chunk) => appendJobLog(job, chunk.toString().trimEnd())); let streamedBytes = 0; const progressStream = createProgressStream(totalBytes, ({ currentBytes, totalBytes: bytesTotal, percent }) => { streamedBytes = currentBytes; setJobProgress(job, { currentBytes, totalBytes: bytesTotal, percent: 10 + percent * 0.78, detail: `Streaming ${formatBytes(currentBytes)} of ${formatBytes(bytesTotal)}`, }); }); let resticOk = false; try { const result = await spawnCommand('restic', ['backup', '--stdin', '--stdin-filename', `${job.vmName}.zfs`, '--tag', job.vmName, '--tag', 'data', '--tag', 'container'], { env: { ...process.env, ...config.resticEnv }, input: zfs.stdout.pipe(progressStream), log: (line) => appendJobLog(job, line), }); const snapshotId = parseResticSnapshotId(result); await verifyResticFileSize(job, snapshotId, `${job.vmName}.zfs`, streamedBytes || totalBytes); resticOk = true; } finally { if (!resticOk && !zfs.killed) zfs.kill('SIGTERM'); } await zfsClosed; setJobStep(job, 'Deleting temporary Incus snapshot'); setJobProgress(job, { percent: 93 }); await deleteIncusSnapshotWithRetry(job, snapshotName); setJobStep(job, 'Backing up Incus metadata'); setJobProgress(job, { percent: 95 }); await backupInstanceMetadata(job, 'container'); setJobStep(job, 'Applying Restic retention policy'); setJobProgress(job, { percent: 97 }); await runRestic(retentionArgs(job.vmName), { log: (line) => appendJobLog(job, line), }); finishJob(job, 'success'); } catch (error) { appendJobLog(job, error.message); await deleteIncusSnapshotWithRetry(job, snapshotName, { ignoreExitCode: true }); finishJob(job, 'failed', error); } } async function deviceReadStream(path) { const { createReadStream } = await import('node:fs'); return createReadStream(path); } async function zfsVolumeSize(zvol) { const result = await spawnCommand('zfs', ['get', '-Hp', '-o', 'value', 'volsize', zvol]); const size = Number(result.stdout.trim()); return Number.isFinite(size) && size > 0 ? size : null; } async function zfsDatasetUsed(dataset) { const result = await spawnCommand('zfs', ['get', '-Hp', '-o', 'value', 'used', dataset]); const size = Number(result.stdout.trim()); return Number.isFinite(size) && size > 0 ? size : null; } async function backupInstanceMetadata(job, instanceType) { const tempDir = await mkdtemp(path.join(tmpdir(), `incus-backup-${job.vmName}-`)); try { const metadataDir = path.join(tempDir, job.vmName); await writeMetadataFile(metadataDir, 'README.txt', metadataReadme(job.vmName, instanceType)); await writeCommandOutput(metadataDir, 'config.yaml', ['config', 'show', job.vmName]); await writeCommandOutput(metadataDir, 'config-expanded.yaml', ['config', 'show', job.vmName, '--expanded']); await writeCommandOutput(metadataDir, 'info.txt', ['info', job.vmName]); await writeCommandOutput(metadataDir, 'snapshots.json', ['snapshot', 'list', job.vmName, '--format', 'json']); await runRestic(['backup', metadataDir, '--tag', job.vmName, '--tag', 'metadata', '--tag', instanceType], { log: (line) => appendJobLog(job, line), }); } finally { await rm(tempDir, { recursive: true, force: true }); } } async function writeCommandOutput(directory, filename, args) { const result = await spawnCommand('incus', args); await writeMetadataFile(directory, filename, result.stdout); } async function writeMetadataFile(directory, filename, content) { const { mkdir } = await import('node:fs/promises'); await mkdir(directory, { recursive: true }); await writeFile(path.join(directory, filename), content || '', { mode: 0o600 }); } function metadataReadme(instanceName, instanceType) { return [ `Instance: ${instanceName}`, `Type: ${instanceType}`, `Created: ${new Date().toISOString()}`, '', 'This directory contains Incus metadata captured alongside the disk/dataset backup.', 'It is intended for disaster-recovery reconstruction and restore preflight workflows.', '', ].join('\n'); } function waitForProcess(child, label) { return new Promise((resolve, reject) => { child.on('error', reject); child.on('close', (exitCode) => { if (exitCode !== 0 && exitCode !== null) { reject(new Error(`${label} exited with ${exitCode}`)); return; } resolve(); }); }); } function formatBytes(value) { if (!value) return '-'; const units = ['B', 'KiB', 'MiB', 'GiB', 'TiB']; let size = Number(value); let unitIndex = 0; while (size >= 1024 && unitIndex < units.length - 1) { size /= 1024; unitIndex += 1; } return `${size.toFixed(unitIndex === 0 ? 0 : 1)} ${units[unitIndex]}`; } function retentionArgs(vmName) { const args = ['forget', '--tag', vmName, '--prune']; const keepHourly = positiveInteger(config.retention.keepHourly); const keepDaily = positiveInteger(config.retention.keepDaily); const keepWeekly = positiveInteger(config.retention.keepWeekly); const keepMonthly = positiveInteger(config.retention.keepMonthly); if (keepHourly) args.push('--keep-hourly', String(keepHourly)); if (keepDaily) args.push('--keep-daily', String(keepDaily)); if (keepWeekly) args.push('--keep-weekly', String(keepWeekly)); if (keepMonthly) args.push('--keep-monthly', String(keepMonthly)); if (!keepHourly && !keepDaily && !keepWeekly && !keepMonthly) args.push('--keep-daily', '7'); return args; } function positiveInteger(value) { const number = Number(value); return Number.isFinite(number) && number > 0 ? Math.floor(number) : 0; } function parseResticSnapshotId(result) { const output = `${result.stdout || ''}\n${result.stderr || ''}`; const match = output.match(/snapshot\s+([0-9a-f]{8,64})\s+saved/i); if (!match) { throw new Error('Restic backup completed but no snapshot ID could be parsed.'); } return match[1]; } async function verifyResticFileSize(job, snapshotId, filename, expectedBytes) { if (!expectedBytes) { appendJobLog(job, `Skipping size verification for ${snapshotId}: expected size is unknown.`); return; } try { const storedBytes = await resticSnapshotFileSize(snapshotId, filename); if (storedBytes !== expectedBytes) { throw new Error(`Backup verification failed for ${snapshotId}: stored ${formatBytes(storedBytes)}, expected ${formatBytes(expectedBytes)}.`); } appendJobLog(job, `Verified Restic snapshot ${snapshotId}: ${formatBytes(storedBytes)}.`); } catch (error) { appendJobLog(job, error.message); await forgetFailedSnapshot(job, snapshotId); throw error; } } async function forgetFailedSnapshot(job, snapshotId) { appendJobLog(job, `Removing failed Restic snapshot ${snapshotId}.`); await runRestic(['forget', snapshotId, '--prune'], { log: (line) => appendJobLog(job, line), }).catch((error) => appendJobLog(job, `Failed to remove Restic snapshot ${snapshotId}: ${error.message}`)); } async function cleanupBackup(job, zvol, snapshotName) { setJobStep(job, 'Running cleanup'); await spawnCommand('zfs', ['set', 'snapdev=hidden', zvol], { ignoreExitCode: true, log: (line) => appendJobLog(job, line), }).catch((error) => appendJobLog(job, error.message)); await settleUdev(job); await deleteIncusSnapshotWithRetry(job, snapshotName, { ignoreExitCode: true }); } async function settleUdev(job) { await spawnCommand('udevadm', ['trigger'], { ignoreExitCode: true, log: (line) => appendJobLog(job, line), }).catch((error) => appendJobLog(job, error.message)); await spawnCommand('udevadm', ['settle'], { ignoreExitCode: true, log: (line) => appendJobLog(job, line), }).catch((error) => appendJobLog(job, error.message)); } async function deleteIncusSnapshotWithRetry(job, snapshotName, options = {}) { const maxAttempts = 5; for (let attempt = 1; attempt <= maxAttempts; attempt += 1) { try { return await spawnCommand('incus', ['snapshot', 'delete', job.vmName, snapshotName], { ignoreExitCode: options.ignoreExitCode, log: (line) => appendJobLog(job, line), }); } catch (error) { const retryable = error.message.includes('dataset is busy'); if (!retryable || attempt === maxAttempts) { if (options.ignoreExitCode) { appendJobLog(job, error.message); return null; } throw error; } appendJobLog(job, `Snapshot device still busy; retrying delete (${attempt}/${maxAttempts}).`); await delay(2000); } } return null; }