359 lines
14 KiB
JavaScript
359 lines
14 KiB
JavaScript
import { Router } from 'express';
|
|
import { spawn } from 'node:child_process';
|
|
import { mkdtemp, rm, writeFile } from 'node:fs/promises';
|
|
import { tmpdir } from 'node:os';
|
|
import path from 'node:path';
|
|
import { finished } from 'node:stream/promises';
|
|
import { setTimeout as delay } from 'node:timers/promises';
|
|
import { config } from '../config.js';
|
|
import { createProgressStream, resticSnapshotFileSize, spawnCommand, runRestic } from '../executor.js';
|
|
import { appendJobLog, createJob, finishJob, setJobProgress, setJobRunning, setJobStep } from '../jobs.js';
|
|
import { validateVmExists } from '../validators.js';
|
|
|
|
export const backupRouter = Router();
|
|
|
|
backupRouter.post('/:vmName', async (req, res, next) => {
|
|
try {
|
|
const vmName = req.params.vmName;
|
|
const job = await startBackupForVm(vmName);
|
|
res.status(202).json({ jobId: job.id, message: 'Backup job started.' });
|
|
} catch (error) {
|
|
next(error);
|
|
}
|
|
});
|
|
|
|
export async function startBackupForVm(vmName) {
|
|
const instance = await validateVmExists(vmName);
|
|
const job = createJob('backup', vmName);
|
|
job.instanceType = instance.type;
|
|
runBackupJob(job, instance).catch(() => {});
|
|
return job;
|
|
}
|
|
|
|
export async function runBackupJob(job, instance = null) {
|
|
const currentInstance = instance || await validateVmExists(job.vmName);
|
|
if (currentInstance.type === 'container') {
|
|
await runContainerBackupJob(job);
|
|
return;
|
|
}
|
|
|
|
const timestamp = new Date().toISOString().replace(/[-:.TZ]/g, '').slice(0, 14);
|
|
const snapshotName = `s3-backup-${timestamp}`;
|
|
const zvol = `${config.zfsPoolName}/virtual-machines/${job.vmName}.block`;
|
|
const snapshotDevice = `/dev/zvol/${zvol}@snapshot-${snapshotName}`;
|
|
|
|
try {
|
|
const totalBytes = await zfsVolumeSize(zvol);
|
|
|
|
setJobRunning(job, 'Creating Incus snapshot');
|
|
setJobProgress(job, { percent: 2 });
|
|
await spawnCommand('incus', ['snapshot', 'create', job.vmName, snapshotName], { log: (line) => appendJobLog(job, line) });
|
|
|
|
setJobStep(job, 'Making ZFS snapshot device visible');
|
|
setJobProgress(job, { percent: 5 });
|
|
await spawnCommand('zfs', ['set', 'snapdev=visible', zvol], { log: (line) => appendJobLog(job, line) });
|
|
|
|
setJobStep(job, 'Waiting for snapshot device');
|
|
setJobProgress(job, { percent: 8 });
|
|
await new Promise((resolve) => setTimeout(resolve, 2000));
|
|
|
|
setJobStep(job, 'Streaming block device to Restic');
|
|
const snapshotStream = await deviceReadStream(snapshotDevice);
|
|
let streamedBytes = 0;
|
|
const progressStream = createProgressStream(totalBytes, ({ currentBytes, totalBytes: bytesTotal, percent }) => {
|
|
streamedBytes = currentBytes;
|
|
setJobProgress(job, {
|
|
currentBytes,
|
|
totalBytes: bytesTotal,
|
|
percent: 10 + percent * 0.78,
|
|
detail: `Streaming ${formatBytes(currentBytes)} of ${formatBytes(bytesTotal)}`,
|
|
});
|
|
});
|
|
try {
|
|
const result = await spawnCommand('restic', ['backup', '--stdin', '--stdin-filename', `${job.vmName}.raw`, '--tag', job.vmName, '--tag', 'data', '--tag', 'virtual-machine'], {
|
|
env: { ...process.env, ...config.resticEnv },
|
|
input: snapshotStream.pipe(progressStream),
|
|
log: (line) => appendJobLog(job, line),
|
|
});
|
|
const snapshotId = parseResticSnapshotId(result);
|
|
await verifyResticFileSize(job, snapshotId, `${job.vmName}.raw`, streamedBytes || totalBytes);
|
|
} finally {
|
|
if (!snapshotStream.destroyed) {
|
|
snapshotStream.destroy();
|
|
}
|
|
await finished(snapshotStream, { cleanup: true }).catch(() => {});
|
|
}
|
|
|
|
setJobStep(job, 'Hiding ZFS snapshot device');
|
|
setJobProgress(job, { percent: 90 });
|
|
await spawnCommand('zfs', ['set', 'snapdev=hidden', zvol], { log: (line) => appendJobLog(job, line) });
|
|
await settleUdev(job);
|
|
|
|
setJobStep(job, 'Deleting temporary Incus snapshot');
|
|
setJobProgress(job, { percent: 93 });
|
|
await deleteIncusSnapshotWithRetry(job, snapshotName);
|
|
|
|
setJobStep(job, 'Backing up Incus metadata');
|
|
setJobProgress(job, { percent: 95 });
|
|
await backupInstanceMetadata(job, 'virtual-machine');
|
|
|
|
setJobStep(job, 'Applying Restic retention policy');
|
|
setJobProgress(job, { percent: 97 });
|
|
await runRestic(retentionArgs(job.vmName), {
|
|
log: (line) => appendJobLog(job, line),
|
|
});
|
|
|
|
finishJob(job, 'success');
|
|
} catch (error) {
|
|
appendJobLog(job, error.message);
|
|
await cleanupBackup(job, zvol, snapshotName);
|
|
finishJob(job, 'failed', error);
|
|
}
|
|
}
|
|
|
|
async function runContainerBackupJob(job) {
|
|
const timestamp = new Date().toISOString().replace(/[-:.TZ]/g, '').slice(0, 14);
|
|
const snapshotName = `s3-backup-${timestamp}`;
|
|
const dataset = `${config.zfsPoolName}/containers/${job.vmName}`;
|
|
const snapshot = `${dataset}@snapshot-${snapshotName}`;
|
|
|
|
try {
|
|
const totalBytes = await zfsDatasetUsed(dataset);
|
|
|
|
setJobRunning(job, 'Creating Incus snapshot');
|
|
setJobProgress(job, { percent: 2 });
|
|
await spawnCommand('incus', ['snapshot', 'create', job.vmName, snapshotName], { log: (line) => appendJobLog(job, line) });
|
|
|
|
setJobStep(job, 'Streaming ZFS snapshot to Restic');
|
|
const zfs = spawn('zfs', ['send', snapshot], { stdio: ['ignore', 'pipe', 'pipe'] });
|
|
const zfsClosed = waitForProcess(zfs, 'zfs send');
|
|
zfs.stderr.on('data', (chunk) => appendJobLog(job, chunk.toString().trimEnd()));
|
|
let streamedBytes = 0;
|
|
const progressStream = createProgressStream(totalBytes, ({ currentBytes, totalBytes: bytesTotal, percent }) => {
|
|
streamedBytes = currentBytes;
|
|
setJobProgress(job, {
|
|
currentBytes,
|
|
totalBytes: bytesTotal,
|
|
percent: 10 + percent * 0.78,
|
|
detail: `Streaming ${formatBytes(currentBytes)} of ${formatBytes(bytesTotal)}`,
|
|
});
|
|
});
|
|
let resticOk = false;
|
|
try {
|
|
const result = await spawnCommand('restic', ['backup', '--stdin', '--stdin-filename', `${job.vmName}.zfs`, '--tag', job.vmName, '--tag', 'data', '--tag', 'container'], {
|
|
env: { ...process.env, ...config.resticEnv },
|
|
input: zfs.stdout.pipe(progressStream),
|
|
log: (line) => appendJobLog(job, line),
|
|
});
|
|
const snapshotId = parseResticSnapshotId(result);
|
|
await verifyResticFileSize(job, snapshotId, `${job.vmName}.zfs`, streamedBytes || totalBytes);
|
|
resticOk = true;
|
|
} finally {
|
|
if (!resticOk && !zfs.killed) zfs.kill('SIGTERM');
|
|
}
|
|
await zfsClosed;
|
|
|
|
setJobStep(job, 'Deleting temporary Incus snapshot');
|
|
setJobProgress(job, { percent: 93 });
|
|
await deleteIncusSnapshotWithRetry(job, snapshotName);
|
|
|
|
setJobStep(job, 'Backing up Incus metadata');
|
|
setJobProgress(job, { percent: 95 });
|
|
await backupInstanceMetadata(job, 'container');
|
|
|
|
setJobStep(job, 'Applying Restic retention policy');
|
|
setJobProgress(job, { percent: 97 });
|
|
await runRestic(retentionArgs(job.vmName), {
|
|
log: (line) => appendJobLog(job, line),
|
|
});
|
|
|
|
finishJob(job, 'success');
|
|
} catch (error) {
|
|
appendJobLog(job, error.message);
|
|
await deleteIncusSnapshotWithRetry(job, snapshotName, { ignoreExitCode: true });
|
|
finishJob(job, 'failed', error);
|
|
}
|
|
}
|
|
|
|
async function deviceReadStream(path) {
|
|
const { createReadStream } = await import('node:fs');
|
|
return createReadStream(path);
|
|
}
|
|
|
|
async function zfsVolumeSize(zvol) {
|
|
const result = await spawnCommand('zfs', ['get', '-Hp', '-o', 'value', 'volsize', zvol]);
|
|
const size = Number(result.stdout.trim());
|
|
return Number.isFinite(size) && size > 0 ? size : null;
|
|
}
|
|
|
|
async function zfsDatasetUsed(dataset) {
|
|
const result = await spawnCommand('zfs', ['get', '-Hp', '-o', 'value', 'used', dataset]);
|
|
const size = Number(result.stdout.trim());
|
|
return Number.isFinite(size) && size > 0 ? size : null;
|
|
}
|
|
|
|
async function backupInstanceMetadata(job, instanceType) {
|
|
const tempDir = await mkdtemp(path.join(tmpdir(), `incus-backup-${job.vmName}-`));
|
|
try {
|
|
const metadataDir = path.join(tempDir, job.vmName);
|
|
await writeMetadataFile(metadataDir, 'README.txt', metadataReadme(job.vmName, instanceType));
|
|
await writeCommandOutput(metadataDir, 'config.yaml', ['config', 'show', job.vmName]);
|
|
await writeCommandOutput(metadataDir, 'config-expanded.yaml', ['config', 'show', job.vmName, '--expanded']);
|
|
await writeCommandOutput(metadataDir, 'info.txt', ['info', job.vmName]);
|
|
await writeCommandOutput(metadataDir, 'snapshots.json', ['snapshot', 'list', job.vmName, '--format', 'json']);
|
|
await runRestic(['backup', metadataDir, '--tag', job.vmName, '--tag', 'metadata', '--tag', instanceType], {
|
|
log: (line) => appendJobLog(job, line),
|
|
});
|
|
} finally {
|
|
await rm(tempDir, { recursive: true, force: true });
|
|
}
|
|
}
|
|
|
|
async function writeCommandOutput(directory, filename, args) {
|
|
const result = await spawnCommand('incus', args);
|
|
await writeMetadataFile(directory, filename, result.stdout);
|
|
}
|
|
|
|
async function writeMetadataFile(directory, filename, content) {
|
|
const { mkdir } = await import('node:fs/promises');
|
|
await mkdir(directory, { recursive: true });
|
|
await writeFile(path.join(directory, filename), content || '', { mode: 0o600 });
|
|
}
|
|
|
|
function metadataReadme(instanceName, instanceType) {
|
|
return [
|
|
`Instance: ${instanceName}`,
|
|
`Type: ${instanceType}`,
|
|
`Created: ${new Date().toISOString()}`,
|
|
'',
|
|
'This directory contains Incus metadata captured alongside the disk/dataset backup.',
|
|
'It is intended for disaster-recovery reconstruction and restore preflight workflows.',
|
|
'',
|
|
].join('\n');
|
|
}
|
|
|
|
function waitForProcess(child, label) {
|
|
return new Promise((resolve, reject) => {
|
|
child.on('error', reject);
|
|
child.on('close', (exitCode) => {
|
|
if (exitCode !== 0 && exitCode !== null) {
|
|
reject(new Error(`${label} exited with ${exitCode}`));
|
|
return;
|
|
}
|
|
resolve();
|
|
});
|
|
});
|
|
}
|
|
|
|
function formatBytes(value) {
|
|
if (!value) return '-';
|
|
const units = ['B', 'KiB', 'MiB', 'GiB', 'TiB'];
|
|
let size = Number(value);
|
|
let unitIndex = 0;
|
|
while (size >= 1024 && unitIndex < units.length - 1) {
|
|
size /= 1024;
|
|
unitIndex += 1;
|
|
}
|
|
return `${size.toFixed(unitIndex === 0 ? 0 : 1)} ${units[unitIndex]}`;
|
|
}
|
|
|
|
function retentionArgs(vmName) {
|
|
const args = ['forget', '--tag', vmName, '--prune'];
|
|
const keepHourly = positiveInteger(config.retention.keepHourly);
|
|
const keepDaily = positiveInteger(config.retention.keepDaily);
|
|
const keepWeekly = positiveInteger(config.retention.keepWeekly);
|
|
const keepMonthly = positiveInteger(config.retention.keepMonthly);
|
|
|
|
if (keepHourly) args.push('--keep-hourly', String(keepHourly));
|
|
if (keepDaily) args.push('--keep-daily', String(keepDaily));
|
|
if (keepWeekly) args.push('--keep-weekly', String(keepWeekly));
|
|
if (keepMonthly) args.push('--keep-monthly', String(keepMonthly));
|
|
if (!keepHourly && !keepDaily && !keepWeekly && !keepMonthly) args.push('--keep-daily', '7');
|
|
|
|
return args;
|
|
}
|
|
|
|
function positiveInteger(value) {
|
|
const number = Number(value);
|
|
return Number.isFinite(number) && number > 0 ? Math.floor(number) : 0;
|
|
}
|
|
|
|
function parseResticSnapshotId(result) {
|
|
const output = `${result.stdout || ''}\n${result.stderr || ''}`;
|
|
const match = output.match(/snapshot\s+([0-9a-f]{8,64})\s+saved/i);
|
|
if (!match) {
|
|
throw new Error('Restic backup completed but no snapshot ID could be parsed.');
|
|
}
|
|
return match[1];
|
|
}
|
|
|
|
async function verifyResticFileSize(job, snapshotId, filename, expectedBytes) {
|
|
if (!expectedBytes) {
|
|
appendJobLog(job, `Skipping size verification for ${snapshotId}: expected size is unknown.`);
|
|
return;
|
|
}
|
|
try {
|
|
const storedBytes = await resticSnapshotFileSize(snapshotId, filename);
|
|
if (storedBytes !== expectedBytes) {
|
|
throw new Error(`Backup verification failed for ${snapshotId}: stored ${formatBytes(storedBytes)}, expected ${formatBytes(expectedBytes)}.`);
|
|
}
|
|
appendJobLog(job, `Verified Restic snapshot ${snapshotId}: ${formatBytes(storedBytes)}.`);
|
|
} catch (error) {
|
|
appendJobLog(job, error.message);
|
|
await forgetFailedSnapshot(job, snapshotId);
|
|
throw error;
|
|
}
|
|
}
|
|
|
|
async function forgetFailedSnapshot(job, snapshotId) {
|
|
appendJobLog(job, `Removing failed Restic snapshot ${snapshotId}.`);
|
|
await runRestic(['forget', snapshotId, '--prune'], {
|
|
log: (line) => appendJobLog(job, line),
|
|
}).catch((error) => appendJobLog(job, `Failed to remove Restic snapshot ${snapshotId}: ${error.message}`));
|
|
}
|
|
|
|
async function cleanupBackup(job, zvol, snapshotName) {
|
|
setJobStep(job, 'Running cleanup');
|
|
await spawnCommand('zfs', ['set', 'snapdev=hidden', zvol], {
|
|
ignoreExitCode: true,
|
|
log: (line) => appendJobLog(job, line),
|
|
}).catch((error) => appendJobLog(job, error.message));
|
|
await settleUdev(job);
|
|
await deleteIncusSnapshotWithRetry(job, snapshotName, { ignoreExitCode: true });
|
|
}
|
|
|
|
async function settleUdev(job) {
|
|
await spawnCommand('udevadm', ['trigger'], {
|
|
ignoreExitCode: true,
|
|
log: (line) => appendJobLog(job, line),
|
|
}).catch((error) => appendJobLog(job, error.message));
|
|
await spawnCommand('udevadm', ['settle'], {
|
|
ignoreExitCode: true,
|
|
log: (line) => appendJobLog(job, line),
|
|
}).catch((error) => appendJobLog(job, error.message));
|
|
}
|
|
|
|
async function deleteIncusSnapshotWithRetry(job, snapshotName, options = {}) {
|
|
const maxAttempts = 5;
|
|
for (let attempt = 1; attempt <= maxAttempts; attempt += 1) {
|
|
try {
|
|
return await spawnCommand('incus', ['snapshot', 'delete', job.vmName, snapshotName], {
|
|
ignoreExitCode: options.ignoreExitCode,
|
|
log: (line) => appendJobLog(job, line),
|
|
});
|
|
} catch (error) {
|
|
const retryable = error.message.includes('dataset is busy');
|
|
if (!retryable || attempt === maxAttempts) {
|
|
if (options.ignoreExitCode) {
|
|
appendJobLog(job, error.message);
|
|
return null;
|
|
}
|
|
throw error;
|
|
}
|
|
appendJobLog(job, `Snapshot device still busy; retrying delete (${attempt}/${maxAttempts}).`);
|
|
await delay(2000);
|
|
}
|
|
}
|
|
return null;
|
|
}
|