added install script for agent
changed backend to agent
This commit is contained in:
Philipp committed
1 parent
7f2785fb05
commit
0b059aec1d
669 files changed
+767
-70582
No files matched your search
@@ -0,0 +1,358 @@
|
||||
import { Router } from 'express';
|
||||
import { spawn } from 'node:child_process';
|
||||
import { mkdtemp, rm, writeFile } from 'node:fs/promises';
|
||||
import { tmpdir } from 'node:os';
|
||||
import path from 'node:path';
|
||||
import { finished } from 'node:stream/promises';
|
||||
import { setTimeout as delay } from 'node:timers/promises';
|
||||
import { config } from '../config.js';
|
||||
import { createProgressStream, resticSnapshotFileSize, spawnCommand, runRestic } from '../executor.js';
|
||||
import { appendJobLog, createJob, finishJob, setJobProgress, setJobRunning, setJobStep } from '../jobs.js';
|
||||
import { validateVmExists } from '../validators.js';
|
||||
|
||||
export const backupRouter = Router();
|
||||
|
||||
backupRouter.post('/:vmName', async (req, res, next) => {
|
||||
try {
|
||||
const vmName = req.params.vmName;
|
||||
const job = await startBackupForVm(vmName);
|
||||
res.status(202).json({ jobId: job.id, message: 'Backup job started.' });
|
||||
} catch (error) {
|
||||
next(error);
|
||||
}
|
||||
});
|
||||
|
||||
export async function startBackupForVm(vmName) {
|
||||
const instance = await validateVmExists(vmName);
|
||||
const job = createJob('backup', vmName);
|
||||
job.instanceType = instance.type;
|
||||
runBackupJob(job, instance).catch(() => {});
|
||||
return job;
|
||||
}
|
||||
|
||||
export async function runBackupJob(job, instance = null) {
|
||||
const currentInstance = instance || await validateVmExists(job.vmName);
|
||||
if (currentInstance.type === 'container') {
|
||||
await runContainerBackupJob(job);
|
||||
return;
|
||||
}
|
||||
|
||||
const timestamp = new Date().toISOString().replace(/[-:.TZ]/g, '').slice(0, 14);
|
||||
const snapshotName = `s3-backup-${timestamp}`;
|
||||
const zvol = `${config.zfsPoolName}/virtual-machines/${job.vmName}.block`;
|
||||
const snapshotDevice = `/dev/zvol/${zvol}@snapshot-${snapshotName}`;
|
||||
|
||||
try {
|
||||
const totalBytes = await zfsVolumeSize(zvol);
|
||||
|
||||
setJobRunning(job, 'Creating Incus snapshot');
|
||||
setJobProgress(job, { percent: 2 });
|
||||
await spawnCommand('incus', ['snapshot', 'create', job.vmName, snapshotName], { log: (line) => appendJobLog(job, line) });
|
||||
|
||||
setJobStep(job, 'Making ZFS snapshot device visible');
|
||||
setJobProgress(job, { percent: 5 });
|
||||
await spawnCommand('zfs', ['set', 'snapdev=visible', zvol], { log: (line) => appendJobLog(job, line) });
|
||||
|
||||
setJobStep(job, 'Waiting for snapshot device');
|
||||
setJobProgress(job, { percent: 8 });
|
||||
await new Promise((resolve) => setTimeout(resolve, 2000));
|
||||
|
||||
setJobStep(job, 'Streaming block device to Restic');
|
||||
const snapshotStream = await deviceReadStream(snapshotDevice);
|
||||
let streamedBytes = 0;
|
||||
const progressStream = createProgressStream(totalBytes, ({ currentBytes, totalBytes: bytesTotal, percent }) => {
|
||||
streamedBytes = currentBytes;
|
||||
setJobProgress(job, {
|
||||
currentBytes,
|
||||
totalBytes: bytesTotal,
|
||||
percent: 10 + percent * 0.78,
|
||||
detail: `Streaming ${formatBytes(currentBytes)} of ${formatBytes(bytesTotal)}`,
|
||||
});
|
||||
});
|
||||
try {
|
||||
const result = await spawnCommand('restic', ['backup', '--stdin', '--stdin-filename', `${job.vmName}.raw`, '--tag', job.vmName, '--tag', 'data', '--tag', 'virtual-machine'], {
|
||||
env: { ...process.env, ...config.resticEnv },
|
||||
input: snapshotStream.pipe(progressStream),
|
||||
log: (line) => appendJobLog(job, line),
|
||||
});
|
||||
const snapshotId = parseResticSnapshotId(result);
|
||||
await verifyResticFileSize(job, snapshotId, `${job.vmName}.raw`, streamedBytes || totalBytes);
|
||||
} finally {
|
||||
if (!snapshotStream.destroyed) {
|
||||
snapshotStream.destroy();
|
||||
}
|
||||
await finished(snapshotStream, { cleanup: true }).catch(() => {});
|
||||
}
|
||||
|
||||
setJobStep(job, 'Hiding ZFS snapshot device');
|
||||
setJobProgress(job, { percent: 90 });
|
||||
await spawnCommand('zfs', ['set', 'snapdev=hidden', zvol], { log: (line) => appendJobLog(job, line) });
|
||||
await settleUdev(job);
|
||||
|
||||
setJobStep(job, 'Deleting temporary Incus snapshot');
|
||||
setJobProgress(job, { percent: 93 });
|
||||
await deleteIncusSnapshotWithRetry(job, snapshotName);
|
||||
|
||||
setJobStep(job, 'Backing up Incus metadata');
|
||||
setJobProgress(job, { percent: 95 });
|
||||
await backupInstanceMetadata(job, 'virtual-machine');
|
||||
|
||||
setJobStep(job, 'Applying Restic retention policy');
|
||||
setJobProgress(job, { percent: 97 });
|
||||
await runRestic(retentionArgs(job.vmName), {
|
||||
log: (line) => appendJobLog(job, line),
|
||||
});
|
||||
|
||||
finishJob(job, 'success');
|
||||
} catch (error) {
|
||||
appendJobLog(job, error.message);
|
||||
await cleanupBackup(job, zvol, snapshotName);
|
||||
finishJob(job, 'failed', error);
|
||||
}
|
||||
}
|
||||
|
||||
async function runContainerBackupJob(job) {
|
||||
const timestamp = new Date().toISOString().replace(/[-:.TZ]/g, '').slice(0, 14);
|
||||
const snapshotName = `s3-backup-${timestamp}`;
|
||||
const dataset = `${config.zfsPoolName}/containers/${job.vmName}`;
|
||||
const snapshot = `${dataset}@snapshot-${snapshotName}`;
|
||||
|
||||
try {
|
||||
const totalBytes = await zfsDatasetUsed(dataset);
|
||||
|
||||
setJobRunning(job, 'Creating Incus snapshot');
|
||||
setJobProgress(job, { percent: 2 });
|
||||
await spawnCommand('incus', ['snapshot', 'create', job.vmName, snapshotName], { log: (line) => appendJobLog(job, line) });
|
||||
|
||||
setJobStep(job, 'Streaming ZFS snapshot to Restic');
|
||||
const zfs = spawn('zfs', ['send', snapshot], { stdio: ['ignore', 'pipe', 'pipe'] });
|
||||
const zfsClosed = waitForProcess(zfs, 'zfs send');
|
||||
zfs.stderr.on('data', (chunk) => appendJobLog(job, chunk.toString().trimEnd()));
|
||||
let streamedBytes = 0;
|
||||
const progressStream = createProgressStream(totalBytes, ({ currentBytes, totalBytes: bytesTotal, percent }) => {
|
||||
streamedBytes = currentBytes;
|
||||
setJobProgress(job, {
|
||||
currentBytes,
|
||||
totalBytes: bytesTotal,
|
||||
percent: 10 + percent * 0.78,
|
||||
detail: `Streaming ${formatBytes(currentBytes)} of ${formatBytes(bytesTotal)}`,
|
||||
});
|
||||
});
|
||||
let resticOk = false;
|
||||
try {
|
||||
const result = await spawnCommand('restic', ['backup', '--stdin', '--stdin-filename', `${job.vmName}.zfs`, '--tag', job.vmName, '--tag', 'data', '--tag', 'container'], {
|
||||
env: { ...process.env, ...config.resticEnv },
|
||||
input: zfs.stdout.pipe(progressStream),
|
||||
log: (line) => appendJobLog(job, line),
|
||||
});
|
||||
const snapshotId = parseResticSnapshotId(result);
|
||||
await verifyResticFileSize(job, snapshotId, `${job.vmName}.zfs`, streamedBytes || totalBytes);
|
||||
resticOk = true;
|
||||
} finally {
|
||||
if (!resticOk && !zfs.killed) zfs.kill('SIGTERM');
|
||||
}
|
||||
await zfsClosed;
|
||||
|
||||
setJobStep(job, 'Deleting temporary Incus snapshot');
|
||||
setJobProgress(job, { percent: 93 });
|
||||
await deleteIncusSnapshotWithRetry(job, snapshotName);
|
||||
|
||||
setJobStep(job, 'Backing up Incus metadata');
|
||||
setJobProgress(job, { percent: 95 });
|
||||
await backupInstanceMetadata(job, 'container');
|
||||
|
||||
setJobStep(job, 'Applying Restic retention policy');
|
||||
setJobProgress(job, { percent: 97 });
|
||||
await runRestic(retentionArgs(job.vmName), {
|
||||
log: (line) => appendJobLog(job, line),
|
||||
});
|
||||
|
||||
finishJob(job, 'success');
|
||||
} catch (error) {
|
||||
appendJobLog(job, error.message);
|
||||
await deleteIncusSnapshotWithRetry(job, snapshotName, { ignoreExitCode: true });
|
||||
finishJob(job, 'failed', error);
|
||||
}
|
||||
}
|
||||
|
||||
async function deviceReadStream(path) {
|
||||
const { createReadStream } = await import('node:fs');
|
||||
return createReadStream(path);
|
||||
}
|
||||
|
||||
async function zfsVolumeSize(zvol) {
|
||||
const result = await spawnCommand('zfs', ['get', '-Hp', '-o', 'value', 'volsize', zvol]);
|
||||
const size = Number(result.stdout.trim());
|
||||
return Number.isFinite(size) && size > 0 ? size : null;
|
||||
}
|
||||
|
||||
async function zfsDatasetUsed(dataset) {
|
||||
const result = await spawnCommand('zfs', ['get', '-Hp', '-o', 'value', 'used', dataset]);
|
||||
const size = Number(result.stdout.trim());
|
||||
return Number.isFinite(size) && size > 0 ? size : null;
|
||||
}
|
||||
|
||||
async function backupInstanceMetadata(job, instanceType) {
|
||||
const tempDir = await mkdtemp(path.join(tmpdir(), `incus-backup-${job.vmName}-`));
|
||||
try {
|
||||
const metadataDir = path.join(tempDir, job.vmName);
|
||||
await writeMetadataFile(metadataDir, 'README.txt', metadataReadme(job.vmName, instanceType));
|
||||
await writeCommandOutput(metadataDir, 'config.yaml', ['config', 'show', job.vmName]);
|
||||
await writeCommandOutput(metadataDir, 'config-expanded.yaml', ['config', 'show', job.vmName, '--expanded']);
|
||||
await writeCommandOutput(metadataDir, 'info.txt', ['info', job.vmName]);
|
||||
await writeCommandOutput(metadataDir, 'snapshots.json', ['snapshot', 'list', job.vmName, '--format', 'json']);
|
||||
await runRestic(['backup', metadataDir, '--tag', job.vmName, '--tag', 'metadata', '--tag', instanceType], {
|
||||
log: (line) => appendJobLog(job, line),
|
||||
});
|
||||
} finally {
|
||||
await rm(tempDir, { recursive: true, force: true });
|
||||
}
|
||||
}
|
||||
|
||||
async function writeCommandOutput(directory, filename, args) {
|
||||
const result = await spawnCommand('incus', args);
|
||||
await writeMetadataFile(directory, filename, result.stdout);
|
||||
}
|
||||
|
||||
async function writeMetadataFile(directory, filename, content) {
|
||||
const { mkdir } = await import('node:fs/promises');
|
||||
await mkdir(directory, { recursive: true });
|
||||
await writeFile(path.join(directory, filename), content || '', { mode: 0o600 });
|
||||
}
|
||||
|
||||
function metadataReadme(instanceName, instanceType) {
|
||||
return [
|
||||
`Instance: ${instanceName}`,
|
||||
`Type: ${instanceType}`,
|
||||
`Created: ${new Date().toISOString()}`,
|
||||
'',
|
||||
'This directory contains Incus metadata captured alongside the disk/dataset backup.',
|
||||
'It is intended for disaster-recovery reconstruction and restore preflight workflows.',
|
||||
'',
|
||||
].join('\n');
|
||||
}
|
||||
|
||||
function waitForProcess(child, label) {
|
||||
return new Promise((resolve, reject) => {
|
||||
child.on('error', reject);
|
||||
child.on('close', (exitCode) => {
|
||||
if (exitCode !== 0 && exitCode !== null) {
|
||||
reject(new Error(`${label} exited with ${exitCode}`));
|
||||
return;
|
||||
}
|
||||
resolve();
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
function formatBytes(value) {
|
||||
if (!value) return '-';
|
||||
const units = ['B', 'KiB', 'MiB', 'GiB', 'TiB'];
|
||||
let size = Number(value);
|
||||
let unitIndex = 0;
|
||||
while (size >= 1024 && unitIndex < units.length - 1) {
|
||||
size /= 1024;
|
||||
unitIndex += 1;
|
||||
}
|
||||
return `${size.toFixed(unitIndex === 0 ? 0 : 1)} ${units[unitIndex]}`;
|
||||
}
|
||||
|
||||
function retentionArgs(vmName) {
|
||||
const args = ['forget', '--tag', vmName, '--prune'];
|
||||
const keepHourly = positiveInteger(config.retention.keepHourly);
|
||||
const keepDaily = positiveInteger(config.retention.keepDaily);
|
||||
const keepWeekly = positiveInteger(config.retention.keepWeekly);
|
||||
const keepMonthly = positiveInteger(config.retention.keepMonthly);
|
||||
|
||||
if (keepHourly) args.push('--keep-hourly', String(keepHourly));
|
||||
if (keepDaily) args.push('--keep-daily', String(keepDaily));
|
||||
if (keepWeekly) args.push('--keep-weekly', String(keepWeekly));
|
||||
if (keepMonthly) args.push('--keep-monthly', String(keepMonthly));
|
||||
if (!keepHourly && !keepDaily && !keepWeekly && !keepMonthly) args.push('--keep-daily', '7');
|
||||
|
||||
return args;
|
||||
}
|
||||
|
||||
function positiveInteger(value) {
|
||||
const number = Number(value);
|
||||
return Number.isFinite(number) && number > 0 ? Math.floor(number) : 0;
|
||||
}
|
||||
|
||||
function parseResticSnapshotId(result) {
|
||||
const output = `${result.stdout || ''}\n${result.stderr || ''}`;
|
||||
const match = output.match(/snapshot\s+([0-9a-f]{8,64})\s+saved/i);
|
||||
if (!match) {
|
||||
throw new Error('Restic backup completed but no snapshot ID could be parsed.');
|
||||
}
|
||||
return match[1];
|
||||
}
|
||||
|
||||
async function verifyResticFileSize(job, snapshotId, filename, expectedBytes) {
|
||||
if (!expectedBytes) {
|
||||
appendJobLog(job, `Skipping size verification for ${snapshotId}: expected size is unknown.`);
|
||||
return;
|
||||
}
|
||||
try {
|
||||
const storedBytes = await resticSnapshotFileSize(snapshotId, filename);
|
||||
if (storedBytes !== expectedBytes) {
|
||||
throw new Error(`Backup verification failed for ${snapshotId}: stored ${formatBytes(storedBytes)}, expected ${formatBytes(expectedBytes)}.`);
|
||||
}
|
||||
appendJobLog(job, `Verified Restic snapshot ${snapshotId}: ${formatBytes(storedBytes)}.`);
|
||||
} catch (error) {
|
||||
appendJobLog(job, error.message);
|
||||
await forgetFailedSnapshot(job, snapshotId);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
async function forgetFailedSnapshot(job, snapshotId) {
|
||||
appendJobLog(job, `Removing failed Restic snapshot ${snapshotId}.`);
|
||||
await runRestic(['forget', snapshotId, '--prune'], {
|
||||
log: (line) => appendJobLog(job, line),
|
||||
}).catch((error) => appendJobLog(job, `Failed to remove Restic snapshot ${snapshotId}: ${error.message}`));
|
||||
}
|
||||
|
||||
async function cleanupBackup(job, zvol, snapshotName) {
|
||||
setJobStep(job, 'Running cleanup');
|
||||
await spawnCommand('zfs', ['set', 'snapdev=hidden', zvol], {
|
||||
ignoreExitCode: true,
|
||||
log: (line) => appendJobLog(job, line),
|
||||
}).catch((error) => appendJobLog(job, error.message));
|
||||
await settleUdev(job);
|
||||
await deleteIncusSnapshotWithRetry(job, snapshotName, { ignoreExitCode: true });
|
||||
}
|
||||
|
||||
async function settleUdev(job) {
|
||||
await spawnCommand('udevadm', ['trigger'], {
|
||||
ignoreExitCode: true,
|
||||
log: (line) => appendJobLog(job, line),
|
||||
}).catch((error) => appendJobLog(job, error.message));
|
||||
await spawnCommand('udevadm', ['settle'], {
|
||||
ignoreExitCode: true,
|
||||
log: (line) => appendJobLog(job, line),
|
||||
}).catch((error) => appendJobLog(job, error.message));
|
||||
}
|
||||
|
||||
async function deleteIncusSnapshotWithRetry(job, snapshotName, options = {}) {
|
||||
const maxAttempts = 5;
|
||||
for (let attempt = 1; attempt <= maxAttempts; attempt += 1) {
|
||||
try {
|
||||
return await spawnCommand('incus', ['snapshot', 'delete', job.vmName, snapshotName], {
|
||||
ignoreExitCode: options.ignoreExitCode,
|
||||
log: (line) => appendJobLog(job, line),
|
||||
});
|
||||
} catch (error) {
|
||||
const retryable = error.message.includes('dataset is busy');
|
||||
if (!retryable || attempt === maxAttempts) {
|
||||
if (options.ignoreExitCode) {
|
||||
appendJobLog(job, error.message);
|
||||
return null;
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
appendJobLog(job, `Snapshot device still busy; retrying delete (${attempt}/${maxAttempts}).`);
|
||||
await delay(2000);
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
Reference in new issue
Block a user