mirror of
https://github.com/advplyr/audiobookshelf.git
synced 2025-07-12 18:35:00 +02:00
Migrate backups manager
This commit is contained in:
advplyr
parent
0a179e4eed
commit
254ba1f089
8 changed files with 202 additions and 107 deletions
|
|
@ -1,6 +1,8 @@
|
|||
const sqlite3 = require('sqlite3')
|
||||
const Path = require('path')
|
||||
const Logger = require('../Logger')
|
||||
const SocketAuthority = require('../SocketAuthority')
|
||||
const Database = require('../Database')
|
||||
|
||||
const cron = require('../libs/nodeCron')
|
||||
const fs = require('../libs/fsExtra')
|
||||
|
|
@ -14,27 +16,32 @@ const filePerms = require('../utils/filePerms')
|
|||
const Backup = require('../objects/Backup')
|
||||
|
||||
class BackupManager {
|
||||
constructor(db) {
|
||||
constructor() {
|
||||
this.BackupPath = Path.join(global.MetadataPath, 'backups')
|
||||
this.ItemsMetadataPath = Path.join(global.MetadataPath, 'items')
|
||||
this.AuthorsMetadataPath = Path.join(global.MetadataPath, 'authors')
|
||||
|
||||
this.db = db
|
||||
|
||||
this.scheduleTask = null
|
||||
|
||||
this.backups = []
|
||||
}
|
||||
|
||||
get serverSettings() {
|
||||
return this.db.serverSettings || {}
|
||||
get backupSchedule() {
|
||||
return global.ServerSettings.backupSchedule
|
||||
}
|
||||
|
||||
get backupsToKeep() {
|
||||
return global.ServerSettings.backupsToKeep || 2
|
||||
}
|
||||
|
||||
get maxBackupSize() {
|
||||
return global.ServerSettings.maxBackupSize || 1
|
||||
}
|
||||
|
||||
async init() {
|
||||
const backupsDirExists = await fs.pathExists(this.BackupPath)
|
||||
if (!backupsDirExists) {
|
||||
await fs.ensureDir(this.BackupPath)
|
||||
await filePerms.setDefault(this.BackupPath)
|
||||
}
|
||||
|
||||
await this.loadBackups()
|
||||
|
|
@ -42,42 +49,42 @@ class BackupManager {
|
|||
}
|
||||
|
||||
scheduleCron() {
|
||||
if (!this.serverSettings.backupSchedule) {
|
||||
if (!this.backupSchedule) {
|
||||
Logger.info(`[BackupManager] Auto Backups are disabled`)
|
||||
return
|
||||
}
|
||||
try {
|
||||
var cronSchedule = this.serverSettings.backupSchedule
|
||||
var cronSchedule = this.backupSchedule
|
||||
this.scheduleTask = cron.schedule(cronSchedule, this.runBackup.bind(this))
|
||||
} catch (error) {
|
||||
Logger.error(`[BackupManager] Failed to schedule backup cron ${this.serverSettings.backupSchedule}`, error)
|
||||
Logger.error(`[BackupManager] Failed to schedule backup cron ${this.backupSchedule}`, error)
|
||||
}
|
||||
}
|
||||
|
||||
updateCronSchedule() {
|
||||
if (this.scheduleTask && !this.serverSettings.backupSchedule) {
|
||||
if (this.scheduleTask && !this.backupSchedule) {
|
||||
Logger.info(`[BackupManager] Disabling backup schedule`)
|
||||
if (this.scheduleTask.stop) this.scheduleTask.stop()
|
||||
this.scheduleTask = null
|
||||
} else if (!this.scheduleTask && this.serverSettings.backupSchedule) {
|
||||
Logger.info(`[BackupManager] Starting backup schedule ${this.serverSettings.backupSchedule}`)
|
||||
} else if (!this.scheduleTask && this.backupSchedule) {
|
||||
Logger.info(`[BackupManager] Starting backup schedule ${this.backupSchedule}`)
|
||||
this.scheduleCron()
|
||||
} else if (this.serverSettings.backupSchedule) {
|
||||
Logger.info(`[BackupManager] Restarting backup schedule ${this.serverSettings.backupSchedule}`)
|
||||
} else if (this.backupSchedule) {
|
||||
Logger.info(`[BackupManager] Restarting backup schedule ${this.backupSchedule}`)
|
||||
if (this.scheduleTask.stop) this.scheduleTask.stop()
|
||||
this.scheduleCron()
|
||||
}
|
||||
}
|
||||
|
||||
async uploadBackup(req, res) {
|
||||
var backupFile = req.files.file
|
||||
const backupFile = req.files.file
|
||||
if (Path.extname(backupFile.name) !== '.audiobookshelf') {
|
||||
Logger.error(`[BackupManager] Invalid backup file uploaded "${backupFile.name}"`)
|
||||
return res.status(500).send('Invalid backup file')
|
||||
}
|
||||
|
||||
var tempPath = Path.join(this.BackupPath, backupFile.name)
|
||||
var success = await backupFile.mv(tempPath).then(() => true).catch((error) => {
|
||||
const tempPath = Path.join(this.BackupPath, backupFile.name)
|
||||
const success = await backupFile.mv(tempPath).then(() => true).catch((error) => {
|
||||
Logger.error('[BackupManager] Failed to move backup file', path, error)
|
||||
return false
|
||||
})
|
||||
|
|
@ -86,10 +93,17 @@ class BackupManager {
|
|||
}
|
||||
|
||||
const zip = new StreamZip.async({ file: tempPath })
|
||||
const data = await zip.entryData('details')
|
||||
var details = data.toString('utf8').split('\n')
|
||||
|
||||
var backup = new Backup({ details, fullPath: tempPath })
|
||||
const entries = await zip.entries()
|
||||
if (!Object.keys(entries).includes('absdatabase.sqlite')) {
|
||||
Logger.error(`[BackupManager] Invalid backup with no absdatabase.sqlite file - might be a backup created on an old Audiobookshelf server.`)
|
||||
return res.status(500).send('Invalid backup with no absdatabase.sqlite file - might be a backup created on an old Audiobookshelf server.')
|
||||
}
|
||||
|
||||
const data = await zip.entryData('details')
|
||||
const details = data.toString('utf8').split('\n')
|
||||
|
||||
const backup = new Backup({ details, fullPath: tempPath })
|
||||
|
||||
if (!backup.serverVersion) {
|
||||
Logger.error(`[BackupManager] Invalid backup with no server version - might be a backup created before version 2.0.0`)
|
||||
|
|
@ -98,7 +112,7 @@ class BackupManager {
|
|||
|
||||
backup.fileSize = await getFileSize(backup.fullPath)
|
||||
|
||||
var existingBackupIndex = this.backups.findIndex(b => b.id === backup.id)
|
||||
const existingBackupIndex = this.backups.findIndex(b => b.id === backup.id)
|
||||
if (existingBackupIndex >= 0) {
|
||||
Logger.warn(`[BackupManager] Backup already exists with id ${backup.id} - overwriting`)
|
||||
this.backups.splice(existingBackupIndex, 1, backup)
|
||||
|
|
@ -122,14 +136,23 @@ class BackupManager {
|
|||
}
|
||||
}
|
||||
|
||||
async requestApplyBackup(backup) {
|
||||
async requestApplyBackup(backup, res) {
|
||||
const zip = new StreamZip.async({ file: backup.fullPath })
|
||||
await zip.extract('config/', global.ConfigPath)
|
||||
if (backup.backupMetadataCovers) {
|
||||
await zip.extract('metadata-items/', this.ItemsMetadataPath)
|
||||
await zip.extract('metadata-authors/', this.AuthorsMetadataPath)
|
||||
|
||||
const entries = await zip.entries()
|
||||
if (!Object.keys(entries).includes('absdatabase.sqlite')) {
|
||||
Logger.error(`[BackupManager] Cannot apply old backup ${backup.fullPath}`)
|
||||
return res.status(500).send('Invalid backup file. Does not include absdatabase.sqlite. This might be from an older Audiobookshelf server.')
|
||||
}
|
||||
await this.db.reinit()
|
||||
|
||||
await Database.disconnect()
|
||||
|
||||
await zip.extract('absdatabase.sqlite', global.ConfigPath)
|
||||
await zip.extract('metadata-items/', this.ItemsMetadataPath)
|
||||
await zip.extract('metadata-authors/', this.AuthorsMetadataPath)
|
||||
|
||||
await Database.reconnect()
|
||||
|
||||
SocketAuthority.emitter('backup_applied')
|
||||
}
|
||||
|
||||
|
|
@ -182,44 +205,52 @@ class BackupManager {
|
|||
async runBackup() {
|
||||
// Check if Metadata Path is inside Config Path (otherwise there will be an infinite loop as the archiver tries to zip itself)
|
||||
Logger.info(`[BackupManager] Running Backup`)
|
||||
var newBackup = new Backup()
|
||||
const newBackup = new Backup()
|
||||
newBackup.setData(this.BackupPath)
|
||||
|
||||
const newBackData = {
|
||||
backupMetadataCovers: this.serverSettings.backupMetadataCovers,
|
||||
backupDirPath: this.BackupPath
|
||||
await fs.ensureDir(this.AuthorsMetadataPath)
|
||||
|
||||
// Create backup sqlite file
|
||||
const sqliteBackupPath = await this.backupSqliteDb(newBackup).catch((error) => {
|
||||
Logger.error(`[BackupManager] Failed to backup sqlite db`, error)
|
||||
return false
|
||||
})
|
||||
|
||||
if (!sqliteBackupPath) {
|
||||
return false
|
||||
}
|
||||
newBackup.setData(newBackData)
|
||||
|
||||
var metadataAuthorsPath = this.AuthorsMetadataPath
|
||||
if (!await fs.pathExists(metadataAuthorsPath)) metadataAuthorsPath = null
|
||||
|
||||
var zipResult = await this.zipBackup(metadataAuthorsPath, newBackup).then(() => true).catch((error) => {
|
||||
// Zip sqlite file, /metadata/items, and /metadata/authors folders
|
||||
const zipResult = await this.zipBackup(sqliteBackupPath, newBackup).catch((error) => {
|
||||
Logger.error(`[BackupManager] Backup Failed ${error}`)
|
||||
return false
|
||||
})
|
||||
if (zipResult) {
|
||||
Logger.info(`[BackupManager] Backup successful ${newBackup.id}`)
|
||||
await filePerms.setDefault(newBackup.fullPath)
|
||||
newBackup.fileSize = await getFileSize(newBackup.fullPath)
|
||||
var existingIndex = this.backups.findIndex(b => b.id === newBackup.id)
|
||||
if (existingIndex >= 0) {
|
||||
this.backups.splice(existingIndex, 1, newBackup)
|
||||
} else {
|
||||
this.backups.push(newBackup)
|
||||
}
|
||||
|
||||
// Check remove oldest backup
|
||||
if (this.backups.length > this.serverSettings.backupsToKeep) {
|
||||
this.backups.sort((a, b) => a.createdAt - b.createdAt)
|
||||
// Remove sqlite backup
|
||||
await fs.remove(sqliteBackupPath)
|
||||
|
||||
var oldBackup = this.backups.shift()
|
||||
Logger.debug(`[BackupManager] Removing old backup ${oldBackup.id}`)
|
||||
this.removeBackup(oldBackup)
|
||||
}
|
||||
return true
|
||||
if (!zipResult) return false
|
||||
|
||||
Logger.info(`[BackupManager] Backup successful ${newBackup.id}`)
|
||||
|
||||
newBackup.fileSize = await getFileSize(newBackup.fullPath)
|
||||
|
||||
const existingIndex = this.backups.findIndex(b => b.id === newBackup.id)
|
||||
if (existingIndex >= 0) {
|
||||
this.backups.splice(existingIndex, 1, newBackup)
|
||||
} else {
|
||||
return false
|
||||
this.backups.push(newBackup)
|
||||
}
|
||||
|
||||
// Check remove oldest backup
|
||||
if (this.backups.length > this.backupsToKeep) {
|
||||
this.backups.sort((a, b) => a.createdAt - b.createdAt)
|
||||
|
||||
const oldBackup = this.backups.shift()
|
||||
Logger.debug(`[BackupManager] Removing old backup ${oldBackup.id}`)
|
||||
this.removeBackup(oldBackup)
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
async removeBackup(backup) {
|
||||
|
|
@ -233,7 +264,35 @@ class BackupManager {
|
|||
}
|
||||
}
|
||||
|
||||
zipBackup(metadataAuthorsPath, backup) {
|
||||
/**
|
||||
* @see https://github.com/TryGhost/node-sqlite3/pull/1116
|
||||
* @param {Backup} backup
|
||||
* @promise
|
||||
*/
|
||||
backupSqliteDb(backup) {
|
||||
const db = new sqlite3.Database(Database.dbPath)
|
||||
const dbFilePath = Path.join(global.ConfigPath, `absdatabase.${backup.id}.sqlite`)
|
||||
return new Promise(async (resolve, reject) => {
|
||||
const backup = db.backup(dbFilePath)
|
||||
backup.step(-1)
|
||||
backup.finish()
|
||||
|
||||
// Max time ~2 mins
|
||||
for (let i = 0; i < 240; i++) {
|
||||
if (backup.completed) {
|
||||
return resolve(dbFilePath)
|
||||
} else if (backup.failed) {
|
||||
return reject(backup.message || 'Unknown failure reason')
|
||||
}
|
||||
await new Promise((r) => setTimeout(r, 500))
|
||||
}
|
||||
|
||||
Logger.error(`[BackupManager] Backup sqlite timed out`)
|
||||
reject('Backup timed out')
|
||||
})
|
||||
}
|
||||
|
||||
zipBackup(sqliteBackupPath, backup) {
|
||||
return new Promise((resolve, reject) => {
|
||||
// create a file to stream archive data to
|
||||
const output = fs.createWriteStream(backup.fullPath)
|
||||
|
|
@ -245,7 +304,7 @@ class BackupManager {
|
|||
// 'close' event is fired only when a file descriptor is involved
|
||||
output.on('close', () => {
|
||||
Logger.info('[BackupManager]', archive.pointer() + ' total bytes')
|
||||
resolve()
|
||||
resolve(true)
|
||||
})
|
||||
|
||||
// This event is fired when the data source is drained no matter what was the data source.
|
||||
|
|
@ -281,7 +340,7 @@ class BackupManager {
|
|||
reject(err)
|
||||
})
|
||||
archive.on('progress', ({ fs: fsobj }) => {
|
||||
const maxBackupSizeInBytes = this.serverSettings.maxBackupSize * 1000 * 1000 * 1000
|
||||
const maxBackupSizeInBytes = this.maxBackupSize * 1000 * 1000 * 1000
|
||||
if (fsobj.processedBytes > maxBackupSizeInBytes) {
|
||||
Logger.error(`[BackupManager] Archiver is too large - aborting to prevent endless loop, Bytes Processed: ${fsobj.processedBytes}`)
|
||||
archive.abort()
|
||||
|
|
@ -295,26 +354,9 @@ class BackupManager {
|
|||
// pipe archive data to the file
|
||||
archive.pipe(output)
|
||||
|
||||
archive.directory(Path.join(this.db.LibraryItemsPath, 'data'), 'config/libraryItems/data')
|
||||
archive.directory(Path.join(this.db.UsersPath, 'data'), 'config/users/data')
|
||||
archive.directory(Path.join(this.db.SessionsPath, 'data'), 'config/sessions/data')
|
||||
archive.directory(Path.join(this.db.LibrariesPath, 'data'), 'config/libraries/data')
|
||||
archive.directory(Path.join(this.db.SettingsPath, 'data'), 'config/settings/data')
|
||||
archive.directory(Path.join(this.db.CollectionsPath, 'data'), 'config/collections/data')
|
||||
archive.directory(Path.join(this.db.AuthorsPath, 'data'), 'config/authors/data')
|
||||
archive.directory(Path.join(this.db.SeriesPath, 'data'), 'config/series/data')
|
||||
archive.directory(Path.join(this.db.PlaylistsPath, 'data'), 'config/playlists/data')
|
||||
archive.directory(Path.join(this.db.FeedsPath, 'data'), 'config/feeds/data')
|
||||
|
||||
if (this.serverSettings.backupMetadataCovers) {
|
||||
Logger.debug(`[BackupManager] Backing up Metadata Items "${this.ItemsMetadataPath}"`)
|
||||
archive.directory(this.ItemsMetadataPath, 'metadata-items')
|
||||
|
||||
if (metadataAuthorsPath) {
|
||||
Logger.debug(`[BackupManager] Backing up Metadata Authors "${metadataAuthorsPath}"`)
|
||||
archive.directory(metadataAuthorsPath, 'metadata-authors')
|
||||
}
|
||||
}
|
||||
archive.file(sqliteBackupPath, { name: 'absdatabase.sqlite' })
|
||||
archive.directory(this.ItemsMetadataPath, 'metadata-items')
|
||||
archive.directory(this.AuthorsMetadataPath, 'metadata-authors')
|
||||
|
||||
archive.append(backup.detailsString, { name: 'details' })
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue