Abort backup if it is getting too large #89, support for ebook only book folders #75

server/BackupManager.js

@@ -24,6 +24,9 @@ class BackupManager {
     this.scheduleTask = null
 
     this.backups = []
+
+    // If backup exceeds this value it will be aborted
+    this.MaxBytesBeforeAbort = 1000000000 // ~ 1GB
   }
 
   get serverSettings() {
@@ -191,6 +194,7 @@ class BackupManager {
   }
 
   async runBackup() {
+    // Check if Metadata Path is inside Config Path (otherwise there will be an infinite loop as the archiver tries to zip itself)
     Logger.info(`[BackupManager] Running Backup`)
     var metadataBooksPath = this.serverSettings.backupMetadataCovers ? Path.join(this.MetadataPath, 'books') : null
 
@@ -233,6 +237,7 @@ class BackupManager {
 
   async removeBackup(backup) {
     try {
+      Logger.debug(`[BackupManager] Removing Backup "${backup.fullPath}"`)
       await fs.remove(backup.fullPath)
       this.backups = this.backups.filter(b => b.id !== backup.id)
       Logger.info(`[BackupManager] Backup "${backup.id}" Removed`)
@@ -263,6 +268,15 @@ class BackupManager {
         Logger.debug('Data has been drained')
       })
 
+      output.on('finish', () => {
+        Logger.debug('Write Stream Finished')
+      })
+
+      output.on('error', (err) => {
+        Logger.debug('Write Stream Error', err)
+        reject(err)
+      })
+
       // good practice to catch warnings (ie stat failures and other non-blocking errors)
       archive.on('warning', function (err) {
         if (err.code === 'ENOENT') {
@@ -279,6 +293,16 @@ class BackupManager {
         Logger.error(`[BackupManager] Archiver error: ${err.message}`)
         reject(err)
       })
+      archive.on('progress', ({ fs: fsobj }) => {
+        if (fsobj.processedBytes > this.MaxBytesBeforeAbort) {
+          Logger.error(`[BackupManager] Archiver is too large - aborting to prevent endless loop, Bytes Processed: ${fsobj.processedBytes}`)
+          archive.abort()
+          setTimeout(() => {
+            this.removeBackup(backup)
+            output.destroy('Backup too large') // Promise is reject in write stream error evt
+          }, 500)
+        }
+      })
 
       // pipe archive data to the file
       archive.pipe(output)

server/Scanner.js

@@ -143,18 +143,21 @@ class Scanner {
       forceAudioFileScan = true
     }
 
-    // ino is now set for every file in scandir
+    // inode is required
     audiobookData.audioFiles = audiobookData.audioFiles.filter(af => af.ino)
 
-    // REMOVE: No valid audio files
-    // TODO: Label as incomplete, do not actually delete
-    if (!audiobookData.audioFiles.length) {
-      Logger.error(`[Scanner] "${existingAudiobook.title}" no valid audio files found - removing audiobook`)
-
-      await this.db.removeEntity('audiobook', existingAudiobook.id)
-      this.emitter('audiobook_removed', existingAudiobook.toJSONMinified())
-
-      return ScanResult.REMOVED
+    // No valid ebook and audio files found, mark as incomplete
+    var ebookFiles = audiobookData.otherFiles.filter(f => f.filetype === 'ebook')
+    if (!audiobookData.audioFiles.length && !ebookFiles.length) {
+      Logger.error(`[Scanner] "${existingAudiobook.title}" no valid book files found - marking as incomplete`)
+      existingAudiobook.setLastScan(version)
+      existingAudiobook.isIncomplete = true
+      await this.db.updateAudiobook(existingAudiobook)
+      this.emitter('audiobook_updated', existingAudiobook.toJSONMinified())
+      return ScanResult.UPDATED
+    } else if (existingAudiobook.isIncomplete) { // Was incomplete but now is not
+      Logger.info(`[Scanner] "${existingAudiobook.title}" was incomplete but now has book files`)
+      existingAudiobook.isIncomplete = false
     }
 
     // Check for audio files that were removed
@@ -219,14 +222,15 @@ class Scanner {
       await audioFileScanner.scanAudioFiles(existingAudiobook, newAudioFiles)
     }
 
-    // If after a scan no valid audio tracks remain
-    // TODO: Label as incomplete, do not actually delete
-    if (!existingAudiobook.tracks.length) {
-      Logger.error(`[Scanner] "${existingAudiobook.title}" has no valid tracks after update - removing audiobook`)
-
-      await this.db.removeEntity('audiobook', existingAudiobook.id)
-      this.emitter('audiobook_removed', existingAudiobook.toJSONMinified())
-      return ScanResult.REMOVED
+    // After scanning audio files, some may no longer be valid
+    //   so make sure the directory still has valid book files
+    if (!existingAudiobook.tracks.length && !ebookFiles.length) {
+      Logger.error(`[Scanner] "${existingAudiobook.title}" no valid book files found after update - marking as incomplete`)
+      existingAudiobook.setLastScan(version)
+      existingAudiobook.isIncomplete = true
+      await this.db.updateAudiobook(existingAudiobook)
+      this.emitter('audiobook_updated', existingAudiobook.toJSONMinified())
+      return ScanResult.UPDATED
     }
 
     var hasUpdates = hasUpdatedIno || hasUpdatedLibraryOrFolder || removedAudioFiles.length || removedAudioTracks.length || newAudioFiles.length || hasUpdatedAudioFiles
@@ -269,8 +273,9 @@ class Scanner {
   }
 
   async scanNewAudiobook(audiobookData) {
-    if (!audiobookData.audioFiles.length) {
-      Logger.error('[Scanner] No valid audio tracks for Audiobook', audiobookData.path)
+    var ebookFiles = audiobookData.otherFiles.map(f => f.filetype === 'ebook')
+    if (!audiobookData.audioFiles.length && !ebookFiles.length) {
+      Logger.error('[Scanner] No valid audio files and ebooks for Audiobook', audiobookData.path)
       return null
     }
 
@@ -279,8 +284,9 @@ class Scanner {
 
     // Scan audio files and set tracks, pulls metadata
     await audioFileScanner.scanAudioFiles(audiobook, audiobookData.audioFiles)
-    if (!audiobook.tracks.length) {
-      Logger.warn('[Scanner] Invalid audiobook, no valid tracks', audiobook.title)
+
+    if (!audiobook.tracks.length && !audiobook.ebooks.length) {
+      Logger.warn('[Scanner] Invalid audiobook, no valid audio tracks and ebook files', audiobook.title)
       return null
     }
 

server/objects/Audiobook.js

@@ -37,6 +37,8 @@ class Audiobook {
 
     // Audiobook was scanned and not found
     this.isMissing = false
+    // Audiobook no longer has "book" files
+    this.isInvalid = false
 
     if (audiobook) {
       this.construct(audiobook)
@@ -70,6 +72,7 @@ class Audiobook {
     }
 
     this.isMissing = !!audiobook.isMissing
+    this.isInvalid = !!audiobook.isInvalid
   }
 
   get title() {
@@ -175,7 +178,8 @@ class Audiobook {
       audioFiles: this._audioFiles.map(audioFile => audioFile.toJSON()),
       otherFiles: this._otherFiles.map(otherFile => otherFile.toJSON()),
       chapters: this.chapters || [],
-      isMissing: !!this.isMissing
+      isMissing: !!this.isMissing,
+      isInvalid: !!this.isInvalid
     }
   }
 
@@ -197,10 +201,12 @@ class Audiobook {
       hasMissingParts: this.missingParts ? this.missingParts.length : 0,
       hasInvalidParts: this.invalidParts ? this.invalidParts.length : 0,
       // numEbooks: this.ebooks.length,
-      numEbooks: this.hasEpub ? 1 : 0,
+      ebooks: this.ebooks.map(ebook => ebook.toJSON()),
+      numEbooks: this.hasEpub ? 1 : 0, // Only supporting epubs in the reader currently
       numTracks: this.tracks.length,
       chapters: this.chapters || [],
-      isMissing: !!this.isMissing
+      isMissing: !!this.isMissing,
+      isInvalid: !!this.isInvalid
     }
   }
 
@@ -220,15 +226,16 @@ class Audiobook {
       sizePretty: this.sizePretty,
       missingParts: this.missingParts,
       invalidParts: this.invalidParts,
-      audioFiles: (this.audioFiles || []).map(audioFile => audioFile.toJSON()),
-      otherFiles: (this.otherFiles || []).map(otherFile => otherFile.toJSON()),
-      ebooks: (this.ebooks || []).map(ebook => ebook.toJSON()),
+      audioFiles: this._audioFiles.map(audioFile => audioFile.toJSON()),
+      otherFiles: this._otherFiles.map(otherFile => otherFile.toJSON()),
+      ebooks: this.ebooks.map(ebook => ebook.toJSON()),
       numEbooks: this.hasEpub ? 1 : 0,
       tags: this.tags,
       book: this.bookToJSON(),
       tracks: this.tracksToJSON(),
       chapters: this.chapters || [],
-      isMissing: !!this.isMissing
+      isMissing: !!this.isMissing,
+      isInvalid: !!this.isInvalid
     }
   }
 

server/utils/scandir.js

@@ -16,11 +16,12 @@ function getPaths(path) {
   })
 }
 
-function isAudioFile(path) {
+function isBookFile(path) {
   if (!path) return false
   var ext = Path.extname(path)
   if (!ext) return false
-  return globals.SupportedAudioTypes.includes(ext.slice(1).toLowerCase())
+  var extclean = ext.slice(1).toLowerCase()
+  return globals.SupportedAudioTypes.includes(extclean) || globals.SupportedEbookTypes.includes(extclean)
 }
 
 // Input: array of relative file paths
@@ -36,17 +37,18 @@ function groupFilesIntoAudiobookPaths(paths, useAllFileTypes = false) {
     return pathsA - pathsB
   })
 
-  // Step 2.5: Seperate audio files and other files (optional)
-  var audioFilePaths = []
+  // Step 2.5: Seperate audio/ebook files and other files (optional)
+  //              - Directories without an audio or ebook file will not be included
+  var bookFilePaths = []
   var otherFilePaths = []
   pathsFiltered.forEach(path => {
-    if (isAudioFile(path) || useAllFileTypes) audioFilePaths.push(path)
+    if (isBookFile(path) || useAllFileTypes) bookFilePaths.push(path)
     else otherFilePaths.push(path)
   })
 
   // Step 3: Group audio files in audiobooks
   var audiobookGroup = {}
-  audioFilePaths.forEach((path) => {
+  bookFilePaths.forEach((path) => {
     var dirparts = Path.dirname(path).split(Path.sep)
     var numparts = dirparts.length
     var _path = ''