Fix: Scanner check path and inode value for removed books, scanner v5 outlined

server/scanner/AudioFileScanner.js

@@ -0,0 +1,22 @@
+const AudioFile = require('../objects/AudioFile')
+const AudioProbeData = require('./AudioProbeData')
+
+const prober = require('../utils/prober')
+const Logger = require('../Logger')
+
+class AudioFileScanner {
+  constructor() { }
+
+  async scan(audioFileData, verbose = false) {
+    var probeData = await prober.probe2(audioFileData.fullPath, verbose)
+    if (probeData.error) {
+      Logger.error(`[AudioFileScanner] ${probeData.error} : "${audioFileData.fullPath}"`)
+      return null
+    }
+
+    var audioFile = new AudioFile()
+    // TODO: Build audio file
+    return audioFile
+  }
+}
+module.exports = new AudioFileScanner()

server/scanner/AudioProbeData.js

@@ -0,0 +1,74 @@
+const AudioFileMetadata = require('../objects/AudioFileMetadata')
+
+class AudioProbeData {
+  constructor() {
+    this.embeddedCoverArt = null
+    this.format = null
+    this.duration = null
+    this.size = null
+    this.bitRate = null
+    this.codec = null
+    this.timeBase = null
+    this.language = null
+    this.channelLayout = null
+    this.channels = null
+    this.sampleRate = null
+    this.chapters = []
+
+    this.audioFileMetadata = null
+
+    this.trackNumber = null
+    this.trackTotal = null
+  }
+
+  getDefaultAudioStream(audioStreams) {
+    if (audioStreams.length === 1) return audioStreams[0]
+    var defaultStream = audioStreams.find(a => a.is_default)
+    if (!defaultStream) return audioStreams[0]
+    return defaultStream
+  }
+
+  getEmbeddedCoverArt(videoStream) {
+    const ImageCodecs = ['mjpeg', 'jpeg', 'png']
+    return ImageCodecs.includes(videoStream.codec) ? videoStream.codec : null
+  }
+
+  setData(data) {
+    var audioStream = getDefaultAudioStream(data.audio_streams)
+
+    this.embeddedCoverArt = data.video_stream ? this.getEmbeddedCoverArt(data.video_stream) : false
+    this.format = data.format
+    this.duration = data.duration
+    this.size = data.size
+    this.bitRate = audioStream.bit_rate || data.bit_rate
+    this.codec = audioStream.codec
+    this.timeBase = audioStream.time_base
+    this.language = audioStream.language
+    this.channelLayout = audioStream.channel_layout
+    this.channels = audioStream.channels
+    this.sampleRate = audioStream.sample_rate
+    this.chapters = data.chapters || []
+
+    var metatags = {}
+    for (const key in data) {
+      if (data[key] && key.startsWith('file_tag')) {
+        metatags[key] = data[key]
+      }
+    }
+
+    this.audioFileMetadata = new AudioFileMetadata()
+    this.audioFileMetadata.setData(metatags)
+
+    // Track ID3 tag might be "3/10" or just "3"
+    if (this.audioFileMetadata.tagTrack) {
+      var trackParts = this.audioFileMetadata.tagTrack.split('/').map(part => Number(part))
+      if (trackParts.length > 0) {
+        this.trackNumber = trackParts[0]
+      }
+      if (trackParts.length > 1) {
+        this.trackTotal = trackParts[1]
+      }
+    }
+  }
+}
+module.exports = AudioProbeData

server/scanner/LibraryScan.js

@@ -0,0 +1,34 @@
+const Folder = require('../objects/Folder')
+
+const { getId } = require('../utils/index')
+
+class LibraryScan {
+  constructor() {
+    this.id = null
+    this.libraryId = null
+    this.libraryName = null
+    this.folders = null
+
+    this.scanOptions = null
+
+    this.startedAt = null
+    this.finishedAt = null
+
+    this.folderScans = []
+  }
+
+  get _scanOptions() { return this.scanOptions || {} }
+  get forceRescan() { return !!this._scanOptions.forceRescan }
+
+  setData(library, scanOptions) {
+    this.id = getId('lscan')
+    this.libraryId = library.id
+    this.libraryName = library.name
+    this.folders = library.folders.map(folder => Folder(folder.toJSON()))
+
+    this.scanOptions = scanOptions
+
+    this.startedAt = Date.now()
+  }
+}
+module.exports = LibraryScan

server/scanner/ScanOptions.js

@@ -0,0 +1,68 @@
+const { CoverDestination } = require('../utils/constants')
+
+class ScanOptions {
+  constructor(options) {
+    this.forceRescan = false
+
+    this.metadataPrecedence = [
+      {
+        id: 'directory',
+        include: true
+      },
+      {
+        id: 'reader-desc-txt',
+        include: true
+      },
+      {
+        id: 'audio-file-metadata',
+        include: true
+      },
+      {
+        id: 'metadata-opf',
+        include: true
+      },
+      {
+        id: 'external-source',
+        include: false
+      }
+    ]
+
+    // Server settings
+    this.parseSubtitles = false
+    this.findCovers = false
+    this.coverDestination = CoverDestination.METADATA
+
+    if (options) {
+      this.construct(options)
+    }
+  }
+
+  construct(options) {
+    for (const key in options) {
+      if (key === 'metadataPrecedence' && options[key].length) {
+        this.metadataPrecedence = [...options[key]]
+      } else if (this[key] !== undefined) {
+        this[key] = options[key]
+      }
+    }
+  }
+
+  toJSON() {
+    return {
+      forceRescan: this.forceRescan,
+      metadataPrecedence: this.metadataPrecedence,
+      parseSubtitles: this.parseSubtitles,
+      findCovers: this.findCovers,
+      coverDestination: this.coverDestination
+    }
+  }
+
+  setData(options, serverSettings) {
+    this.forceRescan = !!options.forceRescan
+
+    this.parseSubtitles = !!serverSettings.scannerParseSubtitle
+    this.findCovers = !!serverSettings.scannerFindCovers
+    this.coverDestination = serverSettings.coverDestination
+  }
+}
+module.exports = ScanOptions

server/scanner/Scanner.js

@@ -0,0 +1,172 @@
+const fs = require('fs-extra')
+const Path = require('path')
+
+// Utils
+const Logger = require('../Logger')
+const { version } = require('../../package.json')
+const audioFileScanner = require('../utils/audioFileScanner')
+const { groupFilesIntoAudiobookPaths, getAudiobookFileData, scanRootDir } = require('../utils/scandir')
+const { comparePaths, getIno, getId } = require('../utils/index')
+const { secondsToTimestamp } = require('../utils/fileUtils')
+const { ScanResult, CoverDestination } = require('../utils/constants')
+
+const AudioFileScanner = require('./AudioFileScanner')
+const BookFinder = require('../BookFinder')
+const Audiobook = require('../objects/Audiobook')
+const LibraryScan = require('./LibraryScan')
+const ScanOptions = require('./ScanOptions')
+
+class Scanner {
+  constructor(AUDIOBOOK_PATH, METADATA_PATH, db, coverController, emitter) {
+    this.AudiobookPath = AUDIOBOOK_PATH
+    this.MetadataPath = METADATA_PATH
+    this.BookMetadataPath = Path.posix.join(this.MetadataPath.replace(/\\/g, '/'), 'books')
+
+    this.db = db
+    this.coverController = coverController
+    this.emitter = emitter
+
+    this.cancelScan = false
+    this.cancelLibraryScan = {}
+    this.librariesScanning = []
+
+    this.bookFinder = new BookFinder()
+  }
+
+  async scan(libraryId, options = {}) {
+    if (this.librariesScanning.includes(libraryId)) {
+      Logger.error(`[Scanner] Already scanning ${libraryId}`)
+      return
+    }
+
+    var library = this.db.libraries.find(lib => lib.id === libraryId)
+    if (!library) {
+      Logger.error(`[Scanner] Library not found for scan ${libraryId}`)
+      return
+    } else if (!library.folders.length) {
+      Logger.warn(`[Scanner] Library has no folders to scan "${library.name}"`)
+      return
+    }
+
+    var scanOptions = new ScanOptions()
+    scanOptions.setData(options, this.db.serverSettings)
+
+    var libraryScan = new LibraryScan()
+    libraryScan.setData(library, scanOptions)
+
+    Logger.info(`[Scanner] Starting library scan ${libraryScan.id} for ${libraryScan.libraryName}`)
+
+    var results = await this.scanLibrary(libraryScan)
+
+    Logger.info(`[Scanner] Library scan ${libraryScan.id} complete`)
+
+    return results
+  }
+
+  async scanLibrary(libraryScan) {
+    var audiobookDataFound = []
+    for (let i = 0; i < libraryScan.folders.length; i++) {
+      var folder = libraryScan.folders[i]
+      var abDataFoundInFolder = await scanRootDir(folder, this.db.serverSettings)
+      Logger.debug(`[Scanner] ${abDataFoundInFolder.length} ab data found in folder "${folder.fullPath}"`)
+      audiobookDataFound = audiobookDataFound.concat(abDataFoundInFolder)
+    }
+
+    // Remove audiobooks with no inode
+    audiobookDataFound = audiobookDataFound.filter(abd => abd.ino)
+
+    var audiobooksInLibrary = this.db.audiobooks.filter(ab => ab.libraryId === libraryScan.libraryId)
+
+    const audiobooksToUpdate = []
+    const audiobooksToRescan = []
+    const newAudiobookData = []
+
+    // Check for existing & removed audiobooks
+    for (let i = 0; i < audiobooksInLibrary.length; i++) {
+      var audiobook = audiobooksInLibrary[i]
+      var dataFound = audiobookDataFound.find(abd => abd.ino === audiobook.ino || comparePaths(abd.path, audiobook.path))
+      if (!dataFound) {
+        Logger.info(`[Scanner] Audiobook "${audiobook.title}" is missing`)
+        audiobook.isMissing = true
+        audiobook.lastUpdate = Date.now()
+        scanResults.missing++
+        audiobooksToUpdate.push(audiobook)
+      } else {
+        var checkRes = audiobook.checkShouldRescan(dataFound)
+        if (checkRes.newAudioFileData.length || checkRes.newOtherFileData.length) {
+          // existing audiobook has new files
+          checkRes.audiobook = audiobook
+          audiobooksToRescan.push(checkRes)
+        } else if (checkRes.updated) {
+          audiobooksToUpdate.push(audiobook)
+        }
+
+        // Remove this abf
+        audiobookDataFound = audiobookDataFound.filter(abf => abf.ino !== dataFound.ino)
+      }
+    }
+
+    // Potential NEW Audiobooks
+    for (let i = 0; i < audiobookDataFound.length; i++) {
+      var dataFound = audiobookDataFound[i]
+      var hasEbook = dataFound.otherFiles.find(otherFile => otherFile.filetype === 'ebook')
+      if (!hasEbook && !dataFound.audioFiles.length) {
+        Logger.info(`[Scanner] Directory found "${audiobookDataFound.path}" has no ebook or audio files`)
+      } else {
+        newAudiobookData.push(dataFound)
+      }
+    }
+
+    var rescans = []
+    for (let i = 0; i < audiobooksToRescan.length; i++) {
+      var rescan = this.rescanAudiobook(audiobooksToRescan[i])
+      rescans.push(rescan)
+    }
+    var newscans = []
+    for (let i = 0; i < newAudiobookData.length; i++) {
+      var newscan = this.scanNewAudiobook(newAudiobookData[i])
+      newscans.push(newscan)
+    }
+
+    var rescanResults = await Promise.all(rescans)
+
+    var newscanResults = await Promise.all(newscans)
+
+    // TODO: Return report
+    return {
+      updates: 0,
+      additions: 0
+    }
+  }
+
+  // Return scan result payload
+  async rescanAudiobook(audiobookCheckData) {
+    const { newAudioFileData, newOtherFileData, audiobook } = audiobookCheckData
+    if (newAudioFileData.length) {
+      var newAudioFiles = await this.scanAudioFiles(newAudioFileData)
+      // TODO: Update audiobook tracks
+    }
+    if (newOtherFileData.length) {
+      // TODO: Check other files
+    }
+
+    return {
+      updated: true
+    }
+  }
+
+  async scanNewAudiobook(audiobookData) {
+    // TODO: Return new audiobook
+    return null
+  }
+
+  async scanAudioFiles(audioFileData) {
+    var proms = []
+    for (let i = 0; i < audioFileData.length; i++) {
+      var prom = AudioFileScanner.scan(audioFileData[i])
+      proms.push(prom)
+    }
+    return Promise.all(proms)
+  }
+}
+module.exports = Scanner