Fixing scanner inodes, select all fix, starting ebook reader

server/EbookReader.js

@@ -0,0 +1,42 @@
+// const express = require('express')
+// const EPub = require('epub')
+// const Logger = require('./Logger')
+
+// class EbookReader {
+//   constructor(db, MetadataPath, AudiobookPath) {
+//     this.db = db
+//     this.MetadataPath = MetadataPath
+//     this.AudiobookPath = AudiobookPath
+
+//     this.router = express()
+//     this.init()
+//   }
+
+//   init() {
+//     this.router.get('/open/:id/:ino', this.openRequest.bind(this))
+//   }
+
+//   openRequest(req, res) {
+//     Logger.info('Open request received', req.params)
+//     var audiobookId = req.params.id
+//     var fileIno = req.params.ino
+//     var audiobook = this.db.audiobooks.find(ab => ab.id === audiobookId)
+//     if (!audiobook) {
+//       return res.sendStatus(404)
+//     }
+//     var ebook = audiobook.ebooks.find(eb => eb.ino === fileIno)
+//     if (!ebook) {
+//       Logger.error('Ebook file not found', fileIno)
+//       return res.sendStatus(404)
+//     }
+//     Logger.info('Ebook found', ebook)
+//     this.open(ebook.fullPath)
+//     res.sendStatus(200)
+//   }
+
+//   open(path) {
+//     var epub = new EPub(path)
+//     console.log('epub', epub)
+//   }
+// }
+// module.exports = EbookReader

server/Scanner.js

@@ -25,25 +25,6 @@ class Scanner {
     return this.db.audiobooks
   }
 
-  async setAudiobookDataInos(audiobookData) {
-    for (let i = 0; i < audiobookData.length; i++) {
-      var abd = audiobookData[i]
-      var matchingAB = this.db.audiobooks.find(_ab => comparePaths(_ab.path, abd.path))
-      if (matchingAB) {
-        if (!matchingAB.ino) {
-          matchingAB.ino = await getIno(matchingAB.fullPath)
-        }
-        abd.ino = matchingAB.ino
-      } else {
-        abd.ino = await getIno(abd.fullPath)
-        if (!abd.ino) {
-          Logger.error('[Scanner] Invalid ino - ignoring audiobook data', abd.path)
-        }
-      }
-    }
-    return audiobookData.filter(abd => !!abd.ino)
-  }
-
   async setAudioFileInos(audiobookDataAudioFiles, audiobookAudioFiles) {
     for (let i = 0; i < audiobookDataAudioFiles.length; i++) {
       var abdFile = audiobookDataAudioFiles[i]
@@ -68,7 +49,6 @@ class Scanner {
     Logger.debug(`[Scanner] Scanning "${audiobookData.title}" (${audiobookData.ino}) - ${!!existingAudiobook ? 'Exists' : 'New'}`)
 
     if (existingAudiobook) {
-
       // REMOVE: No valid audio files
       // TODO: Label as incomplete, do not actually delete
       if (!audiobookData.audioFiles.length) {
@@ -80,7 +60,10 @@ class Scanner {
         return ScanResult.REMOVED
       }
 
-      audiobookData.audioFiles = await this.setAudioFileInos(audiobookData.audioFiles, existingAudiobook.audioFiles)
+      // ino is now set for every file in scandir
+      audiobookData.audioFiles = audiobookData.audioFiles.filter(af => af.ino)
+      // audiobookData.audioFiles = await this.setAudioFileInos(audiobookData.audioFiles, existingAudiobook.audioFiles)
+
 
       // Check for audio files that were removed
       var abdAudioFileInos = audiobookData.audioFiles.map(af => af.ino)
@@ -90,6 +73,14 @@ class Scanner {
         removedAudioFiles.forEach((af) => existingAudiobook.removeAudioFile(af))
       }
 
+      // Check for mismatched audio tracks - tracks with no matching audio file
+      var removedAudioTracks = existingAudiobook.tracks.filter(track => !abdAudioFileInos.includes(track.ino))
+      if (removedAudioTracks.length) {
+        Logger.info(`[Scanner] ${removedAudioTracks.length} tracks removed no matching audio file for audiobook "${existingAudiobook.title}"`)
+        removedAudioTracks.forEach((at) => existingAudiobook.removeAudioTrack(at))
+      }
+
+
       // Check for new audio files and sync existing audio files
       var newAudioFiles = []
       var hasUpdatedAudioFiles = false
@@ -124,7 +115,7 @@ class Scanner {
         return ScanResult.REMOVED
       }
 
-      var hasUpdates = removedAudioFiles.length || newAudioFiles.length || hasUpdatedAudioFiles
+      var hasUpdates = removedAudioFiles.length || removedAudioTracks.length || newAudioFiles.length || hasUpdatedAudioFiles
 
       if (existingAudiobook.checkUpdateMissingParts()) {
         Logger.info(`[Scanner] "${existingAudiobook.title}" missing parts updated`)
@@ -187,27 +178,28 @@ class Scanner {
     // TODO: This temporary fix from pre-release should be removed soon, including the "fixRelativePath" and "checkUpdateInos"
     // TEMP - fix relative file paths
     // TEMP - update ino for each audiobook
-    // if (this.audiobooks.length) {
-    //   for (let i = 0; i < this.audiobooks.length; i++) {
-    //     var ab = this.audiobooks[i]
-    //     var shouldUpdate = ab.fixRelativePath(this.AudiobookPath) || !ab.ino
+    if (this.audiobooks.length) {
+      for (let i = 0; i < this.audiobooks.length; i++) {
+        var ab = this.audiobooks[i]
+        // var shouldUpdate = ab.fixRelativePath(this.AudiobookPath) || !ab.ino
 
-    //     // Update ino if an audio file has the same ino as the audiobook
-    //     var shouldUpdateIno = !ab.ino || (ab.audioFiles || []).find(abf => abf.ino === ab.ino)
-    //     if (shouldUpdateIno) {
-    //       await ab.checkUpdateInos()
-    //     }
-    //     if (shouldUpdate) {
-    //       await this.db.updateAudiobook(ab)
-    //     }
-    //   }
-    // }
+        // Update ino if inos are not set
+        var shouldUpdateIno = ab.hasMissingIno
+        if (shouldUpdateIno) {
+          Logger.debug(`Updating inos for ${ab.title}`)
+          var hasUpdates = await ab.checkUpdateInos()
+          if (hasUpdates) {
+            await this.db.updateAudiobook(ab)
+          }
+        }
+      }
+    }
 
     const scanStart = Date.now()
     var audiobookDataFound = await scanRootDir(this.AudiobookPath, this.db.serverSettings)
 
-    // Set ino for each ab data as a string
-    audiobookDataFound = await this.setAudiobookDataInos(audiobookDataFound)
+    // Remove audiobooks with no inode
+    audiobookDataFound = audiobookDataFound.filter(abd => abd.ino)
 
     if (this.cancelScan) {
       this.cancelScan = false
@@ -323,7 +315,11 @@ class Scanner {
   async filesChanged(filepaths) {
     if (!filepaths.length) return ScanResult.NOTHING
     var relfilepaths = filepaths.map(path => path.replace(this.AudiobookPath, ''))
-    var fileGroupings = groupFilesIntoAudiobookPaths(relfilepaths)
+    var fileGroupings = groupFilesIntoAudiobookPaths(relfilepaths, true)
+
+
+    Logger.debug(`[Scanner] fileGroupings `, filepaths, fileGroupings)
+
 
     var results = []
     for (const dir in fileGroupings) {

server/Server.js

@@ -14,6 +14,7 @@ const HlsController = require('./HlsController')
 const StreamManager = require('./StreamManager')
 const RssFeeds = require('./RssFeeds')
 const DownloadManager = require('./DownloadManager')
+// const EbookReader = require('./EbookReader')
 const Logger = require('./Logger')
 
 class Server {
@@ -37,6 +38,7 @@ class Server {
     this.downloadManager = new DownloadManager(this.db, this.MetadataPath, this.AudiobookPath, this.emitter.bind(this))
     this.apiController = new ApiController(this.MetadataPath, this.db, this.scanner, this.auth, this.streamManager, this.rssFeeds, this.downloadManager, this.emitter.bind(this), this.clientEmitter.bind(this))
     this.hlsController = new HlsController(this.db, this.scanner, this.auth, this.streamManager, this.emitter.bind(this), this.streamManager.StreamsPath)
+    // this.ebookReader = new EbookReader(this.db, this.MetadataPath, this.AudiobookPath)
 
     this.server = null
     this.io = null
@@ -204,7 +206,7 @@ class Server {
 
     app.use('/api', this.authMiddleware.bind(this), this.apiController.router)
     app.use('/hls', this.authMiddleware.bind(this), this.hlsController.router)
-    // app.use('/hls', this.hlsController.router)
+    // app.use('/ebook', this.ebookReader.router)
     app.use('/feeds', this.rssFeeds.router)
 
     app.post('/upload', this.authMiddleware.bind(this), this.handleUpload.bind(this))

server/objects/Audiobook.js

@@ -106,6 +106,14 @@ class Audiobook {
     return (this.audioFiles || []).filter(af => af.invalid).map(af => ({ filename: af.filename, error: af.error || 'Unknown Error' }))
   }
 
+  get ebooks() {
+    return this.otherFiles.filter(file => file.filetype === 'ebook')
+  }
+
+  get hasMissingIno() {
+    return !this.ino || (this.audioFiles || []).find(abf => !abf.ino) || (this.otherFiles || []).find(f => !f.ino) || (this.tracks || []).find(t => !t.ino)
+  }
+
   bookToJSON() {
     return this.book ? this.book.toJSON() : null
   }
@@ -152,6 +160,7 @@ class Audiobook {
       hasBookMatch: !!this.book,
       hasMissingParts: this.missingParts ? this.missingParts.length : 0,
       hasInvalidParts: this.invalidParts ? this.invalidParts.length : 0,
+      numEbooks: this.ebooks.length,
       numTracks: this.tracks.length,
       chapters: this.chapters || [],
       isMissing: !!this.isMissing
@@ -173,6 +182,7 @@ class Audiobook {
       invalidParts: this.invalidParts,
       audioFiles: (this.audioFiles || []).map(audioFile => audioFile.toJSON()),
       otherFiles: (this.otherFiles || []).map(otherFile => otherFile.toJSON()),
+      ebooks: (this.ebooks || []).map(ebook => ebook.toJSON()),
       tags: this.tags,
       book: this.bookToJSON(),
       tracks: this.tracksToJSON(),
@@ -195,7 +205,8 @@ class Audiobook {
     return false
   }
 
-  // Update was made to add ino values, ensure they are set
+  // Originally files did not store the inode value
+  // this function checks all files and sets the inode
   async checkUpdateInos() {
     var hasUpdates = false
     if (!this.ino) {
@@ -204,15 +215,55 @@ class Audiobook {
     }
     for (let i = 0; i < this.audioFiles.length; i++) {
       var af = this.audioFiles[i]
+      var at = this.tracks.find(t => t.ino === af.ino)
+      if (!at) {
+        at = this.tracks.find(t => comparePaths(t.path, af.path))
+      }
       if (!af.ino || af.ino === this.ino) {
         af.ino = await getIno(af.fullPath)
         if (!af.ino) {
           Logger.error('[Audiobook] checkUpdateInos: Failed to set ino for audio file', af.fullPath)
         } else {
-          var track = this.tracks.find(t => comparePaths(t.path, af.path))
-          if (track) {
-            track.ino = af.ino
+          Logger.debug(`[Audiobook] Set INO For audio file ${af.path}`)
+          if (at) at.ino = af.ino
+        }
+        hasUpdates = true
+      } else if (at && at.ino !== af.ino) {
+        at.ino = af.ino
+        hasUpdates = true
+      }
+    }
+    for (let i = 0; i < this.tracks.length; i++) {
+      var at = this.tracks[i]
+      if (!at.ino) {
+        Logger.debug(`[Audiobook] Track ${at.filename} still does not have ino`)
+        var atino = await getIno(at.fullPath)
+        var af = this.audioFiles.find(_af => _af.ino === atino)
+        if (!af) {
+          Logger.debug(`[Audiobook] Track ${at.filename} no matching audio file with ino ${atino}`)
+          af = this.audioFiles.find(_af => _af.filename === at.filename)
+          if (!af) {
+            Logger.debug(`[Audiobook] Track ${at.filename} no matching audio file with filename`)
+          } else {
+            Logger.debug(`[Audiobook] Track ${at.filename} found matching filename but mismatch ino ${atino}/${af.ino}`)
+            // at.ino = af.ino
+            // at.path = af.path
+            // at.fullPath = af.fullPath
+            // hasUpdates = true
           }
+        } else {
+          Logger.debug(`[Audiobook] Track ${at.filename} found audio file with matching ino ${at.path}/${af.path}`)
+        }
+      }
+    }
+    for (let i = 0; i < this.otherFiles.length; i++) {
+      var file = this.otherFiles[i]
+      if (!file.ino || file.ino === this.ino) {
+        file.ino = await getIno(file.fullPath)
+        if (!file.ino) {
+          Logger.error('[Audiobook] checkUpdateInos: Failed to set ino for other file', file.fullPath)
+        } else {
+          Logger.debug(`[Audiobook] Set INO For other file ${file.path}`)
         }
         hasUpdates = true
       }
@@ -329,6 +380,11 @@ class Audiobook {
     this.audioFiles = this.audioFiles.filter(f => f.ino !== audioFile.ino)
   }
 
+  removeAudioTrack(track) {
+    this.tracks = this.tracks.filter(t => t.ino !== track.ino)
+    this.audioFiles = this.audioFiles.filter(f => f.ino !== track.ino)
+  }
+
   checkUpdateMissingParts() {
     var currMissingParts = (this.missingParts || []).join(',') || ''
 
@@ -363,6 +419,7 @@ class Audiobook {
     var newOtherFilePaths = newOtherFiles.map(f => f.path)
     this.otherFiles = this.otherFiles.filter(f => newOtherFilePaths.includes(f.path))
 
+    // TODO: Should use inode
     newOtherFiles.forEach((file) => {
       var existingOtherFile = this.otherFiles.find(f => f.path === file.path)
       if (!existingOtherFile) {

server/utils/audioFileScanner.js

@@ -85,7 +85,7 @@ function getTrackNumberFromFilename(title, author, series, publishYear, filename
 
 async function scanAudioFiles(audiobook, newAudioFiles) {
   if (!newAudioFiles || !newAudioFiles.length) {
-    Logger.error('[AudioFileScanner] Scan Audio Files no files', audiobook.title)
+    Logger.error('[AudioFileScanner] Scan Audio Files no new files', audiobook.title)
     return
   }
   var tracks = []

server/utils/scandir.js

@@ -1,6 +1,7 @@
 const Path = require('path')
 const dir = require('node-dir')
 const Logger = require('../Logger')
+const { getIno } = require('./index')
 
 const AUDIO_FORMATS = ['m4b', 'mp3', 'm4a']
 const INFO_FORMATS = ['nfo']
@@ -26,7 +27,7 @@ function isAudioFile(path) {
   return AUDIO_FORMATS.includes(ext.slice(1).toLowerCase())
 }
 
-function groupFilesIntoAudiobookPaths(paths) {
+function groupFilesIntoAudiobookPaths(paths, useAllFileTypes = false) {
   // Step 1: Normalize path, Remove leading "/", Filter out files in root dir
   var pathsFiltered = paths.map(path => Path.normalize(path.slice(1))).filter(path => Path.parse(path).dir)
 
@@ -37,11 +38,11 @@ function groupFilesIntoAudiobookPaths(paths) {
     return pathsA - pathsB
   })
 
-  // Step 2.5: Seperate audio files and other files
+  // Step 2.5: Seperate audio files and other files (optional)
   var audioFilePaths = []
   var otherFilePaths = []
   pathsFiltered.forEach(path => {
-    if (isAudioFile(path)) audioFilePaths.push(path)
+    if (isAudioFile(path) || useAllFileTypes) audioFilePaths.push(path)
     else otherFilePaths.push(path)
   })
 
@@ -134,7 +135,12 @@ async function scanRootDir(abRootPath, serverSettings = {}) {
     var audiobookData = getAudiobookDataFromDir(abRootPath, audiobookPath, parseSubtitle)
 
     var fileObjs = cleanFileObjects(audiobookData.fullPath, audiobookPath, audiobookGrouping[audiobookPath])
+    for (let i = 0; i < fileObjs.length; i++) {
+      fileObjs[i].ino = await getIno(fileObjs[i].fullPath)
+    }
+    var audiobookIno = await getIno(audiobookData.fullPath)
     audiobooks.push({
+      ino: audiobookIno,
       ...audiobookData,
       audioFiles: fileObjs.filter(f => f.filetype === 'audio'),
       otherFiles: fileObjs.filter(f => f.filetype !== 'audio')
@@ -241,11 +247,15 @@ async function getAudiobookFileData(abRootPath, audiobookPath, serverSettings =
     otherFiles: []
   }
 
-  filepaths.forEach((filepath) => {
+  for (let i = 0; i < filepaths.length; i++) {
+    var filepath = filepaths[i]
+
     var relpath = Path.normalize(filepath).replace(abRootPath, '').slice(1)
     var extname = Path.extname(filepath)
     var basename = Path.basename(filepath)
+    var ino = await getIno(filepath)
     var fileObj = {
+      ino,
       filetype: getFileType(extname),
       filename: basename,
       path: relpath,
@@ -257,7 +267,7 @@ async function getAudiobookFileData(abRootPath, audiobookPath, serverSettings =
     } else {
       audiobook.otherFiles.push(fileObj)
     }
-  })
+  }
   return audiobook
 }
 module.exports.getAudiobookFileData = getAudiobookFileData