Add back in m4b merge downloader in experimental #478

server/Server.js

@@ -23,7 +23,7 @@ const HlsRouter = require('./routers/HlsRouter')
 const StaticRouter = require('./routers/StaticRouter')
 
 const CoverManager = require('./managers/CoverManager')
-const DownloadManager = require('./managers/DownloadManager')
+const AbMergeManager = require('./managers/AbMergeManager')
 const CacheManager = require('./managers/CacheManager')
 const LogManager = require('./managers/LogManager')
 const BackupManager = require('./managers/BackupManager')
@@ -58,7 +58,7 @@ class Server {
     this.backupManager = new BackupManager(this.db, this.emitter.bind(this))
     this.logManager = new LogManager(this.db)
     this.cacheManager = new CacheManager()
-    this.downloadManager = new DownloadManager(this.db)
+    this.abMergeManager = new AbMergeManager(this.db, this.clientEmitter.bind(this))
     this.playbackSessionManager = new PlaybackSessionManager(this.db, this.emitter.bind(this), this.clientEmitter.bind(this))
     this.coverManager = new CoverManager(this.db, this.cacheManager)
     this.podcastManager = new PodcastManager(this.db, this.watcher, this.emitter.bind(this))
@@ -66,7 +66,7 @@ class Server {
     this.scanner = new Scanner(this.db, this.coverManager, this.emitter.bind(this))
 
     // Routers
-    this.apiRouter = new ApiRouter(this.db, this.auth, this.scanner, this.playbackSessionManager, this.downloadManager, this.coverManager, this.backupManager, this.watcher, this.cacheManager, this.podcastManager, this.emitter.bind(this), this.clientEmitter.bind(this))
+    this.apiRouter = new ApiRouter(this.db, this.auth, this.scanner, this.playbackSessionManager, this.abMergeManager, this.coverManager, this.backupManager, this.watcher, this.cacheManager, this.podcastManager, this.emitter.bind(this), this.clientEmitter.bind(this))
     this.hlsRouter = new HlsRouter(this.db, this.auth, this.playbackSessionManager, this.emitter.bind(this))
     this.staticRouter = new StaticRouter(this.db)
 
@@ -112,8 +112,8 @@ class Server {
 
   async init() {
     Logger.info('[Server] Init v' + version)
+    await this.abMergeManager.removeOrphanDownloads()
     await this.playbackSessionManager.removeOrphanStreams()
-    await this.downloadManager.removeOrphanDownloads()
 
     var previousVersion = await this.db.checkPreviousVersion() // Returns null if same server version
     if (previousVersion) {

server/controllers/MiscController.js

@@ -82,15 +82,43 @@ class MiscController {
     res.sendStatus(200)
   }
 
+  // GET: api/audiobook-merge/:id
+  async mergeAudiobook(req, res) {
+    if (!req.user.canDownload) {
+      Logger.error('User attempting to download without permission', req.user)
+      return res.sendStatus(403)
+    }
+
+    var libraryItem = this.db.getLibraryItem(req.params.id)
+    if (!libraryItem || libraryItem.isMissing || libraryItem.isInvalid) {
+      Logger.error(`[MiscController] mergeAudiboook: library item not found or invalid ${req.params.id}`)
+      return res.status(404).send('Audiobook not found')
+    }
+
+    if (libraryItem.mediaType !== 'book') {
+      Logger.error(`[MiscController] mergeAudiboook: Invalid library item ${req.params.id}: not a book`)
+      return res.status(500).send('Invalid library item: not a book')
+    }
+
+    if (libraryItem.media.tracks.length <= 0) {
+      Logger.error(`[MiscController] mergeAudiboook: Invalid audiobook ${req.params.id}: no audio tracks`)
+      return res.status(500).send('Invalid audiobook: no audio tracks')
+    }
+
+    this.abMergeManager.startAudiobookMerge(req.user, libraryItem)
+
+    res.sendStatus(200)
+  }
+
   // GET: api/download/:id
-  async download(req, res) {
+  async getDownload(req, res) {
     if (!req.user.canDownload) {
       Logger.error('User attempting to download without permission', req.user)
       return res.sendStatus(403)
     }
     var downloadId = req.params.id
     Logger.info('Download Request', downloadId)
-    var download = this.downloadManager.getDownload(downloadId)
+    var download = this.abMergeManager.getDownload(downloadId)
     if (!download) {
       Logger.error('Download request not found', downloadId)
       return res.sendStatus(404)
@@ -101,13 +129,36 @@ class MiscController {
         'Content-Type': download.mimeType
       }
     }
-    res.download(download.fullPath, download.filename, options, (err) => {
+    res.download(download.path, download.filename, options, (err) => {
       if (err) {
         Logger.error('Download Error', err)
       }
     })
   }
 
+  // DELETE: api/download/:id
+  async removeDownload(req, res) {
+    if (!req.user.canDownload || !req.user.canDelete) {
+      Logger.error('User attempting to remove download without permission', req.user.username)
+      return res.sendStatus(403)
+    }
+    this.abMergeManager.removeDownloadById(req.params.id)
+    res.sendStatus(200)
+  }
+
+  // GET: api/downloads
+  async getDownloads(req, res) {
+    if (!req.user.canDownload) {
+      Logger.error('User attempting to get downloads without permission', req.user.username)
+      return res.sendStatus(403)
+    }
+    var downloads = {
+      downloads: this.abMergeManager.downloads,
+      pendingDownloads: this.abMergeManager.pendingDownloads
+    }
+    res.json(downloads)
+  }
+
   // PATCH: api/settings (Root)
   async updateServerSettings(req, res) {
     if (!req.user.isRoot) {

server/managers/AbMergeManager.js

@@ -0,0 +1,284 @@
+
+const Path = require('path')
+const fs = require('fs-extra')
+
+const workerThreads = require('worker_threads')
+const Logger = require('../Logger')
+const Download = require('../objects/Download')
+const filePerms = require('../utils/filePerms')
+const { getId } = require('../utils/index')
+const { writeConcatFile, writeMetadataFile } = require('../utils/ffmpegHelpers')
+const { getFileSize } = require('../utils/fileUtils')
+
+class AbMergeManager {
+  constructor(db, clientEmitter) {
+    this.db = db
+    this.clientEmitter = clientEmitter
+
+    this.downloadDirPath = Path.join(global.MetadataPath, 'downloads')
+
+    this.pendingDownloads = []
+    this.downloads = []
+  }
+
+  getDownload(downloadId) {
+    return this.downloads.find(d => d.id === downloadId)
+  }
+
+  removeDownloadById(downloadId) {
+    var download = this.getDownload(downloadId)
+    if (download) {
+      this.removeDownload(download)
+    }
+  }
+
+  async removeOrphanDownloads() {
+    try {
+      var dirs = await fs.readdir(this.downloadDirPath)
+      if (!dirs || !dirs.length) return true
+
+      dirs = dirs.filter(d => d.startsWith('abmerge'))
+
+      await Promise.all(dirs.map(async (dirname) => {
+        var fullPath = Path.join(this.downloadDirPath, dirname)
+        Logger.info(`Removing Orphan Download ${dirname}`)
+        return fs.remove(fullPath)
+      }))
+      return true
+    } catch (error) {
+      return false
+    }
+  }
+
+  async startAudiobookMerge(user, libraryItem) {
+    var downloadId = getId('abmerge')
+    var dlpath = Path.join(this.downloadDirPath, downloadId)
+    Logger.info(`Start audiobook merge for ${libraryItem.id} - DownloadId: ${downloadId} - ${dlpath}`)
+
+    var audiobookDirname = Path.basename(libraryItem.path)
+    var filename = audiobookDirname + '.m4b'
+    var downloadData = {
+      id: downloadId,
+      libraryItemId: libraryItem.id,
+      type: 'abmerge',
+      dirpath: dlpath,
+      path: Path.join(dlpath, filename),
+      filename,
+      ext: '.m4b',
+      userId: user.id
+    }
+    var download = new Download()
+    download.setData(downloadData)
+    download.setTimeoutTimer(this.downloadTimedOut.bind(this))
+
+
+    try {
+      await fs.mkdir(download.dirpath)
+    } catch (error) {
+      Logger.error(`[AbMergeManager] Failed to make directory ${download.dirpath}`)
+      var downloadJson = download.toJSON()
+      this.clientEmitter(user.id, 'abmerge_failed', downloadJson)
+      return
+    }
+
+    this.clientEmitter(user.id, 'abmerge_started', download.toJSON())
+    this.runAudiobookMerge(libraryItem, download)
+  }
+
+  async runAudiobookMerge(libraryItem, download) {
+
+    // If changing audio file type then encoding is needed
+    var audioTracks = libraryItem.media.tracks
+    var audioRequiresEncode = audioTracks[0].metadata.ext !== download.ext
+    var shouldIncludeCover = libraryItem.media.coverPath
+    var firstTrackIsM4b = audioTracks[0].metadata.ext.toLowerCase() === '.m4b'
+    var isOneTrack = audioTracks.length === 1
+
+    const ffmpegInputs = []
+
+    if (!isOneTrack) {
+      var concatFilePath = Path.join(download.dirpath, 'files.txt')
+      console.log('Write files.txt', concatFilePath)
+      await writeConcatFile(audioTracks, concatFilePath)
+      ffmpegInputs.push({
+        input: concatFilePath,
+        options: ['-safe 0', '-f concat']
+      })
+    } else {
+      ffmpegInputs.push({
+        input: audioTracks[0].metadata.path,
+        options: firstTrackIsM4b ? ['-f mp4'] : []
+      })
+    }
+
+    const logLevel = process.env.NODE_ENV === 'production' ? 'error' : 'warning'
+    var ffmpegOptions = [`-loglevel ${logLevel}`]
+    var ffmpegOutputOptions = []
+
+    if (audioRequiresEncode) {
+      ffmpegOptions = ffmpegOptions.concat([
+        '-map 0:a',
+        '-acodec aac',
+        '-ac 2',
+        '-b:a 64k',
+        '-id3v2_version 3'
+      ])
+    } else {
+      ffmpegOptions.push('-max_muxing_queue_size 1000')
+
+      if (isOneTrack && firstTrackIsM4b && !shouldIncludeCover) {
+        ffmpegOptions.push('-c copy')
+      } else {
+        ffmpegOptions.push('-c:a copy')
+      }
+    }
+    if (download.ext === '.m4b') {
+      ffmpegOutputOptions.push('-f mp4')
+    }
+
+    // Create ffmetadata file
+    var metadataFilePath = Path.join(download.dirpath, 'metadata.txt')
+    await writeMetadataFile(libraryItem, metadataFilePath)
+    ffmpegInputs.push({
+      input: metadataFilePath
+    })
+    ffmpegOptions.push('-map_metadata 1')
+
+    // Embed cover art
+    if (shouldIncludeCover) {
+      var coverPath = libraryItem.media.coverPath.replace(/\\/g, '/')
+      ffmpegInputs.push({
+        input: coverPath,
+        options: ['-f image2pipe']
+      })
+      ffmpegOptions.push('-vf [2:v]crop=trunc(iw/2)*2:trunc(ih/2)*2')
+      ffmpegOptions.push('-map 2:v')
+    }
+
+    var workerData = {
+      inputs: ffmpegInputs,
+      options: ffmpegOptions,
+      outputOptions: ffmpegOutputOptions,
+      output: download.path,
+    }
+
+    var worker = null
+    try {
+      var workerPath = Path.join(global.appRoot, 'server/utils/downloadWorker.js')
+      worker = new workerThreads.Worker(workerPath, { workerData })
+    } catch (error) {
+      Logger.error(`[AbMergeManager] Start worker thread failed`, error)
+      if (download.userId) {
+        var downloadJson = download.toJSON()
+        this.clientEmitter(download.userId, 'abmerge_failed', downloadJson)
+      }
+      this.removeDownload(download)
+      return
+    }
+
+    worker.on('message', (message) => {
+      if (message != null && typeof message === 'object') {
+        if (message.type === 'RESULT') {
+          if (!download.isTimedOut) {
+            this.sendResult(download, message)
+          }
+        } else if (message.type === 'FFMPEG') {
+          if (Logger[message.level]) {
+            Logger[message.level](message.log)
+          }
+        }
+      } else {
+        Logger.error('Invalid worker message', message)
+      }
+    })
+    this.pendingDownloads.push({
+      id: download.id,
+      download,
+      worker
+    })
+  }
+
+  async sendResult(download, result) {
+    download.clearTimeoutTimer()
+
+    // Remove pending download
+    this.pendingDownloads = this.pendingDownloads.filter(d => d.id !== download.id)
+
+    if (result.isKilled) {
+      if (download.userId) {
+        this.clientEmitter(download.userId, 'abmerge_killed', download.toJSON())
+      }
+      return
+    }
+
+    if (!result.success) {
+      if (download.userId) {
+        this.clientEmitter(download.userId, 'abmerge_failed', download.toJSON())
+      }
+      this.removeDownload(download)
+      return
+    }
+
+    // Set file permissions and ownership
+    await filePerms.setDefault(download.path)
+
+    var filesize = await getFileSize(download.path)
+    download.setComplete(filesize)
+    if (download.userId) {
+      this.clientEmitter(download.userId, 'abmerge_ready', download.toJSON())
+    }
+    download.setExpirationTimer(this.downloadExpired.bind(this))
+
+    this.downloads.push(download)
+    Logger.info(`[AbMergeManager] Download Ready ${download.id}`)
+  }
+
+  async downloadExpired(download) {
+    Logger.info(`[AbMergeManager] Download ${download.id} expired`)
+
+    if (download.userId) {
+      this.clientEmitter(download.userId, 'abmerge_expired', download.toJSON())
+    }
+    this.removeDownload(download)
+  }
+
+  async downloadTimedOut(download) {
+    Logger.info(`[AbMergeManager] Download ${download.id} timed out (${download.timeoutTimeMs}ms)`)
+
+    if (download.userId) {
+      var downloadJson = download.toJSON()
+      downloadJson.isTimedOut = true
+      this.clientEmitter(download.userId, 'abmerge_failed', downloadJson)
+    }
+    this.removeDownload(download)
+  }
+
+  async removeDownload(download) {
+    Logger.info('[AbMergeManager] Removing download ' + download.id)
+
+    download.clearTimeoutTimer()
+    download.clearExpirationTimer()
+
+    var pendingDl = this.pendingDownloads.find(d => d.id === download.id)
+
+    if (pendingDl) {
+      this.pendingDownloads = this.pendingDownloads.filter(d => d.id !== download.id)
+      Logger.warn(`[AbMergeManager] Removing download in progress - stopping worker`)
+      if (pendingDl.worker) {
+        try {
+          pendingDl.worker.postMessage('STOP')
+        } catch (error) {
+          Logger.error('[AbMergeManager] Error posting stop message to worker', error)
+        }
+      }
+    }
+
+    await fs.remove(download.dirpath).then(() => {
+      Logger.info('[AbMergeManager] Deleted download', download.dirpath)
+    }).catch((err) => {
+      Logger.error('[AbMergeManager] Failed to delete download', err)
+    })
+    this.downloads = this.downloads.filter(d => d.id !== download.id)
+  }
+}
+module.exports = AbMergeManager

server/objects/Download.js

@@ -1,20 +1,18 @@
 const DEFAULT_EXPIRATION = 1000 * 60 * 60 // 60 minutes
-const DEFAULT_TIMEOUT = 1000 * 60 * 15 // 15 minutes
+const DEFAULT_TIMEOUT = 1000 * 60 * 20 // 20 minutes
 class Download {
   constructor(download) {
     this.id = null
-    this.audiobookId = null
+    this.libraryItemId = null
     this.type = null
-    this.options = {}
 
     this.dirpath = null
-    this.fullPath = null
+    this.path = null
     this.ext = null
     this.filename = null
     this.size = 0
 
     this.userId = null
-    this.socket = null // Socket to notify when complete
     this.isReady = false
     this.isTimedOut = false
 
@@ -33,14 +31,6 @@ class Download {
     }
   }
 
-  get includeMetadata() {
-    return !!this.options.includeMetadata
-  }
-
-  get includeCover() {
-    return !!this.options.includeCover
-  }
-
   get mimeType() {
     if (this.ext === '.mp3' || this.ext === '.m4b' || this.ext === '.m4a') {
       return 'audio/mpeg'
@@ -57,11 +47,10 @@ class Download {
   toJSON() {
     return {
       id: this.id,
-      audiobookId: this.audiobookId,
+      libraryItemId: this.libraryItemId,
       type: this.type,
-      options: this.options,
       dirpath: this.dirpath,
-      fullPath: this.fullPath,
+      path: this.path,
       ext: this.ext,
       filename: this.filename,
       size: this.size,
@@ -75,18 +64,16 @@ class Download {
 
   construct(download) {
     this.id = download.id
-    this.audiobookId = download.audiobookId
+    this.libraryItemId = download.libraryItemId
     this.type = download.type
-    this.options = { ...download.options }
 
     this.dirpath = download.dirpath
-    this.fullPath = download.fullPath
+    this.path = download.path
     this.ext = download.ext
     this.filename = download.filename
     this.size = download.size || 0
 
     this.userId = download.userId
-    this.socket = download.socket || null
     this.isReady = !!download.isReady
 
     this.startedAt = download.startedAt

server/routers/ApiRouter.js

@@ -25,12 +25,12 @@ const Series = require('../objects/entities/Series')
 const FileSystemController = require('../controllers/FileSystemController')
 
 class ApiRouter {
-  constructor(db, auth, scanner, playbackSessionManager, downloadManager, coverManager, backupManager, watcher, cacheManager, podcastManager, emitter, clientEmitter) {
+  constructor(db, auth, scanner, playbackSessionManager, abMergeManager, coverManager, backupManager, watcher, cacheManager, podcastManager, emitter, clientEmitter) {
     this.db = db
     this.auth = auth
     this.scanner = scanner
     this.playbackSessionManager = playbackSessionManager
-    this.downloadManager = downloadManager
+    this.abMergeManager = abMergeManager
     this.backupManager = backupManager
     this.coverManager = coverManager
     this.watcher = watcher
@@ -185,7 +185,10 @@ class ApiRouter {
     // Misc Routes
     //
     this.router.post('/upload', MiscController.handleUpload.bind(this))
-    this.router.get('/download/:id', MiscController.download.bind(this))
+    this.router.get('/audiobook-merge/:id', MiscController.mergeAudiobook.bind(this))
+    this.router.get('/download/:id', MiscController.getDownload.bind(this))
+    this.router.delete('/download/:id', MiscController.removeDownload.bind(this))
+    this.router.get('/downloads', MiscController.getDownloads.bind(this))
     this.router.patch('/settings', MiscController.updateServerSettings.bind(this)) // Root only
     this.router.post('/purgecache', MiscController.purgeCache.bind(this)) // Root only
     this.router.post('/authorize', MiscController.authorize.bind(this))

server/utils/ffmpegHelpers.js

@@ -41,20 +41,19 @@ async function writeConcatFile(tracks, outputPath, startTime = 0) {
 module.exports.writeConcatFile = writeConcatFile
 
 
-async function writeMetadataFile(audiobook, outputPath) {
+async function writeMetadataFile(libraryItem, outputPath) {
   var inputstrs = [
     ';FFMETADATA1',
-    `title=${audiobook.title}`,
-    `artist=${audiobook.authorFL}`,
-    `album_artist=${audiobook.authorFL}`,
-    `date=${audiobook.book.publishedYear || ''}`,
-    `description=${audiobook.book.description}`,
-    `genre=${audiobook.book._genres.join(';')}`,
-    `comment=Audiobookshelf v${package.version}`
+    `title=${libraryItem.media.metadata.title}`,
+    `artist=${libraryItem.media.metadata.authorName}`,
+    `album_artist=${libraryItem.media.metadata.authorName}`,
+    `date=${libraryItem.media.metadata.publishedYear || ''}`,
+    `description=${libraryItem.media.metadata.description}`,
+    `genre=${libraryItem.media.metadata.genres.join(';')}`
   ]
 
-  if (audiobook.chapters) {
-    audiobook.chapters.forEach((chap) => {
+  if (libraryItem.media.chapters) {
+    libraryItem.media.chapters.forEach((chap) => {
       const chapterstrs = [
         '[CHAPTER]',
         'TIMEBASE=1/1000',