Merge branch 'master' into sso

server/Database.js

@@ -132,6 +132,11 @@ class Database {
     return this.models.playbackSession
   }
 
+  /** @type {typeof import('./models/CustomMetadataProvider')} */
+  get customMetadataProviderModel() {
+    return this.models.customMetadataProvider
+  }
+
   /**
    * Check if db file exists
    * @returns {boolean}
@@ -245,6 +250,7 @@ class Database {
     require('./models/Feed').init(this.sequelize)
     require('./models/FeedEpisode').init(this.sequelize)
     require('./models/Setting').init(this.sequelize)
+    require('./models/CustomMetadataProvider').init(this.sequelize)
 
     return this.sequelize.sync({ force, alter: false })
   }

server/Logger.js

@@ -3,13 +3,17 @@ const { LogLevel } = require('./utils/constants')
 
 class Logger {
   constructor() {
+    /** @type {import('./managers/LogManager')} */
+    this.logManager = null
+
     this.isDev = process.env.NODE_ENV !== 'production'
     this.logLevel = !this.isDev ? LogLevel.INFO : LogLevel.TRACE
     this.socketListeners = []
-
-    this.logManager = null
   }
 
+  /**
+   * @returns {string}
+   */
   get timestamp() {
     return date.format(new Date(), 'YYYY-MM-DD HH:mm:ss.SSS')
   }
@@ -23,6 +27,9 @@ class Logger {
     return 'UNKNOWN'
   }
 
+  /**
+   * @returns {string}
+   */
   get source() {
     try {
       throw new Error()
@@ -62,7 +69,12 @@ class Logger {
     this.socketListeners = this.socketListeners.filter(s => s.id !== socketId)
   }
 
-  handleLog(level, args) {
+  /**
+   * 
+   * @param {number} level 
+   * @param {string[]} args 
+   */
+  async handleLog(level, args) {
     const logObj = {
       timestamp: this.timestamp,
       source: this.source,
@@ -71,15 +83,17 @@ class Logger {
       level
     }
 
-    if (level >= this.logLevel && this.logManager) {
-      this.logManager.logToFile(logObj)
-    }
-
+    // Emit log to sockets that are listening to log events
     this.socketListeners.forEach((socketListener) => {
       if (socketListener.level <= level) {
         socketListener.socket.emit('log', logObj)
       }
     })
+
+    // Save log to file
+    if (level >= this.logLevel) {
+      await this.logManager.logToFile(logObj)
+    }
   }
 
   setLogLevel(level) {
@@ -117,9 +131,15 @@ class Logger {
     this.handleLog(LogLevel.ERROR, args)
   }
 
+  /**
+   * Fatal errors are ones that exit the process
+   * Fatal logs are saved to crash_logs.txt
+   * 
+   * @param  {...any} args 
+   */
   fatal(...args) {
     console.error(`[${this.timestamp}] FATAL:`, ...args, `(${this.source})`)
-    this.handleLog(LogLevel.FATAL, args)
+    return this.handleLog(LogLevel.FATAL, args)
   }
 
   note(...args) {

server/Server.js

@@ -2,6 +2,7 @@ const Path = require('path')
 const Sequelize = require('sequelize')
 const express = require('express')
 const http = require('http')
+const util = require('util')
 const fs = require('./libs/fsExtra')
 const fileUpload = require('./libs/expressFileupload')
 const rateLimit = require('./libs/expressRateLimit')
@@ -21,11 +22,11 @@ const SocketAuthority = require('./SocketAuthority')
 const ApiRouter = require('./routers/ApiRouter')
 const HlsRouter = require('./routers/HlsRouter')
 
+const LogManager = require('./managers/LogManager')
 const NotificationManager = require('./managers/NotificationManager')
 const EmailManager = require('./managers/EmailManager')
 const AbMergeManager = require('./managers/AbMergeManager')
 const CacheManager = require('./managers/CacheManager')
-const LogManager = require('./managers/LogManager')
 const BackupManager = require('./managers/BackupManager')
 const PlaybackSessionManager = require('./managers/PlaybackSessionManager')
 const PodcastManager = require('./managers/PodcastManager')
@@ -67,7 +68,6 @@ class Server {
     this.notificationManager = new NotificationManager()
     this.emailManager = new EmailManager()
     this.backupManager = new BackupManager()
-    this.logManager = new LogManager()
     this.abMergeManager = new AbMergeManager()
     this.playbackSessionManager = new PlaybackSessionManager()
     this.podcastManager = new PodcastManager(this.watcher, this.notificationManager)
@@ -81,7 +81,7 @@ class Server {
     this.apiRouter = new ApiRouter(this)
     this.hlsRouter = new HlsRouter(this.auth, this.playbackSessionManager)
 
-    Logger.logManager = this.logManager
+    Logger.logManager = new LogManager()
 
     this.server = null
     this.io = null
@@ -102,10 +102,13 @@ class Server {
    */
   async init() {
     Logger.info('[Server] Init v' + version)
+
     await this.playbackSessionManager.removeOrphanStreams()
 
     await Database.init(false)
 
+    await Logger.logManager.init()
+
     // Create token secret if does not exist (Added v2.1.0)
     if (!Database.serverSettings.tokenSecret) {
       await this.auth.initTokenSecret()
@@ -115,7 +118,6 @@ class Server {
     await CacheManager.ensureCachePaths()
 
     await this.backupManager.init()
-    await this.logManager.init()
     await this.rssFeedManager.init()
 
     const libraries = await Database.libraryModel.getAllOldLibraries()
@@ -135,8 +137,41 @@ class Server {
     }
   }
 
+  /**
+   * Listen for SIGINT and uncaught exceptions
+   */
+  initProcessEventListeners() {
+    let sigintAlreadyReceived = false
+    process.on('SIGINT', async () => {
+      if (!sigintAlreadyReceived) {
+        sigintAlreadyReceived = true
+        Logger.info('SIGINT (Ctrl+C) received. Shutting down...')
+        await this.stop()
+        Logger.info('Server stopped. Exiting.')
+      } else {
+        Logger.info('SIGINT (Ctrl+C) received again. Exiting immediately.')
+      }
+      process.exit(0)
+    })
+
+    /**
+     * @see https://nodejs.org/api/process.html#event-uncaughtexceptionmonitor
+     */
+    process.on('uncaughtExceptionMonitor', async (error, origin) => {
+      await Logger.fatal(`[Server] Uncaught exception origin: ${origin}, error:`, util.format('%O', error))
+    })
+    /**
+     * @see https://nodejs.org/api/process.html#event-unhandledrejection
+     */
+    process.on('unhandledRejection', async (reason, promise) => {
+      await Logger.fatal(`[Server] Unhandled rejection: ${reason}, promise:`, util.format('%O', promise))
+      process.exit(1)
+    })
+  }
+
   async start() {
     Logger.info('=== Starting Server ===')
+    this.initProcessEventListeners()
     await this.init()
 
     const app = express()
@@ -284,19 +319,6 @@ class Server {
     })
     app.get('/healthcheck', (req, res) => res.sendStatus(200))
 
-    let sigintAlreadyReceived = false
-    process.on('SIGINT', async () => {
-      if (!sigintAlreadyReceived) {
-        sigintAlreadyReceived = true
-        Logger.info('SIGINT (Ctrl+C) received. Shutting down...')
-        await this.stop()
-        Logger.info('Server stopped. Exiting.')
-      } else {
-        Logger.info('SIGINT (Ctrl+C) received again. Exiting immediately.')
-      }
-      process.exit(0)
-    })
-
     this.server.listen(this.Port, this.Host, () => {
       if (this.Host) Logger.info(`Listening on http://${this.Host}:${this.Port}`)
       else Logger.info(`Listening on port :${this.Port}`)

server/SocketAuthority.js

@@ -116,7 +116,6 @@ class SocketAuthority {
       // Logs
       socket.on('set_log_listener', (level) => Logger.addSocketListener(socket, level))
       socket.on('remove_log_listener', () => Logger.removeSocketListener(socket.id))
-      socket.on('fetch_daily_logs', () => this.Server.logManager.socketRequestDailyLogs(socket))
 
       // Sent automatically from socket.io clients
       socket.on('disconnect', (reason) => {

server/controllers/CustomMetadataProviderController.js

@@ -0,0 +1,117 @@
+const Logger = require('../Logger')
+const SocketAuthority = require('../SocketAuthority')
+const Database = require('../Database')
+
+const { validateUrl } = require('../utils/index')
+
+//
+// This is a controller for routes that don't have a home yet :(
+//
+class CustomMetadataProviderController {
+  constructor() { }
+
+  /**
+   * GET: /api/custom-metadata-providers
+   *
+   * @param {import('express').Request} req
+   * @param {import('express').Response} res
+   */
+  async getAll(req, res) {
+    const providers = await Database.customMetadataProviderModel.findAll()
+
+    res.json({
+      providers
+    })
+  }
+
+  /**
+   * POST: /api/custom-metadata-providers
+   *
+   * @param {import('express').Request} req
+   * @param {import('express').Response} res
+   */
+  async create(req, res) {
+    const { name, url, mediaType, authHeaderValue } = req.body
+
+    if (!name || !url || !mediaType) {
+      return res.status(400).send('Invalid request body')
+    }
+
+    const validUrl = validateUrl(url)
+    if (!validUrl) {
+      Logger.error(`[CustomMetadataProviderController] Invalid url "${url}"`)
+      return res.status(400).send('Invalid url')
+    }
+
+    const provider = await Database.customMetadataProviderModel.create({
+      name,
+      mediaType,
+      url,
+      authHeaderValue: !authHeaderValue ? null : authHeaderValue,
+    })
+
+    // TODO: Necessary to emit to all clients?
+    SocketAuthority.emitter('custom_metadata_provider_added', provider.toClientJson())
+
+    res.json({
+      provider
+    })
+  }
+
+  /**
+   * DELETE: /api/custom-metadata-providers/:id
+   * 
+   * @param {import('express').Request} req
+   * @param {import('express').Response} res
+   */
+  async delete(req, res) {
+    const slug = `custom-${req.params.id}`
+
+    /** @type {import('../models/CustomMetadataProvider')} */
+    const provider = req.customMetadataProvider
+    const providerClientJson = provider.toClientJson()
+
+    const fallbackProvider = provider.mediaType === 'book' ? 'google' : 'itunes'
+
+    await provider.destroy()
+
+    // Libraries using this provider fallback to default provider
+    await Database.libraryModel.update({
+      provider: fallbackProvider
+    }, {
+      where: {
+        provider: slug
+      }
+    })
+
+    // TODO: Necessary to emit to all clients?
+    SocketAuthority.emitter('custom_metadata_provider_removed', providerClientJson)
+
+    res.sendStatus(200)
+  }
+
+  /**
+   * Middleware that requires admin or up
+   * 
+   * @param {import('express').Request} req 
+   * @param {import('express').Response} res 
+   * @param {import('express').NextFunction} next 
+   */
+  async middleware(req, res, next) {
+    if (!req.user.isAdminOrUp) {
+      Logger.warn(`[CustomMetadataProviderController] Non-admin user "${req.user.username}" attempted access route "${req.path}"`)
+      return res.sendStatus(403)
+    }
+
+    // If id param then add req.customMetadataProvider
+    if (req.params.id) {
+      req.customMetadataProvider = await Database.customMetadataProviderModel.findByPk(req.params.id)
+      if (!req.customMetadataProvider) {
+        return res.sendStatus(404)
+      }
+    }
+
+    next()
+  }
+}
+module.exports = new CustomMetadataProviderController()

server/controllers/LibraryController.js

@@ -33,6 +33,14 @@ class LibraryController {
       return res.status(500).send('Invalid request')
     }
 
+    // Validate that the custom provider exists if given any
+    if (newLibraryPayload.provider?.startsWith('custom-')) {
+      if (!await Database.customMetadataProviderModel.checkExistsBySlug(newLibraryPayload.provider)) {
+        Logger.error(`[LibraryController] Custom metadata provider "${newLibraryPayload.provider}" does not exist`)
+        return res.status(400).send('Custom metadata provider does not exist')
+      }
+    }
+
     // Validate folder paths exist or can be created & resolve rel paths
     //   returns 400 if a folder fails to access
     newLibraryPayload.folders = newLibraryPayload.folders.map(f => {
@@ -86,19 +94,27 @@ class LibraryController {
     })
   }
 
+  /**
+   * GET: /api/libraries/:id
+   * 
+   * @param {import('express').Request} req 
+   * @param {import('express').Response} res 
+   */
   async findOne(req, res) {
     const includeArray = (req.query.include || '').split(',')
     if (includeArray.includes('filterdata')) {
       const filterdata = await libraryFilters.getFilterData(req.library.mediaType, req.library.id)
+      const customMetadataProviders = await Database.customMetadataProviderModel.getForClientByMediaType(req.library.mediaType)
 
       return res.json({
         filterdata,
         issues: filterdata.numIssues,
         numUserPlaylists: await Database.playlistModel.getNumPlaylistsForUserAndLibrary(req.user.id, req.library.id),
+        customMetadataProviders,
         library: req.library
       })
     }
-    return res.json(req.library)
+    res.json(req.library)
   }
 
   /**
@@ -115,6 +131,14 @@ class LibraryController {
   async update(req, res) {
     const library = req.library
 
+    // Validate that the custom provider exists if given any
+    if (req.body.provider?.startsWith('custom-')) {
+      if (!await Database.customMetadataProviderModel.checkExistsBySlug(req.body.provider)) {
+        Logger.error(`[LibraryController] Custom metadata provider "${req.body.provider}" does not exist`)
+        return res.status(400).send('Custom metadata provider does not exist')
+      }
+    }
+
     // Validate new folder paths exist or can be created & resolve rel paths
     //   returns 400 if a new folder fails to access
     if (req.body.folders) {

server/controllers/LibraryItemController.js

@@ -124,11 +124,6 @@ class LibraryItemController {
     const libraryItem = req.libraryItem
     const mediaPayload = req.body
 
-    // Item has cover and update is removing cover so purge it from cache
-    if (libraryItem.media.coverPath && (mediaPayload.coverPath === '' || mediaPayload.coverPath === null)) {
-      await CacheManager.purgeCoverCache(libraryItem.id)
-    }
-
     // Book specific
     if (libraryItem.isBook) {
       await this.createAuthorsAndSeriesForItemUpdate(mediaPayload, libraryItem.libraryId)

server/controllers/MeController.js

@@ -336,7 +336,7 @@ class MeController {
   }
 
   /**
-   * GET: /api/stats/year/:year
+   * GET: /api/me/stats/year/:year
    * 
    * @param {import('express').Request} req 
    * @param {import('express').Response} res 

server/controllers/MiscController.js

@@ -699,7 +699,7 @@ class MiscController {
   }
 
   /**
-   * GET: /api/me/stats/year/:year
+   * GET: /api/stats/year/:year
    * 
    * @param {import('express').Request} req 
    * @param {import('express').Response} res 
@@ -717,5 +717,23 @@ class MiscController {
     const stats = await adminStats.getStatsForYear(year)
     res.json(stats)
   }
+
+  /**
+   * GET: /api/logger-data
+   * admin or up
+   * 
+   * @param {import('express').Request} req 
+   * @param {import('express').Response} res 
+   */
+  async getLoggerData(req, res) {
+    if (!req.user.isAdminOrUp) {
+      Logger.error(`[MiscController] Non-admin user "${req.user.username}" attempted to get logger data`)
+      return res.sendStatus(403)
+    }
+
+    res.json({
+      currentDailyLogs: Logger.logManager.getMostRecentCurrentDailyLogs()
+    })
+  }
 }
 module.exports = new MiscController()

server/controllers/SearchController.js

@@ -43,12 +43,15 @@ class SearchController {
    */
   async findPodcasts(req, res) {
     const term = req.query.term
+    const country = req.query.country || 'us'
     if (!term) {
       Logger.error('[SearchController] Invalid request query param "term" is required')
       return res.status(400).send('Invalid request query param "term" is required')
     }
 
-    const results = await PodcastFinder.search(term)
+    const results = await PodcastFinder.search(term, {
+      country
+    })
     res.json(results)
   }
 

server/controllers/SessionController.js

@@ -161,7 +161,7 @@ class SessionController {
    * @typedef batchDeleteReqBody
    * @property {string[]} sessions
    * 
-   * @param {import('express').Request<{}, {}, batchDeleteReqBody, {}} req 
+   * @param {import('express').Request<{}, {}, batchDeleteReqBody, {}>} req 
    * @param {import('express').Response} res 
    */
   async batchDelete(req, res) {

server/finders/BookFinder.js

@@ -5,6 +5,7 @@ const iTunes = require('../providers/iTunes')
 const Audnexus = require('../providers/Audnexus')
 const FantLab = require('../providers/FantLab')
 const AudiobookCovers = require('../providers/AudiobookCovers')
+const CustomProviderAdapter = require('../providers/CustomProviderAdapter')
 const Logger = require('../Logger')
 const { levenshteinDistance, escapeRegExp } = require('../utils/index')
 
@@ -17,6 +18,7 @@ class BookFinder {
     this.audnexus = new Audnexus()
     this.fantLab = new FantLab()
     this.audiobookCovers = new AudiobookCovers()
+    this.customProviderAdapter = new CustomProviderAdapter()
 
     this.providers = ['google', 'itunes', 'openlibrary', 'fantlab', 'audiobookcovers', 'audible', 'audible.ca', 'audible.uk', 'audible.au', 'audible.fr', 'audible.de', 'audible.jp', 'audible.it', 'audible.in', 'audible.es']
 
@@ -147,6 +149,20 @@ class BookFinder {
     return books
   }
 
+  /**
+   * 
+   * @param {string} title 
+   * @param {string} author 
+   * @param {string} providerSlug 
+   * @returns {Promise<Object[]>}
+   */
+  async getCustomProviderResults(title, author, providerSlug) {
+    const books = await this.customProviderAdapter.search(title, author, providerSlug, 'book')
+    if (this.verbose) Logger.debug(`Custom provider '${providerSlug}' Search Results: ${books.length || 0}`)
+
+    return books
+  }
+
   static TitleCandidates = class {
 
     constructor(cleanAuthor) {
@@ -315,6 +331,11 @@ class BookFinder {
     const maxFuzzySearches = !isNaN(options.maxFuzzySearches) ? Number(options.maxFuzzySearches) : 5
     let numFuzzySearches = 0
 
+    // Custom providers are assumed to be correct
+    if (provider.startsWith('custom-')) {
+      return this.getCustomProviderResults(title, author, provider)
+    }
+
     if (!title)
       return books
 
@@ -397,8 +418,7 @@ class BookFinder {
       books = await this.getFantLabResults(title, author)
     } else if (provider === 'audiobookcovers') {
       books = await this.getAudiobookCoversResults(title)
-    }
-    else {
+    } else {
       books = await this.getGoogleBooksResults(title, author)
     }
     return books

server/finders/PodcastFinder.js

@@ -6,10 +6,16 @@ class PodcastFinder {
     this.iTunesApi = new iTunes()
   }
 
+  /**
+   * 
+   * @param {string} term 
+   * @param {{country:string}} options 
+   * @returns {Promise<import('../providers/iTunes').iTunesPodcastSearchResult[]>}
+   */
   async search(term, options = {}) {
     if (!term) return null
     Logger.debug(`[iTunes] Searching for podcast with term "${term}"`)
-    var results = await this.iTunesApi.searchPodcasts(term, options)
+    const results = await this.iTunesApi.searchPodcasts(term, options)
     Logger.debug(`[iTunes] Podcast search for "${term}" returned ${results.length} results`)
     return results
   }

server/managers/LogManager.js

@@ -1,19 +1,34 @@
 const Path = require('path')
 const fs = require('../libs/fsExtra')
 
+const Logger = require('../Logger')
 const DailyLog = require('../objects/DailyLog')
 
-const Logger = require('../Logger')
+const { LogLevel } = require('../utils/constants')
 
 const TAG = '[LogManager]'
 
+/**
+ * @typedef LogObject
+ * @property {string} timestamp
+ * @property {string} source
+ * @property {string} message
+ * @property {string} levelName
+ * @property {number} level
+ */
+
 class LogManager {
   constructor() {
     this.DailyLogPath = Path.posix.join(global.MetadataPath, 'logs', 'daily')
     this.ScanLogPath = Path.posix.join(global.MetadataPath, 'logs', 'scans')
 
+    /** @type {DailyLog} */
     this.currentDailyLog = null
+
+    /** @type {LogObject[]} */
     this.dailyLogBuffer = []
+
+    /** @type {string[]} */
     this.dailyLogFiles = []
   }
 
@@ -26,12 +41,12 @@ class LogManager {
     await fs.ensureDir(this.ScanLogPath)
   }
 
-  async ensureScanLogDir() {
-    if (!(await fs.pathExists(this.ScanLogPath))) {
-      await fs.mkdir(this.ScanLogPath)
-    }
-  }
-
+  /**
+   * 1. Ensure log directories exist
+   * 2. Load daily log files
+   * 3. Remove old daily log files
+   * 4. Create/set current daily log file
+   */
   async init() {
     await this.ensureLogDirs()
 
@@ -46,11 +61,11 @@ class LogManager {
       }
     }
 
+    // set current daily log file or create if does not exist
     const currentDailyLogFilename = DailyLog.getCurrentDailyLogFilename()
     Logger.info(TAG, `Init current daily log filename: ${currentDailyLogFilename}`)
 
-    this.currentDailyLog = new DailyLog()
-    this.currentDailyLog.setData({ dailyLogDirPath: this.DailyLogPath })
+    this.currentDailyLog = new DailyLog(this.DailyLogPath)
 
     if (this.dailyLogFiles.includes(currentDailyLogFilename)) {
       Logger.debug(TAG, `Daily log file already exists - set in Logger`)
@@ -59,7 +74,7 @@ class LogManager {
       this.dailyLogFiles.push(this.currentDailyLog.filename)
     }
 
-    // Log buffered Logs
+    // Log buffered daily logs
     if (this.dailyLogBuffer.length) {
       this.dailyLogBuffer.forEach((logObj) => {
         this.currentDailyLog.appendLog(logObj)
@@ -68,9 +83,12 @@ class LogManager {
     }
   }
 
+  /**
+   * Load all daily log filenames in /metadata/logs/daily
+   */
   async scanLogFiles() {
     const dailyFiles = await fs.readdir(this.DailyLogPath)
-    if (dailyFiles && dailyFiles.length) {
+    if (dailyFiles?.length) {
       dailyFiles.forEach((logFile) => {
         if (Path.extname(logFile) === '.txt') {
           Logger.debug('Daily Log file found', logFile)
@@ -83,30 +101,38 @@ class LogManager {
     this.dailyLogFiles.sort()
   }
 
-  async removeOldestLog() {
-    if (!this.dailyLogFiles.length) return
-    const oldestLog = this.dailyLogFiles[0]
-    return this.removeLogFile(oldestLog)
-  }
-
+  /**
+   * 
+   * @param {string} filename 
+   */
   async removeLogFile(filename) {
     const fullPath = Path.join(this.DailyLogPath, filename)
     const exists = await fs.pathExists(fullPath)
     if (!exists) {
       Logger.error(TAG, 'Invalid log dne ' + fullPath)
-      this.dailyLogFiles = this.dailyLogFiles.filter(dlf => dlf.filename !== filename)
+      this.dailyLogFiles = this.dailyLogFiles.filter(dlf => dlf !== filename)
     } else {
       try {
         await fs.unlink(fullPath)
         Logger.info(TAG, 'Removed daily log: ' + filename)
-        this.dailyLogFiles = this.dailyLogFiles.filter(dlf => dlf.filename !== filename)
+        this.dailyLogFiles = this.dailyLogFiles.filter(dlf => dlf !== filename)
       } catch (error) {
         Logger.error(TAG, 'Failed to unlink log file ' + fullPath)
       }
     }
   }
 
-  logToFile(logObj) {
+  /**
+   * 
+   * @param {LogObject} logObj 
+   */
+  async logToFile(logObj) {
+    // Fatal crashes get logged to a separate file
+    if (logObj.level === LogLevel.FATAL) {
+      await this.logCrashToFile(logObj)
+    }
+
+    // Buffer when logging before daily logs have been initialized
     if (!this.currentDailyLog) {
       this.dailyLogBuffer.push(logObj)
       return
@@ -114,25 +140,39 @@ class LogManager {
 
     // Check log rolls to next day
     if (this.currentDailyLog.id !== DailyLog.getCurrentDateString()) {
-      const newDailyLog = new DailyLog()
-      newDailyLog.setData({ dailyLogDirPath: this.DailyLogPath })
-      this.currentDailyLog = newDailyLog
+      this.currentDailyLog = new DailyLog(this.DailyLogPath)
       if (this.dailyLogFiles.length > this.loggerDailyLogsToKeep) {
-        this.removeOldestLog()
+        // Remove oldest log
+        this.removeLogFile(this.dailyLogFiles[0])
       }
     }
 
     // Append log line to log file
-    this.currentDailyLog.appendLog(logObj)
+    return this.currentDailyLog.appendLog(logObj)
   }
 
-  socketRequestDailyLogs(socket) {
-    if (!this.currentDailyLog) {
-      return
-    }
+  /**
+   * 
+   * @param {LogObject} logObj 
+   */
+  async logCrashToFile(logObj) {
+    const line = JSON.stringify(logObj) + '\n'
 
-    const lastLogs = this.currentDailyLog.logs.slice(-5000)
-    socket.emit('daily_logs', lastLogs)
+    const logsDir = Path.join(global.MetadataPath, 'logs')
+    await fs.ensureDir(logsDir)
+    const crashLogPath = Path.join(logsDir, 'crash_logs.txt')
+    return fs.writeFile(crashLogPath, line, { flag: "a+" }).catch((error) => {
+      console.log('[LogManager] Appended crash log', error)
+    })
+  }
+
+  /**
+   * Most recent 5000 daily logs
+   * 
+   * @returns {string}
+   */
+  getMostRecentCurrentDailyLogs() {
+    return this.currentDailyLog?.logs.slice(-5000) || ''
   }
 }
 module.exports = LogManager

server/models/CustomMetadataProvider.js

@@ -0,0 +1,103 @@
+const { DataTypes, Model } = require('sequelize')
+
+/**
+ * @typedef ClientCustomMetadataProvider
+ * @property {UUIDV4} id
+ * @property {string} name
+ * @property {string} url
+ * @property {string} slug
+ */
+
+class CustomMetadataProvider extends Model {
+  constructor(values, options) {
+    super(values, options)
+
+    /** @type {UUIDV4} */
+    this.id
+    /** @type {string} */
+    this.mediaType
+    /** @type {string} */
+    this.name
+    /** @type {string} */
+    this.url
+    /** @type {string} */
+    this.authHeaderValue
+    /** @type {Object} */
+    this.extraData
+    /** @type {Date} */
+    this.createdAt
+    /** @type {Date} */
+    this.updatedAt
+  }
+
+  getSlug() {
+    return `custom-${this.id}`
+  }
+
+  /**
+   * Safe for clients
+   * @returns {ClientCustomMetadataProvider}
+   */
+  toClientJson() {
+    return {
+      id: this.id,
+      name: this.name,
+      mediaType: this.mediaType,
+      slug: this.getSlug()
+    }
+  }
+
+  /**
+   * Get providers for client by media type
+   * Currently only available for "book" media type
+   * 
+   * @param {string} mediaType 
+   * @returns {Promise<ClientCustomMetadataProvider[]>}
+   */
+  static async getForClientByMediaType(mediaType) {
+    if (mediaType !== 'book') return []
+    const customMetadataProviders = await this.findAll({
+      where: {
+        mediaType
+      }
+    })
+    return customMetadataProviders.map(cmp => cmp.toClientJson())
+  }
+
+  /**
+   * Check if provider exists by slug
+   * 
+   * @param {string} providerSlug 
+   * @returns {Promise<boolean>}
+   */
+  static async checkExistsBySlug(providerSlug) {
+    const providerId = providerSlug?.split?.('custom-')[1]
+    if (!providerId) return false
+
+    return (await this.count({ where: { id: providerId } })) > 0
+  }
+
+  /**
+   * Initialize model
+   * @param {import('../Database').sequelize} sequelize 
+   */
+  static init(sequelize) {
+    super.init({
+      id: {
+        type: DataTypes.UUID,
+        defaultValue: DataTypes.UUIDV4,
+        primaryKey: true
+      },
+      name: DataTypes.STRING,
+      mediaType: DataTypes.STRING,
+      url: DataTypes.STRING,
+      authHeaderValue: DataTypes.STRING,
+      extraData: DataTypes.JSON
+    }, {
+      sequelize,
+      modelName: 'customMetadataProvider'
+    })
+  }
+}
+
+module.exports = CustomMetadataProvider

server/models/PlaybackSession.js

@@ -118,7 +118,9 @@ class PlaybackSession extends Model {
 
   static createFromOld(oldPlaybackSession) {
     const playbackSession = this.getFromOld(oldPlaybackSession)
-    return this.create(playbackSession)
+    return this.create(playbackSession, {
+      silent: true
+    })
   }
 
   static updateFromOld(oldPlaybackSession) {
@@ -126,7 +128,8 @@ class PlaybackSession extends Model {
     return this.update(playbackSession, {
       where: {
         id: playbackSession.id
-      }
+      },
+      silent: true
     })
   }
 

server/objects/DailyLog.js

@@ -1,23 +1,28 @@
 const Path = require('path')
 const date = require('../libs/dateAndTime')
 const fs = require('../libs/fsExtra')
-const { readTextFile } = require('../utils/fileUtils')
+const fileUtils = require('../utils/fileUtils')
 const Logger = require('../Logger')
 
 class DailyLog {
-  constructor() {
-    this.id = null
-    this.datePretty = null
+  /**
+   * 
+   * @param {string} dailyLogDirPath Path to daily logs /metadata/logs/daily
+   */
+  constructor(dailyLogDirPath) {
+    this.id = date.format(new Date(), 'YYYY-MM-DD')
 
-    this.dailyLogDirPath = null
-    this.filename = null
-    this.path = null
-    this.fullPath = null
+    this.dailyLogDirPath = dailyLogDirPath
+    this.filename = this.id + '.txt'
+    this.fullPath = Path.join(this.dailyLogDirPath, this.filename)
 
-    this.createdAt = null
+    this.createdAt = Date.now()
 
+    /** @type {import('../managers/LogManager').LogObject[]} */
     this.logs = []
+    /** @type {string[]} */
     this.bufferedLogLines = []
+
     this.locked = false
   }
 
@@ -32,8 +37,6 @@ class DailyLog {
   toJSON() {
     return {
       id: this.id,
-      datePretty: this.datePretty,
-      path: this.path,
       dailyLogDirPath: this.dailyLogDirPath,
       fullPath: this.fullPath,
       filename: this.filename,
@@ -41,36 +44,34 @@ class DailyLog {
     }
   }
 
-  setData(data) {
-    this.id = date.format(new Date(), 'YYYY-MM-DD')
-    this.datePretty = date.format(new Date(), 'ddd, MMM D YYYY')
-
-    this.dailyLogDirPath = data.dailyLogDirPath
-
-    this.filename = this.id + '.txt'
-    this.path = Path.join('backups', this.filename)
-    this.fullPath = Path.join(this.dailyLogDirPath, this.filename)
-
-    this.createdAt = Date.now()
-  }
-
-  async appendBufferedLogs() {
-    var buffered = [...this.bufferedLogLines]
+  /**
+   * Append all buffered lines to daily log file
+   */
+  appendBufferedLogs() {
+    let buffered = [...this.bufferedLogLines]
     this.bufferedLogLines = []
 
-    var oneBigLog = ''
+    let oneBigLog = ''
     buffered.forEach((logLine) => {
       oneBigLog += logLine
     })
-    this.appendLogLine(oneBigLog)
+    return this.appendLogLine(oneBigLog)
   }
 
-  async appendLog(logObj) {
+  /**
+   * 
+   * @param {import('../managers/LogManager').LogObject} logObj 
+   */
+  appendLog(logObj) {
     this.logs.push(logObj)
-    var line = JSON.stringify(logObj) + '\n'
-    this.appendLogLine(line)
+    return this.appendLogLine(JSON.stringify(logObj) + '\n')
   }
 
+  /**
+   * Append log to daily log file
+   * 
+   * @param {string} line 
+   */
   async appendLogLine(line) {
     if (this.locked) {
       this.bufferedLogLines.push(line)
@@ -84,24 +85,29 @@ class DailyLog {
 
     this.locked = false
     if (this.bufferedLogLines.length) {
-      this.appendBufferedLogs()
+      await this.appendBufferedLogs()
     }
   }
 
+  /**
+   * Load all logs from file
+   * Parses lines and re-saves the file if bad lines are removed
+   */
   async loadLogs() {
-    var exists = await fs.pathExists(this.fullPath)
-    if (!exists) {
+    if (!await fs.pathExists(this.fullPath)) {
       console.error('Daily log does not exist')
       return
     }
 
-    var text = await readTextFile(this.fullPath)
+    const text = await fileUtils.readTextFile(this.fullPath)
 
-    var hasFailures = false
+    let hasFailures = false
 
-    var logLines = text.split(/\r?\n/)
+    let logLines = text.split(/\r?\n/)
     // remove last log if empty
     if (logLines.length && !logLines[logLines.length - 1]) logLines = logLines.slice(0, -1)
+
+    // JSON parse log lines
     this.logs = logLines.map(t => {
       if (!t) {
         hasFailures = true
@@ -118,7 +124,7 @@ class DailyLog {
 
     // Rewrite log file to remove errors
     if (hasFailures) {
-      var newLogLines = this.logs.map(l => JSON.stringify(l)).join('\n') + '\n'
+      const newLogLines = this.logs.map(l => JSON.stringify(l)).join('\n') + '\n'
       await fs.writeFile(this.fullPath, newLogLines)
       console.log('Re-Saved log file to remove bad lines')
     }

server/objects/settings/LibrarySettings.js

@@ -10,6 +10,7 @@ class LibrarySettings {
     this.audiobooksOnly = false
     this.hideSingleBookSeries = false // Do not show series that only have 1 book 
     this.metadataPrecedence = ['folderStructure', 'audioMetatags', 'nfoFile', 'txtFiles', 'opfFile', 'absMetadata']
+    this.podcastSearchRegion = 'us'
 
     if (settings) {
       this.construct(settings)
@@ -30,6 +31,7 @@ class LibrarySettings {
       // Added in v2.4.5
       this.metadataPrecedence = ['folderStructure', 'audioMetatags', 'nfoFile', 'txtFiles', 'opfFile', 'absMetadata']
     }
+    this.podcastSearchRegion = settings.podcastSearchRegion || 'us'
   }
 
   toJSON() {
@@ -41,7 +43,8 @@ class LibrarySettings {
       autoScanCronExpression: this.autoScanCronExpression,
       audiobooksOnly: this.audiobooksOnly,
       hideSingleBookSeries: this.hideSingleBookSeries,
-      metadataPrecedence: [...this.metadataPrecedence]
+      metadataPrecedence: [...this.metadataPrecedence],
+      podcastSearchRegion: this.podcastSearchRegion
     }
   }
 

server/objects/settings/ServerSettings.js

@@ -55,7 +55,7 @@ class ServerSettings {
     this.buildNumber = packageJson.buildNumber
 
     // Auth settings
-    // Active auth methodes
+    this.authLoginCustomMessage = null
     this.authActiveAuthMethods = ['local']
 
     // openid settings
@@ -113,6 +113,7 @@ class ServerSettings {
     this.version = settings.version || null
     this.buildNumber = settings.buildNumber || 0 // Added v2.4.5
 
+    this.authLoginCustomMessage = settings.authLoginCustomMessage || null // Added v2.7.3
     this.authActiveAuthMethods = settings.authActiveAuthMethods || ['local']
 
     this.authOpenIDIssuerURL = settings.authOpenIDIssuerURL || null
@@ -201,6 +202,7 @@ class ServerSettings {
       logLevel: this.logLevel,
       version: this.version,
       buildNumber: this.buildNumber,
+      authLoginCustomMessage: this.authLoginCustomMessage,
       authActiveAuthMethods: this.authActiveAuthMethods,
       authOpenIDIssuerURL: this.authOpenIDIssuerURL,
       authOpenIDAuthorizationURL: this.authOpenIDAuthorizationURL,
@@ -213,7 +215,7 @@ class ServerSettings {
       authOpenIDButtonText: this.authOpenIDButtonText,
       authOpenIDAutoLaunch: this.authOpenIDAutoLaunch,
       authOpenIDAutoRegister: this.authOpenIDAutoRegister,
-      authOpenIDMatchExistingBy: this.authOpenIDMatchExistingBy, 
+      authOpenIDMatchExistingBy: this.authOpenIDMatchExistingBy,
       authOpenIDMobileRedirectURIs: this.authOpenIDMobileRedirectURIs // Do not return to client
     }
   }
@@ -246,6 +248,7 @@ class ServerSettings {
 
   get authenticationSettings() {
     return {
+      authLoginCustomMessage: this.authLoginCustomMessage,
       authActiveAuthMethods: this.authActiveAuthMethods,
       authOpenIDIssuerURL: this.authOpenIDIssuerURL,
       authOpenIDAuthorizationURL: this.authOpenIDAuthorizationURL,
@@ -264,7 +267,9 @@ class ServerSettings {
   }
 
   get authFormData() {
-    const clientFormData = {}
+    const clientFormData = {
+      authLoginCustomMessage: this.authLoginCustomMessage
+    }
     if (this.authActiveAuthMethods.includes('openid')) {
       clientFormData.authOpenIDButtonText = this.authOpenIDButtonText
       clientFormData.authOpenIDAutoLaunch = this.authOpenIDAutoLaunch

server/providers/AudiobookCovers.js

@@ -14,7 +14,7 @@ class AudiobookCovers {
       Logger.error('[AudiobookCovers] Cover search error', error)
       return []
     })
-    return items.map(item => ({ cover: item.filename }))
+    return items.map(item => ({ cover: item.versions.png.original }))
   }
 }
 

server/providers/CustomProviderAdapter.js

@@ -0,0 +1,93 @@
+const Database = require('../Database')
+const axios = require('axios')
+const Logger = require('../Logger')
+
+class CustomProviderAdapter {
+    constructor() { }
+
+    /**
+     * 
+     * @param {string} title 
+     * @param {string} author 
+     * @param {string} providerSlug 
+     * @param {string} mediaType
+     * @returns {Promise<Object[]>}
+     */
+    async search(title, author, providerSlug, mediaType) {
+        const providerId = providerSlug.split('custom-')[1]
+        const provider = await Database.customMetadataProviderModel.findByPk(providerId)
+
+        if (!provider) {
+            throw new Error("Custom provider not found for the given id")
+        }
+
+        // Setup query params
+        const queryObj = {
+            mediaType,
+            query: title
+        }
+        if (author) {
+            queryObj.author = author
+        }
+        const queryString = (new URLSearchParams(queryObj)).toString()
+
+        // Setup headers
+        const axiosOptions = {}
+        if (provider.authHeaderValue) {
+            axiosOptions.headers = {
+                'Authorization': provider.authHeaderValue
+            }
+        }
+
+        const matches = await axios.get(`${provider.url}/search?${queryString}}`, axiosOptions).then((res) => {
+            if (!res?.data || !Array.isArray(res.data.matches)) return null
+            return res.data.matches
+        }).catch(error => {
+            Logger.error('[CustomMetadataProvider] Search error', error)
+            return []
+        })
+
+        if (!matches) {
+            throw new Error("Custom provider returned malformed response")
+        }
+
+        // re-map keys to throw out
+        return matches.map(({
+            title,
+            subtitle,
+            author,
+            narrator,
+            publisher,
+            publishedYear,
+            description,
+            cover,
+            isbn,
+            asin,
+            genres,
+            tags,
+            series,
+            language,
+            duration
+        }) => {
+            return {
+                title,
+                subtitle,
+                author,
+                narrator,
+                publisher,
+                publishedYear,
+                description,
+                cover,
+                isbn,
+                asin,
+                genres,
+                tags: tags?.join(',') || null,
+                series: series?.length ? series : null,
+                language,
+                duration
+            }
+        })
+    }
+}
+
+module.exports = CustomProviderAdapter

server/providers/iTunes.js

@@ -2,16 +2,46 @@ const axios = require('axios')
 const Logger = require('../Logger')
 const htmlSanitizer = require('../utils/htmlSanitizer')
 
+/**
+ * @typedef iTunesSearchParams
+ * @property {string} term
+ * @property {string} country
+ * @property {string} media
+ * @property {string} entity
+ * @property {number} limit
+ */
+
+/**
+ * @typedef iTunesPodcastSearchResult
+ * @property {string} id
+ * @property {string} artistId
+ * @property {string} title
+ * @property {string} artistName
+ * @property {string} description
+ * @property {string} descriptionPlain
+ * @property {string} releaseDate
+ * @property {string[]} genres
+ * @property {string} cover
+ * @property {string} feedUrl
+ * @property {string} pageUrl
+ * @property {boolean} explicit
+ */
+
 class iTunes {
   constructor() { }
 
-  // https://developer.apple.com/library/archive/documentation/AudioVideo/Conceptual/iTuneSearchAPI/Searching.html
+  /**
+   * @see https://developer.apple.com/library/archive/documentation/AudioVideo/Conceptual/iTuneSearchAPI/Searching.html
+   * 
+   * @param {iTunesSearchParams} options 
+   * @returns {Promise<Object[]>}
+   */
   search(options) {
     if (!options.term) {
       Logger.error('[iTunes] Invalid search options - no term')
       return []
     }
-    var query = {
+    const query = {
       term: options.term,
       media: options.media,
       entity: options.entity,
@@ -82,6 +112,11 @@ class iTunes {
     })
   }
 
+  /**
+   * 
+   * @param {Object} data 
+   * @returns {iTunesPodcastSearchResult}
+   */
   cleanPodcast(data) {
     return {
       id: data.collectionId,
@@ -100,6 +135,12 @@ class iTunes {
     }
   }
 
+  /**
+   * 
+   * @param {string} term 
+   * @param {{country:string}} options 
+   * @returns {Promise<iTunesPodcastSearchResult[]>}
+   */
   searchPodcasts(term, options = {}) {
     return this.search({ term, entity: 'podcast', media: 'podcast', ...options }).then((results) => {
       return results.map(this.cleanPodcast.bind(this))

server/routers/ApiRouter.js

@@ -28,6 +28,7 @@ const SearchController = require('../controllers/SearchController')
 const CacheController = require('../controllers/CacheController')
 const ToolsController = require('../controllers/ToolsController')
 const RSSFeedController = require('../controllers/RSSFeedController')
+const CustomMetadataProviderController = require('../controllers/CustomMetadataProviderController')
 const MiscController = require('../controllers/MiscController')
 
 const Author = require('../objects/entities/Author')
@@ -299,6 +300,14 @@ class ApiRouter {
     this.router.post('/feeds/series/:seriesId/open', RSSFeedController.middleware.bind(this), RSSFeedController.openRSSFeedForSeries.bind(this))
     this.router.post('/feeds/:id/close', RSSFeedController.middleware.bind(this), RSSFeedController.closeRSSFeed.bind(this))
 
+    //
+    // Custom Metadata Provider routes
+    //
+    this.router.get('/custom-metadata-providers', CustomMetadataProviderController.middleware.bind(this), CustomMetadataProviderController.getAll.bind(this))
+    this.router.post('/custom-metadata-providers', CustomMetadataProviderController.middleware.bind(this), CustomMetadataProviderController.create.bind(this))
+    this.router.delete('/custom-metadata-providers/:id', CustomMetadataProviderController.middleware.bind(this), CustomMetadataProviderController.delete.bind(this))
+
+
     //
     // Misc Routes
     //
@@ -318,6 +327,7 @@ class ApiRouter {
     this.router.patch('/auth-settings', MiscController.updateAuthSettings.bind(this))
     this.router.post('/watcher/update', MiscController.updateWatchedPath.bind(this))
     this.router.get('/stats/year/:year', MiscController.getAdminStatsForYear.bind(this))
+    this.router.get('/logger-data', MiscController.getLoggerData.bind(this))
   }
 
   //

server/scanner/LibraryScan.js

@@ -134,10 +134,13 @@ class LibraryScan {
   }
 
   async saveLog() {
-    await Logger.logManager.ensureScanLogDir()
+    const scanLogDir = Path.join(global.MetadataPath, 'logs', 'scans')
 
-    const logDir = Path.join(global.MetadataPath, 'logs', 'scans')
-    const outputPath = Path.join(logDir, this.logFilename)
+    if (!(await fs.pathExists(scanLogDir))) {
+      await fs.mkdir(scanLogDir)
+    }
+
+    const outputPath = Path.join(scanLogDir, this.logFilename)
     const logLines = [JSON.stringify(this.toJSON())]
     this.logs.forEach(l => {
       logLines.push(JSON.stringify(l))

server/utils/ffmpegHelpers.js

@@ -101,8 +101,8 @@ module.exports.downloadPodcastEpisode = (podcastEpisodeDownload) => {
     })
     if (!response) return resolve(false)
 
-
     const ffmpeg = Ffmpeg(response.data)
+    ffmpeg.addOption('-loglevel debug') // Debug logs printed on error
     ffmpeg.outputOptions(
       '-c', 'copy',
       '-metadata', 'podcast=1'
@@ -110,6 +110,7 @@ module.exports.downloadPodcastEpisode = (podcastEpisodeDownload) => {
 
     const podcastMetadata = podcastEpisodeDownload.libraryItem.media.metadata
     const podcastEpisode = podcastEpisodeDownload.podcastEpisode
+    const finalSizeInBytes = Number(podcastEpisode.enclosure?.length || 0)
 
     const taggings = {
       'album': podcastMetadata.title,
@@ -147,13 +148,30 @@ module.exports.downloadPodcastEpisode = (podcastEpisodeDownload) => {
 
     ffmpeg.addOutput(podcastEpisodeDownload.targetPath)
 
+    const stderrLines = []
+    ffmpeg.on('stderr', (stderrLine) => {
+      if (typeof stderrLine === 'string') {
+        stderrLines.push(stderrLine)
+      }
+    })
     ffmpeg.on('start', (cmd) => {
       Logger.debug(`[FfmpegHelpers] downloadPodcastEpisode: Cmd: ${cmd}`)
     })
-    ffmpeg.on('error', (err, stdout, stderr) => {
-      Logger.error(`[FfmpegHelpers] downloadPodcastEpisode: Error ${err} ${stdout} ${stderr}`)
+    ffmpeg.on('error', (err) => {
+      Logger.error(`[FfmpegHelpers] downloadPodcastEpisode: Error ${err}`)
+      if (stderrLines.length) {
+        Logger.error(`Full stderr dump for episode url "${podcastEpisodeDownload.url}": ${stderrLines.join('\n')}`)
+      }
       resolve(false)
     })
+    ffmpeg.on('progress', (progress) => {
+      let progressPercent = 0
+      if (finalSizeInBytes && progress.targetSize && !isNaN(progress.targetSize)) {
+        const finalSizeInKb = Math.floor(finalSizeInBytes / 1000)
+        progressPercent = Math.min(1, progress.targetSize / finalSizeInKb) * 100
+      }
+      Logger.debug(`[FfmpegHelpers] downloadPodcastEpisode: Progress estimate ${progressPercent.toFixed(0)}% (${progress?.targetSize || 'N/A'} KB) for "${podcastEpisodeDownload.url}"`)
+    })
     ffmpeg.on('end', () => {
       Logger.debug(`[FfmpegHelpers] downloadPodcastEpisode: Complete`)
       resolve(podcastEpisodeDownload.targetPath)

server/utils/probeWorker.js

@@ -1,9 +0,0 @@
-const { parentPort } = require("worker_threads")
-const prober = require('./prober')
-
-parentPort.on("message", async ({ mediaPath }) => {
-  const results = await prober.probe(mediaPath)
-  parentPort.postMessage({
-    data: results,
-  })
-})

server/utils/queries/adminStats.js

@@ -110,7 +110,7 @@ module.exports = {
       })
 
       // Filter out bad genres like "audiobook" and "audio book"
-      const genres = (ls.mediaMetadata.genres || []).filter(g => !g.toLowerCase().includes('audiobook') && !g.toLowerCase().includes('audio book'))
+      const genres = (ls.mediaMetadata.genres || []).filter(g => g && !g.toLowerCase().includes('audiobook') && !g.toLowerCase().includes('audio book'))
       genres.forEach((genre) => {
         if (!genreListeningMap[genre]) genreListeningMap[genre] = 0
         genreListeningMap[genre] += (ls.timeListening || 0)

server/utils/queries/userStats.js

@@ -141,7 +141,7 @@ module.exports = {
         })
 
         // Filter out bad genres like "audiobook" and "audio book"
-        const genres = (ls.mediaMetadata.genres || []).filter(g => !g.toLowerCase().includes('audiobook') && !g.toLowerCase().includes('audio book'))
+        const genres = (ls.mediaMetadata.genres || []).filter(g => g && !g.toLowerCase().includes('audiobook') && !g.toLowerCase().includes('audio book'))
         genres.forEach((genre) => {
           if (!genreListeningMap[genre]) genreListeningMap[genre] = 0
           genreListeningMap[genre] += listeningSessionListeningTime