Merge branch 'advplyr:master' into binary-manager

server/utils/index.js

@@ -1,4 +1,5 @@
 const Path = require('path')
+const uuid = require('uuid')
 const Logger = require('../Logger')
 const { parseString } = require("xml2js")
 const areEquivalent = require('./areEquivalent')
@@ -11,24 +12,24 @@ const levenshteinDistance = (str1, str2, caseSensitive = false) => {
     str2 = str2.toLowerCase()
   }
   const track = Array(str2.length + 1).fill(null).map(() =>
-    Array(str1.length + 1).fill(null));
+    Array(str1.length + 1).fill(null))
   for (let i = 0; i <= str1.length; i += 1) {
-    track[0][i] = i;
+    track[0][i] = i
   }
   for (let j = 0; j <= str2.length; j += 1) {
-    track[j][0] = j;
+    track[j][0] = j
   }
   for (let j = 1; j <= str2.length; j += 1) {
     for (let i = 1; i <= str1.length; i += 1) {
-      const indicator = str1[i - 1] === str2[j - 1] ? 0 : 1;
+      const indicator = str1[i - 1] === str2[j - 1] ? 0 : 1
       track[j][i] = Math.min(
         track[j][i - 1] + 1, // deletion
         track[j - 1][i] + 1, // insertion
         track[j - 1][i - 1] + indicator, // substitution
-      );
+      )
     }
   }
-  return track[str2.length][str1.length];
+  return track[str2.length][str1.length]
 }
 module.exports.levenshteinDistance = levenshteinDistance
 
@@ -204,4 +205,31 @@ module.exports.asciiOnlyToLowerCase = (str) => {
 module.exports.escapeRegExp = (str) => {
   if (typeof str !== 'string') return ''
   return str.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')
+}
+
+/**
+ * Validate url string with URL class
+ * 
+ * @param {string} rawUrl 
+ * @returns {string} null if invalid
+ */
+module.exports.validateUrl = (rawUrl) => {
+  if (!rawUrl || typeof rawUrl !== 'string') return null
+  try {
+    return new URL(rawUrl).toString()
+  } catch (error) {
+    Logger.error(`Invalid URL "${rawUrl}"`, error)
+    return null
+  }
+}
+
+/**
+ * Check if a string is a valid UUID
+ * 
+ * @param {string} str 
+ * @returns {boolean}
+ */
+module.exports.isUUID = (str) => {
+  if (!str || typeof str !== 'string') return false
+  return uuid.validate(str)
 }

server/utils/parsers/parseOpfMetadata.js

@@ -100,13 +100,19 @@ function fetchLanguage(metadata) {
 }
 
 function fetchSeries(metadataMeta) {
-  if (!metadataMeta) return null
-  return fetchTagString(metadataMeta, "calibre:series")
-}
-
-function fetchVolumeNumber(metadataMeta) {
-  if (!metadataMeta) return null
-  return fetchTagString(metadataMeta, "calibre:series_index")
+  if (!metadataMeta) return []
+  const result = []
+  for (let i = 0; i < metadataMeta.length; i++) {
+    if (metadataMeta[i].$?.name === "calibre:series" && metadataMeta[i].$.content?.trim()) {
+      const name = metadataMeta[i].$.content.trim()
+      let sequence = null
+      if (metadataMeta[i + 1]?.$?.name === "calibre:series_index" && metadataMeta[i + 1].$?.content?.trim()) {
+        sequence = metadataMeta[i + 1].$.content.trim()
+      }
+      result.push({ name, sequence })
+    }
+  }
+  return result
 }
 
 function fetchNarrators(creators, metadata) {
@@ -173,8 +179,7 @@ module.exports.parseOpfMetadataXML = async (xml) => {
     description: fetchDescription(metadata),
     genres: fetchGenres(metadata),
     language: fetchLanguage(metadata),
-    series: fetchSeries(metadata.meta),
-    sequence: fetchVolumeNumber(metadata.meta),
+    series: fetchSeries(metadataMeta),
     tags: fetchTags(metadata)
   }
   return data

server/utils/podcastUtils.js

@@ -1,5 +1,6 @@
-const Logger = require('../Logger')
 const axios = require('axios')
+const ssrfFilter = require('ssrf-req-filter')
+const Logger = require('../Logger')
 const { xmlToJSON, levenshteinDistance } = require('./index')
 const htmlSanitizer = require('../utils/htmlSanitizer')
 
@@ -216,9 +217,26 @@ module.exports.parsePodcastRssFeedXml = async (xml, excludeEpisodeMetadata = fal
   }
 }
 
+/**
+ * Get podcast RSS feed as JSON
+ * Uses SSRF filter to prevent internal URLs
+ * 
+ * @param {string} feedUrl 
+ * @param {boolean} [excludeEpisodeMetadata=false]
+ * @returns {Promise}
+ */
 module.exports.getPodcastFeed = (feedUrl, excludeEpisodeMetadata = false) => {
   Logger.debug(`[podcastUtils] getPodcastFeed for "${feedUrl}"`)
-  return axios.get(feedUrl, { timeout: 12000, responseType: 'arraybuffer' }).then(async (data) => {
+
+  return axios({
+    url: feedUrl,
+    method: 'GET',
+    timeout: 12000,
+    responseType: 'arraybuffer',
+    headers: { Accept: 'application/rss+xml, application/xhtml+xml, application/xml' },
+    httpAgent: ssrfFilter(feedUrl),
+    httpsAgent: ssrfFilter(feedUrl)
+  }).then(async (data) => {
 
     // Adding support for ios-8859-1 encoded RSS feeds.
     //  See: https://github.com/advplyr/audiobookshelf/issues/1489
@@ -231,12 +249,12 @@ module.exports.getPodcastFeed = (feedUrl, excludeEpisodeMetadata = false) => {
 
     if (!data?.data) {
       Logger.error(`[podcastUtils] getPodcastFeed: Invalid podcast feed request response (${feedUrl})`)
-      return false
+      return null
     }
     Logger.debug(`[podcastUtils] getPodcastFeed for "${feedUrl}" success - parsing xml`)
     const payload = await this.parsePodcastRssFeedXml(data.data, excludeEpisodeMetadata)
     if (!payload) {
-      return false
+      return null
     }
 
     // RSS feed may be a private RSS feed
@@ -245,7 +263,7 @@ module.exports.getPodcastFeed = (feedUrl, excludeEpisodeMetadata = false) => {
     return payload.podcast
   }).catch((error) => {
     Logger.error('[podcastUtils] getPodcastFeed Error', error)
-    return false
+    return null
   })
 }
 

server/utils/queries/adminStats.js

@@ -0,0 +1,162 @@
+const Sequelize = require('sequelize')
+const Database = require('../../Database')
+const PlaybackSession = require('../../models/PlaybackSession')
+const fsExtra = require('../../libs/fsExtra')
+
+module.exports = {
+  /**
+   * 
+   * @param {number} year YYYY
+   * @returns {Promise<PlaybackSession[]>}
+   */
+  async getListeningSessionsForYear(year) {
+    const sessions = await Database.playbackSessionModel.findAll({
+      where: {
+        createdAt: {
+          [Sequelize.Op.gte]: `${year}-01-01`,
+          [Sequelize.Op.lt]: `${year + 1}-01-01`
+        }
+      }
+    })
+    return sessions
+  },
+
+  /**
+   * 
+   * @param {number} year YYYY
+   * @returns {Promise<number>}
+   */
+  async getNumAuthorsAddedForYear(year) {
+    const count = await Database.authorModel.count({
+      where: {
+        createdAt: {
+          [Sequelize.Op.gte]: `${year}-01-01`,
+          [Sequelize.Op.lt]: `${year + 1}-01-01`
+        }
+      }
+    })
+    return count
+  },
+
+  /**
+   * 
+   * @param {number} year YYYY
+   * @returns {Promise<import('../../models/Book')[]>}
+   */
+  async getBooksAddedForYear(year) {
+    const books = await Database.bookModel.findAll({
+      attributes: ['id', 'title', 'coverPath', 'duration', 'createdAt'],
+      where: {
+        createdAt: {
+          [Sequelize.Op.gte]: `${year}-01-01`,
+          [Sequelize.Op.lt]: `${year + 1}-01-01`
+        }
+      },
+      include: {
+        model: Database.libraryItemModel,
+        attributes: ['id', 'mediaId', 'mediaType', 'size'],
+        required: true
+      },
+      order: Database.sequelize.random()
+    })
+    return books
+  },
+
+  /**
+   * 
+   * @param {number} year YYYY
+   */
+  async getStatsForYear(year) {
+    const booksAdded = await this.getBooksAddedForYear(year)
+
+    let totalBooksAddedSize = 0
+    let totalBooksAddedDuration = 0
+    const booksWithCovers = []
+
+    for (const book of booksAdded) {
+      // Grab first 25 that have a cover
+      if (book.coverPath && !booksWithCovers.includes(book.libraryItem.id) && booksWithCovers.length < 25 && await fsExtra.pathExists(book.coverPath)) {
+        booksWithCovers.push(book.libraryItem.id)
+      }
+      if (book.duration && !isNaN(book.duration)) {
+        totalBooksAddedDuration += book.duration
+      }
+      if (book.libraryItem.size && !isNaN(book.libraryItem.size)) {
+        totalBooksAddedSize += book.libraryItem.size
+      }
+    }
+
+    const numAuthorsAdded = await this.getNumAuthorsAddedForYear(year)
+
+    let authorListeningMap = {}
+    let narratorListeningMap = {}
+    let genreListeningMap = {}
+
+    const listeningSessions = await this.getListeningSessionsForYear(year)
+    let totalListeningTime = 0
+    for (const ls of listeningSessions) {
+      totalListeningTime += (ls.timeListening || 0)
+
+      const authors = ls.mediaMetadata.authors || []
+      authors.forEach((au) => {
+        if (!authorListeningMap[au.name]) authorListeningMap[au.name] = 0
+        authorListeningMap[au.name] += (ls.timeListening || 0)
+      })
+
+      const narrators = ls.mediaMetadata.narrators || []
+      narrators.forEach((narrator) => {
+        if (!narratorListeningMap[narrator]) narratorListeningMap[narrator] = 0
+        narratorListeningMap[narrator] += (ls.timeListening || 0)
+      })
+
+      // Filter out bad genres like "audiobook" and "audio book"
+      const genres = (ls.mediaMetadata.genres || []).filter(g => !g.toLowerCase().includes('audiobook') && !g.toLowerCase().includes('audio book'))
+      genres.forEach((genre) => {
+        if (!genreListeningMap[genre]) genreListeningMap[genre] = 0
+        genreListeningMap[genre] += (ls.timeListening || 0)
+      })
+    }
+
+    let topAuthors = null
+    topAuthors = Object.keys(authorListeningMap).map(authorName => ({
+      name: authorName,
+      time: Math.round(authorListeningMap[authorName])
+    })).sort((a, b) => b.time - a.time).slice(0, 3)
+
+    let topNarrators = null
+    topNarrators = Object.keys(narratorListeningMap).map(narratorName => ({
+      name: narratorName,
+      time: Math.round(narratorListeningMap[narratorName])
+    })).sort((a, b) => b.time - a.time).slice(0, 3)
+
+    let topGenres = null
+    topGenres = Object.keys(genreListeningMap).map(genre => ({
+      genre,
+      time: Math.round(genreListeningMap[genre])
+    })).sort((a, b) => b.time - a.time).slice(0, 3)
+
+    // Stats for total books, size and duration for everything added this year or earlier
+    const [totalStatResultsRow] = await Database.sequelize.query(`SELECT SUM(li.size) AS totalSize, SUM(b.duration) AS totalDuration, COUNT(*) AS totalItems FROM libraryItems li, books b WHERE b.id = li.mediaId AND li.mediaType = 'book' AND li.createdAt < ":nextYear-01-01";`, {
+      replacements: {
+        nextYear: year + 1
+      }
+    })
+    const totalStatResults = totalStatResultsRow[0]
+
+    return {
+      numListeningSessions: listeningSessions.length,
+      numBooksAdded: booksAdded.length,
+      numAuthorsAdded,
+      totalBooksAddedSize,
+      totalBooksAddedDuration: Math.round(totalBooksAddedDuration),
+      booksAddedWithCovers: booksWithCovers,
+      totalBooksSize: totalStatResults?.totalSize || 0,
+      totalBooksDuration: totalStatResults?.totalDuration || 0,
+      totalListeningTime,
+      numBooks: totalStatResults?.totalItems || 0,
+      topAuthors,
+      topNarrators,
+      topGenres
+    }
+  }
+}

server/utils/queries/userStats.js

@@ -0,0 +1,206 @@
+const Sequelize = require('sequelize')
+const Database = require('../../Database')
+const PlaybackSession = require('../../models/PlaybackSession')
+const MediaProgress = require('../../models/MediaProgress')
+const fsExtra = require('../../libs/fsExtra')
+
+module.exports = {
+  /**
+   * 
+   * @param {string} userId 
+   * @param {number} year YYYY
+   * @returns {Promise<PlaybackSession[]>}
+   */
+  async getUserListeningSessionsForYear(userId, year) {
+    const sessions = await Database.playbackSessionModel.findAll({
+      where: {
+        userId,
+        createdAt: {
+          [Sequelize.Op.gte]: `${year}-01-01`,
+          [Sequelize.Op.lt]: `${year + 1}-01-01`
+        }
+      },
+      include: {
+        model: Database.bookModel,
+        attributes: ['id', 'coverPath'],
+        include: {
+          model: Database.libraryItemModel,
+          attributes: ['id', 'mediaId', 'mediaType']
+        },
+        required: false
+      },
+      order: Database.sequelize.random()
+    })
+    return sessions
+  },
+
+  /**
+   * 
+   * @param {string} userId 
+   * @param {number} year YYYY
+   * @returns {Promise<MediaProgress[]>}
+   */
+  async getBookMediaProgressFinishedForYear(userId, year) {
+    const progresses = await Database.mediaProgressModel.findAll({
+      where: {
+        userId,
+        mediaItemType: 'book',
+        finishedAt: {
+          [Sequelize.Op.gte]: `${year}-01-01`,
+          [Sequelize.Op.lt]: `${year + 1}-01-01`
+        }
+      },
+      include: {
+        model: Database.bookModel,
+        attributes: ['id', 'title', 'coverPath'],
+        include: {
+          model: Database.libraryItemModel,
+          attributes: ['id', 'mediaId', 'mediaType']
+        },
+        required: true
+      },
+      order: Database.sequelize.random()
+    })
+    return progresses
+  },
+
+  /**
+   * @param {import('../../objects/user/User')} user
+   * @param {number} year YYYY
+   */
+  async getStatsForYear(user, year) {
+    const userId = user.id
+    const listeningSessions = await this.getUserListeningSessionsForYear(userId, year)
+    const bookProgressesFinished = await this.getBookMediaProgressFinishedForYear(userId, year)
+
+    let totalBookListeningTime = 0
+    let totalPodcastListeningTime = 0
+    let totalListeningTime = 0
+
+    let authorListeningMap = {}
+    let genreListeningMap = {}
+    let narratorListeningMap = {}
+    let monthListeningMap = {}
+    let bookListeningMap = {}
+
+    const booksWithCovers = []
+    const finishedBooksWithCovers = []
+
+    // Get finished book stats
+    const numBooksFinished = bookProgressesFinished.length
+    let longestAudiobookFinished = null
+    for (const mediaProgress of bookProgressesFinished) {
+      // Grab first 5 that have a cover
+      if (mediaProgress.mediaItem?.coverPath && !finishedBooksWithCovers.includes(mediaProgress.mediaItem.libraryItem.id) && finishedBooksWithCovers.length < 5 && await fsExtra.pathExists(mediaProgress.mediaItem.coverPath)) {
+        finishedBooksWithCovers.push(mediaProgress.mediaItem.libraryItem.id)
+      }
+
+      if (mediaProgress.duration && (!longestAudiobookFinished?.duration || mediaProgress.duration > longestAudiobookFinished.duration)) {
+        longestAudiobookFinished = {
+          id: mediaProgress.mediaItem.id,
+          title: mediaProgress.mediaItem.title,
+          duration: Math.round(mediaProgress.duration),
+          finishedAt: mediaProgress.finishedAt
+        }
+      }
+    }
+
+    // Get listening session stats
+    for (const ls of listeningSessions) {
+      // Grab first 25 that have a cover
+      if (ls.mediaItem?.coverPath && !booksWithCovers.includes(ls.mediaItem.libraryItem.id) && !finishedBooksWithCovers.includes(ls.mediaItem.libraryItem.id) && booksWithCovers.length < 25 && await fsExtra.pathExists(ls.mediaItem.coverPath)) {
+        booksWithCovers.push(ls.mediaItem.libraryItem.id)
+      }
+
+      const listeningSessionListeningTime = ls.timeListening || 0
+
+      const lsMonth = ls.createdAt.getMonth()
+      if (!monthListeningMap[lsMonth]) monthListeningMap[lsMonth] = 0
+      monthListeningMap[lsMonth] += listeningSessionListeningTime
+
+      totalListeningTime += listeningSessionListeningTime
+      if (ls.mediaItemType === 'book') {
+        totalBookListeningTime += listeningSessionListeningTime
+
+        if (ls.displayTitle && !bookListeningMap[ls.displayTitle]) {
+          bookListeningMap[ls.displayTitle] = listeningSessionListeningTime
+        } else if (ls.displayTitle) {
+          bookListeningMap[ls.displayTitle] += listeningSessionListeningTime
+        }
+
+        const authors = ls.mediaMetadata.authors || []
+        authors.forEach((au) => {
+          if (!authorListeningMap[au.name]) authorListeningMap[au.name] = 0
+          authorListeningMap[au.name] += listeningSessionListeningTime
+        })
+
+        const narrators = ls.mediaMetadata.narrators || []
+        narrators.forEach((narrator) => {
+          if (!narratorListeningMap[narrator]) narratorListeningMap[narrator] = 0
+          narratorListeningMap[narrator] += listeningSessionListeningTime
+        })
+
+        // Filter out bad genres like "audiobook" and "audio book"
+        const genres = (ls.mediaMetadata.genres || []).filter(g => !g.toLowerCase().includes('audiobook') && !g.toLowerCase().includes('audio book'))
+        genres.forEach((genre) => {
+          if (!genreListeningMap[genre]) genreListeningMap[genre] = 0
+          genreListeningMap[genre] += listeningSessionListeningTime
+        })
+      } else {
+        totalPodcastListeningTime += listeningSessionListeningTime
+      }
+    }
+
+    totalListeningTime = Math.round(totalListeningTime)
+    totalBookListeningTime = Math.round(totalBookListeningTime)
+    totalPodcastListeningTime = Math.round(totalPodcastListeningTime)
+
+    let topAuthors = null
+    topAuthors = Object.keys(authorListeningMap).map(authorName => ({
+      name: authorName,
+      time: Math.round(authorListeningMap[authorName])
+    })).sort((a, b) => b.time - a.time).slice(0, 3)
+
+    let mostListenedNarrator = null
+    for (const narrator in narratorListeningMap) {
+      if (!mostListenedNarrator?.time || narratorListeningMap[narrator] > mostListenedNarrator.time) {
+        mostListenedNarrator = {
+          time: Math.round(narratorListeningMap[narrator]),
+          name: narrator
+        }
+      }
+    }
+
+    let topGenres = null
+    topGenres = Object.keys(genreListeningMap).map(genre => ({
+      genre,
+      time: Math.round(genreListeningMap[genre])
+    })).sort((a, b) => b.time - a.time).slice(0, 3)
+
+    let mostListenedMonth = null
+    for (const month in monthListeningMap) {
+      if (!mostListenedMonth?.time || monthListeningMap[month] > mostListenedMonth.time) {
+        mostListenedMonth = {
+          month: Number(month),
+          time: Math.round(monthListeningMap[month])
+        }
+      }
+    }
+
+    return {
+      totalListeningSessions: listeningSessions.length,
+      totalListeningTime,
+      totalBookListeningTime,
+      totalPodcastListeningTime,
+      topAuthors,
+      topGenres,
+      mostListenedNarrator,
+      mostListenedMonth,
+      numBooksFinished,
+      numBooksListened: Object.keys(bookListeningMap).length,
+      longestAudiobookFinished,
+      booksWithCovers,
+      finishedBooksWithCovers
+    }
+  }
+}