mirror/advplyr.audiobookshelf
1
0
Fork 0
mirror of https://github.com/advplyr/audiobookshelf.git synced 2025-08-04 02:05:06 +02:00
Code Issues Projects Releases Packages Wiki Activity Actions

Update:Remove scanner settings, add library scanner settings tab, add order of precedence

Browse source
This commit is contained in:
advplyr 2023-10-08 17:10:43 -05:00
parent 5ad9f507ba
commit 347b49f564
35 changed files with 764 additions and 809 deletions
Download patch file Download diff file Expand all files Collapse all files

12
server/controllers/LibraryController.js
View file

@ -784,7 +784,14 @@ class LibraryController {
res.sendStatus(200)
}
// POST: api/libraries/:id/scan
/**
* POST: /api/libraries/:id/scan
* Optional query:
* ?force=1
*
* @param {import('express').Request} req
* @param {import('express').Response} res
*/
async scan(req, res) {
if (!req.user.isAdminOrUp) {
Logger.error(`[LibraryController] Non-root user attempted to scan library`, req.user)
@ -792,7 +799,8 @@ class LibraryController {
}
res.sendStatus(200)
await LibraryScanner.scan(req.library)
const forceRescan = req.query.force === '1'
await LibraryScanner.scan(req.library, forceRescan)
await Database.resetLibraryIssuesFilterData(req.library.id)
Logger.info('[LibraryController] Scan complete')

1
server/models/Library.js
View file

@ -11,6 +11,7 @@ const oldLibrary = require('../objects/Library')
* @property {string} autoScanCronExpression
* @property {boolean} audiobooksOnly
* @property {boolean} hideSingleBookSeries Do not show series that only have 1 book
* @property {string[]} metadataPrecedence
*/
class Library extends Model {

211
server/objects/mediaTypes/Book.js
View file

@ -1,9 +1,7 @@
const Path = require('path')
const Logger = require('../../Logger')
const BookMetadata = require('../metadata/BookMetadata')
const { areEquivalent, copyValue, cleanStringForSearch } = require('../../utils/index')
const { parseOpfMetadataXML } = require('../../utils/parsers/parseOpfMetadata')
const { parseOverdriveMediaMarkersAsChapters } = require('../../utils/parsers/parseOverdriveMediaMarkers')
const abmetadataGenerator = require('../../utils/generators/abmetadataGenerator')
const { readTextFile, filePathToPOSIX } = require('../../utils/fileUtils')
const AudioFile = require('../files/AudioFile')
@ -248,108 +246,6 @@ class Book {
}
}
// Look for desc.txt, reader.txt, metadata.abs and opf file then update details if found
async syncMetadataFiles(textMetadataFiles, opfMetadataOverrideDetails) {
let metadataUpdatePayload = {}
let hasUpdated = false
const descTxt = textMetadataFiles.find(lf => lf.metadata.filename === 'desc.txt')
if (descTxt) {
const descriptionText = await readTextFile(descTxt.metadata.path)
if (descriptionText) {
Logger.debug(`[Book] "${this.metadata.title}" found desc.txt updating description with "${descriptionText.slice(0, 20)}..."`)
metadataUpdatePayload.description = descriptionText
}
}
const readerTxt = textMetadataFiles.find(lf => lf.metadata.filename === 'reader.txt')
if (readerTxt) {
const narratorText = await readTextFile(readerTxt.metadata.path)
if (narratorText) {
Logger.debug(`[Book] "${this.metadata.title}" found reader.txt updating narrator with "${narratorText}"`)
metadataUpdatePayload.narrators = this.metadata.parseNarratorsTag(narratorText)
}
}
const metadataIsJSON = global.ServerSettings.metadataFileFormat === 'json'
const metadataAbs = textMetadataFiles.find(lf => lf.metadata.filename === 'metadata.abs')
const metadataJson = textMetadataFiles.find(lf => lf.metadata.filename === 'metadata.json')
const metadataFile = metadataIsJSON ? metadataJson : metadataAbs
if (metadataFile) {
Logger.debug(`[Book] Found ${metadataFile.metadata.filename} file for "${this.metadata.title}"`)
const metadataText = await readTextFile(metadataFile.metadata.path)
const abmetadataUpdates = abmetadataGenerator.parseAndCheckForUpdates(metadataText, this, 'book', metadataIsJSON)
if (abmetadataUpdates && Object.keys(abmetadataUpdates).length) {
Logger.debug(`[Book] "${this.metadata.title}" changes found in metadata.abs file`, abmetadataUpdates)
if (abmetadataUpdates.tags) { // Set media tags if updated
this.tags = abmetadataUpdates.tags
hasUpdated = true
}
if (abmetadataUpdates.chapters) { // Set chapters if updated
this.chapters = abmetadataUpdates.chapters
hasUpdated = true
}
if (abmetadataUpdates.metadata) {
metadataUpdatePayload = {
...metadataUpdatePayload,
...abmetadataUpdates.metadata
}
}
}
} else if (metadataAbs || metadataJson) { // Has different metadata file format so mark as updated
Logger.debug(`[Book] Found different format metadata file ${(metadataAbs || metadataJson).metadata.filename}, expecting .${global.ServerSettings.metadataFileFormat} for "${this.metadata.title}"`)
hasUpdated = true
}
const metadataOpf = textMetadataFiles.find(lf => lf.isOPFFile || lf.metadata.filename === 'metadata.xml')
if (metadataOpf) {
const xmlText = await readTextFile(metadataOpf.metadata.path)
if (xmlText) {
const opfMetadata = await parseOpfMetadataXML(xmlText)
if (opfMetadata) {
for (const key in opfMetadata) {
if (key === 'tags') { // Add tags only if tags are empty
if (opfMetadata.tags.length && (!this.tags.length || opfMetadataOverrideDetails)) {
this.tags = opfMetadata.tags
hasUpdated = true
}
} else if (key === 'genres') { // Add genres only if genres are empty
if (opfMetadata.genres.length && (!this.metadata.genres.length || opfMetadataOverrideDetails)) {
metadataUpdatePayload[key] = opfMetadata.genres
}
} else if (key === 'authors') {
if (opfMetadata.authors && opfMetadata.authors.length && (!this.metadata.authors.length || opfMetadataOverrideDetails)) {
metadataUpdatePayload.authors = opfMetadata.authors.map(authorName => {
return {
id: `new-${Math.floor(Math.random() * 1000000)}`,
name: authorName
}
})
}
} else if (key === 'narrators') {
if (opfMetadata.narrators?.length && (!this.metadata.narrators.length || opfMetadataOverrideDetails)) {
metadataUpdatePayload.narrators = opfMetadata.narrators
}
} else if (key === 'series') {
if (opfMetadata.series && (!this.metadata.series.length || opfMetadataOverrideDetails)) {
metadataUpdatePayload.series = this.metadata.parseSeriesTag(opfMetadata.series, opfMetadata.sequence)
}
} else if (opfMetadata[key] && ((!this.metadata[key] && !metadataUpdatePayload[key]) || opfMetadataOverrideDetails)) {
metadataUpdatePayload[key] = opfMetadata[key]
}
}
}
}
}
if (Object.keys(metadataUpdatePayload).length) {
return this.metadata.update(metadataUpdatePayload) || hasUpdated
}
return hasUpdated
}
searchQuery(query) {
const payload = {
tags: this.tags.filter(t => cleanStringForSearch(t).includes(query)),
@ -426,113 +322,6 @@ class Book {
Logger.debug(`[Book] Tracks being rebuilt...!`)
this.audioFiles.sort((a, b) => a.index - b.index)
this.missingParts = []
this.setChapters()
this.checkUpdateMissingTracks()
}
checkUpdateMissingTracks() {
var currMissingParts = (this.missingParts || []).join(',') || ''
var current_index = 1
var missingParts = []
for (let i = 0; i < this.tracks.length; i++) {
var _track = this.tracks[i]
if (_track.index > current_index) {
var num_parts_missing = _track.index - current_index
for (let x = 0; x < num_parts_missing && x < 9999; x++) {
missingParts.push(current_index + x)
}
}
current_index = _track.index + 1
}
this.missingParts = missingParts
var newMissingParts = (this.missingParts || []).join(',') || ''
var wasUpdated = newMissingParts !== currMissingParts
if (wasUpdated && this.missingParts.length) {
Logger.info(`[Audiobook] "${this.metadata.title}" has ${missingParts.length} missing parts`)
}
return wasUpdated
}
setChapters() {
const preferOverdriveMediaMarker = !!global.ServerSettings.scannerPreferOverdriveMediaMarker
// If 1 audio file without chapters, then no chapters will be set
const includedAudioFiles = this.audioFiles.filter(af => !af.exclude)
if (!includedAudioFiles.length) return
// If overdrive media markers are present and preferred, use those instead
if (preferOverdriveMediaMarker) {
const overdriveChapters = parseOverdriveMediaMarkersAsChapters(includedAudioFiles)
if (overdriveChapters) {
Logger.info('[Book] Overdrive Media Markers and preference found! Using these for chapter definitions')
this.chapters = overdriveChapters
return
}
}
// If first audio file has embedded chapters then use embedded chapters
if (includedAudioFiles[0].chapters?.length) {
// If all files chapters are the same, then only make chapters for the first file
if (
includedAudioFiles.length === 1 ||
includedAudioFiles.length > 1 &&
includedAudioFiles[0].chapters.length === includedAudioFiles[1].chapters?.length &&
includedAudioFiles[0].chapters.every((c, i) => c.title === includedAudioFiles[1].chapters[i].title)
) {
Logger.debug(`[Book] setChapters: Using embedded chapters in first audio file ${includedAudioFiles[0].metadata?.path}`)
this.chapters = includedAudioFiles[0].chapters.map((c) => ({ ...c }))
} else {
Logger.debug(`[Book] setChapters: Using embedded chapters from all audio files ${includedAudioFiles[0].metadata?.path}`)
this.chapters = []
let currChapterId = 0
let currStartTime = 0
includedAudioFiles.forEach((file) => {
if (file.duration) {
const chapters = file.chapters?.map((c) => ({
...c,
id: c.id + currChapterId,
start: c.start + currStartTime,
end: c.end + currStartTime,
})) ?? []
this.chapters = this.chapters.concat(chapters)
currChapterId += file.chapters?.length ?? 0
currStartTime += file.duration
}
})
}
} else if (includedAudioFiles.length > 1) {
const preferAudioMetadata = !!global.ServerSettings.scannerPreferAudioMetadata
// Build chapters from audio files
this.chapters = []
let currChapterId = 0
let currStartTime = 0
includedAudioFiles.forEach((file) => {
if (file.duration) {
let title = file.metadata.filename ? Path.basename(file.metadata.filename, Path.extname(file.metadata.filename)) : `Chapter ${currChapterId}`
// When prefer audio metadata server setting is set then use ID3 title tag as long as it is not the same as the book title
if (preferAudioMetadata && file.metaTags?.tagTitle && file.metaTags?.tagTitle !== this.metadata.title) {
title = file.metaTags.tagTitle
}
this.chapters.push({
id: currChapterId++,
start: currStartTime,
end: currStartTime + file.duration,
title
})
currStartTime += file.duration
}
})
}
}
// Only checks container format

4
server/objects/mediaTypes/Music.js
View file

@ -140,10 +140,6 @@ class Music {
return this.metadata.setDataFromAudioMetaTags(this.audioFile.metaTags, overrideExistingDetails)
}
syncMetadataFiles(textMetadataFiles, opfMetadataOverrideDetails) {
return false
}
searchQuery(query) {
return {}
}

31
server/objects/mediaTypes/Podcast.js
View file

@ -203,37 +203,6 @@ class Podcast {
this.lastEpisodeCheck = Date.now() // Makes sure new episodes are after this
}
async syncMetadataFiles(textMetadataFiles, opfMetadataOverrideDetails) {
let metadataUpdatePayload = {}
let tagsUpdated = false
const metadataAbs = textMetadataFiles.find(lf => lf.metadata.filename === 'metadata.abs' || lf.metadata.filename === 'metadata.json')
if (metadataAbs) {
const isJSON = metadataAbs.metadata.filename === 'metadata.json'
const metadataText = await readTextFile(metadataAbs.metadata.path)
const abmetadataUpdates = abmetadataGenerator.parseAndCheckForUpdates(metadataText, this, 'podcast', isJSON)
if (abmetadataUpdates && Object.keys(abmetadataUpdates).length) {
Logger.debug(`[Podcast] "${this.metadata.title}" changes found in metadata.abs file`, abmetadataUpdates)
if (abmetadataUpdates.tags) { // Set media tags if updated
this.tags = abmetadataUpdates.tags
tagsUpdated = true
}
if (abmetadataUpdates.metadata) {
metadataUpdatePayload = {
...metadataUpdatePayload,
...abmetadataUpdates.metadata
}
}
}
}
if (Object.keys(metadataUpdatePayload).length) {
return this.metadata.update(metadataUpdatePayload) || tagsUpdated
}
return tagsUpdated
}
searchEpisodes(query) {
return this.episodes.filter(ep => ep.searchQuery(query))
}

17
server/objects/settings/LibrarySettings.js
View file

@ -9,6 +9,7 @@ class LibrarySettings {
this.autoScanCronExpression = null
this.audiobooksOnly = false
this.hideSingleBookSeries = false // Do not show series that only have 1 book
this.metadataPrecedence = ['folderStructure', 'audioMetatags', 'txtFiles', 'opfFile', 'absMetadata']
if (settings) {
this.construct(settings)
@ -23,6 +24,12 @@ class LibrarySettings {
this.autoScanCronExpression = settings.autoScanCronExpression || null
this.audiobooksOnly = !!settings.audiobooksOnly
this.hideSingleBookSeries = !!settings.hideSingleBookSeries
if (settings.metadataPrecedence) {
this.metadataPrecedence = [...settings.metadataPrecedence]
} else {
// Added in v2.4.5
this.metadataPrecedence = ['folderStructure', 'audioMetatags', 'txtFiles', 'opfFile', 'absMetadata']
}
}
toJSON() {
@ -33,14 +40,20 @@ class LibrarySettings {
skipMatchingMediaWithIsbn: this.skipMatchingMediaWithIsbn,
autoScanCronExpression: this.autoScanCronExpression,
audiobooksOnly: this.audiobooksOnly,
hideSingleBookSeries: this.hideSingleBookSeries
hideSingleBookSeries: this.hideSingleBookSeries,
metadataPrecedence: [...this.metadataPrecedence]
}
}
update(payload) {
let hasUpdates = false
for (const key in payload) {
if (this[key] !== payload[key]) {
if (key === 'metadataPrecedence') {
if (payload[key] && Array.isArray(payload[key]) && payload[key].join() !== this[key].join()) {
this[key] = payload[key]
hasUpdates = true
}
} else if (this[key] !== payload[key]) {
this[key] = payload[key]
hasUpdates = true
}

9
server/objects/settings/ServerSettings.js
View file

@ -10,11 +10,8 @@ class ServerSettings {
this.scannerParseSubtitle = false
this.scannerFindCovers = false
this.scannerCoverProvider = 'google'
this.scannerPreferAudioMetadata = false
this.scannerPreferOpfMetadata = false
this.scannerPreferMatchedMetadata = false
this.scannerDisableWatcher = false
this.scannerPreferOverdriveMediaMarker = false
// Metadata - choose to store inside users library item folder
this.storeCoverWithItem = false
@ -65,11 +62,8 @@ class ServerSettings {
this.scannerFindCovers = !!settings.scannerFindCovers
this.scannerCoverProvider = settings.scannerCoverProvider || 'google'
this.scannerParseSubtitle = settings.scannerParseSubtitle
this.scannerPreferAudioMetadata = !!settings.scannerPreferAudioMetadata
this.scannerPreferOpfMetadata = !!settings.scannerPreferOpfMetadata
this.scannerPreferMatchedMetadata = !!settings.scannerPreferMatchedMetadata
this.scannerDisableWatcher = !!settings.scannerDisableWatcher
this.scannerPreferOverdriveMediaMarker = !!settings.scannerPreferOverdriveMediaMarker
this.storeCoverWithItem = !!settings.storeCoverWithItem
this.storeMetadataWithItem = !!settings.storeMetadataWithItem
@ -130,11 +124,8 @@ class ServerSettings {
scannerFindCovers: this.scannerFindCovers,
scannerCoverProvider: this.scannerCoverProvider,
scannerParseSubtitle: this.scannerParseSubtitle,
scannerPreferAudioMetadata: this.scannerPreferAudioMetadata,
scannerPreferOpfMetadata: this.scannerPreferOpfMetadata,
scannerPreferMatchedMetadata: this.scannerPreferMatchedMetadata,
scannerDisableWatcher: this.scannerDisableWatcher,
scannerPreferOverdriveMediaMarker: this.scannerPreferOverdriveMediaMarker,
storeCoverWithItem: this.storeCoverWithItem,
storeMetadataWithItem: this.storeMetadataWithItem,
metadataFileFormat: this.metadataFileFormat,

65
server/scanner/AbsMetadataFileScanner.js Normal file
View file

@ -0,0 +1,65 @@
const Path = require('path')
const fsExtra = require('../libs/fsExtra')
const { readTextFile } = require('../utils/fileUtils')
const { LogLevel } = require('../utils/constants')
const abmetadataGenerator = require('../utils/generators/abmetadataGenerator')
class AbsMetadataFileScanner {
constructor() { }
/**
* Check for metadata.json or metadata.abs file and set book metadata
*
* @param {import('./LibraryScan')} libraryScan
* @param {import('./LibraryItemScanData')} libraryItemData
* @param {Object} bookMetadata
* @param {string} [existingLibraryItemId]
*/
async scanBookMetadataFile(libraryScan, libraryItemData, bookMetadata, existingLibraryItemId = null) {
const metadataLibraryFile = libraryItemData.metadataJsonLibraryFile || libraryItemData.metadataAbsLibraryFile
let metadataText = metadataLibraryFile ? await readTextFile(metadataLibraryFile.metadata.path) : null
let metadataFilePath = metadataLibraryFile?.metadata.path
let metadataFileFormat = libraryItemData.metadataJsonLibraryFile ? 'json' : 'abs'
// When metadata file is not stored with library item then check in the /metadata/items folder for it
if (!metadataText && existingLibraryItemId) {
let metadataPath = Path.join(global.MetadataPath, 'items', existingLibraryItemId)
let altFormat = global.ServerSettings.metadataFileFormat === 'json' ? 'abs' : 'json'
// First check the metadata format set in server settings, fallback to the alternate
metadataFilePath = Path.join(metadataPath, `metadata.${global.ServerSettings.metadataFileFormat}`)
metadataFileFormat = global.ServerSettings.metadataFileFormat
if (await fsExtra.pathExists(metadataFilePath)) {
metadataText = await readTextFile(metadataFilePath)
} else if (await fsExtra.pathExists(Path.join(metadataPath, `metadata.${altFormat}`))) {
metadataFilePath = Path.join(metadataPath, `metadata.${altFormat}`)
metadataFileFormat = altFormat
metadataText = await readTextFile(metadataFilePath)
}
}
if (metadataText) {
libraryScan.addLog(LogLevel.INFO, `Found metadata file "${metadataFilePath}" - preferring`)
let abMetadata = null
if (metadataFileFormat === 'json') {
abMetadata = abmetadataGenerator.parseJson(metadataText)
} else {
abMetadata = abmetadataGenerator.parse(metadataText, 'book')
}
if (abMetadata) {
if (abMetadata.tags?.length) {
bookMetadata.tags = abMetadata.tags
}
if (abMetadata.chapters?.length) {
bookMetadata.chapters = abMetadata.chapters
}
for (const key in abMetadata.metadata) {
if (abMetadata.metadata[key] === undefined) continue
bookMetadata[key] = abMetadata.metadata[key]
}
}
}
}
}
module.exports = new AbsMetadataFileScanner()

202
server/scanner/AudioFileScanner.js
View file

@ -1,6 +1,9 @@
const Path = require('path')
const Logger = require('../Logger')
const prober = require('../utils/prober')
const { LogLevel } = require('../utils/constants')
const { parseOverdriveMediaMarkersAsChapters } = require('../utils/parsers/parseOverdriveMediaMarkers')
const parseNameString = require('../utils/parsers/parseNameString')
const LibraryItem = require('../models/LibraryItem')
const AudioFile = require('../objects/files/AudioFile')
@ -205,5 +208,204 @@ class AudioFileScanner {
Logger.debug(`[AudioFileScanner] Running ffprobe for audio file at "${audioFile.metadata.path}"`)
return prober.rawProbe(audioFile.metadata.path)
}
/**
* Set book metadata & chapters from audio file meta tags
*
* @param {string} bookTitle
* @param {import('../models/Book').AudioFileObject} audioFile
* @param {Object} bookMetadata
* @param {LibraryScan} libraryScan
*/
setBookMetadataFromAudioMetaTags(bookTitle, audioFiles, bookMetadata, libraryScan) {
const MetadataMapArray = [
{
tag: 'tagComposer',
key: 'narrators'
},
{
tag: 'tagDescription',
altTag: 'tagComment',
key: 'description'
},
{
tag: 'tagPublisher',
key: 'publisher'
},
{
tag: 'tagDate',
key: 'publishedYear'
},
{
tag: 'tagSubtitle',
key: 'subtitle'
},
{
tag: 'tagAlbum',
altTag: 'tagTitle',
key: 'title',
},
{
tag: 'tagArtist',
altTag: 'tagAlbumArtist',
key: 'authors'
},
{
tag: 'tagGenre',
key: 'genres'
},
{
tag: 'tagSeries',
key: 'series'
},
{
tag: 'tagIsbn',
key: 'isbn'
},
{
tag: 'tagLanguage',
key: 'language'
},
{
tag: 'tagASIN',
key: 'asin'
}
]
const firstScannedFile = audioFiles[0]
const audioFileMetaTags = firstScannedFile.metaTags
MetadataMapArray.forEach((mapping) => {
let value = audioFileMetaTags[mapping.tag]
if (!value && mapping.altTag) {
value = audioFileMetaTags[mapping.altTag]
}
if (value && typeof value === 'string') {
value = value.trim() // Trim whitespace
if (mapping.key === 'narrators') {
bookMetadata.narrators = parseNameString.parse(value)?.names || []
} else if (mapping.key === 'authors') {
bookMetadata.authors = parseNameString.parse(value)?.names || []
} else if (mapping.key === 'genres') {
bookMetadata.genres = this.parseGenresString(value)
} else if (mapping.key === 'series') {
bookMetadata.series = [
{
name: value,
sequence: audioFileMetaTags.tagSeriesPart || null
}
]
} else {
bookMetadata[mapping.key] = value
}
}
})
// Set chapters
const chapters = this.getBookChaptersFromAudioFiles(bookTitle, audioFiles, libraryScan)
if (chapters.length) {
bookMetadata.chapters = chapters
}
}
/**
* @param {string} bookTitle
* @param {AudioFile[]} audioFiles
* @param {LibraryScan} libraryScan
* @returns {import('../models/Book').ChapterObject[]}
*/
getBookChaptersFromAudioFiles(bookTitle, audioFiles, libraryScan) {
// If overdrive media markers are present then use those instead
const overdriveChapters = parseOverdriveMediaMarkersAsChapters(audioFiles)
if (overdriveChapters?.length) {
libraryScan.addLog(LogLevel.DEBUG, 'Overdrive Media Markers and preference found! Using these for chapter definitions')
return overdriveChapters
}
let chapters = []
// If first audio file has embedded chapters then use embedded chapters
if (audioFiles[0].chapters?.length) {
// If all files chapters are the same, then only make chapters for the first file
if (
audioFiles.length === 1 ||
audioFiles.length > 1 &&
audioFiles[0].chapters.length === audioFiles[1].chapters?.length &&
audioFiles[0].chapters.every((c, i) => c.title === audioFiles[1].chapters[i].title)
) {
libraryScan.addLog(LogLevel.DEBUG, `setChapters: Using embedded chapters in first audio file ${audioFiles[0].metadata?.path}`)
chapters = audioFiles[0].chapters.map((c) => ({ ...c }))
} else {
libraryScan.addLog(LogLevel.DEBUG, `setChapters: Using embedded chapters from all audio files ${audioFiles[0].metadata?.path}`)
let currChapterId = 0
let currStartTime = 0
audioFiles.forEach((file) => {
if (file.duration) {
const afChapters = file.chapters?.map((c) => ({
...c,
id: c.id + currChapterId,
start: c.start + currStartTime,
end: c.end + currStartTime,
})) ?? []
chapters = chapters.concat(afChapters)
currChapterId += file.chapters?.length ?? 0
currStartTime += file.duration
}
})
return chapters
}
} else if (audioFiles.length > 1) {
// In some cases the ID3 title tag for each file is the chapter title, the criteria to determine if this will be used
// 1. Every audio file has an ID3 title tag set
// 2. None of the title tags are the same as the book title
// 3. Every ID3 title tag is unique
const metaTagTitlesFound = [...new Set(audioFiles.map(af => af.metaTags?.tagTitle).filter(tagTitle => !!tagTitle && tagTitle !== bookTitle))]
const useMetaTagAsTitle = metaTagTitlesFound.length === audioFiles.length
// Build chapters from audio files
let currChapterId = 0
let currStartTime = 0
audioFiles.forEach((file) => {
if (file.duration) {
let title = file.metadata.filename ? Path.basename(file.metadata.filename, Path.extname(file.metadata.filename)) : `Chapter ${currChapterId}`
if (useMetaTagAsTitle) {
title = file.metaTags.tagTitle
}
chapters.push({
id: currChapterId++,
start: currStartTime,
end: currStartTime + file.duration,
title
})
currStartTime += file.duration
}
})
}
return chapters
}
/**
* Parse a genre string into multiple genres
* @example "Fantasy;Sci-Fi;History" => ["Fantasy", "Sci-Fi", "History"]
*
* @param {string} genreTag
* @returns {string[]}
*/
parseGenresString(genreTag) {
if (!genreTag?.length) return []
const separators = ['/', '//', ';']
for (let i = 0; i < separators.length; i++) {
if (genreTag.includes(separators[i])) {
return genreTag.split(separators[i]).map(genre => genre.trim()).filter(g => !!g)
}
}
return [genreTag]
}
}
module.exports = new AudioFileScanner()

379
server/scanner/BookScanner.js
View file

@ -3,8 +3,6 @@ const Path = require('path')
const sequelize = require('sequelize')
const { LogLevel } = require('../utils/constants')
const { getTitleIgnorePrefix, areEquivalent } = require('../utils/index')
const { parseOpfMetadataXML } = require('../utils/parsers/parseOpfMetadata')
const { parseOverdriveMediaMarkersAsChapters } = require('../utils/parsers/parseOverdriveMediaMarkers')
const abmetadataGenerator = require('../utils/generators/abmetadataGenerator')
const parseNameString = require('../utils/parsers/parseNameString')
const globals = require('../utils/globals')
@ -16,9 +14,12 @@ const CoverManager = require('../managers/CoverManager')
const LibraryFile = require('../objects/files/LibraryFile')
const SocketAuthority = require('../SocketAuthority')
const fsExtra = require("../libs/fsExtra")
const LibraryScan = require("./LibraryScan")
const BookFinder = require('../finders/BookFinder')
const LibraryScan = require("./LibraryScan")
const OpfFileScanner = require('./OpfFileScanner')
const AbsMetadataFileScanner = require('./AbsMetadataFileScanner')
/**
* Metadata for books pulled from files
* @typedef BookMetadataObject
@ -50,7 +51,7 @@ class BookScanner {
* @param {import('./LibraryItemScanData')} libraryItemData
* @param {import('../models/Library').LibrarySettingsObject} librarySettings
* @param {LibraryScan} libraryScan
* @returns {Promise<import('../models/LibraryItem')>}
* @returns {Promise<{libraryItem:import('../models/LibraryItem'), wasUpdated:boolean}>}
*/
async rescanExistingBookLibraryItem(existingLibraryItem, libraryItemData, librarySettings, libraryScan) {
/** @type {import('../models/Book')} */
@ -168,7 +169,7 @@ class BookScanner {
hasMediaChanges = true
}
const bookMetadata = await this.getBookMetadataFromScanData(media.audioFiles, libraryItemData, libraryScan, existingLibraryItem.id)
const bookMetadata = await this.getBookMetadataFromScanData(media.audioFiles, libraryItemData, libraryScan, librarySettings, existingLibraryItem.id)
let authorsUpdated = false
const bookAuthorsRemoved = []
let seriesUpdated = false
@ -360,7 +361,10 @@ class BookScanner {
libraryScan.seriesRemovedFromBooks.push(...bookSeriesRemoved)
libraryScan.authorsRemovedFromBooks.push(...bookAuthorsRemoved)
return existingLibraryItem
return {
libraryItem: existingLibraryItem,
wasUpdated: hasMediaChanges || libraryItemUpdated || seriesUpdated || authorsUpdated
}
}
/**
@ -389,7 +393,7 @@ class BookScanner {
ebookLibraryFile.ebookFormat = ebookLibraryFile.metadata.ext.slice(1).toLowerCase()
}
const bookMetadata = await this.getBookMetadataFromScanData(scannedAudioFiles, libraryItemData, libraryScan)
const bookMetadata = await this.getBookMetadataFromScanData(scannedAudioFiles, libraryItemData, libraryScan, librarySettings)
bookMetadata.explicit = !!bookMetadata.explicit // Ensure boolean
bookMetadata.abridged = !!bookMetadata.abridged // Ensure boolean
@ -548,226 +552,41 @@ class BookScanner {
* @param {import('../models/Book').AudioFileObject[]} audioFiles
* @param {import('./LibraryItemScanData')} libraryItemData
* @param {LibraryScan} libraryScan
* @param {import('../models/Library').LibrarySettingsObject} librarySettings
* @param {string} [existingLibraryItemId]
* @returns {Promise<BookMetadataObject>}
*/
async getBookMetadataFromScanData(audioFiles, libraryItemData, libraryScan, existingLibraryItemId = null) {
async getBookMetadataFromScanData(audioFiles, libraryItemData, libraryScan, librarySettings, existingLibraryItemId = null) {
// First set book metadata from folder/file names
const bookMetadata = {
title: libraryItemData.mediaMetadata.title,
titleIgnorePrefix: getTitleIgnorePrefix(libraryItemData.mediaMetadata.title),
subtitle: libraryItemData.mediaMetadata.subtitle || undefined,
publishedYear: libraryItemData.mediaMetadata.publishedYear || undefined,
title: libraryItemData.mediaMetadata.title, // required
titleIgnorePrefix: undefined,
subtitle: undefined,
publishedYear: undefined,
publisher: undefined,
description: undefined,
isbn: undefined,
asin: undefined,
language: undefined,
narrators: parseNameString.parse(libraryItemData.mediaMetadata.narrators)?.names || [],
narrators: [],
genres: [],
tags: [],
authors: parseNameString.parse(libraryItemData.mediaMetadata.author)?.names || [],
authors: [],
series: [],
chapters: [],
explicit: undefined,
abridged: undefined,
coverPath: undefined
}
if (libraryItemData.mediaMetadata.series) {
bookMetadata.series.push({
name: libraryItemData.mediaMetadata.series,
sequence: libraryItemData.mediaMetadata.sequence || null
})
}
// Fill in or override book metadata from audio file meta tags
if (audioFiles.length) {
const MetadataMapArray = [
{
tag: 'tagComposer',
key: 'narrators'
},
{
tag: 'tagDescription',
altTag: 'tagComment',
key: 'description'
},
{
tag: 'tagPublisher',
key: 'publisher'
},
{
tag: 'tagDate',
key: 'publishedYear'
},
{
tag: 'tagSubtitle',
key: 'subtitle'
},
{
tag: 'tagAlbum',
altTag: 'tagTitle',
key: 'title',
},
{
tag: 'tagArtist',
altTag: 'tagAlbumArtist',
key: 'authors'
},
{
tag: 'tagGenre',
key: 'genres'
},
{
tag: 'tagSeries',
key: 'series'
},
{
tag: 'tagIsbn',
key: 'isbn'
},
{
tag: 'tagLanguage',
key: 'language'
},
{
tag: 'tagASIN',
key: 'asin'
}
]
const overrideExistingDetails = Database.serverSettings.scannerPreferAudioMetadata
const firstScannedFile = audioFiles[0]
const audioFileMetaTags = firstScannedFile.metaTags
MetadataMapArray.forEach((mapping) => {
let value = audioFileMetaTags[mapping.tag]
if (!value && mapping.altTag) {
value = audioFileMetaTags[mapping.altTag]
}
if (value && typeof value === 'string') {
value = value.trim() // Trim whitespace
if (mapping.key === 'narrators' && (!bookMetadata.narrators.length || overrideExistingDetails)) {
bookMetadata.narrators = parseNameString.parse(value)?.names || []
} else if (mapping.key === 'authors' && (!bookMetadata.authors.length || overrideExistingDetails)) {
bookMetadata.authors = parseNameString.parse(value)?.names || []
} else if (mapping.key === 'genres' && (!bookMetadata.genres.length || overrideExistingDetails)) {
bookMetadata.genres = this.parseGenresString(value)
} else if (mapping.key === 'series' && (!bookMetadata.series.length || overrideExistingDetails)) {
bookMetadata.series = [
{
name: value,
sequence: audioFileMetaTags.tagSeriesPart || null
}
]
} else if (!bookMetadata[mapping.key] || overrideExistingDetails) {
bookMetadata[mapping.key] = value
}
}
})
}
// If desc.txt in library item folder then use this for description
if (libraryItemData.descTxtLibraryFile) {
const description = await readTextFile(libraryItemData.descTxtLibraryFile.metadata.path)
if (description.trim()) bookMetadata.description = description.trim()
}
// If reader.txt in library item folder then use this for narrator
if (libraryItemData.readerTxtLibraryFile) {
let narrator = await readTextFile(libraryItemData.readerTxtLibraryFile.metadata.path)
narrator = narrator.split(/\r?\n/)[0]?.trim() || '' // Only use first line
if (narrator) {
bookMetadata.narrators = parseNameString.parse(narrator)?.names || []
}
}
// If opf file is found look for metadata
if (libraryItemData.metadataOpfLibraryFile) {
const xmlText = await readTextFile(libraryItemData.metadataOpfLibraryFile.metadata.path)
const opfMetadata = xmlText ? await parseOpfMetadataXML(xmlText) : null
if (opfMetadata) {
const opfMetadataOverrideDetails = Database.serverSettings.scannerPreferOpfMetadata
for (const key in opfMetadata) {
if (key === 'tags') { // Add tags only if tags are empty
if (opfMetadata.tags.length && (!bookMetadata.tags.length || opfMetadataOverrideDetails)) {
bookMetadata.tags = opfMetadata.tags
}
} else if (key === 'genres') { // Add genres only if genres are empty
if (opfMetadata.genres.length && (!bookMetadata.genres.length || opfMetadataOverrideDetails)) {
bookMetadata.genres = opfMetadata.genres
}
} else if (key === 'authors') {
if (opfMetadata.authors?.length && (!bookMetadata.authors.length || opfMetadataOverrideDetails)) {
bookMetadata.authors = opfMetadata.authors
}
} else if (key === 'narrators') {
if (opfMetadata.narrators?.length && (!bookMetadata.narrators.length || opfMetadataOverrideDetails)) {
bookMetadata.narrators = opfMetadata.narrators
}
} else if (key === 'series') {
if (opfMetadata.series && (!bookMetadata.series.length || opfMetadataOverrideDetails)) {
bookMetadata.series = [{
name: opfMetadata.series,
sequence: opfMetadata.sequence || null
}]
}
} else if (opfMetadata[key] && (!bookMetadata[key] || opfMetadataOverrideDetails)) {
bookMetadata[key] = opfMetadata[key]
}
}
}
}
// If metadata.json or metadata.abs use this for metadata
const metadataLibraryFile = libraryItemData.metadataJsonLibraryFile || libraryItemData.metadataAbsLibraryFile
let metadataText = metadataLibraryFile ? await readTextFile(metadataLibraryFile.metadata.path) : null
let metadataFilePath = metadataLibraryFile?.metadata.path
let metadataFileFormat = libraryItemData.metadataJsonLibraryFile ? 'json' : 'abs'
// When metadata file is not stored with library item then check in the /metadata/items folder for it
if (!metadataText && existingLibraryItemId) {
let metadataPath = Path.join(global.MetadataPath, 'items', existingLibraryItemId)
let altFormat = global.ServerSettings.metadataFileFormat === 'json' ? 'abs' : 'json'
// First check the metadata format set in server settings, fallback to the alternate
metadataFilePath = Path.join(metadataPath, `metadata.${global.ServerSettings.metadataFileFormat}`)
metadataFileFormat = global.ServerSettings.metadataFileFormat
if (await fsExtra.pathExists(metadataFilePath)) {
metadataText = await readTextFile(metadataFilePath)
} else if (await fsExtra.pathExists(Path.join(metadataPath, `metadata.${altFormat}`))) {
metadataFilePath = Path.join(metadataPath, `metadata.${altFormat}`)
metadataFileFormat = altFormat
metadataText = await readTextFile(metadataFilePath)
}
}
if (metadataText) {
libraryScan.addLog(LogLevel.INFO, `Found metadata file "${metadataFilePath}" - preferring`)
let abMetadata = null
if (metadataFileFormat === 'json') {
abMetadata = abmetadataGenerator.parseJson(metadataText)
const bookMetadataSourceHandler = new BookScanner.BookMetadataSourceHandler(bookMetadata, audioFiles, libraryItemData, libraryScan, existingLibraryItemId)
for (const metadataSource of librarySettings.metadataPrecedence) {
if (bookMetadataSourceHandler[metadataSource]) {
libraryScan.addLog(LogLevel.DEBUG, `Getting metadata from source "${metadataSource}"`)
await bookMetadataSourceHandler[metadataSource]()
} else {
abMetadata = abmetadataGenerator.parse(metadataText, 'book')
libraryScan.addLog(LogLevel.ERROR, `Invalid metadata source "${metadataSource}"`)
}
if (abMetadata) {
if (abMetadata.tags?.length) {
bookMetadata.tags = abMetadata.tags
}
if (abMetadata.chapters?.length) {
bookMetadata.chapters = abMetadata.chapters
}
for (const key in abMetadata.metadata) {
if (abMetadata.metadata[key] === undefined) continue
bookMetadata[key] = abMetadata.metadata[key]
}
}
}
// Set chapters from audio files if not already set
if (!bookMetadata.chapters.length) {
bookMetadata.chapters = this.getChaptersFromAudioFiles(bookMetadata.title, audioFiles, libraryScan)
}
// Set cover from library file if one is found otherwise check audiofile
@ -781,102 +600,76 @@ class BookScanner {
return bookMetadata
}
/**
* Parse a genre string into multiple genres
* @example "Fantasy;Sci-Fi;History" => ["Fantasy", "Sci-Fi", "History"]
* @param {string} genreTag
* @returns {string[]}
*/
parseGenresString(genreTag) {
if (!genreTag?.length) return []
const separators = ['/', '//', ';']
for (let i = 0; i < separators.length; i++) {
if (genreTag.includes(separators[i])) {
return genreTag.split(separators[i]).map(genre => genre.trim()).filter(g => !!g)
}
}
return [genreTag]
}
/**
* @param {string} bookTitle
* @param {AudioFile[]} audioFiles
* @param {LibraryScan} libraryScan
* @returns {import('../models/Book').ChapterObject[]}
*/
getChaptersFromAudioFiles(bookTitle, audioFiles, libraryScan) {
if (!audioFiles.length) return []
// If overdrive media markers are present and preferred, use those instead
if (Database.serverSettings.scannerPreferOverdriveMediaMarker) {
const overdriveChapters = parseOverdriveMediaMarkersAsChapters(audioFiles)
if (overdriveChapters) {
libraryScan.addLog(LogLevel.DEBUG, 'Overdrive Media Markers and preference found! Using these for chapter definitions')
return overdriveChapters
}
static BookMetadataSourceHandler = class {
/**
*
* @param {Object} bookMetadata
* @param {import('../models/Book').AudioFileObject[]} audioFiles
* @param {import('./LibraryItemScanData')} libraryItemData
* @param {LibraryScan} libraryScan
* @param {string} existingLibraryItemId
*/
constructor(bookMetadata, audioFiles, libraryItemData, libraryScan, existingLibraryItemId) {
this.bookMetadata = bookMetadata
this.audioFiles = audioFiles
this.libraryItemData = libraryItemData
this.libraryScan = libraryScan
this.existingLibraryItemId = existingLibraryItemId
}
let chapters = []
/**
* Metadata parsed from folder names/structure
*/
folderStructure() {
this.libraryItemData.setBookMetadataFromFilenames(this.bookMetadata)
}
// If first audio file has embedded chapters then use embedded chapters
if (audioFiles[0].chapters?.length) {
// If all files chapters are the same, then only make chapters for the first file
if (
audioFiles.length === 1 ||
audioFiles.length > 1 &&
audioFiles[0].chapters.length === audioFiles[1].chapters?.length &&
audioFiles[0].chapters.every((c, i) => c.title === audioFiles[1].chapters[i].title)
) {
libraryScan.addLog(LogLevel.DEBUG, `setChapters: Using embedded chapters in first audio file ${audioFiles[0].metadata?.path}`)
chapters = audioFiles[0].chapters.map((c) => ({ ...c }))
} else {
libraryScan.addLog(LogLevel.DEBUG, `setChapters: Using embedded chapters from all audio files ${audioFiles[0].metadata?.path}`)
let currChapterId = 0
let currStartTime = 0
/**
* Metadata from audio file meta tags
*/
audioMetatags() {
if (!this.audioFiles.length) return
// Modifies bookMetadata with metadata mapped from audio file meta tags
const bookTitle = this.bookMetadata.title || this.libraryItemData.mediaMetadata.title
AudioFileScanner.setBookMetadataFromAudioMetaTags(bookTitle, this.audioFiles, this.bookMetadata, this.libraryScan)
}
audioFiles.forEach((file) => {
if (file.duration) {
const afChapters = file.chapters?.map((c) => ({
...c,
id: c.id + currChapterId,
start: c.start + currStartTime,
end: c.end + currStartTime,
})) ?? []
chapters = chapters.concat(afChapters)
currChapterId += file.chapters?.length ?? 0
currStartTime += file.duration
}
})
return chapters
/**
* Description from desc.txt and narrator from reader.txt
*/
async txtFiles() {
// If desc.txt in library item folder then use this for description
if (this.libraryItemData.descTxtLibraryFile) {
const description = await readTextFile(this.libraryItemData.descTxtLibraryFile.metadata.path)
if (description.trim()) this.bookMetadata.description = description.trim()
}
} else if (audioFiles.length > 1) {
const preferAudioMetadata = !!Database.serverSettings.scannerPreferAudioMetadata
// Build chapters from audio files
let currChapterId = 0
let currStartTime = 0
audioFiles.forEach((file) => {
if (file.duration) {
let title = file.metadata.filename ? Path.basename(file.metadata.filename, Path.extname(file.metadata.filename)) : `Chapter ${currChapterId}`
// When prefer audio metadata server setting is set then use ID3 title tag as long as it is not the same as the book title
if (preferAudioMetadata && file.metaTags?.tagTitle && file.metaTags?.tagTitle !== bookTitle) {
title = file.metaTags.tagTitle
}
chapters.push({
id: currChapterId++,
start: currStartTime,
end: currStartTime + file.duration,
title
})
currStartTime += file.duration
// If reader.txt in library item folder then use this for narrator
if (this.libraryItemData.readerTxtLibraryFile) {
let narrator = await readTextFile(this.libraryItemData.readerTxtLibraryFile.metadata.path)
narrator = narrator.split(/\r?\n/)[0]?.trim() || '' // Only use first line
if (narrator) {
this.bookMetadata.narrators = parseNameString.parse(narrator)?.names || []
}
})
}
}
/**
* Metadata from opf file
*/
async opfFile() {
if (!this.libraryItemData.metadataOpfLibraryFile) return
await OpfFileScanner.scanBookOpfFile(this.libraryItemData.metadataOpfLibraryFile, this.bookMetadata)
}
/**
* Metadata from metadata.json or metadata.abs
*/
async absMetadata() {
// If metadata.json or metadata.abs use this for metadata
await AbsMetadataFileScanner.scanBookMetadataFile(this.libraryScan, this.libraryItemData, this.bookMetadata, this.existingLibraryItemId)
}
return chapters
}
/**

36
server/scanner/LibraryItemScanData.js
View file

@ -25,7 +25,7 @@ class LibraryItemScanData {
this.relPath = data.relPath
/** @type {boolean} */
this.isFile = data.isFile
/** @type {{author:string, title:string, subtitle:string, series:string, sequence:string, publishedYear:string, narrators:string}} */
/** @type {import('../utils/scandir').LibraryItemFilenameMetadata} */
this.mediaMetadata = data.mediaMetadata
/** @type {import('../objects/files/LibraryFile')[]} */
this.libraryFiles = data.libraryFiles
@ -233,10 +233,9 @@ class LibraryItemScanData {
}
await existingLibraryItem.save()
return true
} else {
libraryScan.addLog(LogLevel.DEBUG, `Library item "${existingLibraryItem.relPath}" is up-to-date`)
return false
}
return false
}
/**
@ -303,5 +302,34 @@ class LibraryItemScanData {
return !this.ebookLibraryFiles.some(lf => lf.ino === ebookFile.ino)
}
/**
* Set data parsed from filenames
*
* @param {Object} bookMetadata
*/
setBookMetadataFromFilenames(bookMetadata) {
const keysToMap = ['title', 'subtitle', 'publishedYear']
for (const key in this.mediaMetadata) {
if (keysToMap.includes(key) && this.mediaMetadata[key]) {
bookMetadata[key] = this.mediaMetadata[key]
}
}
if (this.mediaMetadata.authors?.length) {
bookMetadata.authors = this.mediaMetadata.authors
}
if (this.mediaMetadata.narrators?.length) {
bookMetadata.narrators = this.mediaMetadata.narrators
}
if (this.mediaMetadata.seriesName) {
bookMetadata.series = [
{
name: this.mediaMetadata.seriesName,
sequence: this.mediaMetadata.seriesSequence || null
}
]
}
}
}
module.exports = LibraryItemScanData

13
server/scanner/LibraryItemScanner.js
View file

@ -58,7 +58,7 @@ class LibraryItemScanner {
if (await libraryItemScanData.checkLibraryItemData(libraryItem, scanLogger)) {
if (libraryItemScanData.hasLibraryFileChanges || libraryItemScanData.hasPathChange) {
const expandedLibraryItem = await this.rescanLibraryItem(libraryItem, libraryItemScanData, library.settings, scanLogger)
const { libraryItem: expandedLibraryItem } = await this.rescanLibraryItemMedia(libraryItem, libraryItemScanData, library.settings, scanLogger)
const oldLibraryItem = Database.libraryItemModel.getOldLibraryItem(expandedLibraryItem)
SocketAuthority.emitter('item_updated', oldLibraryItem.toJSONExpanded())
@ -71,6 +71,7 @@ class LibraryItemScanner {
return ScanResult.UPDATED
}
libraryScan.addLog(LogLevel.DEBUG, `Library item "${libraryItem.relPath}" is up-to-date`)
return ScanResult.UPTODATE
}
@ -156,16 +157,14 @@ class LibraryItemScanner {
* @param {LibraryItemScanData} libraryItemData
* @param {import('../models/Library').LibrarySettingsObject} librarySettings
* @param {LibraryScan} libraryScan
* @returns {Promise<LibraryItem>}
* @returns {Promise<{libraryItem:LibraryItem, wasUpdated:boolean}>}
*/
async rescanLibraryItem(existingLibraryItem, libraryItemData, librarySettings, libraryScan) {
let newLibraryItem = null
rescanLibraryItemMedia(existingLibraryItem, libraryItemData, librarySettings, libraryScan) {
if (existingLibraryItem.mediaType === 'book') {
newLibraryItem = await BookScanner.rescanExistingBookLibraryItem(existingLibraryItem, libraryItemData, librarySettings, libraryScan)
return BookScanner.rescanExistingBookLibraryItem(existingLibraryItem, libraryItemData, librarySettings, libraryScan)
} else {
newLibraryItem = await PodcastScanner.rescanExistingPodcastLibraryItem(existingLibraryItem, libraryItemData, librarySettings, libraryScan)
return PodcastScanner.rescanExistingPodcastLibraryItem(existingLibraryItem, libraryItemData, librarySettings, libraryScan)
}
return newLibraryItem
}
/**

33
server/scanner/LibraryScanner.js
View file

@ -44,9 +44,9 @@ class LibraryScanner {
/**
*
* @param {import('../objects/Library')} library
* @param {*} options
* @param {boolean} [forceRescan]
*/
async scan(library, options = {}) {
async scan(library, forceRescan = false) {
if (this.isLibraryScanning(library.id)) {
Logger.error(`[Scanner] Already scanning ${library.id}`)
return
@ -64,9 +64,9 @@ class LibraryScanner {
SocketAuthority.emitter('scan_start', libraryScan.getScanEmitData)
Logger.info(`[Scanner] Starting library scan ${libraryScan.id} for ${libraryScan.libraryName}`)
Logger.info(`[Scanner] Starting${forceRescan ? ' (forced)' : ''} library scan ${libraryScan.id} for ${libraryScan.libraryName}`)
const canceled = await this.scanLibrary(libraryScan)
const canceled = await this.scanLibrary(libraryScan, forceRescan)
if (canceled) {
Logger.info(`[Scanner] Library scan canceled for "${libraryScan.libraryName}"`)
@ -95,9 +95,10 @@ class LibraryScanner {
/**
*
* @param {import('./LibraryScan')} libraryScan
* @returns {boolean} true if scan canceled
* @param {boolean} forceRescan
* @returns {Promise<boolean>} true if scan canceled
*/
async scanLibrary(libraryScan) {
async scanLibrary(libraryScan, forceRescan) {
// Make sure library filter data is set
// this is used to check for existing authors & series
await libraryFilters.getFilterData(libraryScan.library.mediaType, libraryScan.libraryId)
@ -155,17 +156,25 @@ class LibraryScanner {
}
} else {
libraryItemDataFound = libraryItemDataFound.filter(lidf => lidf !== libraryItemData)
if (await libraryItemData.checkLibraryItemData(existingLibraryItem, libraryScan)) {
libraryScan.resultsUpdated++
if (libraryItemData.hasLibraryFileChanges || libraryItemData.hasPathChange) {
const libraryItem = await LibraryItemScanner.rescanLibraryItem(existingLibraryItem, libraryItemData, libraryScan.library.settings, libraryScan)
const oldLibraryItem = Database.libraryItemModel.getOldLibraryItem(libraryItem)
oldLibraryItemsUpdated.push(oldLibraryItem)
let libraryItemDataUpdated = await libraryItemData.checkLibraryItemData(existingLibraryItem, libraryScan)
if (libraryItemDataUpdated || forceRescan) {
if (forceRescan || libraryItemData.hasLibraryFileChanges || libraryItemData.hasPathChange) {
const { libraryItem, wasUpdated } = await LibraryItemScanner.rescanLibraryItemMedia(existingLibraryItem, libraryItemData, libraryScan.library.settings, libraryScan)
if (!forceRescan || wasUpdated) {
libraryScan.resultsUpdated++
const oldLibraryItem = Database.libraryItemModel.getOldLibraryItem(libraryItem)
oldLibraryItemsUpdated.push(oldLibraryItem)
} else {
libraryScan.addLog(LogLevel.DEBUG, `Library item "${existingLibraryItem.relPath}" is up-to-date`)
}
} else {
libraryScan.resultsUpdated++
// TODO: Temporary while using old model to socket emit
const oldLibraryItem = await Database.libraryItemModel.getOldById(existingLibraryItem.id)
oldLibraryItemsUpdated.push(oldLibraryItem)
}
} else {
libraryScan.addLog(LogLevel.DEBUG, `Library item "${existingLibraryItem.relPath}" is up-to-date`)
}
}

48
server/scanner/OpfFileScanner.js Normal file
View file

@ -0,0 +1,48 @@
const { parseOpfMetadataXML } = require('../utils/parsers/parseOpfMetadata')
const { readTextFile } = require('../utils/fileUtils')
class OpfFileScanner {
constructor() { }
/**
* Parse metadata from .opf file found in library scan and update bookMetadata
*
* @param {import('../models/LibraryItem').LibraryFileObject} opfLibraryFileObj
* @param {Object} bookMetadata
*/
async scanBookOpfFile(opfLibraryFileObj, bookMetadata) {
const xmlText = await readTextFile(opfLibraryFileObj.metadata.path)
const opfMetadata = xmlText ? await parseOpfMetadataXML(xmlText) : null
if (opfMetadata) {
for (const key in opfMetadata) {
if (key === 'tags') { // Add tags only if tags are empty
if (opfMetadata.tags.length) {
bookMetadata.tags = opfMetadata.tags
}
} else if (key === 'genres') { // Add genres only if genres are empty
if (opfMetadata.genres.length) {
bookMetadata.genres = opfMetadata.genres
}
} else if (key === 'authors') {
if (opfMetadata.authors?.length) {
bookMetadata.authors = opfMetadata.authors
}
} else if (key === 'narrators') {
if (opfMetadata.narrators?.length) {
bookMetadata.narrators = opfMetadata.narrators
}
} else if (key === 'series') {
if (opfMetadata.series) {
bookMetadata.series = [{
name: opfMetadata.series,
sequence: opfMetadata.sequence || null
}]
}
} else if (opfMetadata[key]) {
bookMetadata[key] = opfMetadata[key]
}
}
}
}
}
module.exports = new OpfFileScanner()

19
server/scanner/PodcastScanner.js
View file

@ -39,7 +39,7 @@ class PodcastScanner {
* @param {import('./LibraryItemScanData')} libraryItemData
* @param {import('../models/Library').LibrarySettingsObject} librarySettings
* @param {import('./LibraryScan')} libraryScan
* @returns {Promise<import('../models/LibraryItem')>}
* @returns {Promise<{libraryItem:import('../models/LibraryItem'), wasUpdated:boolean}>}
*/
async rescanExistingPodcastLibraryItem(existingLibraryItem, libraryItemData, librarySettings, libraryScan) {
/** @type {import('../models/Podcast')} */
@ -201,7 +201,10 @@ class PodcastScanner {
await existingLibraryItem.save()
}
return existingLibraryItem
return {
libraryItem: existingLibraryItem,
wasUpdated: hasMediaChanges || libraryItemUpdated
}
}
/**
@ -335,7 +338,6 @@ class PodcastScanner {
if (podcastEpisodes.length) {
const audioFileMetaTags = podcastEpisodes[0].audioFile.metaTags
const overrideExistingDetails = Database.serverSettings.scannerPreferAudioMetadata
const MetadataMapArray = [
{
@ -376,10 +378,10 @@ class PodcastScanner {
if (value && typeof value === 'string') {
value = value.trim() // Trim whitespace
if (mapping.key === 'genres' && (!podcastMetadata.genres.length || overrideExistingDetails)) {
if (mapping.key === 'genres') {
podcastMetadata.genres = this.parseGenresString(value)
libraryScan.addLog(LogLevel.DEBUG, `Mapping metadata to key ${tagToUse} => ${mapping.key}: ${podcastMetadata.genres.join(', ')}`)
} else if (!podcastMetadata[mapping.key] || overrideExistingDetails) {
} else {
podcastMetadata[mapping.key] = value
libraryScan.addLog(LogLevel.DEBUG, `Mapping metadata to key ${tagToUse} => ${mapping.key}: ${podcastMetadata[mapping.key]}`)
}
@ -628,7 +630,6 @@ class PodcastScanner {
]
const audioFileMetaTags = podcastEpisode.audioFile.metaTags
const overrideExistingDetails = Database.serverSettings.scannerPreferAudioMetadata
MetadataMapArray.forEach((mapping) => {
let value = audioFileMetaTags[mapping.tag]
let tagToUse = mapping.tag
@ -640,7 +641,7 @@ class PodcastScanner {
if (value && typeof value === 'string') {
value = value.trim() // Trim whitespace
if (mapping.key === 'pubDate' && (!podcastEpisode.pubDate || overrideExistingDetails)) {
if (mapping.key === 'pubDate') {
const pubJsDate = new Date(value)
if (pubJsDate && !isNaN(pubJsDate)) {
podcastEpisode.publishedAt = pubJsDate.valueOf()
@ -649,14 +650,14 @@ class PodcastScanner {
} else {
scanLogger.addLog(LogLevel.WARN, `Mapping pubDate with tag ${tagToUse} has invalid date "${value}"`)
}
} else if (mapping.key === 'episodeType' && (!podcastEpisode.episodeType || overrideExistingDetails)) {
} else if (mapping.key === 'episodeType') {
if (['full', 'trailer', 'bonus'].includes(value)) {
podcastEpisode.episodeType = value
scanLogger.addLog(LogLevel.DEBUG, `Mapping metadata to key ${tagToUse} => ${mapping.key}: ${podcastEpisode[mapping.key]}`)
} else {
scanLogger.addLog(LogLevel.WARN, `Mapping episodeType with invalid value "${value}". Must be one of [full, trailer, bonus].`)
}
} else if (!podcastEpisode[mapping.key] || overrideExistingDetails) {
} else {
podcastEpisode[mapping.key] = value
scanLogger.addLog(LogLevel.DEBUG, `Mapping metadata to key ${tagToUse} => ${mapping.key}: ${podcastEpisode[mapping.key]}`)
}

33
server/utils/parsers/parseOverdriveMediaMarkers.js
View file

@ -1,13 +1,6 @@
const xml2js = require('xml2js')
const Logger = require('../../Logger')
// given a list of audio files, extract all of the Overdrive Media Markers metaTags, and return an array of them as XML
function extractOverdriveMediaMarkers(includedAudioFiles) {
Logger.debug('[parseOverdriveMediaMarkers] Extracting overdrive media markers')
var markers = includedAudioFiles.map((af) => af.metaTags.tagOverdriveMediaMarker).filter(af => af) || []
return markers
}
// given the array of Overdrive Media Markers from generateOverdriveMediaMarkers()
// parse and clean them in to something a bit more usable
function cleanOverdriveMediaMarkers(overdriveMediaMarkers) {
@ -29,12 +22,11 @@ function cleanOverdriveMediaMarkers(overdriveMediaMarkers) {
]
*/
var parseString = require('xml2js').parseString // function to convert xml to JSON
var parsedOverdriveMediaMarkers = []
const parsedOverdriveMediaMarkers = []
overdriveMediaMarkers.forEach((item, index) => {
var parsed_result = null
parseString(item, function (err, result) {
let parsed_result = null
// convert xml to JSON
xml2js.parseString(item, function (err, result) {
/*
result.Markers.Marker is the result of parsing the XML for the MediaMarker tags for the MP3 file (Part##.mp3)
it is shaped like this, and needs further cleaning below:
@ -54,7 +46,7 @@ function cleanOverdriveMediaMarkers(overdriveMediaMarkers) {
*/
// The values for Name and Time in results.Markers.Marker are returned as Arrays from parseString and should be strings
if (result && result.Markers && result.Markers.Marker) {
if (result?.Markers?.Marker) {
parsed_result = objectValuesArrayToString(result.Markers.Marker)
}
})
@ -138,22 +130,13 @@ function generateParsedChapters(includedAudioFiles, cleanedOverdriveMediaMarkers
return parsedChapters
}
module.exports.overdriveMediaMarkersExist = (includedAudioFiles) => {
return extractOverdriveMediaMarkers(includedAudioFiles).length > 1
}
module.exports.parseOverdriveMediaMarkersAsChapters = (includedAudioFiles) => {
Logger.info('[parseOverdriveMediaMarkers] Parsing of Overdrive Media Markers started')
var overdriveMediaMarkers = extractOverdriveMediaMarkers(includedAudioFiles)
const overdriveMediaMarkers = includedAudioFiles.map((af) => af.metaTags.tagOverdriveMediaMarker).filter(af => af) || []
if (!overdriveMediaMarkers.length) return null
var cleanedOverdriveMediaMarkers = cleanOverdriveMediaMarkers(overdriveMediaMarkers)
// TODO: generateParsedChapters requires overdrive media markers and included audio files length to be the same
// so if not equal then we must exit
if (cleanedOverdriveMediaMarkers.length !== includedAudioFiles.length) return null
var parsedChapters = generateParsedChapters(includedAudioFiles, cleanedOverdriveMediaMarkers)
return parsedChapters
return generateParsedChapters(includedAudioFiles, cleanedOverdriveMediaMarkers)
}

82
server/utils/scandir.js
View file

@ -2,6 +2,18 @@ const Path = require('path')
const { filePathToPOSIX } = require('./fileUtils')
const globals = require('./globals')
const LibraryFile = require('../objects/files/LibraryFile')
const parseNameString = require('./parsers/parseNameString')
/**
* @typedef LibraryItemFilenameMetadata
* @property {string} title
* @property {string} subtitle Book mediaType only
* @property {string[]} authors Book mediaType only
* @property {string[]} narrators Book mediaType only
* @property {string} seriesName Book mediaType only
* @property {string} seriesSequence Book mediaType only
* @property {string} publishedYear Book mediaType only
*/
function isMediaFile(mediaType, ext, audiobooksOnly = false) {
if (!ext) return false
@ -210,10 +222,15 @@ function buildLibraryFile(libraryItemPath, files) {
}
module.exports.buildLibraryFile = buildLibraryFile
// Input relative filepath, output all details that can be parsed
function getBookDataFromDir(folderPath, relPath, parseSubtitle = false) {
relPath = filePathToPOSIX(relPath)
var splitDir = relPath.split('/')
/**
* Get details parsed from filenames
*
* @param {string} relPath
* @param {boolean} parseSubtitle
* @returns {LibraryItemFilenameMetadata}
*/
function getBookDataFromDir(relPath, parseSubtitle = false) {
const splitDir = relPath.split('/')
var folder = splitDir.pop() // Audio files will always be in the directory named for the title
series = (splitDir.length > 1) ? splitDir.pop() : null // If there are at least 2 more directories, next furthest will be the series
@ -226,17 +243,13 @@ function getBookDataFromDir(folderPath, relPath, parseSubtitle = false) {
var [title, subtitle] = parseSubtitle ? getSubtitle(folder) : [folder, null]
return {
mediaMetadata: {
author,
title,
subtitle,
series,
sequence,
publishedYear,
narrators,
},
relPath: relPath, // relative audiobook path i.e. /Author Name/Book Name/..
path: Path.posix.join(folderPath, relPath) // i.e. /audiobook/Author Name/Book Name/..
title,
subtitle,
authors: parseNameString.parse(author)?.names || [],
narrators: parseNameString.parse(narrators)?.names || [],
seriesName: series,
seriesSequence: sequence,
publishedYear
}
}
module.exports.getBookDataFromDir = getBookDataFromDir
@ -301,28 +314,43 @@ function getSubtitle(folder) {
return [splitTitle.shift(), splitTitle.join(' - ')]
}
function getPodcastDataFromDir(folderPath, relPath) {
relPath = filePathToPOSIX(relPath)
/**
*
* @param {string} relPath
* @returns {LibraryItemFilenameMetadata}
*/
function getPodcastDataFromDir(relPath) {
const splitDir = relPath.split('/')
// Audio files will always be in the directory named for the title
const title = splitDir.pop()
return {
mediaMetadata: {
title
},
relPath: relPath, // relative podcast path i.e. /Podcast Name/..
path: Path.posix.join(folderPath, relPath) // i.e. /podcasts/Podcast Name/..
title
}
}
/**
*
* @param {string} libraryMediaType
* @param {string} folderPath
* @param {string} relPath
* @returns {{ mediaMetadata: LibraryItemFilenameMetadata, relPath: string, path: string}}
*/
function getDataFromMediaDir(libraryMediaType, folderPath, relPath) {
relPath = filePathToPOSIX(relPath)
let fullPath = Path.posix.join(folderPath, relPath)
let mediaMetadata = null
if (libraryMediaType === 'podcast') {
return getPodcastDataFromDir(folderPath, relPath)
} else if (libraryMediaType === 'book') {
return getBookDataFromDir(folderPath, relPath, !!global.ServerSettings.scannerParseSubtitle)
} else {
return getPodcastDataFromDir(folderPath, relPath)
mediaMetadata = getPodcastDataFromDir(relPath)
} else { // book
mediaMetadata = getBookDataFromDir(relPath, !!global.ServerSettings.scannerParseSubtitle)
}
return {
mediaMetadata,
relPath,
path: fullPath
}
}
module.exports.getDataFromMediaDir = getDataFromMediaDir