mirror/advplyr.audiobookshelf
1
0
Fork 0
mirror of https://github.com/advplyr/audiobookshelf.git synced 2025-08-04 18:24:46 +02:00
Code Issues Projects Releases Packages Wiki Activity Actions

Update:Remove scanner settings, add library scanner settings tab, add order of precedence

Browse source
This commit is contained in:
advplyr 2023-10-08 17:10:43 -05:00
parent 5ad9f507ba
commit 347b49f564
35 changed files with 764 additions and 809 deletions
Download patch file Download diff file Expand all files Collapse all files

65
server/scanner/AbsMetadataFileScanner.js Normal file
View file

@ -0,0 +1,65 @@
const Path = require('path')
const fsExtra = require('../libs/fsExtra')
const { readTextFile } = require('../utils/fileUtils')
const { LogLevel } = require('../utils/constants')
const abmetadataGenerator = require('../utils/generators/abmetadataGenerator')
class AbsMetadataFileScanner {
constructor() { }
/**
* Check for metadata.json or metadata.abs file and set book metadata
*
* @param {import('./LibraryScan')} libraryScan
* @param {import('./LibraryItemScanData')} libraryItemData
* @param {Object} bookMetadata
* @param {string} [existingLibraryItemId]
*/
async scanBookMetadataFile(libraryScan, libraryItemData, bookMetadata, existingLibraryItemId = null) {
const metadataLibraryFile = libraryItemData.metadataJsonLibraryFile || libraryItemData.metadataAbsLibraryFile
let metadataText = metadataLibraryFile ? await readTextFile(metadataLibraryFile.metadata.path) : null
let metadataFilePath = metadataLibraryFile?.metadata.path
let metadataFileFormat = libraryItemData.metadataJsonLibraryFile ? 'json' : 'abs'
// When metadata file is not stored with library item then check in the /metadata/items folder for it
if (!metadataText && existingLibraryItemId) {
let metadataPath = Path.join(global.MetadataPath, 'items', existingLibraryItemId)
let altFormat = global.ServerSettings.metadataFileFormat === 'json' ? 'abs' : 'json'
// First check the metadata format set in server settings, fallback to the alternate
metadataFilePath = Path.join(metadataPath, `metadata.${global.ServerSettings.metadataFileFormat}`)
metadataFileFormat = global.ServerSettings.metadataFileFormat
if (await fsExtra.pathExists(metadataFilePath)) {
metadataText = await readTextFile(metadataFilePath)
} else if (await fsExtra.pathExists(Path.join(metadataPath, `metadata.${altFormat}`))) {
metadataFilePath = Path.join(metadataPath, `metadata.${altFormat}`)
metadataFileFormat = altFormat
metadataText = await readTextFile(metadataFilePath)
}
}
if (metadataText) {
libraryScan.addLog(LogLevel.INFO, `Found metadata file "${metadataFilePath}" - preferring`)
let abMetadata = null
if (metadataFileFormat === 'json') {
abMetadata = abmetadataGenerator.parseJson(metadataText)
} else {
abMetadata = abmetadataGenerator.parse(metadataText, 'book')
}
if (abMetadata) {
if (abMetadata.tags?.length) {
bookMetadata.tags = abMetadata.tags
}
if (abMetadata.chapters?.length) {
bookMetadata.chapters = abMetadata.chapters
}
for (const key in abMetadata.metadata) {
if (abMetadata.metadata[key] === undefined) continue
bookMetadata[key] = abMetadata.metadata[key]
}
}
}
}
}
module.exports = new AbsMetadataFileScanner()

202
server/scanner/AudioFileScanner.js
View file

@ -1,6 +1,9 @@
const Path = require('path')
const Logger = require('../Logger')
const prober = require('../utils/prober')
const { LogLevel } = require('../utils/constants')
const { parseOverdriveMediaMarkersAsChapters } = require('../utils/parsers/parseOverdriveMediaMarkers')
const parseNameString = require('../utils/parsers/parseNameString')
const LibraryItem = require('../models/LibraryItem')
const AudioFile = require('../objects/files/AudioFile')
@ -205,5 +208,204 @@ class AudioFileScanner {
Logger.debug(`[AudioFileScanner] Running ffprobe for audio file at "${audioFile.metadata.path}"`)
return prober.rawProbe(audioFile.metadata.path)
}
/**
* Set book metadata & chapters from audio file meta tags
*
* @param {string} bookTitle
* @param {import('../models/Book').AudioFileObject} audioFile
* @param {Object} bookMetadata
* @param {LibraryScan} libraryScan
*/
setBookMetadataFromAudioMetaTags(bookTitle, audioFiles, bookMetadata, libraryScan) {
const MetadataMapArray = [
{
tag: 'tagComposer',
key: 'narrators'
},
{
tag: 'tagDescription',
altTag: 'tagComment',
key: 'description'
},
{
tag: 'tagPublisher',
key: 'publisher'
},
{
tag: 'tagDate',
key: 'publishedYear'
},
{
tag: 'tagSubtitle',
key: 'subtitle'
},
{
tag: 'tagAlbum',
altTag: 'tagTitle',
key: 'title',
},
{
tag: 'tagArtist',
altTag: 'tagAlbumArtist',
key: 'authors'
},
{
tag: 'tagGenre',
key: 'genres'
},
{
tag: 'tagSeries',
key: 'series'
},
{
tag: 'tagIsbn',
key: 'isbn'
},
{
tag: 'tagLanguage',
key: 'language'
},
{
tag: 'tagASIN',
key: 'asin'
}
]
const firstScannedFile = audioFiles[0]
const audioFileMetaTags = firstScannedFile.metaTags
MetadataMapArray.forEach((mapping) => {
let value = audioFileMetaTags[mapping.tag]
if (!value && mapping.altTag) {
value = audioFileMetaTags[mapping.altTag]
}
if (value && typeof value === 'string') {
value = value.trim() // Trim whitespace
if (mapping.key === 'narrators') {
bookMetadata.narrators = parseNameString.parse(value)?.names || []
} else if (mapping.key === 'authors') {
bookMetadata.authors = parseNameString.parse(value)?.names || []
} else if (mapping.key === 'genres') {
bookMetadata.genres = this.parseGenresString(value)
} else if (mapping.key === 'series') {
bookMetadata.series = [
{
name: value,
sequence: audioFileMetaTags.tagSeriesPart || null
}
]
} else {
bookMetadata[mapping.key] = value
}
}
})
// Set chapters
const chapters = this.getBookChaptersFromAudioFiles(bookTitle, audioFiles, libraryScan)
if (chapters.length) {
bookMetadata.chapters = chapters
}
}
/**
* @param {string} bookTitle
* @param {AudioFile[]} audioFiles
* @param {LibraryScan} libraryScan
* @returns {import('../models/Book').ChapterObject[]}
*/
getBookChaptersFromAudioFiles(bookTitle, audioFiles, libraryScan) {
// If overdrive media markers are present then use those instead
const overdriveChapters = parseOverdriveMediaMarkersAsChapters(audioFiles)
if (overdriveChapters?.length) {
libraryScan.addLog(LogLevel.DEBUG, 'Overdrive Media Markers and preference found! Using these for chapter definitions')
return overdriveChapters
}
let chapters = []
// If first audio file has embedded chapters then use embedded chapters
if (audioFiles[0].chapters?.length) {
// If all files chapters are the same, then only make chapters for the first file
if (
audioFiles.length === 1 ||
audioFiles.length > 1 &&
audioFiles[0].chapters.length === audioFiles[1].chapters?.length &&
audioFiles[0].chapters.every((c, i) => c.title === audioFiles[1].chapters[i].title)
) {
libraryScan.addLog(LogLevel.DEBUG, `setChapters: Using embedded chapters in first audio file ${audioFiles[0].metadata?.path}`)
chapters = audioFiles[0].chapters.map((c) => ({ ...c }))
} else {
libraryScan.addLog(LogLevel.DEBUG, `setChapters: Using embedded chapters from all audio files ${audioFiles[0].metadata?.path}`)
let currChapterId = 0
let currStartTime = 0
audioFiles.forEach((file) => {
if (file.duration) {
const afChapters = file.chapters?.map((c) => ({
...c,
id: c.id + currChapterId,
start: c.start + currStartTime,
end: c.end + currStartTime,
})) ?? []
chapters = chapters.concat(afChapters)
currChapterId += file.chapters?.length ?? 0
currStartTime += file.duration
}
})
return chapters
}
} else if (audioFiles.length > 1) {
// In some cases the ID3 title tag for each file is the chapter title, the criteria to determine if this will be used
// 1. Every audio file has an ID3 title tag set
// 2. None of the title tags are the same as the book title
// 3. Every ID3 title tag is unique
const metaTagTitlesFound = [...new Set(audioFiles.map(af => af.metaTags?.tagTitle).filter(tagTitle => !!tagTitle && tagTitle !== bookTitle))]
const useMetaTagAsTitle = metaTagTitlesFound.length === audioFiles.length
// Build chapters from audio files
let currChapterId = 0
let currStartTime = 0
audioFiles.forEach((file) => {
if (file.duration) {
let title = file.metadata.filename ? Path.basename(file.metadata.filename, Path.extname(file.metadata.filename)) : `Chapter ${currChapterId}`
if (useMetaTagAsTitle) {
title = file.metaTags.tagTitle
}
chapters.push({
id: currChapterId++,
start: currStartTime,
end: currStartTime + file.duration,
title
})
currStartTime += file.duration
}
})
}
return chapters
}
/**
* Parse a genre string into multiple genres
* @example "Fantasy;Sci-Fi;History" => ["Fantasy", "Sci-Fi", "History"]
*
* @param {string} genreTag
* @returns {string[]}
*/
parseGenresString(genreTag) {
if (!genreTag?.length) return []
const separators = ['/', '//', ';']
for (let i = 0; i < separators.length; i++) {
if (genreTag.includes(separators[i])) {
return genreTag.split(separators[i]).map(genre => genre.trim()).filter(g => !!g)
}
}
return [genreTag]
}
}
module.exports = new AudioFileScanner()

379
server/scanner/BookScanner.js
View file

@ -3,8 +3,6 @@ const Path = require('path')
const sequelize = require('sequelize')
const { LogLevel } = require('../utils/constants')
const { getTitleIgnorePrefix, areEquivalent } = require('../utils/index')
const { parseOpfMetadataXML } = require('../utils/parsers/parseOpfMetadata')
const { parseOverdriveMediaMarkersAsChapters } = require('../utils/parsers/parseOverdriveMediaMarkers')
const abmetadataGenerator = require('../utils/generators/abmetadataGenerator')
const parseNameString = require('../utils/parsers/parseNameString')
const globals = require('../utils/globals')
@ -16,9 +14,12 @@ const CoverManager = require('../managers/CoverManager')
const LibraryFile = require('../objects/files/LibraryFile')
const SocketAuthority = require('../SocketAuthority')
const fsExtra = require("../libs/fsExtra")
const LibraryScan = require("./LibraryScan")
const BookFinder = require('../finders/BookFinder')
const LibraryScan = require("./LibraryScan")
const OpfFileScanner = require('./OpfFileScanner')
const AbsMetadataFileScanner = require('./AbsMetadataFileScanner')
/**
* Metadata for books pulled from files
* @typedef BookMetadataObject
@ -50,7 +51,7 @@ class BookScanner {
* @param {import('./LibraryItemScanData')} libraryItemData
* @param {import('../models/Library').LibrarySettingsObject} librarySettings
* @param {LibraryScan} libraryScan
* @returns {Promise<import('../models/LibraryItem')>}
* @returns {Promise<{libraryItem:import('../models/LibraryItem'), wasUpdated:boolean}>}
*/
async rescanExistingBookLibraryItem(existingLibraryItem, libraryItemData, librarySettings, libraryScan) {
/** @type {import('../models/Book')} */
@ -168,7 +169,7 @@ class BookScanner {
hasMediaChanges = true
}
const bookMetadata = await this.getBookMetadataFromScanData(media.audioFiles, libraryItemData, libraryScan, existingLibraryItem.id)
const bookMetadata = await this.getBookMetadataFromScanData(media.audioFiles, libraryItemData, libraryScan, librarySettings, existingLibraryItem.id)
let authorsUpdated = false
const bookAuthorsRemoved = []
let seriesUpdated = false
@ -360,7 +361,10 @@ class BookScanner {
libraryScan.seriesRemovedFromBooks.push(...bookSeriesRemoved)
libraryScan.authorsRemovedFromBooks.push(...bookAuthorsRemoved)
return existingLibraryItem
return {
libraryItem: existingLibraryItem,
wasUpdated: hasMediaChanges || libraryItemUpdated || seriesUpdated || authorsUpdated
}
}
/**
@ -389,7 +393,7 @@ class BookScanner {
ebookLibraryFile.ebookFormat = ebookLibraryFile.metadata.ext.slice(1).toLowerCase()
}
const bookMetadata = await this.getBookMetadataFromScanData(scannedAudioFiles, libraryItemData, libraryScan)
const bookMetadata = await this.getBookMetadataFromScanData(scannedAudioFiles, libraryItemData, libraryScan, librarySettings)
bookMetadata.explicit = !!bookMetadata.explicit // Ensure boolean
bookMetadata.abridged = !!bookMetadata.abridged // Ensure boolean
@ -548,226 +552,41 @@ class BookScanner {
* @param {import('../models/Book').AudioFileObject[]} audioFiles
* @param {import('./LibraryItemScanData')} libraryItemData
* @param {LibraryScan} libraryScan
* @param {import('../models/Library').LibrarySettingsObject} librarySettings
* @param {string} [existingLibraryItemId]
* @returns {Promise<BookMetadataObject>}
*/
async getBookMetadataFromScanData(audioFiles, libraryItemData, libraryScan, existingLibraryItemId = null) {
async getBookMetadataFromScanData(audioFiles, libraryItemData, libraryScan, librarySettings, existingLibraryItemId = null) {
// First set book metadata from folder/file names
const bookMetadata = {
title: libraryItemData.mediaMetadata.title,
titleIgnorePrefix: getTitleIgnorePrefix(libraryItemData.mediaMetadata.title),
subtitle: libraryItemData.mediaMetadata.subtitle || undefined,
publishedYear: libraryItemData.mediaMetadata.publishedYear || undefined,
title: libraryItemData.mediaMetadata.title, // required
titleIgnorePrefix: undefined,
subtitle: undefined,
publishedYear: undefined,
publisher: undefined,
description: undefined,
isbn: undefined,
asin: undefined,
language: undefined,
narrators: parseNameString.parse(libraryItemData.mediaMetadata.narrators)?.names || [],
narrators: [],
genres: [],
tags: [],
authors: parseNameString.parse(libraryItemData.mediaMetadata.author)?.names || [],
authors: [],
series: [],
chapters: [],
explicit: undefined,
abridged: undefined,
coverPath: undefined
}
if (libraryItemData.mediaMetadata.series) {
bookMetadata.series.push({
name: libraryItemData.mediaMetadata.series,
sequence: libraryItemData.mediaMetadata.sequence || null
})
}
// Fill in or override book metadata from audio file meta tags
if (audioFiles.length) {
const MetadataMapArray = [
{
tag: 'tagComposer',
key: 'narrators'
},
{
tag: 'tagDescription',
altTag: 'tagComment',
key: 'description'
},
{
tag: 'tagPublisher',
key: 'publisher'
},
{
tag: 'tagDate',
key: 'publishedYear'
},
{
tag: 'tagSubtitle',
key: 'subtitle'
},
{
tag: 'tagAlbum',
altTag: 'tagTitle',
key: 'title',
},
{
tag: 'tagArtist',
altTag: 'tagAlbumArtist',
key: 'authors'
},
{
tag: 'tagGenre',
key: 'genres'
},
{
tag: 'tagSeries',
key: 'series'
},
{
tag: 'tagIsbn',
key: 'isbn'
},
{
tag: 'tagLanguage',
key: 'language'
},
{
tag: 'tagASIN',
key: 'asin'
}
]
const overrideExistingDetails = Database.serverSettings.scannerPreferAudioMetadata
const firstScannedFile = audioFiles[0]
const audioFileMetaTags = firstScannedFile.metaTags
MetadataMapArray.forEach((mapping) => {
let value = audioFileMetaTags[mapping.tag]
if (!value && mapping.altTag) {
value = audioFileMetaTags[mapping.altTag]
}
if (value && typeof value === 'string') {
value = value.trim() // Trim whitespace
if (mapping.key === 'narrators' && (!bookMetadata.narrators.length || overrideExistingDetails)) {
bookMetadata.narrators = parseNameString.parse(value)?.names || []
} else if (mapping.key === 'authors' && (!bookMetadata.authors.length || overrideExistingDetails)) {
bookMetadata.authors = parseNameString.parse(value)?.names || []
} else if (mapping.key === 'genres' && (!bookMetadata.genres.length || overrideExistingDetails)) {
bookMetadata.genres = this.parseGenresString(value)
} else if (mapping.key === 'series' && (!bookMetadata.series.length || overrideExistingDetails)) {
bookMetadata.series = [
{
name: value,
sequence: audioFileMetaTags.tagSeriesPart || null
}
]
} else if (!bookMetadata[mapping.key] || overrideExistingDetails) {
bookMetadata[mapping.key] = value
}
}
})
}
// If desc.txt in library item folder then use this for description
if (libraryItemData.descTxtLibraryFile) {
const description = await readTextFile(libraryItemData.descTxtLibraryFile.metadata.path)
if (description.trim()) bookMetadata.description = description.trim()
}
// If reader.txt in library item folder then use this for narrator
if (libraryItemData.readerTxtLibraryFile) {
let narrator = await readTextFile(libraryItemData.readerTxtLibraryFile.metadata.path)
narrator = narrator.split(/\r?\n/)[0]?.trim() || '' // Only use first line
if (narrator) {
bookMetadata.narrators = parseNameString.parse(narrator)?.names || []
}
}
// If opf file is found look for metadata
if (libraryItemData.metadataOpfLibraryFile) {
const xmlText = await readTextFile(libraryItemData.metadataOpfLibraryFile.metadata.path)
const opfMetadata = xmlText ? await parseOpfMetadataXML(xmlText) : null
if (opfMetadata) {
const opfMetadataOverrideDetails = Database.serverSettings.scannerPreferOpfMetadata
for (const key in opfMetadata) {
if (key === 'tags') { // Add tags only if tags are empty
if (opfMetadata.tags.length && (!bookMetadata.tags.length || opfMetadataOverrideDetails)) {
bookMetadata.tags = opfMetadata.tags
}
} else if (key === 'genres') { // Add genres only if genres are empty
if (opfMetadata.genres.length && (!bookMetadata.genres.length || opfMetadataOverrideDetails)) {
bookMetadata.genres = opfMetadata.genres
}
} else if (key === 'authors') {
if (opfMetadata.authors?.length && (!bookMetadata.authors.length || opfMetadataOverrideDetails)) {
bookMetadata.authors = opfMetadata.authors
}
} else if (key === 'narrators') {
if (opfMetadata.narrators?.length && (!bookMetadata.narrators.length || opfMetadataOverrideDetails)) {
bookMetadata.narrators = opfMetadata.narrators
}
} else if (key === 'series') {
if (opfMetadata.series && (!bookMetadata.series.length || opfMetadataOverrideDetails)) {
bookMetadata.series = [{
name: opfMetadata.series,
sequence: opfMetadata.sequence || null
}]
}
} else if (opfMetadata[key] && (!bookMetadata[key] || opfMetadataOverrideDetails)) {
bookMetadata[key] = opfMetadata[key]
}
}
}
}
// If metadata.json or metadata.abs use this for metadata
const metadataLibraryFile = libraryItemData.metadataJsonLibraryFile || libraryItemData.metadataAbsLibraryFile
let metadataText = metadataLibraryFile ? await readTextFile(metadataLibraryFile.metadata.path) : null
let metadataFilePath = metadataLibraryFile?.metadata.path
let metadataFileFormat = libraryItemData.metadataJsonLibraryFile ? 'json' : 'abs'
// When metadata file is not stored with library item then check in the /metadata/items folder for it
if (!metadataText && existingLibraryItemId) {
let metadataPath = Path.join(global.MetadataPath, 'items', existingLibraryItemId)
let altFormat = global.ServerSettings.metadataFileFormat === 'json' ? 'abs' : 'json'
// First check the metadata format set in server settings, fallback to the alternate
metadataFilePath = Path.join(metadataPath, `metadata.${global.ServerSettings.metadataFileFormat}`)
metadataFileFormat = global.ServerSettings.metadataFileFormat
if (await fsExtra.pathExists(metadataFilePath)) {
metadataText = await readTextFile(metadataFilePath)
} else if (await fsExtra.pathExists(Path.join(metadataPath, `metadata.${altFormat}`))) {
metadataFilePath = Path.join(metadataPath, `metadata.${altFormat}`)
metadataFileFormat = altFormat
metadataText = await readTextFile(metadataFilePath)
}
}
if (metadataText) {
libraryScan.addLog(LogLevel.INFO, `Found metadata file "${metadataFilePath}" - preferring`)
let abMetadata = null
if (metadataFileFormat === 'json') {
abMetadata = abmetadataGenerator.parseJson(metadataText)
const bookMetadataSourceHandler = new BookScanner.BookMetadataSourceHandler(bookMetadata, audioFiles, libraryItemData, libraryScan, existingLibraryItemId)
for (const metadataSource of librarySettings.metadataPrecedence) {
if (bookMetadataSourceHandler[metadataSource]) {
libraryScan.addLog(LogLevel.DEBUG, `Getting metadata from source "${metadataSource}"`)
await bookMetadataSourceHandler[metadataSource]()
} else {
abMetadata = abmetadataGenerator.parse(metadataText, 'book')
libraryScan.addLog(LogLevel.ERROR, `Invalid metadata source "${metadataSource}"`)
}
if (abMetadata) {
if (abMetadata.tags?.length) {
bookMetadata.tags = abMetadata.tags
}
if (abMetadata.chapters?.length) {
bookMetadata.chapters = abMetadata.chapters
}
for (const key in abMetadata.metadata) {
if (abMetadata.metadata[key] === undefined) continue
bookMetadata[key] = abMetadata.metadata[key]
}
}
}
// Set chapters from audio files if not already set
if (!bookMetadata.chapters.length) {
bookMetadata.chapters = this.getChaptersFromAudioFiles(bookMetadata.title, audioFiles, libraryScan)
}
// Set cover from library file if one is found otherwise check audiofile
@ -781,102 +600,76 @@ class BookScanner {
return bookMetadata
}
/**
* Parse a genre string into multiple genres
* @example "Fantasy;Sci-Fi;History" => ["Fantasy", "Sci-Fi", "History"]
* @param {string} genreTag
* @returns {string[]}
*/
parseGenresString(genreTag) {
if (!genreTag?.length) return []
const separators = ['/', '//', ';']
for (let i = 0; i < separators.length; i++) {
if (genreTag.includes(separators[i])) {
return genreTag.split(separators[i]).map(genre => genre.trim()).filter(g => !!g)
}
}
return [genreTag]
}
/**
* @param {string} bookTitle
* @param {AudioFile[]} audioFiles
* @param {LibraryScan} libraryScan
* @returns {import('../models/Book').ChapterObject[]}
*/
getChaptersFromAudioFiles(bookTitle, audioFiles, libraryScan) {
if (!audioFiles.length) return []
// If overdrive media markers are present and preferred, use those instead
if (Database.serverSettings.scannerPreferOverdriveMediaMarker) {
const overdriveChapters = parseOverdriveMediaMarkersAsChapters(audioFiles)
if (overdriveChapters) {
libraryScan.addLog(LogLevel.DEBUG, 'Overdrive Media Markers and preference found! Using these for chapter definitions')
return overdriveChapters
}
static BookMetadataSourceHandler = class {
/**
*
* @param {Object} bookMetadata
* @param {import('../models/Book').AudioFileObject[]} audioFiles
* @param {import('./LibraryItemScanData')} libraryItemData
* @param {LibraryScan} libraryScan
* @param {string} existingLibraryItemId
*/
constructor(bookMetadata, audioFiles, libraryItemData, libraryScan, existingLibraryItemId) {
this.bookMetadata = bookMetadata
this.audioFiles = audioFiles
this.libraryItemData = libraryItemData
this.libraryScan = libraryScan
this.existingLibraryItemId = existingLibraryItemId
}
let chapters = []
/**
* Metadata parsed from folder names/structure
*/
folderStructure() {
this.libraryItemData.setBookMetadataFromFilenames(this.bookMetadata)
}
// If first audio file has embedded chapters then use embedded chapters
if (audioFiles[0].chapters?.length) {
// If all files chapters are the same, then only make chapters for the first file
if (
audioFiles.length === 1 ||
audioFiles.length > 1 &&
audioFiles[0].chapters.length === audioFiles[1].chapters?.length &&
audioFiles[0].chapters.every((c, i) => c.title === audioFiles[1].chapters[i].title)
) {
libraryScan.addLog(LogLevel.DEBUG, `setChapters: Using embedded chapters in first audio file ${audioFiles[0].metadata?.path}`)
chapters = audioFiles[0].chapters.map((c) => ({ ...c }))
} else {
libraryScan.addLog(LogLevel.DEBUG, `setChapters: Using embedded chapters from all audio files ${audioFiles[0].metadata?.path}`)
let currChapterId = 0
let currStartTime = 0
/**
* Metadata from audio file meta tags
*/
audioMetatags() {
if (!this.audioFiles.length) return
// Modifies bookMetadata with metadata mapped from audio file meta tags
const bookTitle = this.bookMetadata.title || this.libraryItemData.mediaMetadata.title
AudioFileScanner.setBookMetadataFromAudioMetaTags(bookTitle, this.audioFiles, this.bookMetadata, this.libraryScan)
}
audioFiles.forEach((file) => {
if (file.duration) {
const afChapters = file.chapters?.map((c) => ({
...c,
id: c.id + currChapterId,
start: c.start + currStartTime,
end: c.end + currStartTime,
})) ?? []
chapters = chapters.concat(afChapters)
currChapterId += file.chapters?.length ?? 0
currStartTime += file.duration
}
})
return chapters
/**
* Description from desc.txt and narrator from reader.txt
*/
async txtFiles() {
// If desc.txt in library item folder then use this for description
if (this.libraryItemData.descTxtLibraryFile) {
const description = await readTextFile(this.libraryItemData.descTxtLibraryFile.metadata.path)
if (description.trim()) this.bookMetadata.description = description.trim()
}
} else if (audioFiles.length > 1) {
const preferAudioMetadata = !!Database.serverSettings.scannerPreferAudioMetadata
// Build chapters from audio files
let currChapterId = 0
let currStartTime = 0
audioFiles.forEach((file) => {
if (file.duration) {
let title = file.metadata.filename ? Path.basename(file.metadata.filename, Path.extname(file.metadata.filename)) : `Chapter ${currChapterId}`
// When prefer audio metadata server setting is set then use ID3 title tag as long as it is not the same as the book title
if (preferAudioMetadata && file.metaTags?.tagTitle && file.metaTags?.tagTitle !== bookTitle) {
title = file.metaTags.tagTitle
}
chapters.push({
id: currChapterId++,
start: currStartTime,
end: currStartTime + file.duration,
title
})
currStartTime += file.duration
// If reader.txt in library item folder then use this for narrator
if (this.libraryItemData.readerTxtLibraryFile) {
let narrator = await readTextFile(this.libraryItemData.readerTxtLibraryFile.metadata.path)
narrator = narrator.split(/\r?\n/)[0]?.trim() || '' // Only use first line
if (narrator) {
this.bookMetadata.narrators = parseNameString.parse(narrator)?.names || []
}
})
}
}
/**
* Metadata from opf file
*/
async opfFile() {
if (!this.libraryItemData.metadataOpfLibraryFile) return
await OpfFileScanner.scanBookOpfFile(this.libraryItemData.metadataOpfLibraryFile, this.bookMetadata)
}
/**
* Metadata from metadata.json or metadata.abs
*/
async absMetadata() {
// If metadata.json or metadata.abs use this for metadata
await AbsMetadataFileScanner.scanBookMetadataFile(this.libraryScan, this.libraryItemData, this.bookMetadata, this.existingLibraryItemId)
}
return chapters
}
/**

36
server/scanner/LibraryItemScanData.js
View file

@ -25,7 +25,7 @@ class LibraryItemScanData {
this.relPath = data.relPath
/** @type {boolean} */
this.isFile = data.isFile
/** @type {{author:string, title:string, subtitle:string, series:string, sequence:string, publishedYear:string, narrators:string}} */
/** @type {import('../utils/scandir').LibraryItemFilenameMetadata} */
this.mediaMetadata = data.mediaMetadata
/** @type {import('../objects/files/LibraryFile')[]} */
this.libraryFiles = data.libraryFiles
@ -233,10 +233,9 @@ class LibraryItemScanData {
}
await existingLibraryItem.save()
return true
} else {
libraryScan.addLog(LogLevel.DEBUG, `Library item "${existingLibraryItem.relPath}" is up-to-date`)
return false
}
return false
}
/**
@ -303,5 +302,34 @@ class LibraryItemScanData {
return !this.ebookLibraryFiles.some(lf => lf.ino === ebookFile.ino)
}
/**
* Set data parsed from filenames
*
* @param {Object} bookMetadata
*/
setBookMetadataFromFilenames(bookMetadata) {
const keysToMap = ['title', 'subtitle', 'publishedYear']
for (const key in this.mediaMetadata) {
if (keysToMap.includes(key) && this.mediaMetadata[key]) {
bookMetadata[key] = this.mediaMetadata[key]
}
}
if (this.mediaMetadata.authors?.length) {
bookMetadata.authors = this.mediaMetadata.authors
}
if (this.mediaMetadata.narrators?.length) {
bookMetadata.narrators = this.mediaMetadata.narrators
}
if (this.mediaMetadata.seriesName) {
bookMetadata.series = [
{
name: this.mediaMetadata.seriesName,
sequence: this.mediaMetadata.seriesSequence || null
}
]
}
}
}
module.exports = LibraryItemScanData

13
server/scanner/LibraryItemScanner.js
View file

@ -58,7 +58,7 @@ class LibraryItemScanner {
if (await libraryItemScanData.checkLibraryItemData(libraryItem, scanLogger)) {
if (libraryItemScanData.hasLibraryFileChanges || libraryItemScanData.hasPathChange) {
const expandedLibraryItem = await this.rescanLibraryItem(libraryItem, libraryItemScanData, library.settings, scanLogger)
const { libraryItem: expandedLibraryItem } = await this.rescanLibraryItemMedia(libraryItem, libraryItemScanData, library.settings, scanLogger)
const oldLibraryItem = Database.libraryItemModel.getOldLibraryItem(expandedLibraryItem)
SocketAuthority.emitter('item_updated', oldLibraryItem.toJSONExpanded())
@ -71,6 +71,7 @@ class LibraryItemScanner {
return ScanResult.UPDATED
}
libraryScan.addLog(LogLevel.DEBUG, `Library item "${libraryItem.relPath}" is up-to-date`)
return ScanResult.UPTODATE
}
@ -156,16 +157,14 @@ class LibraryItemScanner {
* @param {LibraryItemScanData} libraryItemData
* @param {import('../models/Library').LibrarySettingsObject} librarySettings
* @param {LibraryScan} libraryScan
* @returns {Promise<LibraryItem>}
* @returns {Promise<{libraryItem:LibraryItem, wasUpdated:boolean}>}
*/
async rescanLibraryItem(existingLibraryItem, libraryItemData, librarySettings, libraryScan) {
let newLibraryItem = null
rescanLibraryItemMedia(existingLibraryItem, libraryItemData, librarySettings, libraryScan) {
if (existingLibraryItem.mediaType === 'book') {
newLibraryItem = await BookScanner.rescanExistingBookLibraryItem(existingLibraryItem, libraryItemData, librarySettings, libraryScan)
return BookScanner.rescanExistingBookLibraryItem(existingLibraryItem, libraryItemData, librarySettings, libraryScan)
} else {
newLibraryItem = await PodcastScanner.rescanExistingPodcastLibraryItem(existingLibraryItem, libraryItemData, librarySettings, libraryScan)
return PodcastScanner.rescanExistingPodcastLibraryItem(existingLibraryItem, libraryItemData, librarySettings, libraryScan)
}
return newLibraryItem
}
/**

33
server/scanner/LibraryScanner.js
View file

@ -44,9 +44,9 @@ class LibraryScanner {
/**
*
* @param {import('../objects/Library')} library
* @param {*} options
* @param {boolean} [forceRescan]
*/
async scan(library, options = {}) {
async scan(library, forceRescan = false) {
if (this.isLibraryScanning(library.id)) {
Logger.error(`[Scanner] Already scanning ${library.id}`)
return
@ -64,9 +64,9 @@ class LibraryScanner {
SocketAuthority.emitter('scan_start', libraryScan.getScanEmitData)
Logger.info(`[Scanner] Starting library scan ${libraryScan.id} for ${libraryScan.libraryName}`)
Logger.info(`[Scanner] Starting${forceRescan ? ' (forced)' : ''} library scan ${libraryScan.id} for ${libraryScan.libraryName}`)
const canceled = await this.scanLibrary(libraryScan)
const canceled = await this.scanLibrary(libraryScan, forceRescan)
if (canceled) {
Logger.info(`[Scanner] Library scan canceled for "${libraryScan.libraryName}"`)
@ -95,9 +95,10 @@ class LibraryScanner {
/**
*
* @param {import('./LibraryScan')} libraryScan
* @returns {boolean} true if scan canceled
* @param {boolean} forceRescan
* @returns {Promise<boolean>} true if scan canceled
*/
async scanLibrary(libraryScan) {
async scanLibrary(libraryScan, forceRescan) {
// Make sure library filter data is set
// this is used to check for existing authors & series
await libraryFilters.getFilterData(libraryScan.library.mediaType, libraryScan.libraryId)
@ -155,17 +156,25 @@ class LibraryScanner {
}
} else {
libraryItemDataFound = libraryItemDataFound.filter(lidf => lidf !== libraryItemData)
if (await libraryItemData.checkLibraryItemData(existingLibraryItem, libraryScan)) {
libraryScan.resultsUpdated++
if (libraryItemData.hasLibraryFileChanges || libraryItemData.hasPathChange) {
const libraryItem = await LibraryItemScanner.rescanLibraryItem(existingLibraryItem, libraryItemData, libraryScan.library.settings, libraryScan)
const oldLibraryItem = Database.libraryItemModel.getOldLibraryItem(libraryItem)
oldLibraryItemsUpdated.push(oldLibraryItem)
let libraryItemDataUpdated = await libraryItemData.checkLibraryItemData(existingLibraryItem, libraryScan)
if (libraryItemDataUpdated || forceRescan) {
if (forceRescan || libraryItemData.hasLibraryFileChanges || libraryItemData.hasPathChange) {
const { libraryItem, wasUpdated } = await LibraryItemScanner.rescanLibraryItemMedia(existingLibraryItem, libraryItemData, libraryScan.library.settings, libraryScan)
if (!forceRescan || wasUpdated) {
libraryScan.resultsUpdated++
const oldLibraryItem = Database.libraryItemModel.getOldLibraryItem(libraryItem)
oldLibraryItemsUpdated.push(oldLibraryItem)
} else {
libraryScan.addLog(LogLevel.DEBUG, `Library item "${existingLibraryItem.relPath}" is up-to-date`)
}
} else {
libraryScan.resultsUpdated++
// TODO: Temporary while using old model to socket emit
const oldLibraryItem = await Database.libraryItemModel.getOldById(existingLibraryItem.id)
oldLibraryItemsUpdated.push(oldLibraryItem)
}
} else {
libraryScan.addLog(LogLevel.DEBUG, `Library item "${existingLibraryItem.relPath}" is up-to-date`)
}
}

48
server/scanner/OpfFileScanner.js Normal file
View file

@ -0,0 +1,48 @@
const { parseOpfMetadataXML } = require('../utils/parsers/parseOpfMetadata')
const { readTextFile } = require('../utils/fileUtils')
class OpfFileScanner {
constructor() { }
/**
* Parse metadata from .opf file found in library scan and update bookMetadata
*
* @param {import('../models/LibraryItem').LibraryFileObject} opfLibraryFileObj
* @param {Object} bookMetadata
*/
async scanBookOpfFile(opfLibraryFileObj, bookMetadata) {
const xmlText = await readTextFile(opfLibraryFileObj.metadata.path)
const opfMetadata = xmlText ? await parseOpfMetadataXML(xmlText) : null
if (opfMetadata) {
for (const key in opfMetadata) {
if (key === 'tags') { // Add tags only if tags are empty
if (opfMetadata.tags.length) {
bookMetadata.tags = opfMetadata.tags
}
} else if (key === 'genres') { // Add genres only if genres are empty
if (opfMetadata.genres.length) {
bookMetadata.genres = opfMetadata.genres
}
} else if (key === 'authors') {
if (opfMetadata.authors?.length) {
bookMetadata.authors = opfMetadata.authors
}
} else if (key === 'narrators') {
if (opfMetadata.narrators?.length) {
bookMetadata.narrators = opfMetadata.narrators
}
} else if (key === 'series') {
if (opfMetadata.series) {
bookMetadata.series = [{
name: opfMetadata.series,
sequence: opfMetadata.sequence || null
}]
}
} else if (opfMetadata[key]) {
bookMetadata[key] = opfMetadata[key]
}
}
}
}
}
module.exports = new OpfFileScanner()

19
server/scanner/PodcastScanner.js
View file

@ -39,7 +39,7 @@ class PodcastScanner {
* @param {import('./LibraryItemScanData')} libraryItemData
* @param {import('../models/Library').LibrarySettingsObject} librarySettings
* @param {import('./LibraryScan')} libraryScan
* @returns {Promise<import('../models/LibraryItem')>}
* @returns {Promise<{libraryItem:import('../models/LibraryItem'), wasUpdated:boolean}>}
*/
async rescanExistingPodcastLibraryItem(existingLibraryItem, libraryItemData, librarySettings, libraryScan) {
/** @type {import('../models/Podcast')} */
@ -201,7 +201,10 @@ class PodcastScanner {
await existingLibraryItem.save()
}
return existingLibraryItem
return {
libraryItem: existingLibraryItem,
wasUpdated: hasMediaChanges || libraryItemUpdated
}
}
/**
@ -335,7 +338,6 @@ class PodcastScanner {
if (podcastEpisodes.length) {
const audioFileMetaTags = podcastEpisodes[0].audioFile.metaTags
const overrideExistingDetails = Database.serverSettings.scannerPreferAudioMetadata
const MetadataMapArray = [
{
@ -376,10 +378,10 @@ class PodcastScanner {
if (value && typeof value === 'string') {
value = value.trim() // Trim whitespace
if (mapping.key === 'genres' && (!podcastMetadata.genres.length || overrideExistingDetails)) {
if (mapping.key === 'genres') {
podcastMetadata.genres = this.parseGenresString(value)
libraryScan.addLog(LogLevel.DEBUG, `Mapping metadata to key ${tagToUse} => ${mapping.key}: ${podcastMetadata.genres.join(', ')}`)
} else if (!podcastMetadata[mapping.key] || overrideExistingDetails) {
} else {
podcastMetadata[mapping.key] = value
libraryScan.addLog(LogLevel.DEBUG, `Mapping metadata to key ${tagToUse} => ${mapping.key}: ${podcastMetadata[mapping.key]}`)
}
@ -628,7 +630,6 @@ class PodcastScanner {
]
const audioFileMetaTags = podcastEpisode.audioFile.metaTags
const overrideExistingDetails = Database.serverSettings.scannerPreferAudioMetadata
MetadataMapArray.forEach((mapping) => {
let value = audioFileMetaTags[mapping.tag]
let tagToUse = mapping.tag
@ -640,7 +641,7 @@ class PodcastScanner {
if (value && typeof value === 'string') {
value = value.trim() // Trim whitespace
if (mapping.key === 'pubDate' && (!podcastEpisode.pubDate || overrideExistingDetails)) {
if (mapping.key === 'pubDate') {
const pubJsDate = new Date(value)
if (pubJsDate && !isNaN(pubJsDate)) {
podcastEpisode.publishedAt = pubJsDate.valueOf()
@ -649,14 +650,14 @@ class PodcastScanner {
} else {
scanLogger.addLog(LogLevel.WARN, `Mapping pubDate with tag ${tagToUse} has invalid date "${value}"`)
}
} else if (mapping.key === 'episodeType' && (!podcastEpisode.episodeType || overrideExistingDetails)) {
} else if (mapping.key === 'episodeType') {
if (['full', 'trailer', 'bonus'].includes(value)) {
podcastEpisode.episodeType = value
scanLogger.addLog(LogLevel.DEBUG, `Mapping metadata to key ${tagToUse} => ${mapping.key}: ${podcastEpisode[mapping.key]}`)
} else {
scanLogger.addLog(LogLevel.WARN, `Mapping episodeType with invalid value "${value}". Must be one of [full, trailer, bonus].`)
}
} else if (!podcastEpisode[mapping.key] || overrideExistingDetails) {
} else {
podcastEpisode[mapping.key] = value
scanLogger.addLog(LogLevel.DEBUG, `Mapping metadata to key ${tagToUse} => ${mapping.key}: ${podcastEpisode[mapping.key]}`)
}