Missing audiobooks flagged not deleted, fix close progress loop on stream errors, clickable download toast, consolidate duplicate track error log, improved scanner to ignore non-audio files

This commit is contained in:
Mark Cooper 2021-09-17 18:40:30 -05:00
parent 0851a1e71e
commit db01db3a2b
15 changed files with 156 additions and 59 deletions

View file

@ -9,7 +9,6 @@ const { comparePaths, getIno } = require('./utils/index')
const { secondsToTimestamp } = require('./utils/fileUtils')
const { ScanResult } = require('./utils/constants')
class Scanner {
constructor(AUDIOBOOK_PATH, METADATA_PATH, db, emitter) {
this.AudiobookPath = AUDIOBOOK_PATH
@ -71,6 +70,7 @@ class Scanner {
if (existingAudiobook) {
// REMOVE: No valid audio files
// TODO: Label as incomplete, do not actually delete
if (!audiobookData.audioFiles.length) {
Logger.error(`[Scanner] "${existingAudiobook.title}" no valid audio files found - removing audiobook`)
@ -109,8 +109,8 @@ class Scanner {
await audioFileScanner.scanAudioFiles(existingAudiobook, newAudioFiles)
}
// REMOVE: No valid audio tracks
// TODO: Label as incomplete, do not actually delete
if (!existingAudiobook.tracks.length) {
Logger.error(`[Scanner] "${existingAudiobook.title}" has no valid tracks after update - removing audiobook`)
@ -135,6 +135,12 @@ class Scanner {
hasUpdates = true
}
if (existingAudiobook.isMissing) {
existingAudiobook.isMissing = false
hasUpdates = true
Logger.info(`[Scanner] "${existingAudiobook.title}" was missing but now it is found`)
}
if (hasUpdates) {
existingAudiobook.setChapters()
@ -173,23 +179,24 @@ class Scanner {
}
async scan() {
// TODO: This temporary fix from pre-release should be removed soon, including the "fixRelativePath" and "checkUpdateInos"
// TEMP - fix relative file paths
// TEMP - update ino for each audiobook
if (this.audiobooks.length) {
for (let i = 0; i < this.audiobooks.length; i++) {
var ab = this.audiobooks[i]
var shouldUpdate = ab.fixRelativePath(this.AudiobookPath) || !ab.ino
// if (this.audiobooks.length) {
// for (let i = 0; i < this.audiobooks.length; i++) {
// var ab = this.audiobooks[i]
// var shouldUpdate = ab.fixRelativePath(this.AudiobookPath) || !ab.ino
// Update ino if an audio file has the same ino as the audiobook
var shouldUpdateIno = !ab.ino || (ab.audioFiles || []).find(abf => abf.ino === ab.ino)
if (shouldUpdateIno) {
await ab.checkUpdateInos()
}
if (shouldUpdate) {
await this.db.updateAudiobook(ab)
}
}
}
// // Update ino if an audio file has the same ino as the audiobook
// var shouldUpdateIno = !ab.ino || (ab.audioFiles || []).find(abf => abf.ino === ab.ino)
// if (shouldUpdateIno) {
// await ab.checkUpdateInos()
// }
// if (shouldUpdate) {
// await this.db.updateAudiobook(ab)
// }
// }
// }
const scanStart = Date.now()
var audiobookDataFound = await scanRootDir(this.AudiobookPath, this.db.serverSettings)
@ -205,18 +212,21 @@ class Scanner {
var scanResults = {
removed: 0,
updated: 0,
added: 0
added: 0,
missing: 0
}
// Check for removed audiobooks
for (let i = 0; i < this.audiobooks.length; i++) {
var dataFound = audiobookDataFound.find(abd => abd.ino === this.audiobooks[i].ino)
var audiobook = this.audiobooks[i]
var dataFound = audiobookDataFound.find(abd => abd.ino === audiobook.ino)
if (!dataFound) {
Logger.info(`[Scanner] Removing audiobook "${this.audiobooks[i].title}" - no longer in dir`)
var audiobookJSON = this.audiobooks[i].toJSONMinified()
await this.db.removeEntity('audiobook', this.audiobooks[i].id)
scanResults.removed++
this.emitter('audiobook_removed', audiobookJSON)
Logger.info(`[Scanner] Audiobook "${audiobook.title}" is missing`)
audiobook.isMissing = true
audiobook.lastUpdate = Date.now()
scanResults.missing++
await this.db.updateAudiobook(audiobook)
this.emitter('audiobook_updated', audiobook.toJSONMinified())
}
if (this.cancelScan) {
this.cancelScan = false
@ -247,7 +257,7 @@ class Scanner {
}
}
const scanElapsed = Math.floor((Date.now() - scanStart) / 1000)
Logger.info(`[Scanned] Finished | ${scanResults.added} added | ${scanResults.updated} updated | ${scanResults.removed} removed | elapsed: ${secondsToTimestamp(scanElapsed)}`)
Logger.info(`[Scanned] Finished | ${scanResults.added} added | ${scanResults.updated} updated | ${scanResults.removed} removed | ${scanResults.missing} missing | elapsed: ${secondsToTimestamp(scanElapsed)}`)
return scanResults
}

View file

@ -197,7 +197,6 @@ class Server {
res.json({ success: true })
})
// Used in development to set-up streams without authentication
if (process.env.NODE_ENV !== 'production') {
app.use('/test-hls', this.hlsController.router)

View file

@ -28,6 +28,9 @@ class Audiobook {
this.book = null
this.chapters = []
// Audiobook was scanned and not found
this.isMissing = false
if (audiobook) {
this.construct(audiobook)
}
@ -55,6 +58,8 @@ class Audiobook {
if (audiobook.chapters) {
this.chapters = audiobook.chapters.map(c => ({ ...c }))
}
this.isMissing = !!audiobook.isMissing
}
get title() {
@ -127,7 +132,8 @@ class Audiobook {
tracks: this.tracksToJSON(),
audioFiles: (this.audioFiles || []).map(audioFile => audioFile.toJSON()),
otherFiles: (this.otherFiles || []).map(otherFile => otherFile.toJSON()),
chapters: this.chapters || []
chapters: this.chapters || [],
isMissing: !!this.isMissing
}
}
@ -147,7 +153,8 @@ class Audiobook {
hasMissingParts: this.missingParts ? this.missingParts.length : 0,
hasInvalidParts: this.invalidParts ? this.invalidParts.length : 0,
numTracks: this.tracks.length,
chapters: this.chapters || []
chapters: this.chapters || [],
isMissing: !!this.isMissing
}
}
@ -169,7 +176,8 @@ class Audiobook {
tags: this.tags,
book: this.bookToJSON(),
tracks: this.tracksToJSON(),
chapters: this.chapters || []
chapters: this.chapters || [],
isMissing: !!this.isMissing
}
}

View file

@ -288,6 +288,7 @@ class Stream extends EventEmitter {
} else {
Logger.error('Ffmpeg Err', err.message)
}
clearInterval(this.loop)
})
this.ffmpeg.on('end', (stdout, stderr) => {
@ -300,6 +301,7 @@ class Stream extends EventEmitter {
}
this.isTranscodeComplete = true
this.ffmpeg = null
clearInterval(this.loop)
})
this.ffmpeg.run()

View file

@ -89,6 +89,8 @@ async function scanAudioFiles(audiobook, newAudioFiles) {
return
}
var tracks = []
var numDuplicateTracks = 0
var numInvalidTracks = 0
for (let i = 0; i < newAudioFiles.length; i++) {
var audioFile = newAudioFiles[i]
var scanData = await scan(audioFile.fullPath)
@ -118,17 +120,19 @@ async function scanAudioFiles(audiobook, newAudioFiles) {
if (newAudioFiles.length > 1) {
trackNumber = isNumber(trackNumFromMeta) ? trackNumFromMeta : trackNumFromFilename
if (trackNumber === null) {
Logger.error('[AudioFileScanner] Invalid track number for', audioFile.filename)
Logger.debug('[AudioFileScanner] Invalid track number for', audioFile.filename)
audioFile.invalid = true
audioFile.error = 'Failed to get track number'
numInvalidTracks++
continue;
}
}
if (tracks.find(t => t.index === trackNumber)) {
Logger.error('[AudioFileScanner] Duplicate track number for', audioFile.filename)
Logger.debug('[AudioFileScanner] Duplicate track number for', audioFile.filename)
audioFile.invalid = true
audioFile.error = 'Duplicate track number'
numDuplicateTracks++
continue;
}
@ -141,6 +145,13 @@ async function scanAudioFiles(audiobook, newAudioFiles) {
return
}
if (numDuplicateTracks > 0) {
Logger.warn(`[AudioFileScanner] ${numDuplicateTracks} Duplicate tracks for "${audiobook.title}"`)
}
if (numInvalidTracks > 0) {
Logger.error(`[AudioFileScanner] ${numDuplicateTracks} Invalid tracks for "${audiobook.title}"`)
}
tracks.sort((a, b) => a.index - b.index)
audiobook.audioFiles.sort((a, b) => {
var aNum = isNumber(a.trackNumFromMeta) ? a.trackNumFromMeta : isNumber(a.trackNumFromFilename) ? a.trackNumFromFilename : 0

View file

@ -19,11 +19,17 @@ function getPaths(path) {
})
}
function isAudioFile(path) {
if (!path) return false
var ext = Path.extname(path)
if (!ext) return false
return AUDIO_FORMATS.includes(ext.slice(1).toLowerCase())
}
function groupFilesIntoAudiobookPaths(paths) {
// Step 1: Normalize path, Remove leading "/", Filter out files in root dir
var pathsFiltered = paths.map(path => Path.normalize(path.slice(1))).filter(path => Path.parse(path).dir)
// Step 2: Sort by least number of directories
pathsFiltered.sort((a, b) => {
var pathsA = Path.dirname(a).split(Path.sep).length
@ -31,25 +37,55 @@ function groupFilesIntoAudiobookPaths(paths) {
return pathsA - pathsB
})
// Step 3: Group into audiobooks
// Step 2.5: Seperate audio files and other files
var audioFilePaths = []
var otherFilePaths = []
pathsFiltered.forEach(path => {
if (isAudioFile(path)) audioFilePaths.push(path)
else otherFilePaths.push(path)
})
// Step 3: Group audio files in audiobooks
var audiobookGroup = {}
pathsFiltered.forEach((path) => {
audioFilePaths.forEach((path) => {
var dirparts = Path.dirname(path).split(Path.sep)
var numparts = dirparts.length
var _path = ''
// Iterate over directories in path
for (let i = 0; i < numparts; i++) {
var dirpart = dirparts.shift()
_path = Path.join(_path, dirpart)
if (audiobookGroup[_path]) {
if (audiobookGroup[_path]) { // Directory already has files, add file
var relpath = Path.join(dirparts.join(Path.sep), Path.basename(path))
audiobookGroup[_path].push(relpath)
return
} else if (!dirparts.length) {
} else if (!dirparts.length) { // This is the last directory, create group
audiobookGroup[_path] = [Path.basename(path)]
return
}
}
})
// Step 4: Add other files into audiobook groups
otherFilePaths.forEach((path) => {
var dirparts = Path.dirname(path).split(Path.sep)
var numparts = dirparts.length
var _path = ''
// Iterate over directories in path
for (let i = 0; i < numparts; i++) {
var dirpart = dirparts.shift()
_path = Path.join(_path, dirpart)
if (audiobookGroup[_path]) { // Directory is audiobook group
var relpath = Path.join(dirparts.join(Path.sep), Path.basename(path))
audiobookGroup[_path].push(relpath)
return
}
}
})
return audiobookGroup
}
module.exports.groupFilesIntoAudiobookPaths = groupFilesIntoAudiobookPaths