Podcast episode downloader, update podcast data model

This commit is contained in:
advplyr 2022-03-21 19:24:38 -05:00
parent 28d76d21f1
commit 920ca683b9
19 changed files with 407 additions and 49 deletions

View file

@ -61,7 +61,7 @@ class Server {
this.downloadManager = new DownloadManager(this.db)
this.playbackSessionManager = new PlaybackSessionManager(this.db, this.emitter.bind(this), this.clientEmitter.bind(this))
this.coverManager = new CoverManager(this.db, this.cacheManager)
this.podcastManager = new PodcastManager(this.db)
this.podcastManager = new PodcastManager(this.db, this.watcher, this.emitter.bind(this))
this.scanner = new Scanner(this.db, this.coverManager, this.emitter.bind(this))

View file

@ -14,6 +14,7 @@ class FolderWatcher extends EventEmitter {
this.pendingDelay = 4000
this.pendingTimeout = null
this.ignoreDirs = []
this.disabled = false
}
@ -115,11 +116,17 @@ class FolderWatcher extends EventEmitter {
}
onNewFile(libraryId, path) {
if (this.checkShouldIgnorePath(path)) {
return
}
Logger.debug('[Watcher] File Added', path)
this.addFileUpdate(libraryId, path, 'added')
}
onFileRemoved(libraryId, path) {
if (this.checkShouldIgnorePath(path)) {
return
}
Logger.debug('[Watcher] File Removed', path)
this.addFileUpdate(libraryId, path, 'deleted')
}
@ -129,6 +136,9 @@ class FolderWatcher extends EventEmitter {
}
onRename(libraryId, pathFrom, pathTo) {
if (this.checkShouldIgnorePath(pathTo)) {
return
}
Logger.debug(`[Watcher] Rename ${pathFrom} => ${pathTo}`)
this.addFileUpdate(libraryId, pathFrom, 'renamed')
this.addFileUpdate(libraryId, pathTo, 'renamed')
@ -185,5 +195,31 @@ class FolderWatcher extends EventEmitter {
this.pendingFileUpdates = []
}, this.pendingDelay)
}
checkShouldIgnorePath(path) {
return !!this.ignoreDirs.find(dirpath => {
return path.replace(/\\/g, '/').startsWith(dirpath)
})
}
cleanDirPath(path) {
var path = path.replace(/\\/g, '/')
if (path.endsWith('/')) path = path.slice(0, -1)
return path
}
addIgnoreDir(path) {
path = this.cleanDirPath(path)
if (this.ignoreDirs.includes(path)) return
Logger.debug(`[Watcher] Ignoring directory "${path}"`)
this.ignoreDirs.push(path)
}
removeIgnoreDir(path) {
path = this.cleanDirPath(path)
if (!this.ignoreDirs.includes(path)) return
Logger.debug(`[Watcher] No longer ignoring directory "${path}"`)
this.ignoreDirs = this.ignoreDirs.filter(p => p !== path)
}
}
module.exports = FolderWatcher

View file

@ -3,6 +3,8 @@ const fs = require('fs-extra')
const Logger = require('../Logger')
const { parsePodcastRssFeedXml } = require('../utils/podcastUtils')
const LibraryItem = require('../objects/LibraryItem')
const { getFileTimestampsWithIno } = require('../utils/fileUtils')
const filePerms = require('../utils/filePerms')
class PodcastController {
@ -13,28 +15,72 @@ class PodcastController {
}
const payload = req.body
if (await fs.pathExists(payload.path)) {
Logger.error(`[PodcastController] Attempt to create podcast when folder path already exists "${payload.path}"`)
const library = this.db.libraries.find(lib => lib.id === payload.libraryId)
if (!library) {
Logger.error(`[PodcastController] Create: Library not found "${payload.libraryId}"`)
return res.status(400).send('Library not found')
}
const folder = library.folders.find(fold => fold.id === payload.folderId)
if (!folder) {
Logger.error(`[PodcastController] Create: Folder not found "${payload.folderId}"`)
return res.status(400).send('Folder not found')
}
var podcastPath = payload.path.replace(/\\/g, '/')
if (await fs.pathExists(podcastPath)) {
Logger.error(`[PodcastController] Attempt to create podcast when folder path already exists "${podcastPath}"`)
return res.status(400).send('Path already exists')
}
var success = await fs.ensureDir(payload.path).then(() => true).catch((error) => {
Logger.error(`[PodcastController] Failed to ensure podcast dir "${payload.path}"`, error)
var success = await fs.ensureDir(podcastPath).then(() => true).catch((error) => {
Logger.error(`[PodcastController] Failed to ensure podcast dir "${podcastPath}"`, error)
return false
})
if (!success) return res.status(400).send('Invalid podcast path')
await filePerms.setDefault(podcastPath)
if (payload.mediaMetadata.imageUrl) {
// TODO: Download image
var libraryItemFolderStats = await getFileTimestampsWithIno(podcastPath)
var relPath = payload.path.replace(folder.fullPath, '')
if (relPath.startsWith('/')) relPath = relPath.slice(1)
const libraryItemPayload = {
path: podcastPath,
relPath,
folderId: payload.folderId,
libraryId: payload.libraryId,
ino: libraryItemFolderStats.ino,
mtimeMs: libraryItemFolderStats.mtimeMs || 0,
ctimeMs: libraryItemFolderStats.ctimeMs || 0,
birthtimeMs: libraryItemFolderStats.birthtimeMs || 0,
media: payload.media
}
var libraryItem = new LibraryItem()
libraryItem.setData('podcast', payload)
libraryItem.setData('podcast', libraryItemPayload)
// Download and save cover image
if (payload.media.metadata.imageUrl) {
var coverResponse = await this.coverManager.downloadCoverFromUrl(libraryItem, payload.media.metadata.imageUrl)
if (coverResponse) {
if (coverResponse.error) {
Logger.error(`[PodcastController] Download cover error from "${payload.media.metadata.imageUrl}": ${coverResponse.error}`)
} else if (coverResponse.cover) {
libraryItem.media.coverPath = coverResponse.cover
}
}
}
await this.db.insertLibraryItem(libraryItem)
this.emitter('item_added', libraryItem.toJSONExpanded())
res.json(libraryItem.toJSONExpanded())
if (payload.episodesToDownload && payload.episodesToDownload.length) {
Logger.info(`[PodcastController] Podcast created now starting ${payload.episodesToDownload.length} episode downloads`)
this.podcastManager.downloadPodcastEpisodes(libraryItem, payload.episodesToDownload)
}
}
getPodcastFeed(req, res) {

View file

@ -1,12 +1,101 @@
const fs = require('fs-extra')
const Logger = require('../Logger')
const { downloadFile } = require('../utils/fileUtils')
const prober = require('../utils/prober')
const LibraryFile = require('../objects/files/LibraryFile')
const PodcastEpisodeDownload = require('../objects/PodcastEpisodeDownload')
const PodcastEpisode = require('../objects/entities/PodcastEpisode')
const AudioFile = require('../objects/files/AudioFile')
class PodcastManager {
constructor(db) {
constructor(db, watcher, emitter) {
this.db = db
this.watcher = watcher
this.emitter = emitter
this.downloadQueue = []
this.currentDownload = null
}
async downloadPodcasts(podcasts, targetDir) {
async downloadPodcastEpisodes(libraryItem, episodesToDownload) {
var index = 1
episodesToDownload.forEach((ep) => {
var newPe = new PodcastEpisode()
newPe.setData(ep, index++)
var newPeDl = new PodcastEpisodeDownload()
newPeDl.setData(newPe, libraryItem)
this.startPodcastEpisodeDownload(newPeDl)
})
}
async startPodcastEpisodeDownload(podcastEpisodeDownload) {
if (this.currentDownload) {
this.downloadQueue.push(podcastEpisodeDownload)
return
}
this.currentDownload = podcastEpisodeDownload
// Ignores all added files to this dir
this.watcher.addIgnoreDir(this.currentDownload.libraryItem.path)
var success = await downloadFile(this.currentDownload.url, this.currentDownload.targetPath).then(() => true).catch((error) => {
Logger.error(`[PodcastManager] Podcast Episode download failed`, error)
return false
})
if (success) {
success = await this.scanAddPodcastEpisodeAudioFile()
if (!success) {
await fs.remove(this.currentDownload.targetPath)
} else {
Logger.info(`[PodcastManager] Successfully downloaded podcast episode "${this.currentDownload.podcastEpisode.title}"`)
}
}
this.watcher.removeIgnoreDir(this.currentDownload.libraryItem.path)
this.currentDownload = null
if (this.downloadQueue.length) {
this.startPodcastEpisodeDownload(this.downloadQueue.shift())
}
}
async scanAddPodcastEpisodeAudioFile() {
var libraryFile = await this.getLibraryFile(this.currentDownload.targetPath, this.currentDownload.targetRelPath)
var audioFile = await this.probeAudioFile(libraryFile)
if (!audioFile) {
return false
}
var libraryItem = this.db.libraryItems.find(li => li.id === this.currentDownload.libraryItem.id)
if (!libraryItem) {
Logger.error(`[PodcastManager] Podcast Episode finished but library item was not found ${this.currentDownload.libraryItem.id}`)
return false
}
var podcastEpisode = this.currentDownload.podcastEpisode
podcastEpisode.audioFile = audioFile
libraryItem.media.addPodcastEpisode(podcastEpisode)
libraryItem.updatedAt = Date.now()
await this.db.updateLibraryItem(libraryItem)
this.emitter('item_updated', libraryItem.toJSONExpanded())
return true
}
async getLibraryFile(path, relPath) {
var newLibFile = new LibraryFile()
await newLibFile.setDataFromPath(path, relPath)
return newLibFile
}
async probeAudioFile(libraryFile) {
var path = libraryFile.metadata.path
var audioProbeData = await prober.probe(path)
if (audioProbeData.error) {
Logger.error(`[PodcastManager] Podcast Episode downloaded but failed to probe "${path}"`, audioProbeData.error)
return false
}
var newAudioFile = new AudioFile()
newAudioFile.setDataFromProbe(libraryFile, audioProbeData)
return newAudioFile
}
}
module.exports = PodcastManager

View file

@ -166,8 +166,10 @@ class LibraryItem {
} else {
this.mediaType = 'book'
this.media = new Book()
}
for (const key in payload) {
if (key === 'libraryFiles') {
this.libraryFiles = payload.libraryFiles.map(lf => lf.clone())
@ -175,13 +177,13 @@ class LibraryItem {
// Use first image library file as cover
var firstImageFile = this.libraryFiles.find(lf => lf.fileType === 'image')
if (firstImageFile) this.media.coverPath = firstImageFile.metadata.path
} else if (this[key] !== undefined) {
} else if (this[key] !== undefined && key !== 'media') {
this[key] = payload[key]
}
}
if (payload.mediaMetadata) {
this.media.setData(payload.mediaMetadata)
if (payload.media) {
this.media.setData(payload.media)
}
this.addedAt = Date.now()

View file

@ -0,0 +1,38 @@
const Path = require('path')
const { getId } = require('../utils/index')
const { sanitizeFilename } = require('../utils/fileUtils')
class PodcastEpisodeDownload {
constructor() {
this.id = null
this.podcastEpisode = null
this.url = null
this.libraryItem = null
this.isDownloading = false
this.startedAt = null
this.createdAt = null
this.finishedAt = null
}
get targetFilename() {
return sanitizeFilename(`${this.podcastEpisode.bestFilename}.mp3`)
}
get targetPath() {
return Path.join(this.libraryItem.path, this.targetFilename)
}
get targetRelPath() {
return Path.join(this.libraryItem.relPath, this.targetFilename)
}
setData(podcastEpisode, libraryItem) {
this.id = getId('epdl')
this.podcastEpisode = podcastEpisode
this.url = podcastEpisode.enclosure.url
this.libraryItem = libraryItem
this.createdAt = Date.now()
}
}
module.exports = PodcastEpisodeDownload

View file

@ -7,13 +7,16 @@ class PodcastEpisode {
this.id = null
this.index = null
this.episodeNumber = null
this.episode = null
this.episodeType = null
this.title = null
this.subtitle = null
this.description = null
this.enclosure = null
this.pubDate = null
this.audioFile = null
this.publishedAt = null
this.addedAt = null
this.updatedAt = null
@ -25,12 +28,15 @@ class PodcastEpisode {
construct(episode) {
this.id = episode.id
this.index = episode.index
this.episodeNumber = episode.episodeNumber
this.episode = episode.episode
this.episodeType = episode.episodeType
this.title = episode.title
this.subtitle = episode.subtitle
this.description = episode.description
this.enclosure = episode.enclosure ? { ...episode.enclosure } : null
this.pubDate = episode.pubDate
this.audioFile = new AudioFile(episode.audioFile)
this.publishedAt = episode.publishedAt
this.addedAt = episode.addedAt
this.updatedAt = episode.updatedAt
}
@ -39,12 +45,15 @@ class PodcastEpisode {
return {
id: this.id,
index: this.index,
episodeNumber: this.episodeNumber,
episode: this.episode,
episodeType: this.episodeType,
title: this.title,
subtitle: this.subtitle,
description: this.description,
enclosure: this.enclosure ? { ...this.enclosure } : null,
pubDate: this.pubDate,
audioFile: this.audioFile.toJSON(),
publishedAt: this.publishedAt,
addedAt: this.addedAt,
updatedAt: this.updatedAt
}
@ -58,15 +67,22 @@ class PodcastEpisode {
return this.audioFile.duration
}
get size() { return this.audioFile.metadata.size }
get bestFilename() {
if (this.episode) return `${this.episode} - ${this.title}`
return this.title
}
setData(data, index = 1) {
this.id = getId('ep')
this.index = index
this.title = data.title
this.subtitle = data.subtitle || ''
this.pubDate = data.pubDate || ''
this.description = data.description || ''
this.enclosure = data.enclosure ? { ...data.enclosure } : null
this.episodeNumber = data.episodeNumber || ''
this.episode = data.episode || ''
this.episodeType = data.episodeType || ''
this.publishedAt = data.publishedAt || 0
this.addedAt = Date.now()
this.updatedAt = Date.now()
}

View file

@ -176,9 +176,11 @@ class Book {
return this.metadata.setDataFromAudioMetaTags(audioFile.metaTags, overrideExistingDetails)
}
setData(scanMediaMetadata) {
setData(mediaPayload) {
this.metadata = new BookMetadata()
this.metadata.setData(scanMediaMetadata)
if (mediaPayload.metadata) {
this.metadata.setData(mediaPayload.metadata)
}
}
// Look for desc.txt, reader.txt, metadata.abs and opf file then update details if found

View file

@ -118,10 +118,14 @@ class Podcast {
return this.episodes[0]
}
setData(metadata, coverPath = null, autoDownload = false) {
this.metadata = new PodcastMetadata(metadata)
this.coverPath = coverPath
this.autoDownloadEpisodes = autoDownload
setData(mediaMetadata) {
this.metadata = new PodcastMetadata()
if (mediaMetadata.metadata) {
this.metadata.setData(mediaMetadata.metadata)
}
this.coverPath = mediaMetadata.coverPath || null
this.autoDownloadEpisodes = !!mediaMetadata.autoDownloadEpisodes
}
async syncMetadataFiles(textMetadataFiles, opfMetadataOverrideDetails) {
@ -150,5 +154,9 @@ class Podcast {
this.episodes.forEach((ep) => total += ep.duration)
return total
}
addPodcastEpisode(podcastEpisode) {
this.episodes.push(podcastEpisode)
}
}
module.exports = Podcast

View file

@ -70,5 +70,22 @@ class PodcastMetadata {
}
return null
}
setData(mediaMetadata = {}) {
this.title = mediaMetadata.title || null
this.author = mediaMetadata.author || null
this.description = mediaMetadata.description || null
this.releaseDate = mediaMetadata.releaseDate || null
this.feedUrl = mediaMetadata.feedUrl || null
this.imageUrl = mediaMetadata.imageUrl || null
this.itunesPageUrl = mediaMetadata.itunesPageUrl || null
this.itunesId = mediaMetadata.itunesId || null
this.itunesArtistId = mediaMetadata.itunesArtistId || null
this.explicit = !!mediaMetadata.explicit
this.language = mediaMetadata.language || null
if (mediaMetadata.genres && mediaMetadata.genres.length) {
this.genres = [...mediaMetadata.genres]
}
}
}
module.exports = PodcastMetadata

View file

@ -54,7 +54,7 @@ class AudioFileScanner {
return Math.floor(total / results.length)
}
async scan(audioLibraryFile, mediaMetadataFromScan, verbose = false) {
async scan(mediaType, audioLibraryFile, mediaMetadataFromScan, verbose = false) {
var probeStart = Date.now()
var probeData = await prober.probe(audioLibraryFile.metadata.path, verbose)
if (probeData.error) {
@ -65,11 +65,11 @@ class AudioFileScanner {
var audioFile = new AudioFile()
audioFile.trackNumFromMeta = probeData.trackNumber
audioFile.discNumFromMeta = probeData.discNumber
const { trackNumber, discNumber } = this.getTrackAndDiscNumberFromFilename(mediaMetadataFromScan, audioLibraryFile)
audioFile.trackNumFromFilename = trackNumber
audioFile.discNumFromFilename = discNumber
if (mediaType === 'book') {
const { trackNumber, discNumber } = this.getTrackAndDiscNumberFromFilename(mediaMetadataFromScan, audioLibraryFile)
audioFile.trackNumFromFilename = trackNumber
audioFile.discNumFromFilename = discNumber
}
audioFile.setDataFromProbe(audioLibraryFile, probeData)
return {
@ -79,11 +79,11 @@ class AudioFileScanner {
}
// Returns array of { AudioFile, elapsed, averageScanDuration } from audio file scan objects
async executeAudioFileScans(audioLibraryFiles, scanData) {
async executeAudioFileScans(mediaType, audioLibraryFiles, scanData) {
var mediaMetadataFromScan = scanData.mediaMetadata || null
var proms = []
for (let i = 0; i < audioLibraryFiles.length; i++) {
proms.push(this.scan(audioLibraryFiles[i], mediaMetadataFromScan))
proms.push(this.scan(mediaType, audioLibraryFiles[i], mediaMetadataFromScan))
}
var scanStart = Date.now()
var results = await Promise.all(proms).then((scanResults) => scanResults.filter(sr => sr))
@ -178,7 +178,7 @@ class AudioFileScanner {
async scanAudioFiles(audioLibraryFiles, scanData, libraryItem, preferAudioMetadata, libraryScan = null) {
var hasUpdated = false
var audioScanResult = await this.executeAudioFileScans(audioLibraryFiles, scanData)
var audioScanResult = await this.executeAudioFileScans(libraryItem.mediaType, audioLibraryFiles, scanData)
if (audioScanResult.audioFiles.length) {
if (libraryScan) {
libraryScan.addLog(LogLevel.DEBUG, `Library Item "${scanData.path}" Audio file scan took ${audioScanResult.elapsed}ms for ${audioScanResult.audioFiles.length} with average time of ${audioScanResult.averageScanDuration}ms`)

View file

@ -502,6 +502,7 @@ class Scanner {
async scanFolderUpdates(library, folder, fileUpdateGroup) {
Logger.debug(`[Scanner] Scanning file update groups in folder "${folder.id}" of library "${library.name}"`)
Logger.debug(`[Scanner] scanFolderUpdates fileUpdateGroup`, fileUpdateGroup)
// First pass - Remove files in parent dirs of items and remap the fileupdate group
// Test Case: Moving audio files from library item folder to author folder should trigger a re-scan of the item

View file

@ -150,4 +150,23 @@ module.exports.downloadFile = async (url, filepath) => {
writer.on('finish', resolve)
writer.on('error', reject)
})
}
module.exports.sanitizeFilename = (filename, replacement = '') => {
if (typeof filename !== 'string') {
return false
}
var illegalRe = /[\/\?<>\\:\*\|"]/g;
var controlRe = /[\x00-\x1f\x80-\x9f]/g;
var reservedRe = /^\.+$/;
var windowsReservedRe = /^(con|prn|aux|nul|com[0-9]|lpt[0-9])(\..*)?$/i;
var windowsTrailingRe = /[\. ]+$/;
var sanitized = filename
.replace(illegalRe, replacement)
.replace(controlRe, replacement)
.replace(reservedRe, replacement)
.replace(windowsReservedRe, replacement)
.replace(windowsTrailingRe, replacement);
return sanitized
}

View file

@ -59,12 +59,12 @@ module.exports = {
}
libraryItems.forEach((li) => {
var mediaMetadata = li.media.metadata
if (mediaMetadata.authors.length) {
if (mediaMetadata.authors && mediaMetadata.authors.length) {
mediaMetadata.authors.forEach((author) => {
if (author && !data.authors.find(au => au.id === author.id)) data.authors.push({ id: author.id, name: author.name })
})
}
if (mediaMetadata.series.length) {
if (mediaMetadata.series && mediaMetadata.series.length) {
mediaMetadata.series.forEach((series) => {
if (series && !data.series.find(se => se.id === series.id)) data.series.push({ id: series.id, name: series.name })
})
@ -79,7 +79,7 @@ module.exports = {
if (tag && !data.tags.includes(tag)) data.tags.push(tag)
})
}
if (mediaMetadata.narrators.length) {
if (mediaMetadata.narrators && mediaMetadata.narrators.length) {
mediaMetadata.narrators.forEach((narrator) => {
if (narrator && !data.narrators.includes(narrator)) data.narrators.push(narrator)
})

View file

@ -81,7 +81,8 @@ function cleanEpisodeData(data) {
author: data.author || '',
duration: data.duration || '',
explicit: data.explicit || '',
publishedAt: (new Date(data.pubDate)).valueOf()
publishedAt: (new Date(data.pubDate)).valueOf(),
enclosure: data.enclosure
}
}

View file

@ -161,7 +161,11 @@ async function scanFolder(libraryMediaType, folder, serverSettings = {}) {
mtimeMs: libraryItemFolderStats.mtimeMs || 0,
ctimeMs: libraryItemFolderStats.ctimeMs || 0,
birthtimeMs: libraryItemFolderStats.birthtimeMs || 0,
...libraryItemData,
path: libraryItemData.path,
relPath: libraryItemData.relPath,
media: {
metadata: libraryItemData.mediaMetadata || null
},
libraryFiles: fileObjs
})
}
@ -262,9 +266,21 @@ function getBookDataFromDir(folderPath, relPath, parseSubtitle = false) {
}
}
function getPodcastDataFromDir(folderPath, relPath) {
relPath = relPath.replace(/\\/g, '/')
return {
relPath: relPath, // relative audiobook path i.e. /Author Name/Book Name/..
path: Path.posix.join(folderPath, relPath) // i.e. /audiobook/Author Name/Book Name/..
}
}
function getDataFromMediaDir(libraryMediaType, folderPath, relPath, serverSettings) {
var parseSubtitle = !!serverSettings.scannerParseSubtitle
return getBookDataFromDir(folderPath, relPath, parseSubtitle)
if (libraryMediaType === 'podcast') {
return getPodcastDataFromDir(folderPath, relPath, parseSubtitle)
} else {
return getBookDataFromDir(folderPath, relPath, parseSubtitle)
}
}
@ -284,7 +300,11 @@ async function getLibraryItemFileData(libraryMediaType, folder, libraryItemPath,
birthtimeMs: libraryItemDirStats.birthtimeMs || 0,
folderId: folder.id,
libraryId: folder.libraryId,
...libraryItemData,
path: libraryItemData.path,
relPath: libraryItemData.relPath,
media: {
metadata: libraryItemData.mediaMetadata || null
},
libraryFiles: []
}