mirror/advplyr.audiobookshelf
1
0
Fork 0
mirror of https://github.com/advplyr/audiobookshelf.git synced 2025-08-04 10:14:36 +02:00
Code Issues Projects Releases Packages Wiki Activity Actions

Merge pull request #3468 from mikiher/nunicode-intergration

Browse source 
Nunicode integration
This commit is contained in:
advplyr 2024-10-01 15:17:54 -05:00 • committed by GitHub
commit 1b1b71a9b6
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
9 changed files with 364 additions and 99 deletions
Download patch file Download diff file Expand all files Collapse all files

114
server/Database.js
View file

@ -28,6 +28,9 @@ class Database {
this.notificationSettings = null
/** @type {import('./objects/settings/EmailSettings')} */
this.emailSettings = null
this.supportsUnaccent = false
this.supportsUnicodeFoldings = false
}
get models() {
@ -223,6 +226,12 @@ class Database {
try {
await this.sequelize.authenticate()
if (process.env.NUSQLITE3_PATH) {
await this.loadExtension(process.env.NUSQLITE3_PATH)
Logger.info(`[Database] Db supports unaccent and unicode foldings`)
this.supportsUnaccent = true
this.supportsUnicodeFoldings = true
}
Logger.info(`[Database] Db connection was successful`)
return true
} catch (error) {
@ -232,10 +241,9 @@ class Database {
}
/**
* TODO: Temporarily disabled
* @param {string[]} extensions paths to extension binaries
* @param {string} extension paths to extension binary
*/
async loadExtensions(extensions) {
async loadExtension(extension) {
// This is a hack to get the db connection for loading extensions.
// The proper way would be to use the 'afterConnect' hook, but that hook is never called for sqlite due to a bug in sequelize.
// See https://github.com/sequelize/sequelize/issues/12487
@ -243,20 +251,18 @@ class Database {
const db = await this.sequelize.dialect.connectionManager.getConnection()
if (typeof db?.loadExtension !== 'function') throw new Error('Failed to get db connection for loading extensions')
for (const ext of extensions) {
Logger.info(`[Database] Loading extension ${ext}`)
await new Promise((resolve, reject) => {
db.loadExtension(ext, (err) => {
if (err) {
Logger.error(`[Database] Failed to load extension ${ext}`, err)
reject(err)
return
}
Logger.info(`[Database] Successfully loaded extension ${ext}`)
resolve()
})
Logger.info(`[Database] Loading extension ${extension}`)
await new Promise((resolve, reject) => {
db.loadExtension(extension, (err) => {
if (err) {
Logger.error(`[Database] Failed to load extension ${extension}`, err)
reject(err)
return
}
Logger.info(`[Database] Successfully loaded extension ${extension}`)
resolve()
})
}
})
}
/**
@ -745,37 +751,57 @@ class Database {
}
}
/**
* TODO: Temporarily unused
* @param {string} value
* @returns {string}
*/
normalize(value) {
return `lower(unaccent(${value}))`
async createTextSearchQuery(query) {
const textQuery = new this.TextSearchQuery(this.sequelize, this.supportsUnaccent, query)
await textQuery.init()
return textQuery
}
/**
* TODO: Temporarily unused
* @param {string} query
* @returns {Promise<string>}
*/
async getNormalizedQuery(query) {
const escapedQuery = this.sequelize.escape(query)
const normalizedQuery = this.normalize(escapedQuery)
const normalizedQueryResult = await this.sequelize.query(`SELECT ${normalizedQuery} as normalized_query`)
return normalizedQueryResult[0][0].normalized_query
}
TextSearchQuery = class {
constructor(sequelize, supportsUnaccent, query) {
this.sequelize = sequelize
this.supportsUnaccent = supportsUnaccent
this.query = query
this.hasAccents = false
}
/**
*
* @param {string} column
* @param {string} normalizedQuery
* @returns {string}
*/
matchExpression(column, normalizedQuery) {
const normalizedPattern = this.sequelize.escape(`%${normalizedQuery}%`)
const normalizedColumn = column
return `${normalizedColumn} LIKE ${normalizedPattern}`
/**
* Returns a normalized (accents-removed) expression for the specified value.
*
* @param {string} value
* @returns {string}
*/
normalize(value) {
return `unaccent(${value})`
}
/**
* Initialize the text query.
*
*/
async init() {
if (!this.supportsUnaccent) return
const escapedQuery = this.sequelize.escape(this.query)
const normalizedQueryExpression = this.normalize(escapedQuery)
const normalizedQueryResult = await this.sequelize.query(`SELECT ${normalizedQueryExpression} as normalized_query`)
const normalizedQuery = normalizedQueryResult[0][0].normalized_query
this.hasAccents = escapedQuery !== this.sequelize.escape(normalizedQuery)
}
/**
* Get match expression for the specified column.
* If the query contains accents, match against the column as-is (case-insensitive exact match).
* otherwise match against a normalized column (case-insensitive match with accents removed).
*
* @param {string} column
* @returns {string}
*/
matchExpression(column) {
const pattern = this.sequelize.escape(`%${this.query}%`)
if (!this.supportsUnaccent) return `${column} LIKE ${pattern}`
const normalizedColumn = this.hasAccents ? column : this.normalize(column)
return `${normalizedColumn} LIKE ${pattern}`
}
}
}

180
server/managers/BinaryManager.js
View file

@ -76,18 +76,27 @@ class ZippedAssetDownloader {
async extractFiles(zipPath, filesToExtract, destDir) {
const zip = new StreamZip.async({ file: zipPath })
for (const file of filesToExtract) {
const outputPath = path.join(destDir, file.outputFileName)
await zip.extract(file.pathInsideZip, outputPath)
Logger.debug(`[ZippedAssetDownloader] Extracted file ${file.pathInsideZip} to ${outputPath}`)
try {
for (const file of filesToExtract) {
const outputPath = path.join(destDir, file.outputFileName)
if (!(await zip.entry(file.pathInsideZip))) {
Logger.error(`[ZippedAssetDownloader] File ${file.pathInsideZip} not found in zip file ${zipPath}`)
continue
}
await zip.extract(file.pathInsideZip, outputPath)
Logger.debug(`[ZippedAssetDownloader] Extracted file ${file.pathInsideZip} to ${outputPath}`)
// Set executable permission for Linux
if (process.platform !== 'win32') {
await fs.chmod(outputPath, 0o755)
// Set executable permission for Linux
if (process.platform !== 'win32') {
await fs.chmod(outputPath, 0o755)
}
}
} catch (error) {
Logger.error('[ZippedAssetDownloader] Error extracting files:', error)
throw error
} finally {
await zip.close()
}
await zip.close()
}
async downloadAndExtractFiles(releaseTag, assetName, filesToExtract, destDir) {
@ -99,7 +108,6 @@ class ZippedAssetDownloader {
await this.extractFiles(zipPath, filesToExtract, destDir)
} catch (error) {
Logger.error(`[ZippedAssetDownloader] Error downloading or extracting files: ${error.message}`)
throw error
} finally {
if (zipPath) await fs.remove(zipPath)
}
@ -164,14 +172,67 @@ class FFBinariesDownloader extends ZippedAssetDownloader {
}
}
class NunicodeDownloader extends ZippedAssetDownloader {
constructor() {
super()
this.platformSuffix = this.getPlatformSuffix()
}
getPlatformSuffix() {
const platform = process.platform
const arch = process.arch
if (platform === 'win32' && arch === 'x64') {
return 'win-x64'
} else if (platform === 'darwin' && (arch === 'x64' || arch === 'arm64')) {
return 'osx-arm64'
} else if (platform === 'linux' && arch === 'x64') {
return 'linux-x64'
} else if (platform === 'linux' && arch === 'arm64') {
return 'linux-arm64'
}
return null
}
async getAssetUrl(releaseTag, assetName) {
return `https://github.com/mikiher/nunicode-sqlite/releases/download/v${releaseTag}/${assetName}`
}
getAssetName(binaryName, releaseTag) {
if (!this.platformSuffix) {
throw new Error(`[NunicodeDownloader] Platform ${process.platform}-${process.arch} not supported`)
}
return `${binaryName}-${this.platformSuffix}.zip`
}
getAssetFileName(binaryName) {
if (process.platform === 'win32') {
return `${binaryName}.dll`
} else if (process.platform === 'darwin') {
return `${binaryName}.dylib`
} else if (process.platform === 'linux') {
return `${binaryName}.so`
}
throw new Error(`[NunicodeDownloader] Platform ${process.platform} not supported`)
}
}
class Binary {
constructor(name, type, envVariable, validVersions, source) {
constructor(name, type, envVariable, validVersions, source, required = true) {
if (!name) throw new Error('Binary name is required')
this.name = name
if (!type) throw new Error('Binary type is required')
this.type = type
if (!envVariable) throw new Error('Binary environment variable name is required')
this.envVariable = envVariable
if (!validVersions || !validVersions.length) throw new Error(`No valid versions specified for ${type} ${name}. At least one version is required.`)
this.validVersions = validVersions
if (!source || !(source instanceof ZippedAssetDownloader)) throw new Error('Binary source is required, and must be an instance of ZippedAssetDownloader')
this.source = source
this.fileName = this.getFileName()
this.required = required
this.exec = exec
}
@ -205,37 +266,65 @@ class Binary {
}
}
async isGood(binaryPath) {
if (!binaryPath || !(await fs.pathExists(binaryPath))) return false
if (!this.validVersions.length) return true
if (this.type === 'library') return true
async isLibraryVersionValid(libraryPath) {
try {
const { stdout } = await this.exec('"' + binaryPath + '"' + ' -version')
const versionFilePath = libraryPath + '.ver'
if (!(await fs.pathExists(versionFilePath))) return false
const version = (await fs.readFile(versionFilePath, 'utf8')).trim()
return this.validVersions.some((validVersion) => version.startsWith(validVersion))
} catch (err) {
Logger.error(`[Binary] Failed to check version of ${libraryPath}`, err)
return false
}
}
async isExecutableVersionValid(executablePath) {
try {
const { stdout } = await this.exec('"' + executablePath + '"' + ' -version')
const version = stdout.match(/version\s([\d\.]+)/)?.[1]
if (!version) return false
return this.validVersions.some((validVersion) => version.startsWith(validVersion))
} catch (err) {
Logger.error(`[Binary] Failed to check version of ${binaryPath}`)
Logger.error(`[Binary] Failed to check version of ${executablePath}`, err)
return false
}
}
async isGood(binaryPath) {
try {
if (!binaryPath || !(await fs.pathExists(binaryPath))) return false
if (this.type === 'library') return await this.isLibraryVersionValid(binaryPath)
else if (this.type === 'executable') return await this.isExecutableVersionValid(binaryPath)
else return true
} catch (err) {
Logger.error(`[Binary] Failed to check ${this.type} ${this.name} at ${binaryPath}`, err)
return false
}
}
async download(destination) {
await this.source.downloadBinary(this.name, this.validVersions[0], destination)
const version = this.validVersions[0]
try {
await this.source.downloadBinary(this.name, version, destination)
// if it's a library, write the version string to a file
if (this.type === 'library') {
const libraryPath = path.join(destination, this.fileName)
await fs.writeFile(libraryPath + '.ver', version)
}
} catch (err) {
Logger.error(`[Binary] Failed to download ${this.type} ${this.name} version ${version} to ${destination}`, err)
}
}
}
const ffbinaries = new FFBinariesDownloader()
module.exports.ffbinaries = ffbinaries // for testing
//const sqlean = new SQLeanDownloader()
//module.exports.sqlean = sqlean // for testing
const nunicode = new NunicodeDownloader()
class BinaryManager {
defaultRequiredBinaries = [
new Binary('ffmpeg', 'executable', 'FFMPEG_PATH', ['5.1'], ffbinaries), // ffmpeg executable
new Binary('ffprobe', 'executable', 'FFPROBE_PATH', ['5.1'], ffbinaries) // ffprobe executable
// TODO: Temporarily disabled due to db corruption issues
// new Binary('unicode', 'library', 'SQLEAN_UNICODE_PATH', ['0.24.2'], sqlean) // sqlean unicode extension
new Binary('ffprobe', 'executable', 'FFPROBE_PATH', ['5.1'], ffbinaries), // ffprobe executable
new Binary('libnusqlite3', 'library', 'NUSQLITE3_PATH', ['1.1'], nunicode, false) // nunicode sqlite3 extension
]
constructor(requiredBinaries = this.defaultRequiredBinaries) {
@ -249,7 +338,7 @@ class BinaryManager {
// Optional skip binaries check
if (process.env.SKIP_BINARIES_CHECK === '1') {
for (const binary of this.requiredBinaries) {
if (!process.env[binary.envVariable]) {
if (!process.env[binary.envVariable] && binary.required) {
await Logger.fatal(`[BinaryManager] Environment variable ${binary.envVariable} must be set`)
process.exit(1)
}
@ -265,21 +354,37 @@ class BinaryManager {
await this.removeOldBinaries(missingBinaries)
await this.install(missingBinaries)
const missingBinariesAfterInstall = await this.findRequiredBinaries()
if (missingBinariesAfterInstall.length) {
Logger.error(`[BinaryManager] Failed to find or install required binaries: ${missingBinariesAfterInstall.join(', ')}`)
const missingRequiredBinryNames = missingBinariesAfterInstall.filter((binary) => binary.required).map((binary) => binary.name)
if (missingRequiredBinryNames.length) {
Logger.error(`[BinaryManager] Failed to find or install required binaries: ${missingRequiredBinryNames.join(', ')}`)
process.exit(1)
}
this.initialized = true
}
/**
* Remove binary
*
* @param {string} destination
* @param {Binary} binary
*/
async removeBinary(destination, binary) {
const binaryPath = path.join(destination, binary.fileName)
if (await fs.pathExists(binaryPath)) {
Logger.debug(`[BinaryManager] Removing binary: ${binaryPath}`)
await fs.remove(binaryPath)
try {
const binaryPath = path.join(destination, binary.fileName)
if (await fs.pathExists(binaryPath)) {
Logger.debug(`[BinaryManager] Removing binary: ${binaryPath}`)
await fs.remove(binaryPath)
}
} catch (err) {
Logger.error(`[BinaryManager] Error removing binary: ${binaryPath}`)
}
}
/**
* Remove old binaries
*
* @param {Binary[]} binaries
*/
async removeOldBinaries(binaries) {
for (const binary of binaries) {
await this.removeBinary(this.mainInstallDir, binary)
@ -290,26 +395,31 @@ class BinaryManager {
/**
* Find required binaries and return array of binary names that are missing
*
* @returns {Promise<string[]>}
* @returns {Promise<Binary[]>} Array of missing binaries
*/
async findRequiredBinaries() {
const missingBinaries = []
for (const binary of this.requiredBinaries) {
const binaryPath = await binary.find(this.mainInstallDir, this.altInstallDir)
if (binaryPath) {
Logger.info(`[BinaryManager] Found valid binary ${binary.name} at ${binaryPath}`)
Logger.info(`[BinaryManager] Found valid ${binary.type} ${binary.name} at ${binaryPath}`)
if (process.env[binary.envVariable] !== binaryPath) {
Logger.info(`[BinaryManager] Updating process.env.${binary.envVariable}`)
process.env[binary.envVariable] = binaryPath
}
} else {
Logger.info(`[BinaryManager] ${binary.name} not found or version too old`)
Logger.info(`[BinaryManager] ${binary.name} not found or not a valid version`)
missingBinaries.push(binary)
}
}
return missingBinaries
}
/**
* Install missing binaries
*
* @param {Binary[]} binaries
*/
async install(binaries) {
if (!binaries.length) return
Logger.info(`[BinaryManager] Installing binaries: ${binaries.map((binary) => binary.name).join(', ')}`)
@ -323,3 +433,5 @@ class BinaryManager {
module.exports = BinaryManager
module.exports.Binary = Binary // for testing
module.exports.ffbinaries = ffbinaries // for testing
module.exports.nunicode = nunicode // for testing

4
server/utils/queries/authorFilters.js
View file

@ -54,13 +54,13 @@ module.exports = {
* Search authors
*
* @param {string} libraryId
* @param {string} query
* @param {Database.TextQuery} query
* @param {number} limit
* @param {number} offset
* @returns {Promise<Object[]>} oldAuthor with numBooks
*/
async search(libraryId, query, limit, offset) {
const matchAuthor = Database.matchExpression('name', query)
const matchAuthor = query.matchExpression('name')
const authors = await Database.authorModel.findAll({
where: {
[Sequelize.Op.and]: [Sequelize.literal(matchAuthor), { libraryId }]

12
server/utils/queries/libraryItemsBookFilters.js
View file

@ -975,10 +975,10 @@ module.exports = {
async search(user, library, query, limit, offset) {
const userPermissionBookWhere = this.getUserPermissionBookWhereQuery(user)
const normalizedQuery = query
const textSearchQuery = await Database.createTextSearchQuery(query)
const matchTitle = Database.matchExpression('title', normalizedQuery)
const matchSubtitle = Database.matchExpression('subtitle', normalizedQuery)
const matchTitle = textSearchQuery.matchExpression('title')
const matchSubtitle = textSearchQuery.matchExpression('subtitle')
// Search title, subtitle, asin, isbn
const books = await Database.bookModel.findAll({
@ -1041,7 +1041,7 @@ module.exports = {
})
}
const matchJsonValue = Database.matchExpression('json_each.value', normalizedQuery)
const matchJsonValue = textSearchQuery.matchExpression('json_each.value')
// Search narrators
const narratorMatches = []
@ -1095,7 +1095,7 @@ module.exports = {
}
// Search series
const matchName = Database.matchExpression('name', normalizedQuery)
const matchName = textSearchQuery.matchExpression('name')
const allSeries = await Database.seriesModel.findAll({
where: {
[Sequelize.Op.and]: [
@ -1136,7 +1136,7 @@ module.exports = {
}
// Search authors
const authorMatches = await authorFilters.search(library.id, normalizedQuery, limit, offset)
const authorMatches = await authorFilters.search(library.id, textSearchQuery, limit, offset)
return {
book: itemMatches,

9
server/utils/queries/libraryItemsPodcastFilters.js
View file

@ -315,9 +315,10 @@ module.exports = {
async search(user, library, query, limit, offset) {
const userPermissionPodcastWhere = this.getUserPermissionPodcastWhereQuery(user)
const normalizedQuery = query
const matchTitle = Database.matchExpression('title', normalizedQuery)
const matchAuthor = Database.matchExpression('author', normalizedQuery)
const textSearchQuery = await Database.createTextSearchQuery(query)
const matchTitle = textSearchQuery.matchExpression('title')
const matchAuthor = textSearchQuery.matchExpression('author')
// Search title, author, itunesId, itunesArtistId
const podcasts = await Database.podcastModel.findAll({
@ -366,7 +367,7 @@ module.exports = {
})
}
const matchJsonValue = Database.matchExpression('json_each.value', normalizedQuery)
const matchJsonValue = textSearchQuery.matchExpression('json_each.value')
// Search tags
const tagMatches = []