Init sqlite take 2

server/utils/migrations/oldDbFiles.js

@@ -0,0 +1,189 @@
+const { once } = require('events')
+const { createInterface } = require('readline')
+const Path = require('path')
+const Logger = require('../../Logger')
+const fs = require('../../libs/fsExtra')
+const archiver = require('../../libs/archiver')
+const StreamZip = require('../../libs/nodeStreamZip')
+
+async function processDbFile(filepath) {
+  if (!fs.pathExistsSync(filepath)) {
+    Logger.error(`[oldDbFiles] Db file does not exist at "${filepath}"`)
+    return []
+  }
+
+  const entities = []
+
+  try {
+    const fileStream = fs.createReadStream(filepath)
+
+    const rl = createInterface({
+      input: fileStream,
+      crlfDelay: Infinity,
+    })
+
+    rl.on('line', (line) => {
+      if (line && line.trim()) {
+        try {
+          const entity = JSON.parse(line)
+          if (entity && Object.keys(entity).length) entities.push(entity)
+        } catch (jsonParseError) {
+          Logger.error(`[oldDbFiles] Failed to parse line "${line}" in db file "${filepath}"`, jsonParseError)
+        }
+      }
+    })
+
+    await once(rl, 'close')
+
+    console.log(`[oldDbFiles] Db file "${filepath}" processed`)
+
+    return entities
+  } catch (error) {
+    Logger.error(`[oldDbFiles] Failed to read db file "${filepath}"`, error)
+    return []
+  }
+}
+
+async function loadDbData(dbpath) {
+  try {
+    Logger.info(`[oldDbFiles] Loading db data at "${dbpath}"`)
+    const files = await fs.readdir(dbpath)
+
+    const entities = []
+    for (const filename of files) {
+      if (Path.extname(filename).toLowerCase() !== '.json') {
+        Logger.warn(`[oldDbFiles] Ignoring filename "${filename}" in db folder "${dbpath}"`)
+        continue
+      }
+
+      const filepath = Path.join(dbpath, filename)
+      Logger.info(`[oldDbFiles] Loading db data file "${filepath}"`)
+      const someEntities = await processDbFile(filepath)
+      Logger.info(`[oldDbFiles] Processed db data file with ${someEntities.length} entities`)
+      entities.push(...someEntities)
+    }
+
+    Logger.info(`[oldDbFiles] Finished loading db data with ${entities.length} entities`)
+    return entities
+  } catch (error) {
+    Logger.error(`[oldDbFiles] Failed to load db data "${dbpath}"`, error)
+    return null
+  }
+}
+
+module.exports.init = async () => {
+  const dbs = {
+    libraryItems: Path.join(global.ConfigPath, 'libraryItems', 'data'),
+    users: Path.join(global.ConfigPath, 'users', 'data'),
+    sessions: Path.join(global.ConfigPath, 'sessions', 'data'),
+    libraries: Path.join(global.ConfigPath, 'libraries', 'data'),
+    settings: Path.join(global.ConfigPath, 'settings', 'data'),
+    collections: Path.join(global.ConfigPath, 'collections', 'data'),
+    playlists: Path.join(global.ConfigPath, 'playlists', 'data'),
+    authors: Path.join(global.ConfigPath, 'authors', 'data'),
+    series: Path.join(global.ConfigPath, 'series', 'data'),
+    feeds: Path.join(global.ConfigPath, 'feeds', 'data')
+  }
+
+  const data = {}
+  for (const key in dbs) {
+    data[key] = await loadDbData(dbs[key])
+    Logger.info(`[oldDbFiles] ${data[key].length} ${key} loaded`)
+  }
+
+  return data
+}
+
+module.exports.zipWrapOldDb = async () => {
+  const dbs = {
+    libraryItems: Path.join(global.ConfigPath, 'libraryItems'),
+    users: Path.join(global.ConfigPath, 'users'),
+    sessions: Path.join(global.ConfigPath, 'sessions'),
+    libraries: Path.join(global.ConfigPath, 'libraries'),
+    settings: Path.join(global.ConfigPath, 'settings'),
+    collections: Path.join(global.ConfigPath, 'collections'),
+    playlists: Path.join(global.ConfigPath, 'playlists'),
+    authors: Path.join(global.ConfigPath, 'authors'),
+    series: Path.join(global.ConfigPath, 'series'),
+    feeds: Path.join(global.ConfigPath, 'feeds')
+  }
+
+  return new Promise((resolve) => {
+    const oldDbPath = Path.join(global.ConfigPath, 'oldDb.zip')
+    const output = fs.createWriteStream(oldDbPath)
+    const archive = archiver('zip', {
+      zlib: { level: 9 } // Sets the compression level.
+    })
+
+    // listen for all archive data to be written
+    // 'close' event is fired only when a file descriptor is involved
+    output.on('close', async () => {
+      Logger.info(`[oldDbFiles] Old db files have been zipped in ${oldDbPath}. ${archive.pointer()} total bytes`)
+
+      // Remove old db folders have successful zip
+      for (const db in dbs) {
+        await fs.remove(dbs[db])
+      }
+
+      resolve(true)
+    })
+
+    // This event is fired when the data source is drained no matter what was the data source.
+    // It is not part of this library but rather from the NodeJS Stream API.
+    // @see: https://nodejs.org/api/stream.html#stream_event_end
+    output.on('end', () => {
+      Logger.debug('[oldDbFiles] Data has been drained')
+    })
+
+    // good practice to catch this error explicitly
+    archive.on('error', (err) => {
+      Logger.error(`[oldDbFiles] Failed to zip old db folders`, err)
+      resolve(false)
+    })
+
+    // pipe archive data to the file
+    archive.pipe(output)
+
+    for (const db in dbs) {
+      archive.directory(dbs[db], db)
+    }
+
+    // finalize the archive (ie we are done appending files but streams have to finish yet)
+    // 'close', 'end' or 'finish' may be fired right after calling this method so register to them beforehand
+    archive.finalize()
+  })
+}
+
+module.exports.checkHasOldDb = async () => {
+  const dbs = {
+    libraryItems: Path.join(global.ConfigPath, 'libraryItems'),
+    users: Path.join(global.ConfigPath, 'users'),
+    sessions: Path.join(global.ConfigPath, 'sessions'),
+    libraries: Path.join(global.ConfigPath, 'libraries'),
+    settings: Path.join(global.ConfigPath, 'settings'),
+    collections: Path.join(global.ConfigPath, 'collections'),
+    playlists: Path.join(global.ConfigPath, 'playlists'),
+    authors: Path.join(global.ConfigPath, 'authors'),
+    series: Path.join(global.ConfigPath, 'series'),
+    feeds: Path.join(global.ConfigPath, 'feeds')
+  }
+  for (const db in dbs) {
+    if (await fs.pathExists(dbs[db])) {
+      return true
+    }
+  }
+  return false
+}
+
+module.exports.checkHasOldDbZip = async () => {
+  const oldDbPath = Path.join(global.ConfigPath, 'oldDb.zip')
+  if (!await fs.pathExists(oldDbPath)) {
+    return false
+  }
+
+  // Extract oldDb.zip
+  const zip = new StreamZip.async({ file: oldDbPath })
+  await zip.extract(null, global.ConfigPath)
+
+  return this.checkHasOldDb()
+}