Merge pull request #1469 from nichwall/download_manager_cleanup

Download manager cleanup
This commit is contained in:
advplyr 2025-02-07 17:13:51 -06:00 committed by GitHub
commit e194df455b
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
2 changed files with 293 additions and 161 deletions

View file

@ -18,18 +18,26 @@ import com.audiobookshelf.app.models.DownloadItemPart
import com.fasterxml.jackson.core.json.JsonReadFeature
import com.fasterxml.jackson.module.kotlin.jacksonObjectMapper
import com.getcapacitor.JSObject
import java.io.File
import java.io.FileOutputStream
import java.util.*
import kotlinx.coroutines.Dispatchers
import kotlinx.coroutines.GlobalScope
import kotlinx.coroutines.delay
import kotlinx.coroutines.launch
import java.io.File
import java.io.FileOutputStream
import java.util.*
class DownloadItemManager(var downloadManager:DownloadManager, private var folderScanner: FolderScanner, var mainActivity: MainActivity, private var clientEventEmitter:DownloadEventEmitter) {
/** Manages download items and their parts. */
class DownloadItemManager(
var downloadManager: DownloadManager,
private var folderScanner: FolderScanner,
var mainActivity: MainActivity,
private var clientEventEmitter: DownloadEventEmitter
) {
val tag = "DownloadItemManager"
private val maxSimultaneousDownloads = 3
private var jacksonMapper = jacksonObjectMapper().enable(JsonReadFeature.ALLOW_UNESCAPED_CONTROL_CHARS.mappedFeature())
private var jacksonMapper =
jacksonObjectMapper()
.enable(JsonReadFeature.ALLOW_UNESCAPED_CONTROL_CHARS.mappedFeature())
enum class DownloadCheckStatus {
InProgress,
@ -37,25 +45,28 @@ class DownloadItemManager(var downloadManager:DownloadManager, private var folde
Failed
}
var downloadItemQueue: MutableList<DownloadItem> = mutableListOf() // All pending and downloading items
var currentDownloadItemParts: MutableList<DownloadItemPart> = mutableListOf() // Item parts currently being downloaded
var downloadItemQueue: MutableList<DownloadItem> =
mutableListOf() // All pending and downloading items
var currentDownloadItemParts: MutableList<DownloadItemPart> =
mutableListOf() // Item parts currently being downloaded
interface DownloadEventEmitter {
fun onDownloadItem(downloadItem:DownloadItem)
fun onDownloadItemPartUpdate(downloadItemPart:DownloadItemPart)
fun onDownloadItemComplete(jsobj:JSObject)
fun onDownloadItem(downloadItem: DownloadItem)
fun onDownloadItemPartUpdate(downloadItemPart: DownloadItemPart)
fun onDownloadItemComplete(jsobj: JSObject)
}
interface InternalProgressCallback {
fun onProgress(totalBytesWritten:Long, progress: Long)
fun onProgress(totalBytesWritten: Long, progress: Long)
fun onComplete(failed: Boolean)
}
companion object {
var isDownloading:Boolean = false
var isDownloading: Boolean = false
}
fun addDownloadItem(downloadItem:DownloadItem) {
/** Adds a download item to the queue and starts processing the queue. */
fun addDownloadItem(downloadItem: DownloadItem) {
DeviceManager.dbManager.saveDownloadItem(downloadItem)
Log.i(tag, "Add download item ${downloadItem.media.metadata.title}")
@ -64,42 +75,18 @@ class DownloadItemManager(var downloadManager:DownloadManager, private var folde
checkUpdateDownloadQueue()
}
/** Checks and updates the download queue. */
private fun checkUpdateDownloadQueue() {
for (downloadItem in downloadItemQueue) {
val numPartsToGet = maxSimultaneousDownloads - currentDownloadItemParts.size
val nextDownloadItemParts = downloadItem.getNextDownloadItemParts(numPartsToGet)
Log.d(tag, "checkUpdateDownloadQueue: numPartsToGet=$numPartsToGet, nextDownloadItemParts=${nextDownloadItemParts.size}")
Log.d(
tag,
"checkUpdateDownloadQueue: numPartsToGet=$numPartsToGet, nextDownloadItemParts=${nextDownloadItemParts.size}"
)
if (nextDownloadItemParts.size > 0) {
nextDownloadItemParts.forEach {
if (it.isInternalStorage) {
val file = File(it.finalDestinationPath)
file.parentFile?.mkdirs()
val fileOutputStream = FileOutputStream(it.finalDestinationPath)
val internalProgressCallback = (object : InternalProgressCallback {
override fun onProgress(totalBytesWritten:Long, progress: Long) {
it.bytesDownloaded = totalBytesWritten
it.progress = progress
}
override fun onComplete(failed:Boolean) {
it.failed = failed
it.completed = true
}
})
Log.d(tag, "Start internal download to destination path ${it.finalDestinationPath} from ${it.serverUrl}")
InternalDownloadManager(fileOutputStream, internalProgressCallback).download(it.serverUrl)
it.downloadId = 1
currentDownloadItemParts.add(it)
} else {
val dlRequest = it.getDownloadRequest()
val downloadId = downloadManager.enqueue(dlRequest)
it.downloadId = downloadId
Log.d(tag, "checkUpdateDownloadQueue: Starting download item part, downloadId=$downloadId")
currentDownloadItemParts.add(it)
}
}
if (nextDownloadItemParts.isNotEmpty()) {
processDownloadItemParts(nextDownloadItemParts)
}
if (currentDownloadItemParts.size >= maxSimultaneousDownloads) {
@ -107,9 +94,59 @@ class DownloadItemManager(var downloadManager:DownloadManager, private var folde
}
}
if (currentDownloadItemParts.size > 0) startWatchingDownloads()
if (currentDownloadItemParts.isNotEmpty()) startWatchingDownloads()
}
/** Processes the download item parts. */
private fun processDownloadItemParts(nextDownloadItemParts: List<DownloadItemPart>) {
nextDownloadItemParts.forEach {
if (it.isInternalStorage) {
startInternalDownload(it)
} else {
startExternalDownload(it)
}
}
}
/** Starts an internal download. */
private fun startInternalDownload(downloadItemPart: DownloadItemPart) {
val file = File(downloadItemPart.finalDestinationPath)
file.parentFile?.mkdirs()
val fileOutputStream = FileOutputStream(downloadItemPart.finalDestinationPath)
val internalProgressCallback =
object : InternalProgressCallback {
override fun onProgress(totalBytesWritten: Long, progress: Long) {
downloadItemPart.bytesDownloaded = totalBytesWritten
downloadItemPart.progress = progress
}
override fun onComplete(failed: Boolean) {
downloadItemPart.failed = failed
downloadItemPart.completed = true
}
}
Log.d(
tag,
"Start internal download to destination path ${downloadItemPart.finalDestinationPath} from ${downloadItemPart.serverUrl}"
)
InternalDownloadManager(fileOutputStream, internalProgressCallback)
.download(downloadItemPart.serverUrl)
downloadItemPart.downloadId = 1
currentDownloadItemParts.add(downloadItemPart)
}
/** Starts an external download. */
private fun startExternalDownload(downloadItemPart: DownloadItemPart) {
val dlRequest = downloadItemPart.getDownloadRequest()
val downloadId = downloadManager.enqueue(dlRequest)
downloadItemPart.downloadId = downloadId
Log.d(tag, "checkUpdateDownloadQueue: Starting download item part, downloadId=$downloadId")
currentDownloadItemParts.add(downloadItemPart)
}
/** Starts watching the downloads. */
private fun startWatchingDownloads() {
if (isDownloading) return // Already watching
@ -117,25 +154,13 @@ class DownloadItemManager(var downloadManager:DownloadManager, private var folde
Log.d(tag, "Starting watching downloads")
isDownloading = true
while (currentDownloadItemParts.size > 0) {
val itemParts = currentDownloadItemParts.filter { !it.isMoving }.map { it }
while (currentDownloadItemParts.isNotEmpty()) {
val itemParts = currentDownloadItemParts.filter { !it.isMoving }
for (downloadItemPart in itemParts) {
if (downloadItemPart.isInternalStorage) {
clientEventEmitter.onDownloadItemPartUpdate(downloadItemPart)
if (downloadItemPart.completed) {
val downloadItem = downloadItemQueue.find { it.id == downloadItemPart.downloadItemId }
downloadItem?.let {
checkDownloadItemFinished(it)
}
currentDownloadItemParts.remove(downloadItemPart)
}
handleInternalDownloadPart(downloadItemPart)
} else {
val downloadCheckStatus = checkDownloadItemPart(downloadItemPart)
clientEventEmitter.onDownloadItemPartUpdate(downloadItemPart)
// Will move to final destination, remove current item parts, and check if download item is finished
handleDownloadItemPartCheck(downloadCheckStatus, downloadItemPart)
handleExternalDownloadPart(downloadItemPart)
}
}
@ -151,7 +176,29 @@ class DownloadItemManager(var downloadManager:DownloadManager, private var folde
}
}
private fun checkDownloadItemPart(downloadItemPart:DownloadItemPart):DownloadCheckStatus {
/** Handles an internal download part. */
private fun handleInternalDownloadPart(downloadItemPart: DownloadItemPart) {
clientEventEmitter.onDownloadItemPartUpdate(downloadItemPart)
if (downloadItemPart.completed) {
val downloadItem = downloadItemQueue.find { it.id == downloadItemPart.downloadItemId }
downloadItem?.let { checkDownloadItemFinished(it) }
currentDownloadItemParts.remove(downloadItemPart)
}
}
/** Handles an external download part. */
private fun handleExternalDownloadPart(downloadItemPart: DownloadItemPart) {
val downloadCheckStatus = checkDownloadItemPart(downloadItemPart)
clientEventEmitter.onDownloadItemPartUpdate(downloadItemPart)
// Will move to final destination, remove current item parts, and check if download item is
// finished
handleDownloadItemPartCheck(downloadCheckStatus, downloadItemPart)
}
/** Checks the status of a download item part. */
private fun checkDownloadItemPart(downloadItemPart: DownloadItemPart): DownloadCheckStatus {
val downloadId = downloadItemPart.downloadId ?: return DownloadCheckStatus.Failed
val query = DownloadManager.Query().setFilterById(downloadId)
@ -159,12 +206,17 @@ class DownloadItemManager(var downloadManager:DownloadManager, private var folde
if (it.moveToFirst()) {
val bytesColumnIndex = it.getColumnIndex(DownloadManager.COLUMN_TOTAL_SIZE_BYTES)
val statusColumnIndex = it.getColumnIndex(DownloadManager.COLUMN_STATUS)
val bytesDownloadedColumnIndex = it.getColumnIndex(DownloadManager.COLUMN_BYTES_DOWNLOADED_SO_FAR)
val bytesDownloadedColumnIndex =
it.getColumnIndex(DownloadManager.COLUMN_BYTES_DOWNLOADED_SO_FAR)
val totalBytes = if (bytesColumnIndex >= 0) it.getInt(bytesColumnIndex) else 0
val downloadStatus = if (statusColumnIndex >= 0) it.getInt(statusColumnIndex) else 0
val bytesDownloadedSoFar = if (bytesDownloadedColumnIndex >= 0) it.getLong(bytesDownloadedColumnIndex) else 0
Log.d(tag, "checkDownloads Download ${downloadItemPart.filename} bytes $totalBytes | bytes dled $bytesDownloadedSoFar | downloadStatus $downloadStatus")
val bytesDownloadedSoFar =
if (bytesDownloadedColumnIndex >= 0) it.getLong(bytesDownloadedColumnIndex) else 0
Log.d(
tag,
"checkDownloads Download ${downloadItemPart.filename} bytes $totalBytes | bytes dled $bytesDownloadedSoFar | downloadStatus $downloadStatus"
)
return when (downloadStatus) {
DownloadManager.STATUS_SUCCESSFUL -> {
@ -183,8 +235,12 @@ class DownloadItemManager(var downloadManager:DownloadManager, private var folde
DownloadCheckStatus.Failed
}
else -> {
val percentProgress = if (totalBytes > 0) ((bytesDownloadedSoFar * 100L) / totalBytes) else 0
Log.d(tag, "checkDownloads Download ${downloadItemPart.filename} Progress = $percentProgress%")
val percentProgress =
if (totalBytes > 0) ((bytesDownloadedSoFar * 100L) / totalBytes) else 0
Log.d(
tag,
"checkDownloads Download ${downloadItemPart.filename} Progress = $percentProgress%"
)
downloadItemPart.progress = percentProgress
downloadItemPart.bytesDownloaded = bytesDownloadedSoFar
@ -200,84 +256,120 @@ class DownloadItemManager(var downloadManager:DownloadManager, private var folde
}
}
private fun handleDownloadItemPartCheck(downloadCheckStatus:DownloadCheckStatus, downloadItemPart:DownloadItemPart) {
/** Handles the result of a download item part check. */
private fun handleDownloadItemPartCheck(
downloadCheckStatus: DownloadCheckStatus,
downloadItemPart: DownloadItemPart
) {
val downloadItem = downloadItemQueue.find { it.id == downloadItemPart.downloadItemId }
if (downloadItem == null) {
Log.e(tag, "Download item part finished but download item not found ${downloadItemPart.filename}")
Log.e(
tag,
"Download item part finished but download item not found ${downloadItemPart.filename}"
)
currentDownloadItemParts.remove(downloadItemPart)
} else if (downloadCheckStatus == DownloadCheckStatus.Successful) {
val file = DocumentFileCompat.fromUri(mainActivity, downloadItemPart.destinationUri)
Log.d(tag, "DOWNLOAD: DESTINATION URI ${downloadItemPart.destinationUri}")
val fcb = object : FileCallback() {
override fun onPrepare() {
Log.d(tag, "DOWNLOAD: PREPARING MOVE FILE")
}
override fun onFailed(errorCode: ErrorCode) {
Log.e(tag, "DOWNLOAD: FAILED TO MOVE FILE $errorCode")
downloadItemPart.failed = true
downloadItemPart.isMoving = false
file?.delete()
checkDownloadItemFinished(downloadItem)
currentDownloadItemParts.remove(downloadItemPart)
}
override fun onCompleted(result:Any) {
Log.d(tag, "DOWNLOAD: FILE MOVE COMPLETED")
val resultDocFile = result as DocumentFile
Log.d(tag, "DOWNLOAD: COMPLETED FILE INFO (name=${resultDocFile.name}) ${resultDocFile.getAbsolutePath(mainActivity)}")
// Rename to fix appended .mp3 on m4b/m4a files
// REF: https://github.com/anggrayudi/SimpleStorage/issues/94
val docNameLowerCase = resultDocFile.name?.lowercase(Locale.getDefault()) ?: ""
if (docNameLowerCase.endsWith(".m4b.mp3")|| docNameLowerCase.endsWith(".m4a.mp3")) {
resultDocFile.renameTo(downloadItemPart.filename)
}
downloadItemPart.moved = true
downloadItemPart.isMoving = false
checkDownloadItemFinished(downloadItem)
currentDownloadItemParts.remove(downloadItemPart)
}
}
val localFolderFile = DocumentFileCompat.fromUri(mainActivity, Uri.parse(downloadItemPart.localFolderUrl))
if (localFolderFile == null) {
// fAILED
downloadItemPart.failed = true
Log.e(tag, "Local Folder File from uri is null")
checkDownloadItemFinished(downloadItem)
currentDownloadItemParts.remove(downloadItemPart)
} else {
downloadItemPart.isMoving = true
val mimetype = if (downloadItemPart.audioTrack != null) MimeType.AUDIO else MimeType.IMAGE
val fileDescription = FileDescription(downloadItemPart.filename, downloadItemPart.finalDestinationSubfolder, mimetype)
file?.moveFileTo(mainActivity, localFolderFile, fileDescription, fcb)
}
moveDownloadedFile(downloadItem, downloadItemPart)
} else if (downloadCheckStatus != DownloadCheckStatus.InProgress) {
checkDownloadItemFinished(downloadItem)
currentDownloadItemParts.remove(downloadItemPart)
}
}
private fun checkDownloadItemFinished(downloadItem:DownloadItem) {
/** Moves the downloaded file to its final destination. */
private fun moveDownloadedFile(downloadItem: DownloadItem, downloadItemPart: DownloadItemPart) {
val file = DocumentFileCompat.fromUri(mainActivity, downloadItemPart.destinationUri)
Log.d(tag, "DOWNLOAD: DESTINATION URI ${downloadItemPart.destinationUri}")
val fcb =
object : FileCallback() {
override fun onPrepare() {
Log.d(tag, "DOWNLOAD: PREPARING MOVE FILE")
}
override fun onFailed(errorCode: ErrorCode) {
Log.e(tag, "DOWNLOAD: FAILED TO MOVE FILE $errorCode")
downloadItemPart.failed = true
downloadItemPart.isMoving = false
file?.delete()
checkDownloadItemFinished(downloadItem)
currentDownloadItemParts.remove(downloadItemPart)
}
override fun onCompleted(result: Any) {
Log.d(tag, "DOWNLOAD: FILE MOVE COMPLETED")
val resultDocFile = result as DocumentFile
Log.d(
tag,
"DOWNLOAD: COMPLETED FILE INFO (name=${resultDocFile.name}) ${resultDocFile.getAbsolutePath(mainActivity)}"
)
// Rename to fix appended .mp3 on m4b/m4a files
// REF: https://github.com/anggrayudi/SimpleStorage/issues/94
val docNameLowerCase = resultDocFile.name?.lowercase(Locale.getDefault()) ?: ""
if (docNameLowerCase.endsWith(".m4b.mp3") || docNameLowerCase.endsWith(".m4a.mp3")
) {
resultDocFile.renameTo(downloadItemPart.filename)
}
downloadItemPart.moved = true
downloadItemPart.isMoving = false
checkDownloadItemFinished(downloadItem)
currentDownloadItemParts.remove(downloadItemPart)
}
}
val localFolderFile =
DocumentFileCompat.fromUri(mainActivity, Uri.parse(downloadItemPart.localFolderUrl))
if (localFolderFile == null) {
// Failed
downloadItemPart.failed = true
Log.e(tag, "Local Folder File from uri is null")
checkDownloadItemFinished(downloadItem)
currentDownloadItemParts.remove(downloadItemPart)
} else {
downloadItemPart.isMoving = true
val mimetype = if (downloadItemPart.audioTrack != null) MimeType.AUDIO else MimeType.IMAGE
val fileDescription =
FileDescription(
downloadItemPart.filename,
downloadItemPart.finalDestinationSubfolder,
mimetype
)
file?.moveFileTo(mainActivity, localFolderFile, fileDescription, fcb)
}
}
/** Checks if a download item is finished and processes it. */
private fun checkDownloadItemFinished(downloadItem: DownloadItem) {
if (downloadItem.isDownloadFinished) {
Log.i(tag, "Download Item finished ${downloadItem.media.metadata.title}")
GlobalScope.launch(Dispatchers.IO) {
folderScanner.scanDownloadItem(downloadItem) { downloadItemScanResult ->
Log.d(tag, "Item download complete ${downloadItem.itemTitle} | local library item id: ${downloadItemScanResult?.localLibraryItem?.id}")
Log.d(
tag,
"Item download complete ${downloadItem.itemTitle} | local library item id: ${downloadItemScanResult?.localLibraryItem?.id}"
)
val jsobj = JSObject()
jsobj.put("libraryItemId", downloadItem.id)
jsobj.put("localFolderId", downloadItem.localFolder.id)
val jsobj =
JSObject().apply {
put("libraryItemId", downloadItem.id)
put("localFolderId", downloadItem.localFolder.id)
downloadItemScanResult?.localLibraryItem?.let { localLibraryItem ->
jsobj.put("localLibraryItem", JSObject(jacksonMapper.writeValueAsString(localLibraryItem)))
}
downloadItemScanResult?.localMediaProgress?.let { localMediaProgress ->
jsobj.put("localMediaProgress", JSObject(jacksonMapper.writeValueAsString(localMediaProgress)))
}
downloadItemScanResult?.localLibraryItem?.let { localLibraryItem ->
put(
"localLibraryItem",
JSObject(jacksonMapper.writeValueAsString(localLibraryItem))
)
}
downloadItemScanResult?.localMediaProgress?.let { localMediaProgress ->
put(
"localMediaProgress",
JSObject(jacksonMapper.writeValueAsString(localMediaProgress))
)
}
}
launch(Dispatchers.Main) {
clientEventEmitter.onDownloadItemComplete(jsobj)

View file

@ -1,60 +1,95 @@
package com.audiobookshelf.app.managers
import android.util.Log
import com.google.common.net.HttpHeaders.CONTENT_LENGTH
import okhttp3.*
import java.io.*
import java.util.*
import java.util.concurrent.TimeUnit
import okhttp3.*
/**
* Manages the internal download process.
*
* @property outputStream The output stream to write the downloaded data.
* @property progressCallback The callback to report download progress.
*/
class InternalDownloadManager(
private val outputStream: FileOutputStream,
private val progressCallback: DownloadItemManager.InternalProgressCallback
) : AutoCloseable {
class InternalDownloadManager(outputStream:FileOutputStream, private val progressCallback: DownloadItemManager.InternalProgressCallback) : AutoCloseable {
private val tag = "InternalDownloadManager"
private val client: OkHttpClient = OkHttpClient()
private val client: OkHttpClient =
OkHttpClient.Builder().connectTimeout(30, TimeUnit.SECONDS).build()
private val writer = BinaryFileWriter(outputStream, progressCallback)
/**
* Downloads a file from the given URL.
*
* @param url The URL to download the file from.
* @throws IOException If an I/O error occurs.
*/
@Throws(IOException::class)
fun download(url:String) {
fun download(url: String) {
val request: Request = Request.Builder().url(url).build()
client.newCall(request).enqueue(object : Callback {
override fun onFailure(call: Call, e: IOException) {
Log.e(tag, "download URL $url FAILED")
progressCallback.onComplete(true)
}
client.newCall(request)
.enqueue(
object : Callback {
override fun onFailure(call: Call, e: IOException) {
Log.e(tag, "Download URL $url FAILED", e)
progressCallback.onComplete(true)
}
override fun onResponse(call: Call, response: Response) {
val responseBody: ResponseBody = response.body
?: throw IllegalStateException("Response doesn't contain a file")
val length: Long = (response.header(CONTENT_LENGTH, "1") ?: "0").toLong()
writer.write(responseBody.byteStream(), length)
}
})
override fun onResponse(call: Call, response: Response) {
response.body?.let { responseBody ->
val length: Long = response.header("Content-Length")?.toLongOrNull() ?: 0L
writer.write(responseBody.byteStream(), length)
}
?: run {
Log.e(tag, "Response doesn't contain a file")
progressCallback.onComplete(true)
}
}
}
)
}
/**
* Closes the download manager and releases resources.
*
* @throws Exception If an error occurs during closing.
*/
@Throws(Exception::class)
override fun close() {
writer.close()
}
}
class BinaryFileWriter(outputStream: OutputStream, progressCallback: DownloadItemManager.InternalProgressCallback) :
AutoCloseable {
private val outputStream: OutputStream
private val progressCallback: DownloadItemManager.InternalProgressCallback
init {
this.outputStream = outputStream
this.progressCallback = progressCallback
}
/**
* Writes binary data to an output stream.
*
* @property outputStream The output stream to write the data to.
* @property progressCallback The callback to report write progress.
*/
class BinaryFileWriter(
private val outputStream: OutputStream,
private val progressCallback: DownloadItemManager.InternalProgressCallback
) : AutoCloseable {
/**
* Writes data from the input stream to the output stream.
*
* @param inputStream The input stream to read the data from.
* @param length The total length of the data to be written.
* @return The total number of bytes written.
* @throws IOException If an I/O error occurs.
*/
@Throws(IOException::class)
fun write(inputStream: InputStream?, length: Long): Long {
fun write(inputStream: InputStream, length: Long): Long {
BufferedInputStream(inputStream).use { input ->
val dataBuffer = ByteArray(CHUNK_SIZE)
var readBytes: Int
var totalBytes: Long = 0
var readBytes: Int
while (input.read(dataBuffer).also { readBytes = it } != -1) {
totalBytes += readBytes.toLong()
totalBytes += readBytes
outputStream.write(dataBuffer, 0, readBytes)
progressCallback.onProgress(totalBytes, (totalBytes * 100L) / length)
}
@ -63,12 +98,17 @@ class BinaryFileWriter(outputStream: OutputStream, progressCallback: DownloadIte
}
}
/**
* Closes the writer and releases resources.
*
* @throws IOException If an error occurs during closing.
*/
@Throws(IOException::class)
override fun close() {
outputStream.close()
}
companion object {
private const val CHUNK_SIZE = 1024
private const val CHUNK_SIZE = 8192 // Increased chunk size for better performance
}
}