Fix crashes related to Realm threading

This commit is contained in:
ronaldheft 2022-08-16 12:32:22 -04:00
parent 8ce0d9ce56
commit b0905d0270
8 changed files with 236 additions and 75 deletions

View file

@ -7,6 +7,7 @@
import Foundation
import Capacitor
import RealmSwift
@objc(AbsAudioPlayer)
public class AbsAudioPlayer: CAPPlugin {
@ -47,7 +48,11 @@ public class AbsAudioPlayer: CAPPlugin {
if (isLocalItem) {
let item = Database.shared.getLocalLibraryItem(localLibraryItemId: libraryItemId!)
let episode = item?.getPodcastEpisode(episodeId: episodeId)
let playbackSession = item?.getPlaybackSession(episode: episode)
guard let playbackSession = item?.getPlaybackSession(episode: episode) else {
NSLog("Failed to get local playback session")
return call.resolve([:])
}
playbackSession.save()
sendPrepareMetadataEvent(itemId: libraryItemId!, playWhenReady: playWhenReady)
do {
self.sendPlaybackSession(session: try playbackSession.asDictionary())
@ -57,7 +62,7 @@ public class AbsAudioPlayer: CAPPlugin {
debugPrint(exception)
call.resolve([:])
}
PlayerHandler.startPlayback(session: playbackSession!, playWhenReady: playWhenReady, playbackRate: playbackRate)
PlayerHandler.startPlayback(sessionId: playbackSession.id, playWhenReady: playWhenReady, playbackRate: playbackRate)
self.sendMetadata()
} else { // Playing from the server
sendPrepareMetadataEvent(itemId: libraryItemId!, playWhenReady: playWhenReady)
@ -71,7 +76,8 @@ public class AbsAudioPlayer: CAPPlugin {
call.resolve([:])
}
PlayerHandler.startPlayback(session: session, playWhenReady: playWhenReady, playbackRate: playbackRate)
session.save()
PlayerHandler.startPlayback(sessionId: session.id, playWhenReady: playWhenReady, playbackRate: playbackRate)
self.sendMetadata()
}
}
@ -197,14 +203,15 @@ public class AbsAudioPlayer: CAPPlugin {
@objc func onPlaybackFailed() {
if (PlayerHandler.getPlayMethod() == PlayMethod.directplay.rawValue) {
let playbackSession = PlayerHandler.getPlaybackSession()
let libraryItemId = playbackSession?.libraryItemId ?? ""
let episodeId = playbackSession?.episodeId ?? nil
let session = PlayerHandler.getPlaybackSession()
let libraryItemId = session?.libraryItemId ?? ""
let episodeId = session?.episodeId ?? nil
NSLog("Forcing Transcode")
// If direct playing then fallback to transcode
ApiClient.startPlaybackSession(libraryItemId: libraryItemId, episodeId: episodeId, forceTranscode: true) { session in
PlayerHandler.startPlayback(session: session, playWhenReady: self.initialPlayWhenReady, playbackRate: self.initialPlaybackRate)
session.save()
PlayerHandler.startPlayback(sessionId: session.id, playWhenReady: self.initialPlayWhenReady, playbackRate: self.initialPlaybackRate)
do {
self.sendPlaybackSession(session: try session.asDictionary())

View file

@ -74,14 +74,14 @@ extension LocalLibraryItem {
let mediaProgress = Database.shared.getLocalMediaProgress(localMediaProgressId: mediaProgressId)
let mediaMetadata = self.media?.metadata
let chapters = Array(self.media?.chapters ?? List<Chapter>())
let chapters = self.media?.chapters ?? List<Chapter>()
let authorName = mediaMetadata?.authorDisplayName
var audioTracks = [AudioTrack]()
let audioTracks = List<AudioTrack>()
if let episode = episode, let track = episode.audioTrack {
audioTracks.append(track)
} else if let tracks = self.media?.tracks {
audioTracks.append(contentsOf: tracks)
audioTracks.append(objectsIn: tracks)
}
let dateNow = Date().timeIntervalSince1970
@ -175,35 +175,41 @@ extension LocalMediaProgress {
}
func updateIsFinished(_ finished: Bool) {
if self.isFinished != finished {
self.progress = finished ? 1.0 : 0.0
}
try! Realm().write {
if self.isFinished != finished {
self.progress = finished ? 1.0 : 0.0
}
if self.startedAt == 0 && finished {
self.startedAt = Int(Date().timeIntervalSince1970)
if self.startedAt == 0 && finished {
self.startedAt = Int(Date().timeIntervalSince1970)
}
self.isFinished = finished
self.lastUpdate = Int(Date().timeIntervalSince1970)
self.finishedAt = finished ? lastUpdate : nil
}
self.isFinished = finished
self.lastUpdate = Int(Date().timeIntervalSince1970)
self.finishedAt = finished ? lastUpdate : nil
}
func updateFromPlaybackSession(_ playbackSession: PlaybackSession) {
self.currentTime = playbackSession.currentTime
self.progress = playbackSession.progress
self.lastUpdate = Int(Date().timeIntervalSince1970)
self.isFinished = playbackSession.progress >= 100.0
self.finishedAt = self.isFinished ? self.lastUpdate : nil
try! Realm().write {
self.currentTime = playbackSession.currentTime
self.progress = playbackSession.progress
self.lastUpdate = Int(Date().timeIntervalSince1970)
self.isFinished = playbackSession.progress >= 100.0
self.finishedAt = self.isFinished ? self.lastUpdate : nil
}
}
func updateFromServerMediaProgress(_ serverMediaProgress: MediaProgress) {
self.isFinished = serverMediaProgress.isFinished
self.progress = serverMediaProgress.progress
self.currentTime = serverMediaProgress.currentTime
self.duration = serverMediaProgress.duration
self.lastUpdate = serverMediaProgress.lastUpdate
self.finishedAt = serverMediaProgress.finishedAt
self.startedAt = serverMediaProgress.startedAt
try! Realm().write {
self.isFinished = serverMediaProgress.isFinished
self.progress = serverMediaProgress.progress
self.currentTime = serverMediaProgress.currentTime
self.duration = serverMediaProgress.duration
self.lastUpdate = serverMediaProgress.lastUpdate
self.finishedAt = serverMediaProgress.finishedAt
self.startedAt = serverMediaProgress.startedAt
}
}
static func fetchOrCreateLocalMediaProgress(localMediaProgressId: String?, localLibraryItemId: String?, localEpisodeId: String?) -> LocalMediaProgress? {

View file

@ -6,29 +6,30 @@
//
import Foundation
import RealmSwift
struct PlaybackSession: Codable {
var id: String
class PlaybackSession: Object, Codable {
@Persisted(primaryKey: true) var id: String = ""
var userId: String?
var libraryItemId: String?
var episodeId: String?
var mediaType: String
@Persisted var libraryItemId: String?
@Persisted var episodeId: String?
@Persisted var mediaType: String = ""
// var mediaMetadata: MediaTypeMetadata - It is not implemented in android?
var chapters: [Chapter]
var displayTitle: String?
var displayAuthor: String?
var coverPath: String?
var duration: Double
var playMethod: Int
var startedAt: Double?
var updatedAt: Double?
var timeListening: Double
var audioTracks: [AudioTrack]
var currentTime: Double
var libraryItem: LibraryItem?
var localLibraryItem: LocalLibraryItem?
var serverConnectionConfigId: String?
var serverAddress: String?
@Persisted var chapters = List<Chapter>()
@Persisted var displayTitle: String?
@Persisted var displayAuthor: String?
@Persisted var coverPath: String?
@Persisted var duration: Double = 0
@Persisted var playMethod: Int = 1
@Persisted var startedAt: Double?
@Persisted var updatedAt: Double?
@Persisted var timeListening: Double = 0
@Persisted var audioTracks = List<AudioTrack>()
@Persisted var currentTime: Double = 0
@Persisted var libraryItem: LibraryItem?
@Persisted var localLibraryItem: LocalLibraryItem?
@Persisted var serverConnectionConfigId: String?
@Persisted var serverAddress: String?
var isLocal: Bool { self.localLibraryItem != nil }
@ -47,4 +48,88 @@ struct PlaybackSession: Codable {
}
var progress: Double { self.currentTime / self.totalDuration }
internal init(id: String, userId: String? = nil, libraryItemId: String? = nil, episodeId: String? = nil, mediaType: String, chapters: List<Chapter> = List<Chapter>(), displayTitle: String? = nil, displayAuthor: String? = nil, coverPath: String? = nil, duration: Double, playMethod: Int, startedAt: Double? = nil, updatedAt: Double? = nil, timeListening: Double, audioTracks: List<AudioTrack> = List<AudioTrack>(), currentTime: Double, libraryItem: LibraryItem? = nil, localLibraryItem: LocalLibraryItem? = nil, serverConnectionConfigId: String? = nil, serverAddress: String? = nil) {
self.id = id
self.userId = userId
self.libraryItemId = libraryItemId
self.episodeId = episodeId
self.mediaType = mediaType
self.chapters = chapters
self.displayTitle = displayTitle
self.displayAuthor = displayAuthor
self.coverPath = coverPath
self.duration = duration
self.playMethod = playMethod
self.startedAt = startedAt
self.updatedAt = updatedAt
self.timeListening = timeListening
self.audioTracks = audioTracks
self.currentTime = currentTime
self.libraryItem = libraryItem
self.localLibraryItem = localLibraryItem
self.serverConnectionConfigId = serverConnectionConfigId
self.serverAddress = serverAddress
}
private enum CodingKeys : String, CodingKey {
case id, userId, libraryItemId, episodeId, mediaType, chapters, displayTitle, displayAuthor, coverPath, duration, playMethod, startedAt, updatedAt, timeListening, audioTracks, currentTime, libraryItem, localLibraryItem, serverConnectionConfigId, serverAddress, isLocal, localMediaProgressId
}
override init() {}
required init(from decoder: Decoder) throws {
super.init()
let values = try decoder.container(keyedBy: CodingKeys.self)
id = try values.decode(String.self, forKey: .id)
userId = try values.decodeIfPresent(String.self, forKey: .userId)
libraryItemId = try values.decodeIfPresent(String.self, forKey: .libraryItemId)
episodeId = try values.decodeIfPresent(String.self, forKey: .episodeId)
mediaType = try values.decode(String.self, forKey: .mediaType)
if let chapterList = try values.decodeIfPresent([Chapter].self, forKey: .chapters) {
chapters.append(objectsIn: chapterList)
}
displayTitle = try values.decodeIfPresent(String.self, forKey: .displayTitle)
displayAuthor = try values.decodeIfPresent(String.self, forKey: .displayAuthor)
coverPath = try values.decodeIfPresent(String.self, forKey: .coverPath)
duration = try values.decode(Double.self, forKey: .duration)
playMethod = try values.decode(Int.self, forKey: .playMethod)
startedAt = try values.decodeIfPresent(Double.self, forKey: .startedAt)
updatedAt = try values.decodeIfPresent(Double.self, forKey: .updatedAt)
timeListening = try values.decode(Double.self, forKey: .timeListening)
if let trackList = try values.decodeIfPresent([AudioTrack].self, forKey: .audioTracks) {
audioTracks.append(objectsIn: trackList)
}
currentTime = try values.decode(Double.self, forKey: .currentTime)
libraryItem = try values.decodeIfPresent(LibraryItem.self, forKey: .libraryItem)
localLibraryItem = try values.decodeIfPresent(LocalLibraryItem.self, forKey: .localLibraryItem)
serverConnectionConfigId = try values.decodeIfPresent(String.self, forKey: .serverConnectionConfigId)
serverAddress = try values.decodeIfPresent(String.self, forKey: .serverAddress)
}
func encode(to encoder: Encoder) throws {
var container = encoder.container(keyedBy: CodingKeys.self)
try container.encode(id, forKey: .id)
try container.encode(userId, forKey: .userId)
try container.encode(libraryItemId, forKey: .libraryItemId)
try container.encode(episodeId, forKey: .episodeId)
try container.encode(mediaType, forKey: .mediaType)
try container.encode(Array(chapters), forKey: .chapters)
try container.encode(displayTitle, forKey: .displayTitle)
try container.encode(displayAuthor, forKey: .displayAuthor)
try container.encode(coverPath, forKey: .coverPath)
try container.encode(duration, forKey: .duration)
try container.encode(playMethod, forKey: .playMethod)
try container.encode(startedAt, forKey: .startedAt)
try container.encode(updatedAt, forKey: .updatedAt)
try container.encode(timeListening, forKey: .timeListening)
try container.encode(Array(audioTracks), forKey: .audioTracks)
try container.encode(currentTime, forKey: .currentTime)
try container.encode(libraryItem, forKey: .libraryItem)
try container.encode(localLibraryItem, forKey: .localLibraryItem)
try container.encode(serverConnectionConfigId, forKey: .serverConnectionConfigId)
try container.encode(serverAddress, forKey: .serverAddress)
try container.encode(isLocal, forKey: .isLocal)
try container.encode(localMediaProgressId, forKey: .localMediaProgressId)
}
}

View file

@ -0,0 +1,41 @@
//
// ActivePlaybackSession.swift
// App
//
// Created by Ron Heft on 8/16/22.
//
import Foundation
import RealmSwift
class ActivePlaybackSession {
static let shared = ActivePlaybackSession()
private let queue = DispatchQueue(label: "ABSActivePlaybackSession")
private var _session: PlaybackSession?
private init() {
// Singleton
}
func startSession(_ session: ThreadSafeReference<PlaybackSession>) {
queue.sync {
_session = try? Realm().resolve(session)
}
}
// This is a funky method, but it ensures the accessing thread gets a live reference to session properly resolved
func get() -> PlaybackSession? {
var activeSession: ThreadSafeReference<PlaybackSession>?
queue.sync {
let realm = try! Realm()
guard let session = _session else { return }
r
activeSession = ThreadSafeReference(to: session)
}
guard let activeSession = activeSession else { return nil }
return try? Realm().resolve(activeSession)
}
}

View file

@ -32,7 +32,7 @@ class AudioPlayer: NSObject {
private var initialPlaybackRate: Float
private var audioPlayer: AVQueuePlayer
private var playbackSession: PlaybackSession
private var sessionId: String
private var queueObserver:NSKeyValueObservation?
private var queueItemStatusObserver:NSKeyValueObservation?
@ -41,12 +41,12 @@ class AudioPlayer: NSObject {
private var allPlayerItems:[AVPlayerItem] = []
// MARK: - Constructor
init(playbackSession: PlaybackSession, playWhenReady: Bool = false, playbackRate: Float = 1) {
init(sessionId: String, playWhenReady: Bool = false, playbackRate: Float = 1) {
self.playWhenReady = playWhenReady
self.initialPlaybackRate = playbackRate
self.audioPlayer = AVQueuePlayer()
self.audioPlayer.automaticallyWaitsToMinimizeStalling = false
self.playbackSession = playbackSession
self.sessionId = sessionId
self.status = -1
self.rate = 0.0
self.tmpRate = playbackRate
@ -56,6 +56,8 @@ class AudioPlayer: NSObject {
initAudioSession()
setupRemoteTransportControls()
let playbackSession = Database.shared.getPlaybackSession(id: self.sessionId)!
// Listen to player events
self.audioPlayer.addObserver(self, forKeyPath: #keyPath(AVPlayer.rate), options: .new, context: &playerContext)
self.audioPlayer.addObserver(self, forKeyPath: #keyPath(AVPlayer.currentItem), options: .new, context: &playerContext)
@ -106,6 +108,7 @@ class AudioPlayer: NSObject {
}
func getItemIndexForTime(time:Double) -> Int {
let playbackSession = Database.shared.getPlaybackSession(id: self.sessionId)!
for index in 0..<self.allPlayerItems.count {
let startOffset = playbackSession.audioTracks[index].startOffset ?? 0.0
let duration = playbackSession.audioTracks[index].duration
@ -132,6 +135,7 @@ class AudioPlayer: NSObject {
func setupQueueItemStatusObserver() {
self.queueItemStatusObserver?.invalidate()
self.queueItemStatusObserver = self.audioPlayer.currentItem?.observe(\.status, options: [.new, .old], changeHandler: { (playerItem, change) in
let playbackSession = Database.shared.getPlaybackSession(id: self.sessionId)!
if (playerItem.status == .readyToPlay) {
NSLog("queueStatusObserver: Current Item Ready to play. PlayWhenReady: \(self.playWhenReady)")
self.updateNowPlaying()
@ -139,11 +143,11 @@ class AudioPlayer: NSObject {
let firstReady = self.status < 0
self.status = 0
if self.playWhenReady {
self.seek(self.playbackSession.currentTime, from: "queueItemStatusObserver")
self.seek(playbackSession.currentTime, from: "queueItemStatusObserver")
self.playWhenReady = false
self.play()
} else if (firstReady) { // Only seek on first readyToPlay
self.seek(self.playbackSession.currentTime, from: "queueItemStatusObserver")
self.seek(playbackSession.currentTime, from: "queueItemStatusObserver")
}
} else if (playerItem.status == .failed) {
NSLog("queueStatusObserver: FAILED \(playerItem.error?.localizedDescription ?? "")")
@ -205,7 +209,9 @@ class AudioPlayer: NSObject {
NSLog("Seek to \(to) from \(from)")
let currentTrack = self.playbackSession.audioTracks[self.currentTrackIndex]
let playbackSession = Database.shared.getPlaybackSession(id: self.sessionId)!
let currentTrack = playbackSession.audioTracks[self.currentTrackIndex]
let ctso = currentTrack.startOffset ?? 0.0
let trackEnd = ctso + currentTrack.duration
NSLog("Seek current track END = \(trackEnd)")
@ -218,7 +224,9 @@ class AudioPlayer: NSObject {
if (self.currentTrackIndex != indexOfSeek) {
self.currentTrackIndex = indexOfSeek
self.playbackSession.currentTime = to
playbackSession.update {
playbackSession.currentTime = to
}
self.playWhenReady = continuePlaying // Only playWhenReady if already playing
self.status = -1
@ -232,7 +240,7 @@ class AudioPlayer: NSObject {
setupQueueItemStatusObserver()
} else {
NSLog("Seeking in current item \(to)")
let currentTrackStartOffset = self.playbackSession.audioTracks[self.currentTrackIndex].startOffset ?? 0.0
let currentTrackStartOffset = playbackSession.audioTracks[self.currentTrackIndex].startOffset ?? 0.0
let seekTime = to - currentTrackStartOffset
self.audioPlayer.seek(to: CMTime(seconds: seekTime, preferredTimescale: 1000)) { completed in
@ -262,6 +270,7 @@ class AudioPlayer: NSObject {
}
public func getCurrentTime() -> Double {
let playbackSession = Database.shared.getPlaybackSession(id: self.sessionId)!
let currentTrackTime = self.audioPlayer.currentTime().seconds
let audioTrack = playbackSession.audioTracks[currentTrackIndex]
let startOffset = audioTrack.startOffset ?? 0.0
@ -269,17 +278,20 @@ class AudioPlayer: NSObject {
}
public func getPlayMethod() -> Int {
return self.playbackSession.playMethod
let playbackSession = Database.shared.getPlaybackSession(id: self.sessionId)!
return playbackSession.playMethod
}
public func getPlaybackSession() -> PlaybackSession {
return self.playbackSession
public func getPlaybackSessionId() -> String {
return self.sessionId
}
public func getDuration() -> Double {
let playbackSession = Database.shared.getPlaybackSession(id: self.sessionId)!
return playbackSession.duration
}
// MARK: - Private
private func createAsset(itemId:String, track:AudioTrack) -> AVAsset {
let playbackSession = Database.shared.getPlaybackSession(id: self.sessionId)!
if (playbackSession.playMethod == PlayMethod.directplay.rawValue) {
// The only reason this is separate is because the filename needs to be encoded
let filename = track.metadata?.filename ?? ""

View file

@ -6,10 +6,10 @@
//
import Foundation
import RealmSwift
class PlayerHandler {
private static var player: AudioPlayer?
private static var session: PlaybackSession?
private static var timer: Timer?
private static var lastSyncTime: Double = 0.0
@ -68,7 +68,8 @@ class PlayerHandler {
timer = nil
}
public static func startPlayback(session: PlaybackSession, playWhenReady: Bool, playbackRate: Float) {
public static func startPlayback(sessionId: String, playWhenReady: Bool, playbackRate: Float) {
guard let session = Database.shared.getPlaybackSession(id: sessionId) else { return }
if player != nil {
player?.destroy()
player = nil
@ -76,8 +77,7 @@ class PlayerHandler {
NowPlayingInfo.shared.setSessionMetadata(metadata: NowPlayingMetadata(id: session.id, itemId: session.libraryItemId!, artworkUrl: session.coverPath, title: session.displayTitle ?? "Unknown title", author: session.displayAuthor, series: nil))
self.session = session
player = AudioPlayer(playbackSession: session, playWhenReady: playWhenReady, playbackRate: playbackRate)
player = AudioPlayer(sessionId: sessionId, playWhenReady: playWhenReady, playbackRate: playbackRate)
startTickTimer()
}
@ -108,7 +108,9 @@ class PlayerHandler {
}
public static func getPlaybackSession() -> PlaybackSession? {
self.player?.getPlaybackSession()
guard let player = player else { return nil }
guard let session = Database.shared.getPlaybackSession(id: player.getPlaybackSessionId()) else { return nil }
return session
}
public static func seekForward(amount: Double) {
@ -164,8 +166,8 @@ class PlayerHandler {
}
public static func syncProgress() {
if session == nil { return }
guard let player = player else { return }
guard let session = getPlaybackSession() else { return }
// Prevent a sync at the current time
let playerCurrentTime = player.getCurrentTime()
@ -185,15 +187,17 @@ class PlayerHandler {
lastSyncTime = Date().timeIntervalSince1970 // seconds
let report = PlaybackReport(currentTime: playerCurrentTime, duration: player.getDuration(), timeListened: listeningTimePassedSinceLastSync)
session!.currentTime = playerCurrentTime
session.update {
session.currentTime = playerCurrentTime
}
listeningTimePassedSinceLastSync = 0
lastSyncReport = report
let sessionIsLocal = session!.isLocal
let sessionIsLocal = session.isLocal
if !sessionIsLocal {
if Connectivity.isConnectedToInternet {
NSLog("sending playback report")
ApiClient.reportPlaybackProgress(report: report, sessionId: session!.id)
ApiClient.reportPlaybackProgress(report: report, sessionId: session.id)
}
} else {
if let localMediaProgress = syncLocalProgress() {
@ -207,10 +211,10 @@ class PlayerHandler {
}
private static func syncLocalProgress() -> LocalMediaProgress? {
guard let session = session else { return nil }
guard let session = getPlaybackSession() else { return nil }
let localMediaProgress = LocalMediaProgress.fetchOrCreateLocalMediaProgress(localMediaProgressId: session.localMediaProgressId, localLibraryItemId: session.localLibraryItem?.id, localEpisodeId: session.episodeId)
guard var localMediaProgress = localMediaProgress else {
guard let localMediaProgress = localMediaProgress else {
// Local media progress should have been created
// If we're here, it means a library id is invalid
return nil

View file

@ -167,13 +167,14 @@ class ApiClient {
}
public static func reportLocalMediaProgress(_ localMediaProgress: LocalMediaProgress, callback: @escaping (_ success: Bool) -> Void) {
postResource(endpoint: "api/session/local", parameters: localMediaProgress, callback: callback)
let progress = localMediaProgress.freeze()
postResource(endpoint: "api/session/local", parameters: progress, callback: callback)
}
public static func syncMediaProgress(callback: @escaping (_ results: LocalMediaProgressSyncResultsPayload) -> Void) {
let localMediaProgressList = Database.shared.getAllLocalMediaProgress().filter {
$0.serverConnectionConfigId == Store.serverConfig?.id
}
}.map { $0.freeze() }
if ( !localMediaProgressList.isEmpty ) {
let payload = LocalMediaProgressSyncPayload(localMediaProgress: localMediaProgressList)

View file

@ -197,4 +197,9 @@ class Database {
realm.delete(progress!)
}
}
public func getPlaybackSession(id: String) -> PlaybackSession? {
let realm = try! Realm()
return realm.object(ofType: PlaybackSession.self, forPrimaryKey: id)
}
}