Add:Fallback to transcode when direct play fails, and send playback failed event to client

ios/App/App.xcodeproj/project.pbxproj

@@ -460,7 +460,7 @@
 				CLANG_ENABLE_MODULES = YES;
 				CODE_SIGN_STYLE = Automatic;
 				CURRENT_PROJECT_VERSION = 6;
-				DEVELOPMENT_TEAM = N8AA4S3S96;
+				DEVELOPMENT_TEAM = 7UFJ7D8V6A;
 				INFOPLIST_FILE = App/Info.plist;
 				IPHONEOS_DEPLOYMENT_TARGET = 12.0;
 				LD_RUNPATH_SEARCH_PATHS = "$(inherited) @executable_path/Frameworks";

ios/App/App/plugins/AbsAudioPlayer.swift

@@ -10,6 +10,9 @@ import Capacitor
 
 @objc(AbsAudioPlayer)
 public class AbsAudioPlayer: CAPPlugin {
+    private var initialPlayWhenReady = false
+    private var initialPlaybackRate:Float = 1
+    
     override public func load() {
         NotificationCenter.default.addObserver(self, selector: #selector(sendMetadata), name: NSNotification.Name(PlayerEvents.update.rawValue), object: nil)
         NotificationCenter.default.addObserver(self, selector: #selector(sendPlaybackClosedEvent), name: NSNotification.Name(PlayerEvents.closed.rawValue), object: nil)
@@ -17,6 +20,7 @@ public class AbsAudioPlayer: CAPPlugin {
         NotificationCenter.default.addObserver(self, selector: #selector(sendMetadata), name: UIApplication.willEnterForegroundNotification, object: nil)
         NotificationCenter.default.addObserver(self, selector: #selector(sendSleepTimerSet), name: NSNotification.Name(PlayerEvents.sleepSet.rawValue), object: nil)
         NotificationCenter.default.addObserver(self, selector: #selector(sendSleepTimerEnded), name: NSNotification.Name(PlayerEvents.sleepEnded.rawValue), object: nil)
+        NotificationCenter.default.addObserver(self, selector: #selector(onPlaybackFailed), name: NSNotification.Name(PlayerEvents.failed.rawValue), object: nil)
         
         self.bridge?.webView?.allowsBackForwardNavigationGestures = true;
     }
@@ -36,8 +40,11 @@ public class AbsAudioPlayer: CAPPlugin {
             return call.resolve()
         }
         
+        initialPlayWhenReady = playWhenReady
+        initialPlaybackRate = playbackRate
+        
         sendPrepareMetadataEvent(itemId: libraryItemId!, playWhenReady: playWhenReady)
-        ApiClient.startPlaybackSession(libraryItemId: libraryItemId!, episodeId: episodeId) { session in
+        ApiClient.startPlaybackSession(libraryItemId: libraryItemId!, episodeId: episodeId, forceTranscode: false) { session in
             PlayerHandler.startPlayback(session: session, playWhenReady: playWhenReady, playbackRate: playbackRate)
             
             do {
@@ -46,7 +53,6 @@ public class AbsAudioPlayer: CAPPlugin {
             } catch(let exception) {
                 NSLog("failed to convert session to json")
                 debugPrint(exception)
-                
                 call.resolve([:])
             }
             
@@ -162,6 +168,34 @@ public class AbsAudioPlayer: CAPPlugin {
         ])
     }
     
+    @objc func onPlaybackFailed() {
+        if (PlayerHandler.getPlayMethod() == PlayMethod.directplay.rawValue) {
+            let playbackSession = PlayerHandler.getPlaybackSession()
+            let libraryItemId = playbackSession?.libraryItemId ?? ""
+            let episodeId = playbackSession?.episodeId ?? nil
+            NSLog("TEST: Forcing Transcode")
+            
+            // If direct playing then fallback to transcode
+            ApiClient.startPlaybackSession(libraryItemId: libraryItemId, episodeId: episodeId, forceTranscode: true) { session in
+                PlayerHandler.startPlayback(session: session, playWhenReady: self.initialPlayWhenReady, playbackRate: self.initialPlaybackRate)
+                
+                do {
+                    self.sendPlaybackSession(session: try session.asDictionary())
+                } catch(let exception) {
+                    NSLog("failed to convert session to json")
+                    debugPrint(exception)
+                }
+                
+                self.sendMetadata()
+            }
+        } else {
+            self.notifyListeners("onPlaybackFailed", data: [
+                "value": "Playback Error"
+            ])
+        }
+        
+    }
+    
     @objc func sendPrepareMetadataEvent(itemId: String, playWhenReady: Bool) {
         self.notifyListeners("onPrepareMedia", data: [
             "audiobookId": itemId,

ios/App/Podfile

@@ -9,13 +9,13 @@ install! 'cocoapods', :disable_input_output_paths => true
 def capacitor_pods
   pod 'Capacitor', :path => '../../node_modules/@capacitor/ios'
   pod 'CapacitorCordova', :path => '../../node_modules/@capacitor/ios'
-  pod 'CapacitorApp', :path => '..\..\node_modules\@capacitor\app'
+  pod 'CapacitorApp', :path => '../../node_modules/@capacitor/app'
-  pod 'CapacitorDialog', :path => '..\..\node_modules\@capacitor\dialog'
+  pod 'CapacitorDialog', :path => '../../node_modules/@capacitor/dialog'
-  pod 'CapacitorHaptics', :path => '..\..\node_modules\@capacitor\haptics'
+  pod 'CapacitorHaptics', :path => '../../node_modules/@capacitor/haptics'
-  pod 'CapacitorNetwork', :path => '..\..\node_modules\@capacitor\network'
+  pod 'CapacitorNetwork', :path => '../../node_modules/@capacitor/network'
-  pod 'CapacitorStatusBar', :path => '..\..\node_modules\@capacitor\status-bar'
+  pod 'CapacitorStatusBar', :path => '../../node_modules/@capacitor/status-bar'
-  pod 'CapacitorStorage', :path => '..\..\node_modules\@capacitor\storage'
+  pod 'CapacitorStorage', :path => '../../node_modules/@capacitor/storage'
-  pod 'RobingenzCapacitorAppUpdate', :path => '..\..\node_modules\@robingenz\capacitor-app-update'
+  pod 'RobingenzCapacitorAppUpdate', :path => '../../node_modules/@robingenz/capacitor-app-update'
 end
 
 target 'App' do

ios/App/Shared/player/AudioPlayer.swift

@@ -10,6 +10,13 @@ import AVFoundation
 import UIKit
 import MediaPlayer
 
+enum PlayMethod:Int {
+    case directplay = 0
+    case directstream = 1
+    case transcode = 2
+    case local = 3
+}
+
 class AudioPlayer: NSObject {
     // enums and @objc are not compatible
     @objc dynamic var status: Int
@@ -137,6 +144,10 @@ class AudioPlayer: NSObject {
                 } else if (firstReady) { // Only seek on first readyToPlay
                     self.seek(self.playbackSession.currentTime, from: "queueItemStatusObserver")
                 }
+            } else if (playerItem.status == .failed) {
+                NSLog("TEST: queueStatusObserver: FAILED \(playerItem.error?.localizedDescription ?? "")")
+                
+                NotificationCenter.default.post(name: NSNotification.Name(PlayerEvents.failed.rawValue), object: nil)
             }
         })
     }
@@ -255,24 +266,31 @@ class AudioPlayer: NSObject {
         let startOffset = audioTrack.startOffset ?? 0.0
         return startOffset + currentTrackTime
     }
+    public func getPlayMethod() -> Int {
+        return self.playbackSession.playMethod
+    }
+    public func getPlaybackSession() -> PlaybackSession {
+        return self.playbackSession
+    }
     public func getDuration() -> Double {
         return playbackSession.duration
     }
     
     // MARK: - Private
     private func createAsset(itemId:String, track:AudioTrack) -> AVAsset {
+        if (playbackSession.playMethod == PlayMethod.directplay.rawValue) {
+            // The only reason this is separate is because the filename needs to be encoded
             let filename = track.metadata?.filename ?? ""
             let filenameEncoded = filename.addingPercentEncoding(withAllowedCharacters: NSCharacterSet.urlQueryAllowed)
             let urlstr = "\(Store.serverConfig!.address)/s/item/\(itemId)/\(filenameEncoded ?? "")?token=\(Store.serverConfig!.token)"
             let url = URL(string: urlstr)!
             return AVURLAsset(url: url)
-        
+        } else { // HLS Transcode
-        // Method for HLS
+            let headers: [String: String] = [
-//        let headers: [String: String] = [
+                "Authorization": "Bearer \(Store.serverConfig!.token)"
-//            "Authorization": "Bearer \(Store.serverConfig!.token)"
+            ]
-//        ]
+            return AVURLAsset(url: URL(string: "\(Store.serverConfig!.address)\(track.contentUrl ?? "")")!, options: ["AVURLAssetHTTPHeaderFieldsKey": headers])
-//
+        }
-//        return AVURLAsset(url: URL(string: "\(Store.serverConfig!.address)\(activeAudioTrack.contentUrl ?? "")")!, options: ["AVURLAssetHTTPHeaderFieldsKey": headers])
     }
     
     private func initAudioSession() {

ios/App/Shared/player/PlayerHandler.swift

@@ -84,6 +84,12 @@ class PlayerHandler {
     public static func setPlaybackSpeed(speed: Float) {
         self.player?.setPlaybackRate(speed)
     }
+    public static func getPlayMethod() -> Int? {
+        self.player?.getPlayMethod()
+    }
+    public static func getPlaybackSession() -> PlaybackSession? {
+        self.player?.getPlaybackSession()
+    }
     
     public static func seekForward(amount: Double) {
         guard let player = player else {

ios/App/Shared/util/ApiClient.swift

@@ -61,15 +61,15 @@ class ApiClient {
         }
     }
     
-    public static func startPlaybackSession(libraryItemId: String, episodeId: String?, callback: @escaping (_ param: PlaybackSession) -> Void) {
+    public static func startPlaybackSession(libraryItemId: String, episodeId: String?, forceTranscode:Bool, callback: @escaping (_ param: PlaybackSession) -> Void) {
         var endpoint = "api/items/\(libraryItemId)/play"
         if episodeId != nil {
             endpoint += "/\(episodeId!)"
         }
         
         ApiClient.postResource(endpoint: endpoint, parameters: [
-            "forceDirectPlay": "true",
+            "forceDirectPlay": !forceTranscode ? "1" : "",
-            "forceTranscode": "false", // TODO: direct play
+            "forceTranscode": forceTranscode ? "1" : "",
             "mediaPlayer": "AVPlayer",
         ], decodable: PlaybackSession.self) { obj in
             var session = obj

ios/App/Shared/util/PlayerEvents.swift

@@ -12,4 +12,5 @@ enum PlayerEvents: String {
     case closed = "com.audiobookshelf.app.player.closed"
     case sleepSet = "com.audiobookshelf.app.player.sleep.set"
     case sleepEnded = "com.audiobookshelf.app.player.sleep.ended"
+    case failed = "com.audiobookshelf.app.player.failed"
 }