fix(MP4): MP4 restructure improvements and some other fixes

AudioStreaming/Streaming/Audio Entry/AudioEntry.swift

@@ -108,6 +108,9 @@ class AudioEntry {
 
     func calculatedBitrate() -> Double {
         lock.lock(); defer { lock.unlock() }
+        if let explicitBitRate = audioStreamState.bitRate, explicitBitRate > 0 {
+            return explicitBitRate
+        }
         let packets = processedPacketsState
         if packetDuration > 0 {
             let packetsCount = packets.count

AudioStreaming/Streaming/Audio Entry/Models/AudioStreamState.swift

@@ -12,4 +12,5 @@ final class AudioStreamState {
     var dataPacketOffset: UInt64?
     var dataPacketCount: Double = 0
     var streamFormat = AudioStreamBasicDescription()
+    var bitRate: Double?
 }

AudioStreaming/Streaming/Audio Source/FileAudioSource.swift

@@ -120,11 +120,15 @@ final class FileAudioSource: NSObject, CoreAudioStreamSource {
             if isMp4, !mp4IsAlreadyOptimized {
                 if !mp4Restructure.dataOptimized {
                     do {
-                        if let mp4OptimizeInfo = try mp4Restructure.checkIsOptimized(data: data) {
-                            try performMp4Restructure(inputStream: inputStream, mp4OptimizeInfo: mp4OptimizeInfo)
-                        } else {
+                        switch try mp4Restructure.checkIsOptimized(data: data) {
+                        case .undetermined:
+                            // Not enough bytes yet; wait for more data before deciding
+                            break
+                        case .optimized:
                             mp4IsAlreadyOptimized = true
                             delegate?.dataAvailable(source: self, data: data)
+                        case let .needsRestructure(moovOffset):
+                            try performMp4Restructure(inputStream: inputStream, moovOffset: moovOffset)
                         }
                     } catch {
                         delegate?.errorOccurred(source: self, error: error)
@@ -141,24 +145,71 @@ final class FileAudioSource: NSObject, CoreAudioStreamSource {
         }
     }
 
-    func performMp4Restructure(inputStream: InputStream, mp4OptimizeInfo: Mp4OptimizeInfo) throws {
-        let offsetAccepted = inputStream.setProperty(mp4OptimizeInfo.moovOffset, forKey: .fileCurrentOffsetKey)
-        if offsetAccepted {
-            let moovDataBuffer = UnsafeMutablePointer.uint8pointer(of: mp4OptimizeInfo.moovSize)
-            defer { moovDataBuffer.deallocate() }
-            let moovRead = inputStream.read(moovDataBuffer, maxLength: mp4OptimizeInfo.moovSize)
-            if moovRead > 0 {
-                let data = Data(bytes: moovDataBuffer, count: moovRead)
-                let moovData = try mp4Restructure.restructureMoov(data: data)
-                delegate?.dataAvailable(source: self, data: moovData.initialData)
-                if !inputStream.setProperty(moovData.mdatOffset, forKey: .fileCurrentOffsetKey) {
+    func performMp4Restructure(inputStream: InputStream, moovOffset: Int) throws {
+        let offsetAccepted = inputStream.setProperty(moovOffset, forKey: .fileCurrentOffsetKey)
+        if !offsetAccepted {
+            delegate?.errorOccurred(source: self, error: inputStream.streamError ?? AudioSystemError.playerStartError)
+            return
+        }
+
+        // Read moov header (8 bytes)
+        var header = [UInt8](repeating: 0, count: 8)
+        let headerRead = inputStream.read(&header, maxLength: 8)
+        guard headerRead == 8 else {
+            delegate?.errorOccurred(source: self, error: AudioSystemError.playerStartError)
+            return
+        }
+
+        // Parse size and type (big endian)
+        let size32 = Data(header[0 ..< 4]).withUnsafeBytes { $0.load(as: UInt32.self) }.bigEndian
+        let type32 = Data(header[4 ..< 8]).withUnsafeBytes { $0.load(as: UInt32.self) }.bigEndian
+        guard Int(type32) == Atoms.moov else {
+            delegate?.errorOccurred(source: self, error: Mp4RestructureError.missingMoovAtom)
+            return
+        }
+
+        var moovSize = Int(size32)
+        var moovData = Data(header)
+
+        // Extended size (64-bit)
+        if moovSize == 1 {
+            var ext = [UInt8](repeating: 0, count: 8)
+            let extRead = inputStream.read(&ext, maxLength: 8)
+            guard extRead == 8 else {
+                delegate?.errorOccurred(source: self, error: AudioSystemError.playerStartError)
+                return
+            }
+            let ext64 = Data(ext).withUnsafeBytes { $0.load(as: UInt64.self) }.bigEndian
+            moovSize = Int(ext64)
+            moovData.append(contentsOf: ext)
+        }
+
+        let remaining = moovSize - moovData.count
+        if remaining < 0 {
+            delegate?.errorOccurred(source: self, error: AudioSystemError.playerStartError)
+            return
+        }
+        if remaining > 0 {
+            var buffer = [UInt8](repeating: 0, count: remaining)
+            var total = 0
+            while total < remaining {
+                let readBytes = buffer.withUnsafeMutableBytes { ptr -> Int in
+                    let base = ptr.baseAddress!.assumingMemoryBound(to: UInt8.self).advanced(by: total)
+                    return inputStream.read(base, maxLength: remaining - total)
+                }
+                guard readBytes > 0 else {
                     delegate?.errorOccurred(source: self, error: AudioSystemError.playerStartError)
+                    return
                 }
-            } else {
-                delegate?.errorOccurred(source: self, error: AudioSystemError.playerStartError)
+                total += readBytes
             }
-        } else {
-            delegate?.errorOccurred(source: self, error: inputStream.streamError ?? AudioSystemError.playerStartError)
+            moovData.append(contentsOf: buffer)
+        }
+
+        let moovResult = try mp4Restructure.restructureMoov(data: moovData)
+        delegate?.dataAvailable(source: self, data: moovResult.initialData)
+        if !inputStream.setProperty(moovResult.mdatOffset, forKey: .fileCurrentOffsetKey) {
+            delegate?.errorOccurred(source: self, error: AudioSystemError.playerStartError)
         }
     }
 

AudioStreaming/Streaming/Audio Source/Mp4/Mp4Restructure.swift

@@ -36,7 +36,7 @@ enum Atoms {
 
     static var cmov: Int { fourCcToInt("cmov") }
     static var stco: Int { fourCcToInt("stco") }
-    static var co64: Int { fourCcToInt("c064") }
+    static var co64: Int { fourCcToInt("co64") }
 
     static var atomPreampleSize: Int = 8
 
@@ -75,6 +75,12 @@ enum Mp4RestructureError: Error {
     case networkError(Error)
 }
 
+enum OptimizeCheckResult: Equatable {
+    case optimized
+    case needsRestructure(moovOffset: Int)
+    case undetermined
+}
+
 final class Mp4Restructure {
 
     private var atomOffset: Int = 0
@@ -129,24 +135,36 @@ final class Mp4Restructure {
         return (initialData, mdatOffset)
     }
 
-    /// Returns `nil` if the data is optimized otherwise `Mp4OptimizeInfo`
-    func checkIsOptimized(data: Data) throws -> Mp4OptimizeInfo? {
-        while atomOffset < UInt64(data.count) {
-            var atomSize = try Int(getInteger(data: data, offset: atomOffset) as UInt32)
-            let atomType = try Int(getInteger(data: data, offset: atomOffset + 4) as UInt32)
+    /// Incrementally checks if the MP4 is optimized. Returns tri-state result.
+    func checkIsOptimized(data: Data) throws -> OptimizeCheckResult {
+        while atomOffset + 8 <= data.count {
+            var atomSize: Int = try Int(getInteger(data: data, offset: atomOffset) as UInt32)
+            let atomType: Int = try Int(getInteger(data: data, offset: atomOffset + 4) as UInt32)
+            var headerSize = 8
+
+            // Handle extended size (64-bit)
+            if atomSize == 1 {
+                if atomOffset + 16 > data.count { break }
+                let ext: UInt64 = try getInteger(data: data, offset: atomOffset + 8)
+                atomSize = Int(ext)
+                headerSize = 16
+            } else if atomSize == 0 {
+                // Size extends to EOF; with partial data we can't determine full box
+                break
+            }
+
+            // Bounds and sanity checks
+            if atomSize < headerSize || atomOffset + atomSize > data.count { break }
+
             switch atomType {
             case Atoms.ftyp:
-                let ftypData = data[Int(atomOffset) ..< atomSize]
+                let start = atomOffset
+                let end = atomOffset + atomSize
+                let ftypData = data[start ..< end]
                 let ftyp = MP4Atom(type: atomType, size: atomSize, offset: atomOffset, data: ftypData)
                 self.ftyp = ftyp
                 atoms.append(ftyp)
             case Atoms.mdat:
-                // ref: https://developer.apple.com/documentation/quicktime-file-format/movie_data_atom
-                // This atom can be quite large, and may exceed 2^32 bytes, in which case the size field will be set to 1, 
-                // and the header will contain a 64-bit extended size field.
-                if atomSize == 1 {
-                    atomSize = Int(try getInteger(data: data, offset: atomOffset + 8) as UInt64)
-                }
                 let mdat = MP4Atom(type: atomType, size: atomSize, offset: atomOffset)
                 atoms.append(mdat)
                 foundMdat = true
@@ -158,19 +176,21 @@ final class Mp4Restructure {
                 let atom = MP4Atom(type: atomType, size: atomSize, offset: atomOffset)
                 atoms.append(atom)
             }
+
             if ftyp != nil {
                 if foundMoov && !foundMdat {
                     Logger.debug("🕵️ detected an optimized mp4", category: .generic)
-                    return nil
+                    return .optimized
                 } else if !foundMoov && foundMdat {
-                    Logger.debug("🕵️ detected an non-optimized mp4", category: .generic)
-                    let possibleMoovOffset = Int(atomOffset) + atomSize
-                    return Mp4OptimizeInfo(moovOffset: possibleMoovOffset, moovSize: atomSize)
+                    Logger.debug("🕵️ detected a non-optimized mp4", category: .generic)
+                    let possibleMoovOffset = atomOffset + atomSize
+                    return .needsRestructure(moovOffset: possibleMoovOffset)
                 }
             }
+
             atomOffset += atomSize
         }
-        return nil
+        return .undetermined
     }
 
     /// logic taken from qt-faststart.c over at ffmpeg
@@ -236,6 +256,8 @@ final class Mp4Restructure {
             // the next integer determines the `Number of entries`
             // https://developer.apple.com/documentation/quicktime-file-format/chunk_offset_atom/number_of_entries
             let numberOfOffsetEntries = try Int(moovAtom.getInteger() as UInt32)
+            // Adjust by moov size
+            let adjustDelta = moovAtomSize
             if atomType == Atoms.stco {
                 Logger.debug("🏗️ patching stco atom...", category: .generic)
                 if moovAtom.bytesAvailable < numberOfOffsetEntries * 4 {
@@ -246,7 +268,7 @@ final class Mp4Restructure {
                 for _ in 0 ..< numberOfOffsetEntries {
                     let currentOffset = try Int(moovAtom.getInteger(moovAtom.offset) as UInt32)
                     // adjust the offset by adding the size of moov atom
-                    let adjustOffset = currentOffset + moovAtomSize
+                    let adjustOffset = currentOffset + adjustDelta
 
                     if currentOffset < 0, adjustOffset >= 0 {
                         throw Mp4RestructureError.unableToRestructureData
@@ -261,8 +283,8 @@ final class Mp4Restructure {
                 }
                 for _ in 0 ..< numberOfOffsetEntries {
                     let currentOffset: Int = try moovAtom.getInteger(moovAtom.offset)
-                    // adjust the offset by adding the size of moov atom
-                    moovAtom.put(currentOffset + moovAtomSize)
+                    // adjust the offset by adding the size of moov atom (write as big-endian 64-bit)
+                    moovAtom.put(UInt64(currentOffset + adjustDelta).bigEndian)
                 }
             }
         }
@@ -271,10 +293,10 @@ final class Mp4Restructure {
 
     func getInteger<T: FixedWidthInteger>(data: Data, offset: Int) throws -> T {
         let sizeOfInteger = MemoryLayout<T>.size
-        guard sizeOfInteger <= data.count else {
+        guard offset >= 0, offset + sizeOfInteger <= data.count else {
             throw ByteBuffer.Error.eof
         }
-        let _offset = offset + sizeOfInteger
-        return T(data: data[_offset - sizeOfInteger ..< _offset]).bigEndian
+        let end = offset + sizeOfInteger
+        return T(data: data[offset ..< end]).bigEndian
     }
 }

AudioStreaming/Streaming/Audio Source/Mp4/RemoteMp4Restructure.swift

@@ -75,8 +75,15 @@ final class RemoteMp4Restructure {
                     }
                     self.audioData.append(data)
                     do {
-                        let value = try self.mp4Restructure.checkIsOptimized(data: self.audioData)
-                        if let value {
+                        switch try self.mp4Restructure.checkIsOptimized(data: self.audioData) {
+                        case .undetermined:
+                            break // keep streaming until decision can be made
+                        case .optimized:
+                            self.audioData = Data()
+                            self.task?.cancel()
+                            self.task = nil
+                            completion(.success(nil))
+                        case let .needsRestructure(moovOffset):
                             guard response.response?.statusCode == 206 else {
                                 Logger.error("⛔️ mp4 error: no moov before mdat and the stream is not seekable", category: .networking)
                                 completion(.failure(Mp4RestructureError.nonOptimizedMp4AndServerCannotSeek))
@@ -86,22 +93,15 @@ final class RemoteMp4Restructure {
                             self.audioData = Data()
                             self.task?.cancel()
                             self.task = nil
-                            self.fetchAndRestructureMoovAtom(offset: value.moovOffset) { result in
+                            self.fetchAndRestructureMoovAtom(offset: moovOffset) { result in
                                 switch result {
                                 case let .success(value):
-                                    let data = value.data
-                                    let offset = value.offset
                                     self.dataOptimized = true
-                                    completion(.success(RestructuredData(initialData: data, mdatOffset: offset)))
+                                    completion(.success(RestructuredData(initialData: value.data, mdatOffset: value.offset)))
                                 case let .failure(error):
                                     completion(.failure(Mp4RestructureError.networkError(error)))
                                 }
                             }
-                        } else {
-                            self.audioData = Data()
-                            self.task?.cancel()
-                            self.task = nil
-                            completion(.success(nil))
                         }
                     } catch {
                         completion(.failure(Mp4RestructureError.invalidAtomSize))
@@ -132,6 +132,8 @@ final class RemoteMp4Restructure {
         }
     }
 
+    // removed warmup range helper
+
     private func urlForPartialContent(with url: URL, offset: Int) -> URLRequest {
         var urlRequest = URLRequest(url: url)
         urlRequest.networkServiceType = .avStreaming

AudioStreaming/Streaming/AudioPlayer/AudioPlayer.swift

@@ -651,18 +651,22 @@ open class AudioPlayer {
 
         guard playerContext.internalState != .paused else { return }
 
+        let snapshot = playerContext.entriesLock.withLock {
+            (reading: playerContext.audioReadingEntry, playing: playerContext.audioPlayingEntry)
+        }
+
         if playerContext.internalState == .pendingNext {
             let entry = entriesQueue.dequeue(type: .upcoming)
             playerContext.setInternalState(to: .waitingForData)
             setCurrentReading(entry: entry, startPlaying: true, shouldClearQueue: true)
             rendererContext.resetBuffers()
-        } else if let playingEntry = playerContext.audioPlayingEntry,
+        } else if let playingEntry = snapshot.playing,
                   playingEntry.seekRequest.requested,
-                  playingEntry != playerContext.audioReadingEntry
+                  playingEntry != snapshot.reading
         {
             playingEntry.audioStreamState.processedDataFormat = false
             playingEntry.reset()
-            if let readingEntry = playerContext.audioReadingEntry {
+            if let readingEntry = snapshot.reading {
                 readingEntry.delegate = nil
                 readingEntry.close()
             }
@@ -677,20 +681,20 @@ open class AudioPlayer {
                 setCurrentReading(entry: playingEntry, startPlaying: true, shouldClearQueue: false)
             }
 
-        } else if playerContext.audioReadingEntry == nil {
+        } else if snapshot.reading == nil {
             if entriesQueue.count(for: .upcoming) > 0 {
                 let entry = entriesQueue.dequeue(type: .upcoming)
-                let shouldStartPlaying = playerContext.audioPlayingEntry == nil
+                let shouldStartPlaying = snapshot.playing == nil
                 playerContext.setInternalState(to: .waitingForData)
                 setCurrentReading(entry: entry, startPlaying: shouldStartPlaying, shouldClearQueue: false)
-            } else if playerContext.audioPlayingEntry == nil {
+            } else if snapshot.playing == nil {
                 if playerContext.internalState != .stopped {
                     stopEngine(reason: .eof)
                 }
             }
         }
 
-        if let playingEntry = playerContext.audioPlayingEntry,
+        if let playingEntry = snapshot.playing,
            playingEntry.audioStreamState.processedDataFormat,
            playingEntry.calculatedBitrate() > 0.0
         {