diff --git a/Session.xcodeproj/project.pbxproj b/Session.xcodeproj/project.pbxproj index 693291d1cf..a9e45963ff 100644 --- a/Session.xcodeproj/project.pbxproj +++ b/Session.xcodeproj/project.pbxproj @@ -494,6 +494,11 @@ FD1A55412E161AF6003761E4 /* Combine+Utilities.swift in Sources */ = {isa = PBXBuildFile; fileRef = FD1A55402E161AF3003761E4 /* Combine+Utilities.swift */; }; FD1A55432E179AED003761E4 /* ObservableKeyEvent+Utilities.swift in Sources */ = {isa = PBXBuildFile; fileRef = FD1A55422E179AE6003761E4 /* ObservableKeyEvent+Utilities.swift */; }; FD1A94FB2900D1C2000D73D3 /* PersistableRecord+Utilities.swift in Sources */ = {isa = PBXBuildFile; fileRef = FD1A94FA2900D1C2000D73D3 /* PersistableRecord+Utilities.swift */; }; + FD1BDB972E612315008EF998 /* JobExecutor.swift in Sources */ = {isa = PBXBuildFile; fileRef = FD1BDB962E612314008EF998 /* JobExecutor.swift */; }; + FD1BDB992E612372008EF998 /* JobError.swift in Sources */ = {isa = PBXBuildFile; fileRef = FD1BDB982E61236E008EF998 /* JobError.swift */; }; + FD1BDB9B2E6123D8008EF998 /* JobQueue.swift in Sources */ = {isa = PBXBuildFile; fileRef = FD1BDB9A2E6123D6008EF998 /* JobQueue.swift */; }; + FD1BDBA92E617C37008EF998 /* StreamLifecycleManager.swift in Sources */ = {isa = PBXBuildFile; fileRef = FD1BDBA82E617C35008EF998 /* StreamLifecycleManager.swift */; }; + FD1BDBAB2E617C62008EF998 /* CancellationAwareAsyncStream.swift in Sources */ = {isa = PBXBuildFile; fileRef = FD1BDBAA2E617C61008EF998 /* CancellationAwareAsyncStream.swift */; }; FD1C98E4282E3C5B00B76F9E /* UINavigationBar+Utilities.swift in Sources */ = {isa = PBXBuildFile; fileRef = FD1C98E3282E3C5B00B76F9E /* UINavigationBar+Utilities.swift */; }; FD1D732E2A86114600E3F410 /* _015_BlockCommunityMessageRequests.swift in Sources */ = {isa = PBXBuildFile; fileRef = FD1D732D2A86114600E3F410 /* _015_BlockCommunityMessageRequests.swift */; }; FD22726B2C32911C004D8A6C /* SendReadReceiptsJob.swift in Sources */ = {isa = PBXBuildFile; fileRef = FD2272532C32911A004D8A6C /* SendReadReceiptsJob.swift */; }; @@ -1864,6 +1869,11 @@ FD1A55402E161AF3003761E4 /* Combine+Utilities.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "Combine+Utilities.swift"; sourceTree = ""; }; FD1A55422E179AE6003761E4 /* ObservableKeyEvent+Utilities.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "ObservableKeyEvent+Utilities.swift"; sourceTree = ""; }; FD1A94FA2900D1C2000D73D3 /* PersistableRecord+Utilities.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "PersistableRecord+Utilities.swift"; sourceTree = ""; }; + FD1BDB962E612314008EF998 /* JobExecutor.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = JobExecutor.swift; sourceTree = ""; }; + FD1BDB982E61236E008EF998 /* JobError.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = JobError.swift; sourceTree = ""; }; + FD1BDB9A2E6123D6008EF998 /* JobQueue.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = JobQueue.swift; sourceTree = ""; }; + FD1BDBA82E617C35008EF998 /* StreamLifecycleManager.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = StreamLifecycleManager.swift; sourceTree = ""; }; + FD1BDBAA2E617C61008EF998 /* CancellationAwareAsyncStream.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = CancellationAwareAsyncStream.swift; sourceTree = ""; }; FD1C98E3282E3C5B00B76F9E /* UINavigationBar+Utilities.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "UINavigationBar+Utilities.swift"; sourceTree = ""; }; FD1D732D2A86114600E3F410 /* _015_BlockCommunityMessageRequests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = _015_BlockCommunityMessageRequests.swift; sourceTree = ""; }; FD2272532C32911A004D8A6C /* SendReadReceiptsJob.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = SendReadReceiptsJob.swift; sourceTree = ""; }; @@ -4181,10 +4191,12 @@ FDE755042C9BB4ED002A2623 /* Bencode.swift */, FDE755032C9BB4ED002A2623 /* BencodeDecoder.swift */, FD2272D32C34ECE1004D8A6C /* BencodeEncoder.swift */, + FD1BDBAA2E617C61008EF998 /* CancellationAwareAsyncStream.swift */, FDB11A5A2DD1900B00BEF49F /* CurrentValueAsyncStream.swift */, FD3FAB682AF1ADCA00DC5421 /* FileManager.swift */, FD6A38F02C2A66B100762359 /* KeychainStorage.swift */, FD78EA032DDEC3C000D55B50 /* MultiTaskManager.swift */, + FD1BDBA82E617C35008EF998 /* StreamLifecycleManager.swift */, FD2272E92C351CA7004D8A6C /* Threading.swift */, FDAA16752AC28A3B00DDBF77 /* UserDefaultsType.swift */, ); @@ -4693,7 +4705,10 @@ FD9004102818ABB000ABAAF6 /* JobRunner */ = { isa = PBXGroup; children = ( + FD1BDB982E61236E008EF998 /* JobError.swift */, + FD1BDB962E612314008EF998 /* JobExecutor.swift */, FDF0B7432804EF1B004C14C5 /* JobRunner.swift */, + FD1BDB9A2E6123D6008EF998 /* JobQueue.swift */, FDE77F68280F9EDA002CFC5D /* JobRunnerError.swift */, ); path = JobRunner; @@ -6296,6 +6311,7 @@ 945D9C582D6FDBE7003C4C0C /* _005_AddJobUniqueHash.swift in Sources */, FDB3486E2BE8457F00B716C2 /* BackgroundTaskManager.swift in Sources */, 7B1D74B027C365960030B423 /* Timer+MainThread.swift in Sources */, + FD1BDB9B2E6123D8008EF998 /* JobQueue.swift in Sources */, FD428B192B4B576F006D0888 /* AppContext.swift in Sources */, FD2272D42C34ECE1004D8A6C /* BencodeEncoder.swift in Sources */, FDE755052C9BB4EE002A2623 /* BencodeDecoder.swift in Sources */, @@ -6339,6 +6355,7 @@ FDE755192C9BC169002A2623 /* UIImage+Utilities.swift in Sources */, C3BBE0AA2554D4DE0050F1E3 /* Dictionary+Utilities.swift in Sources */, FD97B2402A3FEB050027DD57 /* ARC4RandomNumberGenerator.swift in Sources */, + FD1BDBA92E617C37008EF998 /* StreamLifecycleManager.swift in Sources */, FD5931AB2A8DCB0A0040147D /* ScopeAdapter+Utilities.swift in Sources */, FD78E9FD2DDD97F200D55B50 /* Setting.swift in Sources */, FD4BB22B2D63F20700D0DC3D /* MigrationHelper.swift in Sources */, @@ -6375,6 +6392,7 @@ FDB11A5B2DD1901000BEF49F /* CurrentValueAsyncStream.swift in Sources */, FDB3DA862E1E1F0E00148F8D /* TaskCancellation.swift in Sources */, FDE754CC2C9BAF37002A2623 /* MediaUtils.swift in Sources */, + FD1BDB972E612315008EF998 /* JobExecutor.swift in Sources */, FDE754DE2C9BAF8A002A2623 /* Crypto+SessionUtilitiesKit.swift in Sources */, FDF2220F281B55E6000A4995 /* QueryInterfaceRequest+Utilities.swift in Sources */, FD848B9A28442CE6000E298B /* StorageError.swift in Sources */, @@ -6402,6 +6420,7 @@ FD71160028C8253500B47552 /* UIView+Combine.swift in Sources */, B8856D23256F116B001CE70E /* Weak.swift in Sources */, FDE754D42C9BAF6B002A2623 /* UICollectionView+ReusableView.swift in Sources */, + FD1BDBAB2E617C62008EF998 /* CancellationAwareAsyncStream.swift in Sources */, FDE754C02C9BAEF6002A2623 /* Array+Utilities.swift in Sources */, FD17D7E527F6A09900122BE0 /* Identity.swift in Sources */, FD9004142818AD0B00ABAAF6 /* _002_SetupStandardJobs.swift in Sources */, @@ -6415,6 +6434,7 @@ FD52CB632E13B61700A4DA70 /* ObservableKey.swift in Sources */, FD74434C2D07CA9F00862443 /* CGSize+Utilities.swift in Sources */, FD74434D2D07CA9F00862443 /* CGPoint+Utilities.swift in Sources */, + FD1BDB992E612372008EF998 /* JobError.swift in Sources */, FD74434E2D07CA9F00862443 /* CGRect+Utilities.swift in Sources */, FDC289472C881A3800020BC2 /* MutableIdentifiable.swift in Sources */, C300A60D2554B31900555489 /* Logging.swift in Sources */, diff --git a/Session/Meta/AppDelegate.swift b/Session/Meta/AppDelegate.swift index 6299100d40..1995f30371 100644 --- a/Session/Meta/AppDelegate.swift +++ b/Session/Meta/AppDelegate.swift @@ -516,14 +516,14 @@ class AppDelegate: UIResponder, UIApplicationDelegate, UNUserNotificationCenterD /// Increment the launch count (guaranteed to change which results in the write actually doing something and /// outputting and error if the DB is suspended) db[.activeCounter] = ((db[.activeCounter] ?? 0) + 1) - - /// Now that the migrations are completed schedule config syncs for **all** configs that have pending changes to - /// ensure that any pending local state gets pushed and any jobs waiting for a successful config sync are run - /// - /// **Note:** We only want to do this if the app is active, and the user has completed the Onboarding process - if dependencies[singleton: .appContext].isAppForegroundAndActive && dependencies[cache: .onboarding].state == .completed { - dependencies.mutate(cache: .libSession) { $0.syncAllPendingPushes(db) } - } + } + + /// Now that the migrations are completed schedule config syncs for **all** configs that have pending changes to + /// ensure that any pending local state gets pushed and any jobs waiting for a successful config sync are run + /// + /// **Note:** We only want to do this if the app is active, and the user has completed the Onboarding process + if dependencies[singleton: .appContext].isAppForegroundAndActive && dependencies[cache: .onboarding].state == .completed { + dependencies.mutate(cache: .libSession) { $0.syncAllPendingPushesAsync() } } // Add a log to track the proper startup time of the app so we know whether we need to diff --git a/Session/Notifications/SyncPushTokensJob.swift b/Session/Notifications/SyncPushTokensJob.swift index 6a885dfd2c..ecd37ea11b 100644 --- a/Session/Notifications/SyncPushTokensJob.swift +++ b/Session/Notifications/SyncPushTokensJob.swift @@ -22,42 +22,28 @@ public enum SyncPushTokensJob: JobExecutor { private static let maxFrequency: TimeInterval = (12 * 60 * 60) private static let maxRunFrequency: TimeInterval = 1 - public static func run( - _ job: Job, - scheduler: S, - success: @escaping (Job, Bool) -> Void, - failure: @escaping (Job, Error, Bool) -> Void, - deferred: @escaping (Job) -> Void, - using dependencies: Dependencies - ) { - // Don't run when inactive or not in main app or if the user doesn't exist yet + public static func run(_ job: Job, using dependencies: Dependencies) async throws -> JobExecutionResult { + /// Don't run when inactive or not in main app or if the user doesn't exist yet guard dependencies[defaults: .appGroup, key: .isMainAppActive] else { - return deferred(job) // Don't need to do anything if it's not the main app + return .deferred(job) /// Don't need to do anything if it's not the main app } guard dependencies[cache: .onboarding].state == .completed else { Log.info(.syncPushTokensJob, "Deferred due to incomplete registration") - return deferred(job) + return .deferred(job) } /// Since this job can be dependant on network conditions it's possible for multiple jobs to run at the same time, while this shouldn't cause issues /// it can result in multiple API calls getting made concurrently so to avoid this we defer the job as if the previous one was successful then the /// `lastDeviceTokenUpload` value will prevent the subsequent call being made guard - dependencies[singleton: .jobRunner] - .jobInfoFor(state: .running, variant: .syncPushTokens) - .filter({ key, info in key != job.id }) // Exclude this job + await dependencies[singleton: .jobRunner] + .jobInfoFor(state: .running, filters: SyncPushTokensJob.filters(whenRunning: job)) .isEmpty else { - // Defer the job to run 'maxRunFrequency' from when this one ran (if we don't it'll try start - // it again immediately which is pointless) - let updatedJob: Job? = dependencies[singleton: .storage].write { db in - try job - .with(nextRunTimestamp: dependencies.dateNow.timeIntervalSince1970 + maxRunFrequency) - .upserted(db) - } - + /// Defer the job to run `maxRunFrequency` from when this one ran (if we don't it'll try start it again immediately which is pointless) Log.info(.syncPushTokensJob, "Deferred due to in progress job") - return deferred(updatedJob ?? job) + let nextRunTimestamp: TimeInterval = (dependencies.dateNow.timeIntervalSince1970 + maxRunFrequency) + return .deferred(job.with(nextRunTimestamp: nextRunTimestamp)) } // Determine if the device has 'Fast Mode' (APNS) enabled @@ -66,7 +52,8 @@ public enum SyncPushTokensJob: JobExecutor { // If the job is running and 'Fast Mode' is disabled then we should try to unregister the existing // token guard isUsingFullAPNs else { - dependencies[singleton: .storage] + // FIXME: Refactor this to use async/await + let publisher = dependencies[singleton: .storage] .readPublisher { db in db[.lastRecordedPushToken] } .flatMap { lastRecordedPushToken -> AnyPublisher in // Tell the device to unregister for remote notifications (essentially try to invalidate @@ -93,19 +80,17 @@ public enum SyncPushTokensJob: JobExecutor { .setFailureType(to: Error.self) .eraseToAnyPublisher() } - .subscribe(on: scheduler, using: dependencies) - .sinkUntilComplete( - receiveCompletion: { result in - switch result { - case .finished: Log.info(.syncPushTokensJob, "Unregister Completed") - case .failure: Log.error(.syncPushTokensJob, "Unregister Failed") - } - - // We want to complete this job regardless of success or failure - success(job, false) - } - ) - return + + do { + try await publisher.values.first(where: { _ in true }) + Log.info(.syncPushTokensJob, "Unregister Completed") + return .success(job, stop: false) + } + catch { + // We want to complete this job regardless of success or failure + Log.error(.syncPushTokensJob, "Unregister Failed") + return .success(job, stop: false) + } } /// Perform device registration @@ -113,7 +98,8 @@ public enum SyncPushTokensJob: JobExecutor { /// **Note:** Apple's documentation states that we should re-register for notifications on every launch: /// https://developer.apple.com/library/archive/documentation/NetworkingInternet/Conceptual/RemoteNotificationsPG/HandlingRemoteNotifications.html#//apple_ref/doc/uid/TP40008194-CH6-SW1 Log.info(.syncPushTokensJob, "Re-registering for remote notifications") - dependencies[singleton: .pushRegistrationManager].requestPushTokens() + // FIXME: Refactor this to use async/await + let publisher = dependencies[singleton: .pushRegistrationManager].requestPushTokens() .flatMap { (pushToken: String, voipToken: String) -> AnyPublisher<(String, String)?, Error> in Log.info(.syncPushTokensJob, "Received push and voip tokens, waiting for paths to build") @@ -124,7 +110,7 @@ public enum SyncPushTokensJob: JobExecutor { .setFailureType(to: Error.self) .timeout( .seconds(5), // Give the paths a chance to build on launch - scheduler: scheduler, + scheduler: DispatchQueue.global(qos: .default), customError: { NetworkError.timeout(error: "", rawData: nil) } ) .catch { error -> AnyPublisher<(String, String)?, Error> in @@ -204,51 +190,56 @@ public enum SyncPushTokensJob: JobExecutor { .map { _ in () } .eraseToAnyPublisher() } - .subscribe(on: scheduler, using: dependencies) - .sinkUntilComplete( - // We want to complete this job regardless of success or failure - receiveCompletion: { _ in success(job, false) } - ) + + // We want to complete this job regardless of success or failure + try? await publisher.values.first(where: { _ in true }) + return .success(job, stop: false) } - public static func run(uploadOnlyIfStale: Bool, using dependencies: Dependencies) -> AnyPublisher { - return Deferred { - Future { resolver in - guard let job: Job = Job( - variant: .syncPushTokens, - behaviour: .runOnce, - details: SyncPushTokensJob.Details( - uploadOnlyIfStale: uploadOnlyIfStale - ) - ) - else { return resolver(Result.failure(NetworkError.parsingFailed)) } - - SyncPushTokensJob.run( - job, - scheduler: DispatchQueue.global(qos: .userInitiated), - success: { _, _ in resolver(Result.success(())) }, - failure: { _, error, _ in resolver(Result.failure(error)) }, - deferred: { job in - dependencies[singleton: .jobRunner] - .afterJob(job) - .first() - .sinkUntilComplete( - receiveValue: { result in - switch result { - /// If it gets deferred a second time then we should probably just fail - no use waiting on something - /// that may never run (also means we can avoid another potential defer loop) - case .notFound, .deferred: resolver(Result.failure(NetworkError.unknown)) - case .failed(let error, _): resolver(Result.failure(error)) - case .succeeded: resolver(Result.success(())) - } - } - ) - }, - using: dependencies - ) - } + public static func run(uploadOnlyIfStale: Bool, using dependencies: Dependencies) async throws { + guard let job: Job = Job( + variant: .syncPushTokens, + behaviour: .runOnce, + details: SyncPushTokensJob.Details( + uploadOnlyIfStale: uploadOnlyIfStale + ) + ) + else { throw JobRunnerError.missingRequiredDetails } + + let result: JobExecutionResult = try await SyncPushTokensJob.run(job, using: dependencies) + + /// If the job was deferred it was most likely due to another `SyncPushTokens` job in progress so we should wait + /// for the other job to finish and try again + switch result { + case .success: return + case .deferred: break + } + + let runningJobs: [Int64: JobRunner.JobInfo] = await dependencies[singleton: .jobRunner] + .jobInfoFor(state: .running, filters: SyncPushTokensJob.filters(whenRunning: job)) + + /// If we couldn't find a running job then fail (something else went wrong) + guard !runningJobs.isEmpty else { throw JobRunnerError.missingRequiredDetails } + + let otherJobResult: JobRunner.JobResult = await dependencies[singleton: .jobRunner].awaitResult( + forFirstJobMatching: SyncPushTokensJob.filters(whenRunning: job), + in: .running + ) + + /// If it gets deferred a second time then we should probably just fail - no use waiting on something + /// that may never run (also means we can avoid another potential defer loop) + switch otherJobResult { + case .notFound, .deferred: throw JobRunnerError.missingRequiredDetails + case .failed(let error, _): throw error + case .succeeded: break } - .eraseToAnyPublisher() + } + + private static func filters(whenRunning job: Job) -> JobRunner.Filters { + return JobRunner.Filters( + include: [.variant(.syncPushTokens)], + exclude: [job.id.map { .jobId($0) }].compactMap { $0 } /// Exclude this job + ) } } diff --git a/Session/Settings/NukeDataModal.swift b/Session/Settings/NukeDataModal.swift index 4ade2b621d..b645208cce 100644 --- a/Session/Settings/NukeDataModal.swift +++ b/Session/Settings/NukeDataModal.swift @@ -156,16 +156,15 @@ final class NukeDataModal: Modal { private func clearDeviceOnly() { ModalActivityIndicatorViewController.present(fromViewController: self, canCancel: false) { [weak self, dependencies] _ in - ConfigurationSyncJob - .run(swarmPublicKey: dependencies[cache: .general].sessionId.hexString, using: dependencies) - .subscribe(on: DispatchQueue.global(qos: .userInitiated)) - .receive(on: DispatchQueue.main) - .sinkUntilComplete( - receiveCompletion: { _ in - NukeDataModal.deleteAllLocalData(using: dependencies) - self?.dismiss(animated: true, completion: nil) // Dismiss the loader - } + Task(priority: .userInitiated) { [weak self, dependencies] in + try? await ConfigurationSyncJob.run( + swarmPublicKey: dependencies[cache: .general].sessionId.hexString, + using: dependencies ) + + NukeDataModal.deleteAllLocalData(using: dependencies) + self?.dismiss(animated: true, completion: nil) // Dismiss the loader + } } } diff --git a/SessionMessagingKit/Database/Models/Interaction.swift b/SessionMessagingKit/Database/Models/Interaction.swift index 59373a3201..6010610d3c 100644 --- a/SessionMessagingKit/Database/Models/Interaction.swift +++ b/SessionMessagingKit/Database/Models/Interaction.swift @@ -850,18 +850,17 @@ public extension Interaction { /// If we want to send read receipts and it's a contact thread then try to add the `SendReadReceiptsJob` for and unread /// messages that weren't outgoing if trySendReadReceipt && threadVariant == .contact { - dependencies[singleton: .jobRunner].upsert( - db, - job: SendReadReceiptsJob.createOrUpdateIfNeeded( - db, - threadId: threadId, - interactionIds: interactionInfo - .filter { !$0.wasRead && $0.variant != .standardOutgoing } - .map { $0.id }, - using: dependencies - ), - canStartJob: true - ) + db.afterCommit { [dependencies] in + Task { [dependencies] in + await SendReadReceiptsJob.createOrUpdateIfNeeded( + threadId: threadId, + interactionIds: interactionInfo + .filter { !$0.wasRead && $0.variant != .standardOutgoing } + .map { $0.id }, + using: dependencies + ) + } + } } } } diff --git a/SessionMessagingKit/Jobs/AttachmentDownloadJob.swift b/SessionMessagingKit/Jobs/AttachmentDownloadJob.swift index 0127ebc746..fc543790bf 100644 --- a/SessionMessagingKit/Jobs/AttachmentDownloadJob.swift +++ b/SessionMessagingKit/Jobs/AttachmentDownloadJob.swift @@ -10,22 +10,33 @@ public enum AttachmentDownloadJob: JobExecutor { public static var requiresThreadId: Bool = true public static let requiresInteractionId: Bool = true - public static func run( - _ job: Job, - scheduler: S, - success: @escaping (Job, Bool) -> Void, - failure: @escaping (Job, Error, Bool) -> Void, - deferred: @escaping (Job) -> Void, - using dependencies: Dependencies - ) { + public static func run(_ job: Job, using dependencies: Dependencies) async throws -> JobExecutionResult { guard dependencies[singleton: .appContext].isValid, let threadId: String = job.threadId, let detailsData: Data = job.details, let details: Details = try? JSONDecoder(using: dependencies).decode(Details.self, from: detailsData) - else { return failure(job, JobRunnerError.missingRequiredDetails, true) } + else { throw JobRunnerError.missingRequiredDetails } - dependencies[singleton: .storage] + let otherCurrentJobAttachmentIds: Set = await dependencies[singleton: .jobRunner] + .jobInfoFor( + state: .running, + filters: JobRunner.Filters( + include: [.variant(.attachmentDownload)], + exclude: [job.id.map { .jobId($0) }].compactMap { $0 } + ) + ) + .values + .compactMap { info -> String? in + guard let data: Data = info.detailsData else { return nil } + + return (try? JSONDecoder(using: dependencies).decode(Details.self, from: data))? + .attachmentId + } + .asSet() + + // FIXME: Refactor this to use async/await + let publisher = dependencies[singleton: .storage] .writePublisher { db -> Attachment in guard let attachment: Attachment = try? Attachment.fetchOne(db, id: details.attachmentId) else { throw JobRunnerError.missingRequiredDetails @@ -42,18 +53,6 @@ public enum AttachmentDownloadJob: JobExecutor { // the same attachment multiple times at the same time (it also adds a "clean up" mechanism // if an attachment ends up stuck in a "downloading" state incorrectly guard attachment.state != .downloading else { - let otherCurrentJobAttachmentIds: Set = dependencies[singleton: .jobRunner] - .jobInfoFor(state: .running, variant: .attachmentDownload) - .filter { key, _ in key != job.id } - .values - .compactMap { info -> String? in - guard let data: Data = info.detailsData else { return nil } - - return (try? JSONDecoder(using: dependencies).decode(Details.self, from: data))? - .attachmentId - } - .asSet() - // If there isn't another currently running attachmentDownload job downloading this // attachment then we should update the state of the attachment to be failed to // avoid having attachments appear in an endlessly downloading state @@ -140,8 +139,6 @@ public enum AttachmentDownloadJob: JobExecutor { (response.attachment, response.temporaryFileUrl, response.data) } } - .subscribe(on: scheduler, using: dependencies) - .receive(on: scheduler, using: dependencies) .tryMap { attachment, temporaryFileUrl, data -> Attachment in // Store the encrypted data temporarily try data.write(to: temporaryFileUrl, options: .atomic) @@ -195,69 +192,68 @@ public enum AttachmentDownloadJob: JobExecutor { return updatedAttachment } - .sinkUntilComplete( - receiveCompletion: { result in - switch (result, result.errorOrNull, result.errorOrNull as? JobRunnerError) { - case (.finished, _, _): success(job, false) - case (_, let error as AttachmentDownloadError, _) where error == .alreadyDownloaded: - success(job, false) - - case (_, _, .missingRequiredDetails): - failure(job, JobRunnerError.missingRequiredDetails, true) - - case (_, _, .possibleDuplicateJob(let permanentFailure)): - failure(job, JobRunnerError.possibleDuplicateJob(permanentFailure: permanentFailure), permanentFailure) - - case (.failure(let error), _, _): - let targetState: Attachment.State - let permanentFailure: Bool - - switch error { - /// If we get a 404 then we got a successful response from the server but the attachment doesn't - /// exist, in this case update the attachment to an "invalid" state so the user doesn't get stuck in - /// a retry download loop - case NetworkError.notFound: - targetState = .invalid - permanentFailure = true - - /// If we got a 400 or a 401 then we want to fail the download in a way that has to be manually retried as it's - /// likely something else is going on that caused the failure - case NetworkError.badRequest, NetworkError.unauthorised, - SnodeAPIError.signatureVerificationFailed: - targetState = .failedDownload - permanentFailure = true - - /// For any other error it's likely either the server is down or something weird just happened with the request - /// so we want to automatically retry - default: - targetState = .failedDownload - permanentFailure = false - } + + do { + _ = try await publisher.values.first(where: { _ in true }) + return .success(job, stop: false) + } + catch { + switch error { + case AttachmentDownloadError.alreadyDownloaded: return .success(job, stop: false) + case JobRunnerError.missingRequiredDetails: throw error + + case JobRunnerError.possibleDuplicateJob(let permanentFailure): + throw JobRunnerError.possibleDuplicateJob(permanentFailure: permanentFailure) + + default: + let targetState: Attachment.State + let permanentFailure: Bool + + switch error { + /// If we get a 404 then we got a successful response from the server but the attachment doesn't + /// exist, in this case update the attachment to an "invalid" state so the user doesn't get stuck in + /// a retry download loop + case NetworkError.notFound: + targetState = .invalid + permanentFailure = true - /// To prevent the attachment from showing a state of downloading forever, we need to update the attachment - /// state here based on the type of error that occurred - /// - /// **Note:** We **MUST** use the `'with()` function here as it will update the - /// `isValid` and `duration` values based on the downloaded data and the state - dependencies[singleton: .storage].writeAsync( - updates: { db in - _ = try Attachment - .filter(id: details.attachmentId) - .updateAll(db, Attachment.Columns.state.set(to: targetState)) - db.addAttachmentEvent( - id: details.attachmentId, - messageId: job.interactionId, - type: .updated(.state(targetState)) - ) - }, - completion: { _ in - /// Trigger the failure and provide the `permanentFailure` value defined above - failure(job, error, permanentFailure) - } - ) + /// If we got a 400 or a 401 then we want to fail the download in a way that has to be manually retried as it's + /// likely something else is going on that caused the failure + case NetworkError.badRequest, NetworkError.unauthorised, + SnodeAPIError.signatureVerificationFailed: + targetState = .failedDownload + permanentFailure = true + + /// For any other error it's likely either the server is down or something weird just happened with the request + /// so we want to automatically retry + default: + targetState = .failedDownload + permanentFailure = false } - } - ) + + /// To prevent the attachment from showing a state of downloading forever, we need to update the attachment + /// state here based on the type of error that occurred + /// + /// **Note:** We **MUST** use the `'with()` function here as it will update the + /// `isValid` and `duration` values based on the downloaded data and the state + try? await dependencies[singleton: .storage].writeAsync { db in + _ = try Attachment + .filter(id: details.attachmentId) + .updateAll(db, Attachment.Columns.state.set(to: targetState)) + db.addAttachmentEvent( + id: details.attachmentId, + messageId: job.interactionId, + type: .updated(.state(targetState)) + ) + } + + /// Trigger the failure, but force to a `permanentFailure` if desired + switch permanentFailure { + case true: throw JobRunnerError.permanentFailure(error) + case false: throw error + } + } + } } } diff --git a/SessionMessagingKit/Jobs/AttachmentUploadJob.swift b/SessionMessagingKit/Jobs/AttachmentUploadJob.swift index d1dc85ddea..4aec309cf8 100644 --- a/SessionMessagingKit/Jobs/AttachmentUploadJob.swift +++ b/SessionMessagingKit/Jobs/AttachmentUploadJob.swift @@ -19,22 +19,16 @@ public enum AttachmentUploadJob: JobExecutor { public static var requiresThreadId: Bool = true public static let requiresInteractionId: Bool = true - public static func run( - _ job: Job, - scheduler: S, - success: @escaping (Job, Bool) -> Void, - failure: @escaping (Job, Error, Bool) -> Void, - deferred: @escaping (Job) -> Void, - using dependencies: Dependencies - ) { + public static func run(_ job: Job, using dependencies: Dependencies) async throws -> JobExecutionResult { guard let threadId: String = job.threadId, let interactionId: Int64 = job.interactionId, let detailsData: Data = job.details, let details: Details = try? JSONDecoder(using: dependencies).decode(Details.self, from: detailsData) - else { return failure(job, JobRunnerError.missingRequiredDetails, true) } + else { throw JobRunnerError.missingRequiredDetails } - dependencies[singleton: .storage] + // FIXME: Refactor this to use async/await + let publisher = dependencies[singleton: .storage] .readPublisher { db -> Attachment in guard let attachment: Attachment = try? Attachment.fetchOne(db, id: details.attachmentId) else { throw JobRunnerError.missingRequiredDetails @@ -86,8 +80,6 @@ public enum AttachmentUploadJob: JobExecutor { return (attachment, authMethod) } - .subscribe(on: scheduler, using: dependencies) - .receive(on: scheduler, using: dependencies) .tryMap { attachment, authMethod -> Network.PreparedRequest<(attachment: Attachment, fileId: String)> in try AttachmentUploader.preparedUpload( attachment: attachment, @@ -141,68 +133,68 @@ public enum AttachmentUploadJob: JobExecutor { return updatedAttachment } - .sinkUntilComplete( - receiveCompletion: { result in - switch (result, result.errorOrNull) { - case (.finished, _): success(job, false) - - case (_, let error as JobRunnerError) where error == .missingRequiredDetails: - failure(job, error, true) + + do { + guard let result: Attachment = try await publisher.values.first(where: { _ in true }) else { + throw JobRunnerError.permanentFailure(StorageError.objectNotSaved) + } + + return .success(job, stop: false) + } + catch { + switch error { + case JobRunnerError.missingRequiredDetails: throw error + + case StorageError.objectNotFound: + Log.info(.cat, "Failed due to missing interaction") + throw JobRunnerError.permanentFailure(error) + + case AttachmentError.uploadIsStillPendingDownload: + Log.info(.cat, "Deferred as attachment is still being downloaded") + return .deferred(job) + + default: + let alreadyLoggedError: Bool? = try? await dependencies[singleton: .storage].writeAsync { db -> Bool in + /// Update the attachment state + try Attachment + .filter(id: details.attachmentId) + .updateAll(db, Attachment.Columns.state.set(to: Attachment.State.failedUpload)) + db.addAttachmentEvent( + id: details.attachmentId, + messageId: job.interactionId, + type: .updated(.state(.failedUpload)) + ) + + /// If this upload is related to sending a message then trigger the `handleFailedMessageSend` logic + /// as we want to ensure the message has the correct delivery status + guard + let sendJob: Job = try Job.fetchOne(db, id: details.messageSendJobId), + let sendJobDetails: Data = sendJob.details, + let details: MessageSendJob.Details = try? JSONDecoder(using: dependencies) + .decode(MessageSendJob.Details.self, from: sendJobDetails) + else { return false } - case (_, let error as StorageError) where error == .objectNotFound: - Log.info(.cat, "Failed due to missing interaction") - failure(job, error, true) - - case (_, let error as AttachmentError) where error == .uploadIsStillPendingDownload: - Log.info(.cat, "Deferred as attachment is still being downloaded") - return deferred(job) - - case (.failure(let error), _): - dependencies[singleton: .storage].writeAsync( - updates: { db in - /// Update the attachment state - try Attachment - .filter(id: details.attachmentId) - .updateAll(db, Attachment.Columns.state.set(to: Attachment.State.failedUpload)) - db.addAttachmentEvent( - id: details.attachmentId, - messageId: job.interactionId, - type: .updated(.state(.failedUpload)) - ) - - /// If this upload is related to sending a message then trigger the `handleFailedMessageSend` logic - /// as we want to ensure the message has the correct delivery status - guard - let sendJob: Job = try Job.fetchOne(db, id: details.messageSendJobId), - let sendJobDetails: Data = sendJob.details, - let details: MessageSendJob.Details = try? JSONDecoder(using: dependencies) - .decode(MessageSendJob.Details.self, from: sendJobDetails) - else { return false } - - MessageSender.handleFailedMessageSend( - db, - threadId: threadId, - message: details.message, - destination: nil, - error: .other(.cat, "Failed", error), - interactionId: interactionId, - using: dependencies - ) - return true - }, - completion: { result in - /// If we didn't log an error above then log it now - switch result { - case .failure, .success(true): break - case .success(false): Log.error(.cat, "Failed due to error: \(error)") - } - - failure(job, error, false) - } - ) + MessageSender.handleFailedMessageSend( + db, + threadId: threadId, + message: details.message, + destination: nil, + error: .other(.cat, "Failed", error), + interactionId: interactionId, + using: dependencies + ) + return true } - } - ) + + /// If we didn't log an error above then log it now + switch alreadyLoggedError { + case .some(true): break + case .none, .some(false): Log.error(.cat, "Failed due to error: \(error)") + } + + throw error + } + } } } diff --git a/SessionMessagingKit/Jobs/CheckForAppUpdatesJob.swift b/SessionMessagingKit/Jobs/CheckForAppUpdatesJob.swift index aba703204e..69be655d1c 100644 --- a/SessionMessagingKit/Jobs/CheckForAppUpdatesJob.swift +++ b/SessionMessagingKit/Jobs/CheckForAppUpdatesJob.swift @@ -19,15 +19,8 @@ public enum CheckForAppUpdatesJob: JobExecutor { public static var requiresThreadId: Bool = false public static let requiresInteractionId: Bool = false - public static func run( - _ job: Job, - scheduler: S, - success: @escaping (Job, Bool) -> Void, - failure: @escaping (Job, Error, Bool) -> Void, - deferred: @escaping (Job) -> Void, - using dependencies: Dependencies - ) { - // Just defer the update check when running tests or in the simulator + public static func run(_ job: Job, using dependencies: Dependencies) async throws -> JobExecutionResult { + /// Just defer the update check when running tests or in the simulator #if targetEnvironment(simulator) let shouldCheckForUpdates: Bool = false #else @@ -39,40 +32,33 @@ public enum CheckForAppUpdatesJob: JobExecutor { failureCount: 0, nextRunTimestamp: (dependencies.dateNow.timeIntervalSince1970 + updateCheckFrequency) ) - dependencies[singleton: .storage].write { db in + try? await dependencies[singleton: .storage].writeAsync { db in try updatedJob.save(db) } Log.info(.cat, "Deferred due to test/simulator build.") - return deferred(updatedJob) + return .deferred(updatedJob) } - dependencies[singleton: .network] - .checkClientVersion(ed25519SecretKey: dependencies[cache: .general].ed25519SecretKey) - .subscribe(on: scheduler, using: dependencies) - .receive(on: scheduler, using: dependencies) - .sinkUntilComplete( - receiveCompletion: { _ in - var updatedJob: Job = job.with( - failureCount: 0, - nextRunTimestamp: (dependencies.dateNow.timeIntervalSince1970 + updateCheckFrequency) - ) - - dependencies[singleton: .storage].write { db in - try updatedJob.save(db) - } - - success(updatedJob, false) - }, - receiveValue: { _, versionInfo in - switch versionInfo.prerelease { - case .none: - Log.info(.cat, "Latest version: \(versionInfo.version) (Current: \(dependencies[cache: .appVersion].versionInfo))") - - case .some(let prerelease): - Log.info(.cat, "Latest version: \(versionInfo.version), pre-release version: \(prerelease.version) (Current: \(dependencies[cache: .appVersion].versionInfo))") - } - } - ) + // FIXME: Refactor this to use async/await + let publisher = dependencies[singleton: .network].checkClientVersion( + ed25519SecretKey: dependencies[cache: .general].ed25519SecretKey + ) + + let versionInfo = try? await publisher.values.first(where: { _ in true }) + + switch versionInfo?.1.prerelease { + case .none: + Log.info(.cat, "Latest version: \(versionInfo?.1.version ?? "Unknown (error)") (Current: \(dependencies[cache: .appVersion].versionInfo))") + + case .some(let prerelease): + Log.info(.cat, "Latest version: \(versionInfo?.1.version ?? "Unknown (error)"), pre-release version: \(prerelease.version) (Current: \(dependencies[cache: .appVersion].versionInfo))") + } + + let updatedJob: Job = job.with( + failureCount: 0, + nextRunTimestamp: (dependencies.dateNow.timeIntervalSince1970 + updateCheckFrequency) + ) + return .success(updatedJob, stop: false) } } diff --git a/SessionMessagingKit/Jobs/ConfigMessageReceiveJob.swift b/SessionMessagingKit/Jobs/ConfigMessageReceiveJob.swift index 5720b9acfb..dbf72a35f4 100644 --- a/SessionMessagingKit/Jobs/ConfigMessageReceiveJob.swift +++ b/SessionMessagingKit/Jobs/ConfigMessageReceiveJob.swift @@ -19,23 +19,16 @@ public enum ConfigMessageReceiveJob: JobExecutor { public static var requiresThreadId: Bool = true public static let requiresInteractionId: Bool = false - public static func run( - _ job: Job, - scheduler: S, - success: @escaping (Job, Bool) -> Void, - failure: @escaping (Job, Error, Bool) -> Void, - deferred: @escaping (Job) -> Void, - using dependencies: Dependencies - ) { + public static func run(_ job: Job, using dependencies: Dependencies) async throws -> JobExecutionResult { /// When the `configMessageReceive` job fails we want to unblock any `messageReceive` jobs it was blocking /// to ensure the user isn't losing any messages - this generally _shouldn't_ happen but if it does then having a temporary /// "outdated" state due to standard messages which would have been invalidated by a config change incorrectly being /// processed is less severe then dropping a bunch on messages just because they were processed in the same poll as /// invalid config messages - let removeDependencyOnMessageReceiveJobs: () -> () = { + let removeDependencyOnMessageReceiveJobs: () async -> () = { guard let jobId: Int64 = job.id else { return } - dependencies[singleton: .storage].write { db in + _ = try? await dependencies[singleton: .storage].writeAsync { db in try JobDependencies .filter(JobDependencies.Columns.dependantId == jobId) .joining( @@ -51,12 +44,12 @@ public enum ConfigMessageReceiveJob: JobExecutor { let detailsData: Data = job.details, let details: Details = try? JSONDecoder(using: dependencies).decode(Details.self, from: detailsData) else { - removeDependencyOnMessageReceiveJobs() - return failure(job, JobRunnerError.missingRequiredDetails, true) + await removeDependencyOnMessageReceiveJobs() + throw JobRunnerError.missingRequiredDetails } - dependencies[singleton: .storage].writeAsync( - updates: { db in + do { + try await dependencies[singleton: .storage].writeAsync { db in try dependencies.mutate(cache: .libSession) { cache in try cache.handleConfigMessages( db, @@ -64,19 +57,15 @@ public enum ConfigMessageReceiveJob: JobExecutor { messages: details.messages ) } - }, - completion: { result in - // Handle the result - switch result { - case .failure(let error): - Log.error(.cat, "Couldn't receive config message due to error: \(error)") - removeDependencyOnMessageReceiveJobs() - failure(job, error, true) - - case .success: success(job, false) - } } - ) + + return .success(job, stop: false) + } + catch { + Log.error(.cat, "Couldn't receive config message due to error: \(error)") + await removeDependencyOnMessageReceiveJobs() + throw JobRunnerError.permanentFailure(error) + } } } diff --git a/SessionMessagingKit/Jobs/ConfigurationSyncJob.swift b/SessionMessagingKit/Jobs/ConfigurationSyncJob.swift index dc37ac0c02..ac43b2b910 100644 --- a/SessionMessagingKit/Jobs/ConfigurationSyncJob.swift +++ b/SessionMessagingKit/Jobs/ConfigurationSyncJob.swift @@ -21,16 +21,9 @@ public enum ConfigurationSyncJob: JobExecutor { private static let maxRunFrequency: TimeInterval = 3 private static let waitTimeForExpirationUpdate: TimeInterval = 1 - public static func run( - _ job: Job, - scheduler: S, - success: @escaping (Job, Bool) -> Void, - failure: @escaping (Job, Error, Bool) -> Void, - deferred: @escaping (Job) -> Void, - using dependencies: Dependencies - ) { + public static func run(_ job: Job, using dependencies: Dependencies) async throws -> JobExecutionResult { guard !dependencies[cache: .libSession].isEmpty else { - return success(job, true) + return .success(job, stop: true) } /// It's possible for multiple ConfigSyncJob's with the same target (user/group) to try to run at the same time since as soon as @@ -40,18 +33,22 @@ public enum ConfigurationSyncJob: JobExecutor { /// /// **Note:** The one exception to this rule is when the job has `AdditionalTransientData` because if we don't /// run it immediately then the `AdditionalTransientData` may not get run at all - guard - job.transientData != nil || - dependencies[singleton: .jobRunner] - .jobInfoFor(state: .running, variant: .configurationSync) - .filter({ key, info in - key != job.id && // Exclude this job - info.threadId == job.threadId // Exclude jobs for different config stores - }) - .isEmpty - else { - // Defer the job to run 'maxRunFrequency' from when this one ran (if we don't it'll try start - // it again immediately which is pointless) + let hasExistingJob: Bool = await dependencies[singleton: .jobRunner] + .jobInfoFor( + state: .running, + filters: JobRunner.Filters( + include: [.variant(.configurationSync)], + exclude: [ + job.id.map { .jobId($0) }, /// Exclude this job + job.threadId.map { .threadId($0) } /// Exclude jobs for different config stores + ].compactMap { $0 } + ) + ) + .isEmpty + + guard job.transientData != nil || hasExistingJob else { + /// Defer the job to run `maxRunFrequency` from when this one ran (if we don't it'll try start it again immediately which + /// is pointless) let updatedJob: Job? = dependencies[singleton: .storage].write { db in try job .with(nextRunTimestamp: dependencies.dateNow.timeIntervalSince1970 + maxRunFrequency) @@ -59,12 +56,11 @@ public enum ConfigurationSyncJob: JobExecutor { } Log.info(.cat, "For \(job.threadId ?? "UnknownId") deferred due to in progress job") - return deferred(updatedJob ?? job) + return .deferred(updatedJob ?? job) } - // If we don't have a userKeyPair yet then there is no need to sync the configuration - // as the user doesn't exist yet (this will get triggered on the first launch of a - // fresh install due to the migrations getting run) + /// If we don't have a userKeyPair yet then there is no need to sync the configuration as the user doesn't exist yet (this will get + /// triggered on the first launch of a fresh install due to the migrations getting run) guard let swarmPublicKey: String = job.threadId, let pendingPushes: LibSession.PendingPushes = try? dependencies.mutate(cache: .libSession, { @@ -72,7 +68,7 @@ public enum ConfigurationSyncJob: JobExecutor { }) else { Log.info(.cat, "For \(job.threadId ?? "UnknownId") failed due to invalid data") - return failure(job, StorageError.generic, false) + throw StorageError.generic } /// If there is no `pushData` or additional sequence requests then the job can just complete (next time something is updated @@ -83,7 +79,7 @@ public enum ConfigurationSyncJob: JobExecutor { else { Log.info(.cat, "For \(swarmPublicKey) completed with no pending changes") ConfigurationSyncJob.startJobsWaitingOnConfigSync(swarmPublicKey, using: dependencies) - return success(job, true) + return .success(job, stop: true) } let jobStartTimestamp: TimeInterval = dependencies.dateNow.timeIntervalSince1970 @@ -91,53 +87,53 @@ public enum ConfigurationSyncJob: JobExecutor { let additionalTransientData: AdditionalTransientData? = (job.transientData as? AdditionalTransientData) Log.info(.cat, "For \(swarmPublicKey) started with changes: \(pendingPushes.pushData.count), old hashes: \(pendingPushes.obsoleteHashes.count)") - dependencies[singleton: .storage] - .readPublisher { db -> AuthenticationMethod in - try Authentication.with(db, swarmPublicKey: swarmPublicKey, using: dependencies) - } - .tryFlatMap { authMethod -> AnyPublisher<(ResponseInfoType, Network.BatchResponse), Error> in - try SnodeAPI.preparedSequence( - requests: [] - .appending(contentsOf: additionalTransientData?.beforeSequenceRequests) - .appending( - contentsOf: try pendingPushes.pushData - .flatMap { pushData -> [ErasedPreparedRequest] in - try pushData.data.map { data -> ErasedPreparedRequest in - try SnodeAPI - .preparedSendMessage( - message: SnodeMessage( - recipient: swarmPublicKey, - data: data, - ttl: pushData.variant.ttl, - timestampMs: UInt64(messageSendTimestamp) - ), - in: pushData.variant.namespace, - authMethod: authMethod, - using: dependencies - ) - } + let authMethod: AuthenticationMethod = try await dependencies[singleton: .storage].readAsync { db in + try Authentication.with(db, swarmPublicKey: swarmPublicKey, using: dependencies) + } + + let request: Network.PreparedRequest = try SnodeAPI.preparedSequence( + requests: [] + .appending(contentsOf: additionalTransientData?.beforeSequenceRequests) + .appending( + contentsOf: try pendingPushes.pushData + .flatMap { pushData -> [ErasedPreparedRequest] in + try pushData.data.map { data -> ErasedPreparedRequest in + try SnodeAPI + .preparedSendMessage( + message: SnodeMessage( + recipient: swarmPublicKey, + data: data, + ttl: pushData.variant.ttl, + timestampMs: UInt64(messageSendTimestamp) + ), + in: pushData.variant.namespace, + authMethod: authMethod, + using: dependencies + ) } - ) - .appending(try { - guard !pendingPushes.obsoleteHashes.isEmpty else { return nil } - - return try SnodeAPI.preparedDeleteMessages( - serverHashes: Array(pendingPushes.obsoleteHashes), - requireSuccessfulDeletion: false, - authMethod: authMethod, - using: dependencies - ) - }()) - .appending(contentsOf: additionalTransientData?.afterSequenceRequests), - requireAllBatchResponses: (additionalTransientData?.requireAllBatchResponses == true), - swarmPublicKey: swarmPublicKey, - snodeRetrievalRetryCount: 0, // This job has it's own retry mechanism - requestAndPathBuildTimeout: Network.defaultTimeout, - using: dependencies - ).send(using: dependencies) - } - .subscribe(on: scheduler, using: dependencies) - .receive(on: scheduler, using: dependencies) + } + ) + .appending(try { + guard !pendingPushes.obsoleteHashes.isEmpty else { return nil } + + return try SnodeAPI.preparedDeleteMessages( + serverHashes: Array(pendingPushes.obsoleteHashes), + requireSuccessfulDeletion: false, + authMethod: authMethod, + using: dependencies + ) + }()) + .appending(contentsOf: additionalTransientData?.afterSequenceRequests), + requireAllBatchResponses: (additionalTransientData?.requireAllBatchResponses == true), + swarmPublicKey: swarmPublicKey, + snodeRetrievalRetryCount: 0, // This job has it's own retry mechanism + requestAndPathBuildTimeout: Network.defaultTimeout, + using: dependencies + ) + + // FIXME: Refactor this to use async/await + let publisher = request + .send(using: dependencies) .tryMap { (_: ResponseInfoType, response: Network.BatchResponse) -> [ConfigDump] in /// The number of responses returned might not match the number of changes sent but they will be returned /// in the same order, this means we can just `zip` the two arrays as it will take the smaller of the two and @@ -184,106 +180,100 @@ public enum ConfigurationSyncJob: JobExecutor { ) } } - .sinkUntilComplete( - receiveCompletion: { result in - switch result { - case .finished: Log.info(.cat, "For \(swarmPublicKey) completed") - case .failure(let error): - Log.error(.cat, "For \(swarmPublicKey) failed due to error: \(error)") - - // If the failure is due to being offline then we should automatically - // retry if the connection is re-established - dependencies[cache: .libSessionNetwork].networkStatus - .first() - .sinkUntilComplete( - receiveValue: { status in - switch status { - // If we are currently connected then use the standard - // retry behaviour - case .connected: failure(job, error, false) - - // If not then permanently fail the job and reschedule it - // to run again if we re-establish the connection - default: - failure(job, error, true) - - dependencies[cache: .libSessionNetwork].networkStatus - .filter { $0 == .connected } - .first() - .sinkUntilComplete( - receiveCompletion: { _ in - dependencies[singleton: .storage].writeAsync { db in - ConfigurationSyncJob.enqueue( - db, - swarmPublicKey: swarmPublicKey, - using: dependencies - ) - } - } - ) - } - } - ) - } - }, - receiveValue: { (configDumps: [ConfigDump]) in - // Flag to indicate whether the job should be finished or will run again - var shouldFinishCurrentJob: Bool = false - - // Lastly we need to save the updated dumps to the database - let updatedJob: Job? = dependencies[singleton: .storage].write { db in - // Save the updated dumps to the database - try configDumps.forEach { dump in - try dump.upsert(db) - Task.detached(priority: .medium) { [extensionHelper = dependencies[singleton: .extensionHelper]] in - extensionHelper.replicate(dump: dump) - } - } - - // When we complete the 'ConfigurationSync' job we want to immediately schedule - // another one with a 'nextRunTimestamp' set to the 'maxRunFrequency' value to - // throttle the config sync requests - let nextRunTimestamp: TimeInterval = (jobStartTimestamp + maxRunFrequency) + + do { + let configDumps: [ConfigDump] = (try await publisher.values.first(where: { _ in true }) ?? []) + + /// Lastly we need to save the updated dumps to the database + let currentlyRunningJobs: [JobRunner.JobInfo] = await Array(dependencies[singleton: .jobRunner] + .jobInfoFor( + state: .running, + filters: ConfigurationSyncJob.filters( + swarmPublicKey: swarmPublicKey, + whenRunning: job + ) + ) + .values) + let updatedJob: Job? = try await dependencies[singleton: .storage].writeAsync { db in + /// Save the updated dumps to the database + try configDumps.forEach { dump in + try dump.upsert(db) - // If another 'ConfigurationSync' job was scheduled then update that one - // to run at 'nextRunTimestamp' and make the current job stop - if - let existingJob: Job = try? Job - .filter(Job.Columns.id != job.id) - .filter(Job.Columns.variant == Job.Variant.configurationSync) - .filter(Job.Columns.threadId == swarmPublicKey) - .order(Job.Columns.nextRunTimestamp.asc) - .fetchOne(db) - { - // If the next job isn't currently running then delay it's start time - // until the 'nextRunTimestamp' unless it was manually triggered (in which - // case we want it to run immediately as some thread is likely waiting on - // it to return) - if !dependencies[singleton: .jobRunner].isCurrentlyRunning(existingJob) { - let jobWasManualTrigger: Bool = (existingJob.details - .map { try? JSONDecoder(using: dependencies).decode(OptionalDetails.self, from: $0) } - .map { $0.wasManualTrigger }) - .defaulting(to: false) - - try existingJob - .with(nextRunTimestamp: (jobWasManualTrigger ? 0 : nextRunTimestamp)) - .upserted(db) - } - - // If there is another job then we should finish this one - shouldFinishCurrentJob = true - return job + Task.detached(priority: .medium) { [extensionHelper = dependencies[singleton: .extensionHelper]] in + extensionHelper.replicate(dump: dump) } + } + + /// When we complete the `ConfigurationSync` job we want to immediately schedule another one with a + /// `nextRunTimestamp` set to the `maxRunFrequency` value to throttle the config sync requests + let nextRunTimestamp: TimeInterval = (jobStartTimestamp + maxRunFrequency) + + /// If another `ConfigurationSync` job was scheduled then we can just update that one to run at `nextRunTimestamp` + /// and make the current job stop + if + let existingJobInfo: JobRunner.JobInfo = currentlyRunningJobs + .sorted(by: { lhs, rhs in lhs.nextRunTimestamp < rhs.nextRunTimestamp }) + .first, + let jobId: Int64 = existingJobInfo.id, + let existingJob: Job = try? Job.fetchOne(db, id: jobId) + { + /// If the next job isn't currently running then delay it's start time until the `nextRunTimestamp` unless + /// it was manually triggered (in which case we want it to run immediately as some thread is likely waiting on + /// it to return) + let jobWasManualTrigger: Bool = (existingJob.details + .map { try? JSONDecoder(using: dependencies).decode(OptionalDetails.self, from: $0) } + .map { $0.wasManualTrigger }) + .defaulting(to: false) - return try job - .with(nextRunTimestamp: nextRunTimestamp) + try existingJob + .with(nextRunTimestamp: (jobWasManualTrigger ? 0 : nextRunTimestamp)) .upserted(db) + + return nil } - ConfigurationSyncJob.startJobsWaitingOnConfigSync(swarmPublicKey, using: dependencies) - success((updatedJob ?? job), shouldFinishCurrentJob) + return try job + .with(nextRunTimestamp: nextRunTimestamp) + .upserted(db) } - ) + + /// If we returned no `updatedJob` above then we want to stop the current job (because there is an existing job + /// that we've already rescueduled) + ConfigurationSyncJob.startJobsWaitingOnConfigSync(swarmPublicKey, using: dependencies) + Log.info(.cat, "For \(swarmPublicKey) completed") + return .success((updatedJob ?? job), stop: (updatedJob == nil)) + } + catch { + Log.error(.cat, "For \(swarmPublicKey) failed due to error: \(error)") + + /// If the failure is due to being offline then we should automatically retry if the connection is re-established + // FIXME: Refactor this to use async/await + let publisher2 = dependencies[cache: .libSessionNetwork].networkStatus + .first() + + let status: NetworkStatus? = await publisher2.values.first(where: { _ in true }) + switch status { + /// If we are currently connected then use the standard retry behaviour + case .connected: throw error + + /// If not then spin up a task to reschedule the config sync if we re-establish the connection and permanently + /// fail this job + default: + Task.detached(priority: .background) { [dependencies] in + // FIXME: Refactor this to use async/await + let publisher3 = dependencies[cache: .libSessionNetwork].networkStatus + .filter { $0 == .connected } + .first() + _ = await publisher3.values.first(where: { _ in true }) + await ConfigurationSyncJob.enqueue( + swarmPublicKey: swarmPublicKey, + using: dependencies + ) + } + + throw JobRunnerError.permanentFailure(error) + } + } } private static func startJobsWaitingOnConfigSync( @@ -360,40 +350,43 @@ extension ConfigurationSyncJob { public extension ConfigurationSyncJob { static func enqueue( - _ db: ObservingDatabase, swarmPublicKey: String, using dependencies: Dependencies - ) { - // Upsert a config sync job if needed - dependencies[singleton: .jobRunner].upsert( - db, - job: ConfigurationSyncJob.createIfNeeded(db, swarmPublicKey: swarmPublicKey, using: dependencies), - canStartJob: true - ) + ) async { + /// Upsert a config sync job if needed + guard let job: Job = await ConfigurationSyncJob.createIfNeeded(swarmPublicKey: swarmPublicKey, using: dependencies) else { + return + } + + _ = try? await dependencies[singleton: .storage].writeAsync { db in + dependencies[singleton: .jobRunner].upsert(db, job: job, canStartJob: true) + } } @discardableResult static func createIfNeeded( - _ db: ObservingDatabase, swarmPublicKey: String, using dependencies: Dependencies - ) -> Job? { + ) async -> Job? { /// The ConfigurationSyncJob will automatically reschedule itself to run again after 3 seconds so if there is an existing /// job then there is no need to create another instance /// /// **Note:** Jobs with different `threadId` values can run concurrently guard - dependencies[singleton: .jobRunner] - .jobInfoFor(state: .running, variant: .configurationSync) - .filter({ _, info in info.threadId == swarmPublicKey }) + await dependencies[singleton: .jobRunner] + .jobInfoFor( + state: .running, + filters: ConfigurationSyncJob.filters(swarmPublicKey: swarmPublicKey) + ) .isEmpty, - (try? Job - .filter(Job.Columns.variant == Job.Variant.configurationSync) - .filter(Job.Columns.threadId == swarmPublicKey) - .isEmpty(db)) - .defaulting(to: false) + (try? await dependencies[singleton: .storage].readAsync(value: { db -> Bool in + try Job + .filter(Job.Columns.variant == Job.Variant.configurationSync) + .filter(Job.Columns.threadId == swarmPublicKey) + .isEmpty(db) + })) == true else { return nil } - // Otherwise create a new job + /// Otherwise create a new job return Job( variant: .configurationSync, behaviour: .recurring, @@ -412,49 +405,61 @@ public extension ConfigurationSyncJob { requireAllBatchResponses: Bool = false, requireAllRequestsSucceed: Bool = false, using dependencies: Dependencies - ) -> AnyPublisher { - return Deferred { - Future { resolver in - guard - let job: Job = Job( - variant: .configurationSync, - behaviour: .recurring, - threadId: swarmPublicKey, - details: OptionalDetails(wasManualTrigger: true), - transientData: AdditionalTransientData( - beforeSequenceRequests: beforeSequenceRequests, - afterSequenceRequests: afterSequenceRequests, - requireAllBatchResponses: requireAllBatchResponses, - requireAllRequestsSucceed: requireAllRequestsSucceed - ) - ) - else { return resolver(Result.failure(NetworkError.parsingFailed)) } - - ConfigurationSyncJob.run( - job, - scheduler: DispatchQueue.global(qos: .userInitiated), - success: { _, _ in resolver(Result.success(())) }, - failure: { _, error, _ in resolver(Result.failure(error)) }, - deferred: { job in - dependencies[singleton: .jobRunner] - .afterJob(job) - .first() - .sinkUntilComplete( - receiveValue: { result in - switch result { - /// If it gets deferred a second time then we should probably just fail - no use waiting on something - /// that may never run (also means we can avoid another potential defer loop) - case .notFound, .deferred: resolver(Result.failure(NetworkError.unknown)) - case .failed(let error, _): resolver(Result.failure(error)) - case .succeeded: resolver(Result.success(())) - } - } - ) - }, - using: dependencies + ) async throws { + guard + let job: Job = Job( + variant: .configurationSync, + behaviour: .recurring, + threadId: swarmPublicKey, + details: OptionalDetails(wasManualTrigger: true), + transientData: AdditionalTransientData( + beforeSequenceRequests: beforeSequenceRequests, + afterSequenceRequests: afterSequenceRequests, + requireAllBatchResponses: requireAllBatchResponses, + requireAllRequestsSucceed: requireAllRequestsSucceed ) - } + ) + else { throw JobRunnerError.missingRequiredDetails } + + let result: JobExecutionResult = try await ConfigurationSyncJob.run(job, using: dependencies) + + /// If the job was deferred it was most likely due to another `SyncPushTokens` job in progress so we should wait + /// for the other job to finish and try again + switch result { + case .success: return + case .deferred: break + } + + let runningJobs: [Int64: JobRunner.JobInfo] = await dependencies[singleton: .jobRunner] + .jobInfoFor( + state: .running, + filters: ConfigurationSyncJob.filters(swarmPublicKey: swarmPublicKey, whenRunning: job) + ) + + /// If we couldn't find a running job then fail (something else went wrong) + guard !runningJobs.isEmpty else { throw JobRunnerError.missingRequiredDetails } + + let otherJobResult: JobRunner.JobResult = await dependencies[singleton: .jobRunner].awaitResult( + forFirstJobMatching: ConfigurationSyncJob.filters(swarmPublicKey: swarmPublicKey, whenRunning: job), + in: .running + ) + + /// If it gets deferred a second time then we should probably just fail - no use waiting on something + /// that may never run (also means we can avoid another potential defer loop) + switch otherJobResult { + case .notFound, .deferred: throw JobRunnerError.missingRequiredDetails + case .failed(let error, _): throw error + case .succeeded: break } - .eraseToAnyPublisher() + } + + private static func filters(swarmPublicKey: String, whenRunning job: Job? = nil) -> JobRunner.Filters { + return JobRunner.Filters( + include: [ + .variant(.configurationSync), + .threadId(swarmPublicKey) + ], + exclude: [job?.id.map { .jobId($0) }].compactMap { $0 } /// Exclude running job + ) } } diff --git a/SessionMessagingKit/Jobs/DisplayPictureDownloadJob.swift b/SessionMessagingKit/Jobs/DisplayPictureDownloadJob.swift index ec9ee87bc3..af2905000b 100644 --- a/SessionMessagingKit/Jobs/DisplayPictureDownloadJob.swift +++ b/SessionMessagingKit/Jobs/DisplayPictureDownloadJob.swift @@ -21,20 +21,15 @@ public enum DisplayPictureDownloadJob: JobExecutor { public static var requiresThreadId: Bool = false public static var requiresInteractionId: Bool = false - public static func run( - _ job: Job, - scheduler: S, - success: @escaping (Job, Bool) -> Void, - failure: @escaping (Job, Error, Bool) -> Void, - deferred: @escaping (Job) -> Void, - using dependencies: Dependencies - ) { + public static func run(_ job: Job, using dependencies: Dependencies) async throws -> JobExecutionResult { + // TODO: Make the 'shouldBeUnique' part of this job instead. guard let detailsData: Data = job.details, let details: Details = try? JSONDecoder(using: dependencies).decode(Details.self, from: detailsData) - else { return failure(job, JobRunnerError.missingRequiredDetails, true) } + else { throw JobRunnerError.missingRequiredDetails } - dependencies[singleton: .storage] + // FIXME: Refactor this to use async/await + let publisher = dependencies[singleton: .storage] .readPublisher { db -> Network.PreparedRequest in switch details.target { case .profile(_, let url, _), .group(_, let url, _): @@ -80,8 +75,6 @@ public enum DisplayPictureDownloadJob: JobExecutor { } .flatMap { $0.send(using: dependencies) } .map { _, result in result } - .subscribe(on: scheduler, using: dependencies) - .receive(on: scheduler, using: dependencies) .flatMapStorageReadPublisher(using: dependencies) { (db: ObservingDatabase, result: (Data, String, URL?)) -> (Data, String, URL?) in /// Check to make sure this download is still a valid update guard details.isValidUpdate(db, using: dependencies) else { @@ -130,49 +123,48 @@ public enum DisplayPictureDownloadJob: JobExecutor { using: dependencies ) } - .sinkUntilComplete( - receiveCompletion: { result in - switch (result, result.errorOrNull, result.errorOrNull as? DisplayPictureError) { - case (.finished, _, _): success(job, false) - case (_, _, .updateNoLongerValid): success(job, false) - case (_, _, .alreadyDownloaded(let downloadUrl)): - /// If the file already exists then write the changes to the database - dependencies[singleton: .storage].writeAsync( - updates: { db in - try writeChanges( - db, - details: details, - downloadUrl: downloadUrl, - using: dependencies - ) - }, - completion: { result in - switch result { - case .success: success(job, false) - case .failure(let error): failure(job, error, true) - } - } + + do { + try await publisher.values.first(where: { _ in true }) + return .success(job, stop: false) + } + catch { + switch error { + case DisplayPictureError.updateNoLongerValid: return .success(job, stop: false) + case DisplayPictureError.alreadyDownloaded(let downloadUrl): + /// If the file already exists then write the changes to the database + do { + try await dependencies[singleton: .storage].writeAsync { db in + try writeChanges( + db, + details: details, + downloadUrl: downloadUrl, + using: dependencies ) - - case (_, let error as JobRunnerError, _) where error == .missingRequiredDetails: - failure(job, error, true) - - case (_, _, .invalidPath): - Log.error(.cat, "Failed to generate display picture file path for \(details.target)") - failure(job, DisplayPictureError.invalidPath, true) - - case (_, _, .writeFailed): - Log.error(.cat, "Failed to decrypt display picture for \(details.target)") - failure(job, DisplayPictureError.writeFailed, true) - - case (_, _, .loadFailed): - Log.error(.cat, "Failed to load display picture for \(details.target)") - failure(job, DisplayPictureError.loadFailed, true) - - case (.failure(let error), _, _): failure(job, error, true) + } + + return .success(job, stop: false) } - } - ) + catch { + throw JobRunnerError.permanentFailure(error) + } + + case JobRunnerError.missingRequiredDetails: throw error + case DisplayPictureError.invalidPath: + Log.error(.cat, "Failed to generate display picture file path for \(details.target)") + throw error + + case DisplayPictureError.writeFailed: + Log.error(.cat, "Failed to decrypt display picture for \(details.target)") + throw error + + case DisplayPictureError.loadFailed: + Log.error(.cat, "Failed to load display picture for \(details.target)") + throw error + + default: throw JobRunnerError.permanentFailure(error) + } + } } private static func writeChanges( @@ -372,3 +364,16 @@ extension DisplayPictureDownloadJob { } } } + +// MARK: - JobError + +extension DisplayPictureError: JobError { + public var isPermanent: Bool { + switch self { + case .writeFailed: return true + case .loadFailed: return true + case .invalidPath: return true + default: return false + } + } +} diff --git a/SessionMessagingKit/Jobs/ExpirationUpdateJob.swift b/SessionMessagingKit/Jobs/ExpirationUpdateJob.swift index 0ffd051f5a..87961cc3be 100644 --- a/SessionMessagingKit/Jobs/ExpirationUpdateJob.swift +++ b/SessionMessagingKit/Jobs/ExpirationUpdateJob.swift @@ -11,20 +11,14 @@ public enum ExpirationUpdateJob: JobExecutor { public static var requiresThreadId: Bool = true public static var requiresInteractionId: Bool = false - public static func run( - _ job: Job, - scheduler: S, - success: @escaping (Job, Bool) -> Void, - failure: @escaping (Job, Error, Bool) -> Void, - deferred: @escaping (Job) -> Void, - using dependencies: Dependencies - ) { + public static func run(_ job: Job, using dependencies: Dependencies) async throws -> JobExecutionResult { guard let detailsData: Data = job.details, let details: Details = try? JSONDecoder(using: dependencies).decode(Details.self, from: detailsData) - else { return failure(job, JobRunnerError.missingRequiredDetails, true) } + else { throw JobRunnerError.missingRequiredDetails } - dependencies[singleton: .storage] + // FIXME: Refactor this to use async/await + let publisher = dependencies[singleton: .storage] .readPublisher { db in try SnodeAPI .preparedUpdateExpiry( @@ -40,8 +34,6 @@ public enum ExpirationUpdateJob: JobExecutor { ) } .flatMap { $0.send(using: dependencies) } - .subscribe(on: scheduler, using: dependencies) - .receive(on: scheduler, using: dependencies) .map { _, response -> [UInt64: [String]] in guard let results: [UpdateExpiryResponseResult] = response @@ -55,43 +47,43 @@ public enum ExpirationUpdateJob: JobExecutor { return unchangedMessages } - .sinkUntilComplete( - receiveCompletion: { result in - switch result { - case .finished: success(job, false) - case .failure(let error): failure(job, error, true) - } - }, - receiveValue: { unchangedMessages in - guard !unchangedMessages.isEmpty else { return } - - dependencies[singleton: .storage].writeAsync { db in - unchangedMessages.forEach { updatedExpiry, hashes in - hashes.forEach { hash in - guard - let interaction: Interaction = try? Interaction - .filter(Interaction.Columns.serverHash == hash) - .fetchOne(db), - let expiresInSeconds: TimeInterval = interaction.expiresInSeconds - else { return } - - let expiresStartedAtMs: Double = Double(updatedExpiry - UInt64(expiresInSeconds * 1000)) - - dependencies[singleton: .jobRunner].upsert( + + do { + let unchangedMessages = try await publisher.values.first(where: { _ in true }) + + if unchangedMessages?.isEmpty == false { + try? await dependencies[singleton: .storage].writeAsync { db in + unchangedMessages?.forEach { updatedExpiry, hashes in + hashes.forEach { hash in + guard + let interaction: Interaction = try? Interaction + .filter(Interaction.Columns.serverHash == hash) + .fetchOne(db), + let expiresInSeconds: TimeInterval = interaction.expiresInSeconds + else { return } + + let expiresStartedAtMs: Double = Double(updatedExpiry - UInt64(expiresInSeconds * 1000)) + + dependencies[singleton: .jobRunner].upsert( + db, + job: DisappearingMessagesJob.updateNextRunIfNeeded( db, - job: DisappearingMessagesJob.updateNextRunIfNeeded( - db, - interaction: interaction, - startedAtMs: expiresStartedAtMs, - using: dependencies - ), - canStartJob: true - ) - } + interaction: interaction, + startedAtMs: expiresStartedAtMs, + using: dependencies + ), + canStartJob: true + ) } } } - ) + } + + return .success(job, stop: false) + } + catch { + throw JobRunnerError.permanentFailure(error) + } } } diff --git a/SessionMessagingKit/Jobs/FailedAttachmentDownloadsJob.swift b/SessionMessagingKit/Jobs/FailedAttachmentDownloadsJob.swift index 68ab51fc2e..05a8eb028b 100644 --- a/SessionMessagingKit/Jobs/FailedAttachmentDownloadsJob.swift +++ b/SessionMessagingKit/Jobs/FailedAttachmentDownloadsJob.swift @@ -18,60 +18,46 @@ public enum FailedAttachmentDownloadsJob: JobExecutor { public static let requiresThreadId: Bool = false public static let requiresInteractionId: Bool = false - public static func run( - _ job: Job, - scheduler: S, - success: @escaping (Job, Bool) -> Void, - failure: @escaping (Job, Error, Bool) -> Void, - deferred: @escaping (Job) -> Void, - using dependencies: Dependencies - ) { - guard dependencies[cache: .general].userExists else { return success(job, false) } + public static func run(_ job: Job, using dependencies: Dependencies) async throws -> JobExecutionResult { + guard dependencies[cache: .general].userExists else { return .success(job, stop: false) } - var changeCount: Int = -1 - - // Update all 'sending' message states to 'failed' - dependencies[singleton: .storage] - .writePublisher { db in - let attachmentIds: Set = try Attachment - .select(.id) - .filter(Attachment.Columns.state == Attachment.State.downloading) - .asRequest(of: String.self) - .fetchSet(db) - let interactionAttachment: [InteractionAttachment] = try InteractionAttachment - .filter(attachmentIds.contains(InteractionAttachment.Columns.attachmentId)) - .fetchAll(db) - - try Attachment - .filter(attachmentIds.contains(Attachment.Columns.id)) - .updateAll(db, Attachment.Columns.state.set(to: Attachment.State.failedDownload)) - changeCount = attachmentIds.count - - interactionAttachment.forEach { val in - db.addAttachmentEvent( - id: val.attachmentId, - messageId: val.interactionId, - type: .updated(.state(.failedDownload)) - ) - } + /// Update all 'sending' message states to 'failed' + let changeCount: Int = try await dependencies[singleton: .storage].writeAsync { db in + let attachmentIds: Set = try Attachment + .select(.id) + .filter(Attachment.Columns.state == Attachment.State.downloading) + .asRequest(of: String.self) + .fetchSet(db) + let interactionAttachment: [InteractionAttachment] = try InteractionAttachment + .filter(attachmentIds.contains(InteractionAttachment.Columns.attachmentId)) + .fetchAll(db) + + try Attachment + .filter(attachmentIds.contains(Attachment.Columns.id)) + .updateAll(db, Attachment.Columns.state.set(to: Attachment.State.failedDownload)) + + interactionAttachment.forEach { val in + db.addAttachmentEvent( + id: val.attachmentId, + messageId: val.interactionId, + type: .updated(.state(.failedDownload)) + ) + } + + /// Shouldn't be possible but just in case + if attachmentIds.count != interactionAttachment.count { + let remainingIds: Set = attachmentIds + .removing(contentsOf: Set(interactionAttachment.map { $0.attachmentId })) - /// Shouldn't be possible but just in case - if attachmentIds.count != interactionAttachment.count { - let remainingIds: Set = attachmentIds - .removing(contentsOf: Set(interactionAttachment.map { $0.attachmentId })) - - remainingIds.forEach { id in - db.addAttachmentEvent(id: id, messageId: nil, type: .updated(.state(.failedDownload))) - } + remainingIds.forEach { id in + db.addAttachmentEvent(id: id, messageId: nil, type: .updated(.state(.failedDownload))) } } - .subscribe(on: scheduler, using: dependencies) - .receive(on: scheduler, using: dependencies) - .sinkUntilComplete( - receiveCompletion: { _ in - Log.info(.cat, "Marked \(changeCount) attachments as failed") - success(job, false) - } - ) + + return attachmentIds.count + } + + Log.info(.cat, "Marked \(changeCount) attachments as failed") + return .success(job, stop: false) } } diff --git a/SessionMessagingKit/Jobs/FailedMessageSendsJob.swift b/SessionMessagingKit/Jobs/FailedMessageSendsJob.swift index 5e5ccc5cad..eb2c3dd7bb 100644 --- a/SessionMessagingKit/Jobs/FailedMessageSendsJob.swift +++ b/SessionMessagingKit/Jobs/FailedMessageSendsJob.swift @@ -18,75 +18,61 @@ public enum FailedMessageSendsJob: JobExecutor { public static let requiresThreadId: Bool = false public static let requiresInteractionId: Bool = false - public static func run( - _ job: Job, - scheduler: S, - success: @escaping (Job, Bool) -> Void, - failure: @escaping (Job, Error, Bool) -> Void, - deferred: @escaping (Job) -> Void, - using dependencies: Dependencies - ) { - guard dependencies[cache: .general].userExists else { return success(job, false) } + public static func run(_ job: Job, using dependencies: Dependencies) async throws -> JobExecutionResult { + guard dependencies[cache: .general].userExists else { return .success(job, stop: false) } - var changeCount: Int = -1 - var attachmentChangeCount: Int = -1 - - // Update all 'sending' message states to 'failed' - dependencies[singleton: .storage] - .writePublisher { db in - let sendInteractionInfo: Set = try Interaction - .select(.id, .threadId) - .filter(Interaction.Columns.state == Interaction.State.sending) - .asRequest(of: InteractionIdThreadId.self) - .fetchSet(db) - let syncInteractionInfo: Set = try Interaction - .select(.id, .threadId) - .filter(Interaction.Columns.state == Interaction.State.syncing) - .asRequest(of: InteractionIdThreadId.self) - .fetchSet(db) - let attachmentIds: Set = try Attachment - .select(.id) - .filter(Attachment.Columns.state == Attachment.State.uploading) - .asRequest(of: String.self) - .fetchSet(db) - let interactionAttachment: [InteractionAttachment] = try InteractionAttachment - .filter(attachmentIds.contains(InteractionAttachment.Columns.attachmentId)) - .fetchAll(db) - - let sendChangeCount: Int = try Interaction - .filter(sendInteractionInfo.map { $0.id }.contains(Interaction.Columns.id)) - .updateAll(db, Interaction.Columns.state.set(to: Interaction.State.failed)) - let syncChangeCount: Int = try Interaction - .filter(syncInteractionInfo.map { $0.id }.contains(Interaction.Columns.id)) - .updateAll(db, Interaction.Columns.state.set(to: Interaction.State.failedToSync)) - attachmentChangeCount = try Attachment - .filter(attachmentIds.contains(Attachment.Columns.id)) - .updateAll(db, Attachment.Columns.state.set(to: Attachment.State.failedUpload)) - changeCount = (sendChangeCount + syncChangeCount) - - /// Send the database events - sendInteractionInfo.forEach { info in - db.addMessageEvent(id: info.id, threadId: info.threadId, type: .updated(.state(.failed))) - } - syncInteractionInfo.forEach { info in - db.addMessageEvent(id: info.id, threadId: info.threadId, type: .updated(.state(.failedToSync))) - } - interactionAttachment.forEach { val in - db.addAttachmentEvent( - id: val.attachmentId, - messageId: val.interactionId, - type: .updated(.state(.failedUpload)) - ) - } + /// Update all 'sending' message states to 'failed' + let (changeCount, attachmentChangeCount): (Int, Int) = try await dependencies[singleton: .storage].writeAsync { db in + let sendInteractionInfo: Set = try Interaction + .select(.id, .threadId) + .filter(Interaction.Columns.state == Interaction.State.sending) + .asRequest(of: InteractionIdThreadId.self) + .fetchSet(db) + let syncInteractionInfo: Set = try Interaction + .select(.id, .threadId) + .filter(Interaction.Columns.state == Interaction.State.syncing) + .asRequest(of: InteractionIdThreadId.self) + .fetchSet(db) + let attachmentIds: Set = try Attachment + .select(.id) + .filter(Attachment.Columns.state == Attachment.State.uploading) + .asRequest(of: String.self) + .fetchSet(db) + let interactionAttachment: [InteractionAttachment] = try InteractionAttachment + .filter(attachmentIds.contains(InteractionAttachment.Columns.attachmentId)) + .fetchAll(db) + + let sendChangeCount: Int = try Interaction + .filter(sendInteractionInfo.map { $0.id }.contains(Interaction.Columns.id)) + .updateAll(db, Interaction.Columns.state.set(to: Interaction.State.failed)) + let syncChangeCount: Int = try Interaction + .filter(syncInteractionInfo.map { $0.id }.contains(Interaction.Columns.id)) + .updateAll(db, Interaction.Columns.state.set(to: Interaction.State.failedToSync)) + let attachmentChangeCount: Int = try Attachment + .filter(attachmentIds.contains(Attachment.Columns.id)) + .updateAll(db, Attachment.Columns.state.set(to: Attachment.State.failedUpload)) + let changeCount: Int = (sendChangeCount + syncChangeCount) + + /// Send the database events + sendInteractionInfo.forEach { info in + db.addMessageEvent(id: info.id, threadId: info.threadId, type: .updated(.state(.failed))) + } + syncInteractionInfo.forEach { info in + db.addMessageEvent(id: info.id, threadId: info.threadId, type: .updated(.state(.failedToSync))) } - .subscribe(on: scheduler, using: dependencies) - .receive(on: scheduler, using: dependencies) - .sinkUntilComplete( - receiveCompletion: { _ in - Log.info(.cat, "Messages marked as failed: \(changeCount), Uploads cancelled: \(attachmentChangeCount)") - success(job, false) - } - ) + interactionAttachment.forEach { val in + db.addAttachmentEvent( + id: val.attachmentId, + messageId: val.interactionId, + type: .updated(.state(.failedUpload)) + ) + } + + return (changeCount, attachmentChangeCount) + } + + Log.info(.cat, "Messages marked as failed: \(changeCount), Uploads cancelled: \(attachmentChangeCount)") + return .success(job, stop: false) } } diff --git a/SessionMessagingKit/Jobs/GroupInviteMemberJob.swift b/SessionMessagingKit/Jobs/GroupInviteMemberJob.swift index b40827c6e8..8d7af5299e 100644 --- a/SessionMessagingKit/Jobs/GroupInviteMemberJob.swift +++ b/SessionMessagingKit/Jobs/GroupInviteMemberJob.swift @@ -20,14 +20,7 @@ public enum GroupInviteMemberJob: JobExecutor { public static var requiresThreadId: Bool = true public static var requiresInteractionId: Bool = false - public static func run( - _ job: Job, - scheduler: S, - success: @escaping (Job, Bool) -> Void, - failure: @escaping (Job, Error, Bool) -> Void, - deferred: @escaping (Job) -> Void, - using dependencies: Dependencies - ) { + public static func run(_ job: Job, using dependencies: Dependencies) async throws -> JobExecutionResult { guard let threadId: String = job.threadId, let detailsData: Data = job.details, @@ -39,13 +32,14 @@ public enum GroupInviteMemberJob: JobExecutor { .fetchOne(db) }), let details: Details = try? JSONDecoder(using: dependencies).decode(Details.self, from: detailsData) - else { return failure(job, JobRunnerError.missingRequiredDetails, true) } + else { throw JobRunnerError.missingRequiredDetails } let sentTimestampMs: Int64 = dependencies[cache: .snodeAPI].currentOffsetTimestampMs() let adminProfile: Profile = dependencies.mutate(cache: .libSession) { $0.profile } /// Perform the actual message sending - dependencies[singleton: .storage] + // FIXME: Refactor this to use async/await + let publisher = dependencies[singleton: .storage] .writePublisher { db -> (AuthenticationMethod, AuthenticationMethod) in _ = try? GroupMember .filter(GroupMember.Columns.groupId == threadId) @@ -91,71 +85,60 @@ public enum GroupInviteMemberJob: JobExecutor { using: dependencies ).send(using: dependencies) } - .subscribe(on: scheduler, using: dependencies) - .receive(on: scheduler, using: dependencies) - .sinkUntilComplete( - receiveCompletion: { result in - switch result { - case .finished: - dependencies[singleton: .storage].write { db in - try GroupMember - .filter( - GroupMember.Columns.groupId == threadId && - GroupMember.Columns.profileId == details.memberSessionIdHexString && - GroupMember.Columns.role == GroupMember.Role.standard && - GroupMember.Columns.roleStatus != GroupMember.RoleStatus.accepted - ) - .updateAllAndConfig( - db, - GroupMember.Columns.roleStatus.set(to: GroupMember.RoleStatus.pending), - using: dependencies - ) - } - - success(job, false) - - case .failure(let error): - Log.error(.cat, "Couldn't send message due to error: \(error).") - - // Update the invite status of the group member (only if the role is 'standard' and - // the role status isn't already 'accepted') - dependencies[singleton: .storage].write { db in - try GroupMember - .filter( - GroupMember.Columns.groupId == threadId && - GroupMember.Columns.profileId == details.memberSessionIdHexString && - GroupMember.Columns.role == GroupMember.Role.standard && - GroupMember.Columns.roleStatus != GroupMember.RoleStatus.accepted - ) - .updateAllAndConfig( - db, - GroupMember.Columns.roleStatus.set(to: GroupMember.RoleStatus.failed), - using: dependencies - ) - } - - // Notify about the failure - dependencies.mutate(cache: .groupInviteMemberJob) { cache in - cache.addFailure(groupId: threadId, memberId: details.memberSessionIdHexString) - } - - // Register the failure - switch error { - case let senderError as MessageSenderError where !senderError.isRetryable: - failure(job, error, true) - - case SnodeAPIError.rateLimited: - failure(job, error, true) - - case SnodeAPIError.clockOutOfSync: - Log.error(.cat, "Permanently Failing to send due to clock out of sync issue.") - failure(job, error, true) - - default: failure(job, error, false) - } - } - } - ) + + do { + _ = try await publisher.values.first(where: { _ in true }) + try? await dependencies[singleton: .storage].writeAsync { db in + try GroupMember + .filter( + GroupMember.Columns.groupId == threadId && + GroupMember.Columns.profileId == details.memberSessionIdHexString && + GroupMember.Columns.role == GroupMember.Role.standard && + GroupMember.Columns.roleStatus != GroupMember.RoleStatus.accepted + ) + .updateAllAndConfig( + db, + GroupMember.Columns.roleStatus.set(to: GroupMember.RoleStatus.pending), + using: dependencies + ) + } + + return .success(job, stop: false) + } + catch { + Log.error(.cat, "Couldn't send message due to error: \(error).") + + /// Update the invite status of the group member (only if the role is 'standard' and the role status isn't already 'accepted') + try? await dependencies[singleton: .storage].writeAsync { db in + try GroupMember + .filter( + GroupMember.Columns.groupId == threadId && + GroupMember.Columns.profileId == details.memberSessionIdHexString && + GroupMember.Columns.role == GroupMember.Role.standard && + GroupMember.Columns.roleStatus != GroupMember.RoleStatus.accepted + ) + .updateAllAndConfig( + db, + GroupMember.Columns.roleStatus.set(to: GroupMember.RoleStatus.failed), + using: dependencies + ) + } + + /// Notify about the failure + dependencies.mutate(cache: .groupInviteMemberJob) { cache in + cache.addFailure(groupId: threadId, memberId: details.memberSessionIdHexString) + } + + /// Throw the error + switch error { + case is MessageSenderError: throw error + case SnodeAPIError.rateLimited: throw JobRunnerError.permanentFailure(error) + case SnodeAPIError.clockOutOfSync: + Log.error(.cat, "Permanently Failing to send due to clock out of sync issue.") + throw JobRunnerError.permanentFailure(error) + + default: throw error + } } public static func failureMessage(groupName: String, memberIds: [String], profileInfo: [String: Profile]) -> ThemedAttributedString { diff --git a/SessionMessagingKit/Jobs/MessageReceiveJob.swift b/SessionMessagingKit/Jobs/MessageReceiveJob.swift index d1b779bd17..0f52a1f951 100644 --- a/SessionMessagingKit/Jobs/MessageReceiveJob.swift +++ b/SessionMessagingKit/Jobs/MessageReceiveJob.swift @@ -18,21 +18,15 @@ public enum MessageReceiveJob: JobExecutor { public static var requiresThreadId: Bool = true public static let requiresInteractionId: Bool = false - public static func run( - _ job: Job, - scheduler: S, - success: @escaping (Job, Bool) -> Void, - failure: @escaping (Job, Error, Bool) -> Void, - deferred: @escaping (Job) -> Void, - using dependencies: Dependencies - ) { + public static func run(_ job: Job, using dependencies: Dependencies) async throws -> JobExecutionResult { + typealias QueryResult = (updatedJob: Job, lastError: Error?) + guard let threadId: String = job.threadId, let detailsData: Data = job.details, let details: Details = try? JSONDecoder(using: dependencies).decode(Details.self, from: detailsData) - else { return failure(job, JobRunnerError.missingRequiredDetails, true) } + else { throw JobRunnerError.missingRequiredDetails } - var updatedJob: Job = job var lastError: Error? var remainingMessagesToProcess: [Details.MessageInfo] = [] let messageData: [(info: Details.MessageInfo, proto: SNProtoContent)] = details.messages @@ -51,89 +45,78 @@ public enum MessageReceiveJob: JobExecutor { } } - dependencies[singleton: .storage].writeAsync( - updates: { db -> Error? in - for (messageInfo, protoContent) in messageData { - do { - let info: MessageReceiver.InsertedInteractionInfo? = try MessageReceiver.handle( - db, - threadId: threadId, - threadVariant: messageInfo.threadVariant, - message: messageInfo.message, - serverExpirationTimestamp: messageInfo.serverExpirationTimestamp, - associatedWithProto: protoContent, - suppressNotifications: false, - using: dependencies - ) - - /// Notify about the received message - MessageReceiver.prepareNotificationsForInsertedInteractions( - db, - insertedInteractionInfo: info, - isMessageRequest: dependencies.mutate(cache: .libSession) { cache in - cache.isMessageRequest(threadId: threadId, threadVariant: messageInfo.threadVariant) - }, - using: dependencies - ) - } - catch { - // If the current message is a permanent failure then override it with the - // new error (we want to retry if there is a single non-permanent error) - switch error { - // Ignore duplicate and self-send errors (these will usually be caught during - // parsing but sometimes can get past and conflict at database insertion - eg. - // for open group messages) we also don't bother logging as it results in - // excessive logging which isn't useful) - case DatabaseError.SQLITE_CONSTRAINT_UNIQUE, - DatabaseError.SQLITE_CONSTRAINT, // Sometimes thrown for UNIQUE - MessageReceiverError.duplicateMessage, - MessageReceiverError.selfSend: - break - - case let receiverError as MessageReceiverError where !receiverError.isRetryable: - Log.error(.cat, "Permanently failed message due to error: \(error)") - continue - - default: - Log.error(.cat, "Couldn't receive message due to error: \(error)") - lastError = error - - // We failed to process this message but it is a retryable error - // so add it to the list to re-process - remainingMessagesToProcess.append(messageInfo) - } + let queryResult: QueryResult? = try await dependencies[singleton: .storage].writeAsync { db -> QueryResult? in + for (messageInfo, protoContent) in messageData { + do { + let info: MessageReceiver.InsertedInteractionInfo? = try MessageReceiver.handle( + db, + threadId: threadId, + threadVariant: messageInfo.threadVariant, + message: messageInfo.message, + serverExpirationTimestamp: messageInfo.serverExpirationTimestamp, + associatedWithProto: protoContent, + suppressNotifications: false, + using: dependencies + ) + + /// Notify about the received message + MessageReceiver.prepareNotificationsForInsertedInteractions( + db, + insertedInteractionInfo: info, + isMessageRequest: dependencies.mutate(cache: .libSession) { cache in + cache.isMessageRequest(threadId: threadId, threadVariant: messageInfo.threadVariant) + }, + using: dependencies + ) + } + catch { + // If the current message is a permanent failure then override it with the + // new error (we want to retry if there is a single non-permanent error) + switch error { + // Ignore duplicate and self-send errors (these will usually be caught during + // parsing but sometimes can get past and conflict at database insertion - eg. + // for open group messages) we also don't bother logging as it results in + // excessive logging which isn't useful) + case DatabaseError.SQLITE_CONSTRAINT_UNIQUE, + DatabaseError.SQLITE_CONSTRAINT, // Sometimes thrown for UNIQUE + MessageReceiverError.duplicateMessage, + MessageReceiverError.selfSend: + break + + case let receiverError as MessageReceiverError where !receiverError.isRetryable: + Log.error(.cat, "Permanently failed message due to error: \(error)") + continue + + default: + Log.error(.cat, "Couldn't receive message due to error: \(error)") + lastError = error + + // We failed to process this message but it is a retryable error + // so add it to the list to re-process + remainingMessagesToProcess.append(messageInfo) } } - - // If any messages failed to process then we want to update the job to only include - // those failed messages - guard !remainingMessagesToProcess.isEmpty else { return nil } - - updatedJob = try job + } + + // If any messages failed to process then we want to update the job to only include + // those failed messages + guard !remainingMessagesToProcess.isEmpty else { return nil } + + return ( + try job .with(details: Details(messages: remainingMessagesToProcess)) .defaulting(to: job) - .upserted(db) - - return lastError - }, - completion: { result in - // Handle the result - switch result { - case .failure(let error): failure(updatedJob, error, false) - case .success(let lastError): - /// Report the result of the job - switch lastError { - case let error as MessageReceiverError where !error.isRetryable: - failure(updatedJob, error, true) - - case .some(let error): failure(updatedJob, error, false) - case .none: success(updatedJob, false) - } - - success(updatedJob, false) - } - } - ) + .upserted(db), + lastError + ) + } + + // Handle the result + switch (queryResult?.updatedJob, queryResult?.lastError) { + case (_, .some(let error)): throw error + case (.some(let updatedJob), .none): return .success(updatedJob, stop: false) + case (.none, .none): return .success(job, stop: false) + } } } @@ -235,3 +218,9 @@ extension MessageReceiveJob { } } } + +// MARK: - JobError conformance + +extension MessageReceiverError: JobError { + public var isPermanent: Bool { !isRetryable } +} diff --git a/SessionMessagingKit/Jobs/MessageSendJob.swift b/SessionMessagingKit/Jobs/MessageSendJob.swift index 590a660a46..ea381ac226 100644 --- a/SessionMessagingKit/Jobs/MessageSendJob.swift +++ b/SessionMessagingKit/Jobs/MessageSendJob.swift @@ -427,3 +427,9 @@ extension MessageSendJob { } } } + +// MARK: - JobError conformance + +extension MessageSenderError: JobError { + public var isPermanent: Bool { !isRetryable } +} diff --git a/SessionMessagingKit/Jobs/SendReadReceiptsJob.swift b/SessionMessagingKit/Jobs/SendReadReceiptsJob.swift index 1da0c5a95c..6e08ce4476 100644 --- a/SessionMessagingKit/Jobs/SendReadReceiptsJob.swift +++ b/SessionMessagingKit/Jobs/SendReadReceiptsJob.swift @@ -12,28 +12,21 @@ public enum SendReadReceiptsJob: JobExecutor { public static let requiresInteractionId: Bool = false private static let maxRunFrequency: TimeInterval = 3 - public static func run( - _ job: Job, - scheduler: S, - success: @escaping (Job, Bool) -> Void, - failure: @escaping (Job, Error, Bool) -> Void, - deferred: @escaping (Job) -> Void, - using dependencies: Dependencies - ) { + public static func run(_ job: Job, using dependencies: Dependencies) async throws -> JobExecutionResult { guard let threadId: String = job.threadId, let detailsData: Data = job.details, let details: Details = try? JSONDecoder(using: dependencies).decode(Details.self, from: detailsData) - else { return failure(job, JobRunnerError.missingRequiredDetails, true) } + else { throw JobRunnerError.missingRequiredDetails } - // If there are no timestampMs values then the job can just complete (next time - // something is marked as read we want to try and run immediately so don't scuedule - // another run in this case) + /// If there are no timestampMs values then the job can just complete (next time something is marked as read we want to try + /// and run immediately so don't scuedule another run in this case) guard !details.timestampMsValues.isEmpty else { - return success(job, true) + return .success(job, stop: true) } - dependencies[singleton: .storage] + // FIXME: Refactor this to use async/await + let publisher = dependencies[singleton: .storage] .readPublisher { db in try Authentication.with(db, swarmPublicKey: threadId, using: dependencies) } .tryFlatMap { authMethod -> AnyPublisher<(ResponseInfoType, Message), Error> in try MessageSender.preparedSend( @@ -49,48 +42,39 @@ public enum SendReadReceiptsJob: JobExecutor { using: dependencies ).send(using: dependencies) } - .subscribe(on: scheduler, using: dependencies) - .receive(on: scheduler, using: dependencies) - .sinkUntilComplete( - receiveCompletion: { result in - switch result { - case .failure(let error): failure(job, error, false) - case .finished: - // When we complete the 'SendReadReceiptsJob' we want to immediately schedule - // another one for the same thread but with a 'nextRunTimestamp' set to the - // 'maxRunFrequency' value to throttle the read receipt requests - var shouldFinishCurrentJob: Bool = false - let nextRunTimestamp: TimeInterval = (dependencies.dateNow.timeIntervalSince1970 + maxRunFrequency) - - let updatedJob: Job? = dependencies[singleton: .storage].write { db in - // If another 'sendReadReceipts' job was scheduled then update that one - // to run at 'nextRunTimestamp' and make the current job stop - if - let existingJob: Job = try? Job - .filter(Job.Columns.id != job.id) - .filter(Job.Columns.variant == Job.Variant.sendReadReceipts) - .filter(Job.Columns.threadId == threadId) - .fetchOne(db), - !dependencies[singleton: .jobRunner].isCurrentlyRunning(existingJob) - { - try existingJob - .with(nextRunTimestamp: nextRunTimestamp) - .upserted(db) - shouldFinishCurrentJob = true - return job - } - - return try job - .with(details: Details(destination: details.destination, timestampMsValues: [])) - .defaulting(to: job) - .with(nextRunTimestamp: nextRunTimestamp) - .upserted(db) - } - - success(updatedJob ?? job, shouldFinishCurrentJob) - } - } - ) + + _ = try await publisher.values.first(where: { _ in true }) + + /// When we complete the `SendReadReceiptsJob` we want to immediately schedule another one for the same thread + /// but with a `nextRunTimestamp` set to the `maxRunFrequency` value to throttle the read receipt requests + var shouldFinishCurrentJob: Bool = false + let nextRunTimestamp: TimeInterval = (dependencies.dateNow.timeIntervalSince1970 + maxRunFrequency) + + /// If another `sendReadReceipts` job was scheduled then update that one to run at `nextRunTimestamp` + /// and make the current job stop + let existingJob: Job? = try? await dependencies[singleton: .storage].readAsync { db in + try? Job + .filter(Job.Columns.id != job.id) + .filter(Job.Columns.variant == Job.Variant.sendReadReceipts) + .filter(Job.Columns.threadId == threadId) + .fetchOne(db) + } + var targetJob: Job = job + + if + let otherJob: Job = existingJob, + await !dependencies[singleton: .jobRunner].isCurrentlyRunning(otherJob) + { + targetJob = otherJob + shouldFinishCurrentJob = true + } + + let updatedJob: Job = targetJob + .with(details: Details(destination: details.destination, timestampMsValues: [])) + .defaulting(to: targetJob) + .with(nextRunTimestamp: nextRunTimestamp) + + return .success(updatedJob, stop: shouldFinishCurrentJob) } } @@ -111,34 +95,38 @@ public extension SendReadReceiptsJob { /// /// **Note:** This method assumes that the provided `interactionIds` are valid and won't filter out any invalid ids so /// ensure that is done correctly beforehand - @discardableResult static func createOrUpdateIfNeeded( - _ db: ObservingDatabase, + static func createOrUpdateIfNeeded( threadId: String, interactionIds: [Int64], using dependencies: Dependencies - ) -> Job? { - guard dependencies.mutate(cache: .libSession, { $0.get(.areReadReceiptsEnabled) }) else { return nil } - guard !interactionIds.isEmpty else { return nil } + ) async { + guard dependencies.mutate(cache: .libSession, { $0.get(.areReadReceiptsEnabled) }) else { return } + guard !interactionIds.isEmpty else { return } - // Retrieve the timestampMs values for the specified interactions - let timestampMsValues: [Int64] = (try? Interaction - .select(.timestampMs) - .filter(interactionIds.contains(Interaction.Columns.id)) - .distinct() - .asRequest(of: Int64.self) - .fetchAll(db)) - .defaulting(to: []) + /// Retrieve the timestampMs values for the specified interactions + let timestampMsValues: [Int64] = ((try? await dependencies[singleton: .storage].readAsync { db in + try Interaction + .select(.timestampMs) + .filter(interactionIds.contains(Interaction.Columns.id)) + .distinct() + .asRequest(of: Int64.self) + .fetchAll(db) + }) ?? []) - // If there are no timestamp values then do nothing - guard !timestampMsValues.isEmpty else { return nil } + /// If there are no timestamp values then do nothing + guard !timestampMsValues.isEmpty else { return } - // Try to get an existing job (if there is one that's not running) - if - let existingJob: Job = try? Job + /// Try to get an existing job (if there is one that's not running) + let existingJob: Job? = try? await dependencies[singleton: .storage].readAsync { db in + try Job .filter(Job.Columns.variant == Job.Variant.sendReadReceipts) .filter(Job.Columns.threadId == threadId) - .fetchOne(db), - !dependencies[singleton: .jobRunner].isCurrentlyRunning(existingJob), + .fetchOne(db) + } + + if + let existingJob: Job = existingJob, + await !dependencies[singleton: .jobRunner].isCurrentlyRunning(existingJob), let existingDetailsData: Data = existingJob.details, let existingDetails: Details = try? JSONDecoder(using: dependencies) .decode(Details.self, from: existingDetailsData) @@ -152,21 +140,32 @@ public extension SendReadReceiptsJob { ) ) - guard let updatedJob: Job = maybeUpdatedJob else { return nil } + guard let updatedJob: Job = maybeUpdatedJob else { return } - return try? updatedJob - .upserted(db) + _ = try? await dependencies[singleton: .storage].writeAsync { db in + dependencies[singleton: .jobRunner].upsert( + db, + job: try updatedJob.upserted(db), + canStartJob: true + ) + } } - // Otherwise create a new job - return Job( - variant: .sendReadReceipts, - behaviour: .recurring, - threadId: threadId, - details: Details( - destination: .contact(publicKey: threadId), - timestampMsValues: timestampMsValues.asSet() + /// Otherwise create a new job + _ = try? await dependencies[singleton: .storage].writeAsync { db in + dependencies[singleton: .jobRunner].upsert( + db, + job: Job( + variant: .sendReadReceipts, + behaviour: .recurring, + threadId: threadId, + details: Details( + destination: .contact(publicKey: threadId), + timestampMsValues: timestampMsValues.asSet() + ) + ), + canStartJob: true ) - ) + } } } diff --git a/SessionMessagingKit/Jobs/UpdateProfilePictureJob.swift b/SessionMessagingKit/Jobs/UpdateProfilePictureJob.swift index cf1f402590..c111942201 100644 --- a/SessionMessagingKit/Jobs/UpdateProfilePictureJob.swift +++ b/SessionMessagingKit/Jobs/UpdateProfilePictureJob.swift @@ -18,36 +18,21 @@ public enum UpdateProfilePictureJob: JobExecutor { public static let requiresThreadId: Bool = false public static let requiresInteractionId: Bool = false - public static func run( - _ job: Job, - scheduler: S, - success: @escaping (Job, Bool) -> Void, - failure: @escaping (Job, Error, Bool) -> Void, - deferred: @escaping (Job) -> Void, - using dependencies: Dependencies - ) { - // Don't run when inactive or not in main app + public static func run(_ job: Job, using dependencies: Dependencies) async throws -> JobExecutionResult { + /// Don't run when inactive or not in main app guard dependencies[defaults: .appGroup, key: .isMainAppActive] else { - return deferred(job) // Don't need to do anything if it's not the main app + return .deferred(job) /// Don't need to do anything if it's not the main app } - // Only re-upload the profile picture if enough time has passed since the last upload + /// Only re-upload the profile picture if enough time has passed since the last upload guard let lastProfilePictureUpload: Date = dependencies[defaults: .standard, key: .lastProfilePictureUpload], dependencies.dateNow.timeIntervalSince(lastProfilePictureUpload) > (14 * 24 * 60 * 60) else { - // Reset the `nextRunTimestamp` value just in case the last run failed so we don't get stuck - // in a loop endlessly deferring the job - if let jobId: Int64 = job.id { - dependencies[singleton: .storage].write { db in - try Job - .filter(id: jobId) - .updateAll(db, Job.Columns.nextRunTimestamp.set(to: 0)) - } - } - + /// Reset the `nextRunTimestamp` value just in case the last run failed so we don't get stuck in a loop endlessly + /// deferring the job Log.info(.cat, "Deferred as not enough time has passed since the last update") - return deferred(job) + return .deferred(job.with(nextRunTimestamp: 0)) } /// **Note:** The `lastProfilePictureUpload` value is updated in `DisplayPictureManager` @@ -58,22 +43,14 @@ public enum UpdateProfilePictureJob: JobExecutor { .map { .currentUserUploadImageData($0) } .defaulting(to: .none) - Profile - .updateLocal( - displayPictureUpdate: displayPictureUpdate, - using: dependencies - ) - .subscribe(on: scheduler, using: dependencies) - .receive(on: scheduler, using: dependencies) - .sinkUntilComplete( - receiveCompletion: { result in - switch result { - case .failure(let error): failure(job, error, false) - case .finished: - Log.info(.cat, "Profile successfully updated") - success(job, false) - } - } - ) + // FIXME: Refactor this to use async/await + let publisher = Profile.updateLocal( + displayPictureUpdate: displayPictureUpdate, + using: dependencies + ) + + _ = try await publisher.values.first(where: { _ in true }) + Log.info(.cat, "Profile successfully updated") + return .success(job, stop: false) } } diff --git a/SessionMessagingKit/LibSession/LibSession+SessionMessagingKit.swift b/SessionMessagingKit/LibSession/LibSession+SessionMessagingKit.swift index 8bbd82e93b..31c4ff2d7f 100644 --- a/SessionMessagingKit/LibSession/LibSession+SessionMessagingKit.swift +++ b/SessionMessagingKit/LibSession/LibSession+SessionMessagingKit.swift @@ -491,9 +491,14 @@ public extension LibSession { // MARK: - Pushes - public func syncAllPendingPushes(_ db: ObservingDatabase) { - configStore.allIds.forEach { sessionId in - ConfigurationSyncJob.enqueue(db, swarmPublicKey: sessionId.hexString, using: dependencies) + public func syncAllPendingPushesAsync() { + Task { [dependencies] in + for sessionId in configStore.allIds { + await ConfigurationSyncJob.enqueue( + swarmPublicKey: sessionId.hexString, + using: dependencies + ) + } } } @@ -977,7 +982,7 @@ public protocol LibSessionCacheType: LibSessionImmutableCacheType, MutableCacheT // MARK: - Pushes - func syncAllPendingPushes(_ db: ObservingDatabase) + func syncAllPendingPushesAsync() func withCustomBehaviour( _ behaviour: LibSession.CacheBehaviour, for sessionId: SessionId, @@ -1243,7 +1248,7 @@ private final class NoopLibSessionCache: LibSessionCacheType, NoopDependency { // MARK: - Pushes - func syncAllPendingPushes(_ db: ObservingDatabase) {} + func syncAllPendingPushesAsync() {} func withCustomBehaviour( _ behaviour: LibSession.CacheBehaviour, for sessionId: SessionId, diff --git a/SessionMessagingKit/Utilities/Profile+CurrentUser.swift b/SessionMessagingKit/Utilities/Profile+CurrentUser.swift index 0796e4f761..b854be8583 100644 --- a/SessionMessagingKit/Utilities/Profile+CurrentUser.swift +++ b/SessionMessagingKit/Utilities/Profile+CurrentUser.swift @@ -200,7 +200,6 @@ public extension Profile { db, job: Job( variant: .displayPictureDownload, - shouldBeUnique: true, details: DisplayPictureDownloadJob.Details( target: .profile(id: profile.id, url: url, encryptionKey: key), timestamp: sentTimestamp diff --git a/SessionMessagingKitTests/_TestUtilities/MockLibSessionCache.swift b/SessionMessagingKitTests/_TestUtilities/MockLibSessionCache.swift index e27b816544..4cc9261e1c 100644 --- a/SessionMessagingKitTests/_TestUtilities/MockLibSessionCache.swift +++ b/SessionMessagingKitTests/_TestUtilities/MockLibSessionCache.swift @@ -65,8 +65,8 @@ class MockLibSessionCache: Mock, LibSessionCacheType { // MARK: - Pushes - func syncAllPendingPushes(_ db: ObservingDatabase) { - mockNoReturn(untrackedArgs: [db]) + func syncAllPendingPushesAsync() { + mockNoReturn() } func withCustomBehaviour(_ behaviour: LibSession.CacheBehaviour, for sessionId: SessionId, variant: ConfigDump.Variant?, change: @escaping () throws -> ()) throws { diff --git a/SessionShareExtension/ShareNavController.swift b/SessionShareExtension/ShareNavController.swift index 13b036416a..7ba56cda0e 100644 --- a/SessionShareExtension/ShareNavController.swift +++ b/SessionShareExtension/ShareNavController.swift @@ -124,9 +124,7 @@ final class ShareNavController: UINavigationController { /// /// **Note:** We only want to do this if the app is active and ready for app extensions to run if dependencies[singleton: .appContext].isAppForegroundAndActive && userMetadata != nil { - dependencies[singleton: .storage].writeAsync { [dependencies] db in - dependencies.mutate(cache: .libSession) { $0.syncAllPendingPushes(db) } - } + dependencies.mutate(cache: .libSession) { $0.syncAllPendingPushesAsync() } } checkIsAppReady(migrationsCompleted: true, userMetadata: userMetadata) diff --git a/SessionUtilitiesKit/Configuration.swift b/SessionUtilitiesKit/Configuration.swift index 16485d4f3f..39b3171b8f 100644 --- a/SessionUtilitiesKit/Configuration.swift +++ b/SessionUtilitiesKit/Configuration.swift @@ -49,11 +49,14 @@ public enum SNUtilitiesKit: MigratableTarget { // Just to make the external API self.maxValidImageDimension = maxValidImageDimention // Register any recurring jobs to ensure they are actually scheduled - dependencies[singleton: .jobRunner].registerRecurringJobs( - scheduleInfo: [ - (.syncPushTokens, .recurringOnLaunch, false, false), - (.syncPushTokens, .recurringOnActive, false, true) - ] - ) + // FIXME: make async in network refactor + Task { + await dependencies[singleton: .jobRunner].registerRecurringJobs( + scheduleInfo: [ + (.syncPushTokens, .recurringOnLaunch, false, false), + (.syncPushTokens, .recurringOnActive, false, true) + ] + ) + } } } diff --git a/SessionUtilitiesKit/Database/Models/Job.swift b/SessionUtilitiesKit/Database/Models/Job.swift index 1d6a0df046..81fff528a2 100644 --- a/SessionUtilitiesKit/Database/Models/Job.swift +++ b/SessionUtilitiesKit/Database/Models/Job.swift @@ -44,6 +44,8 @@ public struct Job: Codable, Equatable, Hashable, Identifiable, FetchableRecord, case threadId case interactionId case details + + @available(*, deprecated, message: "No longer used, the JobExecuter should handle uniqueness itself") case uniqueHashValue } @@ -234,11 +236,9 @@ public struct Job: Codable, Equatable, Hashable, Identifiable, FetchableRecord, /// JSON encoded data required for the job public let details: Data? - /// When initalizing with `shouldBeUnique` set to `true` this value will be populated with a hash constructed by - /// combining the `variant`, `threadId`, `interactionId` and `details` and if this value is populated - /// adding/inserting a `Job` will fail if there is already a job with the same `uniqueHashValue` in the database or - /// in the `JobRunner` - public let uniqueHashValue: Int? + // TODO: Migration to drop this + @available(*, deprecated, message: "No longer used, the JobExecuter should handle uniqueness itself") + public var uniqueHashValue: Int? { nil } /// Extra data which can be attached to a job that doesn't get persisted to the database (generally used for running /// a job directly which may need some special behaviour) @@ -262,36 +262,6 @@ public struct Job: Codable, Equatable, Hashable, Identifiable, FetchableRecord, // MARK: - Initialization - fileprivate init( - id: Int64?, - priority: Int64, - failureCount: UInt, - variant: Variant, - behaviour: Behaviour, - shouldBlock: Bool, - shouldSkipLaunchBecomeActive: Bool, - nextRunTimestamp: TimeInterval, - threadId: String?, - interactionId: Int64?, - details: Data?, - uniqueHashValue: Int?, - transientData: Any? - ) { - self.id = id - self.priority = priority - self.failureCount = failureCount - self.variant = variant - self.behaviour = behaviour - self.shouldBlock = shouldBlock - self.shouldSkipLaunchBecomeActive = shouldSkipLaunchBecomeActive - self.nextRunTimestamp = nextRunTimestamp - self.threadId = threadId - self.interactionId = interactionId - self.details = details - self.uniqueHashValue = uniqueHashValue - self.transientData = transientData - } - internal init( id: Int64?, priority: Int64 = 0, @@ -299,7 +269,6 @@ public struct Job: Codable, Equatable, Hashable, Identifiable, FetchableRecord, variant: Variant, behaviour: Behaviour, shouldBlock: Bool, - shouldBeUnique: Bool, shouldSkipLaunchBecomeActive: Bool, nextRunTimestamp: TimeInterval, threadId: String?, @@ -324,14 +293,6 @@ public struct Job: Codable, Equatable, Hashable, Identifiable, FetchableRecord, self.threadId = threadId self.interactionId = interactionId self.details = details - self.uniqueHashValue = Job.createUniqueHash( - shouldBeUnique: shouldBeUnique, - customHash: nil, - variant: variant, - threadId: threadId, - interactionId: interactionId, - detailsData: details - ) self.transientData = transientData } @@ -341,7 +302,6 @@ public struct Job: Codable, Equatable, Hashable, Identifiable, FetchableRecord, variant: Variant, behaviour: Behaviour = .runOnce, shouldBlock: Bool = false, - shouldBeUnique: Bool = false, shouldSkipLaunchBecomeActive: Bool = false, nextRunTimestamp: TimeInterval = 0, threadId: String? = nil, @@ -364,14 +324,6 @@ public struct Job: Codable, Equatable, Hashable, Identifiable, FetchableRecord, self.threadId = threadId self.interactionId = interactionId self.details = nil - self.uniqueHashValue = Job.createUniqueHash( - shouldBeUnique: shouldBeUnique, - customHash: nil, - variant: variant, - threadId: threadId, - interactionId: interactionId, - detailsData: nil - ) self.transientData = transientData } @@ -381,7 +333,6 @@ public struct Job: Codable, Equatable, Hashable, Identifiable, FetchableRecord, variant: Variant, behaviour: Behaviour = .runOnce, shouldBlock: Bool = false, - shouldBeUnique: Bool = false, shouldSkipLaunchBecomeActive: Bool = false, nextRunTimestamp: TimeInterval = 0, threadId: String? = nil, @@ -413,14 +364,6 @@ public struct Job: Codable, Equatable, Hashable, Identifiable, FetchableRecord, self.threadId = threadId self.interactionId = interactionId self.details = detailsData - self.uniqueHashValue = Job.createUniqueHash( - shouldBeUnique: shouldBeUnique, - customHash: (details as? UniqueHashable)?.customHash, - variant: variant, - threadId: threadId, - interactionId: interactionId, - detailsData: detailsData - ) self.transientData = transientData } @@ -437,51 +380,6 @@ public struct Job: Codable, Equatable, Hashable, Identifiable, FetchableRecord, precondition(becomeActiveValid, "[Job] Fatal error trying to create a job which skips on 'OnActive' triggered during launch with doesn't run on active") } - private static func createUniqueHash( - shouldBeUnique: Bool, - customHash: Int?, - variant: Variant, - threadId: String?, - interactionId: Int64?, - detailsData: Data? - ) -> Int? { - // Only generate a unique hash if the Job should actually be unique (we don't want to prevent - // all duplicate jobs, just the ones explicitly marked as unique) - guard shouldBeUnique else { return nil } - - switch customHash { - case .some(let customHash): return customHash - default: - var hasher: Hasher = Hasher() - variant.hash(into: &hasher) - threadId?.hash(into: &hasher) - interactionId?.hash(into: &hasher) - detailsData?.hash(into: &hasher) - - return hasher.finalize() - } - } - - private static func createUniqueHash( - shouldBeUnique: Bool, - variant: Variant, - threadId: String?, - interactionId: Int64?, - detailsData: Data? - ) -> Int? { - // Only generate a unique hash if the Job should actually be unique (we don't want to prevent - // all duplicate jobs, just the ones explicitly marked as unique) - guard shouldBeUnique else { return nil } - - var hasher: Hasher = Hasher() - variant.hash(into: &hasher) - threadId?.hash(into: &hasher) - interactionId?.hash(into: &hasher) - detailsData?.hash(into: &hasher) - - return hasher.finalize() - } - // MARK: - Custom Database Interaction public mutating func didInsert(_ inserted: InsertionSuccess) { @@ -507,7 +405,6 @@ public extension Job { threadId: try container.decodeIfPresent(String.self, forKey: .threadId), interactionId: try container.decodeIfPresent(Int64.self, forKey: .interactionId), details: try container.decodeIfPresent(Data.self, forKey: .details), - uniqueHashValue: try container.decodeIfPresent(Int.self, forKey: .uniqueHashValue), transientData: nil ) } @@ -526,7 +423,6 @@ public extension Job { try container.encodeIfPresent(threadId, forKey: .threadId) try container.encodeIfPresent(interactionId, forKey: .interactionId) try container.encodeIfPresent(details, forKey: .details) - try container.encodeIfPresent(uniqueHashValue, forKey: .uniqueHashValue) } } @@ -545,8 +441,7 @@ public extension Job { lhs.nextRunTimestamp == rhs.nextRunTimestamp && lhs.threadId == rhs.threadId && lhs.interactionId == rhs.interactionId && - lhs.details == rhs.details && - lhs.uniqueHashValue == rhs.uniqueHashValue + lhs.details == rhs.details /// `transientData` ignored for equality check ) } @@ -567,7 +462,6 @@ public extension Job { threadId?.hash(into: &hasher) interactionId?.hash(into: &hasher) details?.hash(into: &hasher) - uniqueHashValue?.hash(into: &hasher) /// `transientData` ignored for hashing } } @@ -630,7 +524,6 @@ public extension Job { variant: self.variant, behaviour: self.behaviour, shouldBlock: self.shouldBlock, - shouldBeUnique: (self.uniqueHashValue != nil), shouldSkipLaunchBecomeActive: self.shouldSkipLaunchBecomeActive, nextRunTimestamp: nextRunTimestamp, threadId: self.threadId, @@ -654,7 +547,6 @@ public extension Job { variant: self.variant, behaviour: self.behaviour, shouldBlock: self.shouldBlock, - shouldBeUnique: (self.uniqueHashValue != nil), shouldSkipLaunchBecomeActive: self.shouldSkipLaunchBecomeActive, nextRunTimestamp: self.nextRunTimestamp, threadId: self.threadId, diff --git a/SessionUtilitiesKit/JobRunner/JobError.swift b/SessionUtilitiesKit/JobRunner/JobError.swift new file mode 100644 index 0000000000..3959ecebfb --- /dev/null +++ b/SessionUtilitiesKit/JobRunner/JobError.swift @@ -0,0 +1,8 @@ +// Copyright © 2025 Rangeproof Pty Ltd. All rights reserved. + +import Foundation + +public protocol JobError: Error { + /// Indicates if the failure is permanent and the job should not be retried. + var isPermanent: Bool { get } +} diff --git a/SessionUtilitiesKit/JobRunner/JobExecutor.swift b/SessionUtilitiesKit/JobRunner/JobExecutor.swift new file mode 100644 index 0000000000..f22e0656ba --- /dev/null +++ b/SessionUtilitiesKit/JobRunner/JobExecutor.swift @@ -0,0 +1,43 @@ +// Copyright © 2025 Rangeproof Pty Ltd. All rights reserved. + +import Foundation + +public protocol JobExecutor { + /// The maximum number of times the job can fail before it fails permanently + /// + /// **Note:** A value of `-1` means it will retry indefinitely + static var maxFailureCount: Int { get } + static var requiresThreadId: Bool { get } + static var requiresInteractionId: Bool { get } + + /// This method contains the logic needed to complete a job + /// + /// - Parameters: + /// - job: The job which is being run + /// - dependencies: The application's dependencies + /// - Returns: A `JobExecutionResult` indicating success or deferral + /// - Throws: An error if the job failed, the error should conform to `JobError` to indicate if the failure is permanent + static func run(_ job: Job, using dependencies: Dependencies) async throws -> JobExecutionResult +} + +// MARK: - JobExecutionResult + +public enum JobExecutionResult { + /// The job completed successfully + /// - `updatedJob`: The job instance, potentially with updated details or state for recurring jobs + /// - `stop`: A flag indicating if a recurring job should be permanently stopped and deleted + case success(_ updatedJob: Job, stop: Bool) + + /// The job couldn't be completed and should be run again later + /// - `updatedJob`: The job instance, can include changes like an updated `nextRunTimestamp` value to indicate when the + /// job should be run again + case deferred(_ updatedJob: Job) + + var publicResult: JobRunner.JobResult { + switch self { + case .success: return .succeeded + case .deferred: return .deferred + } + } +} + diff --git a/SessionUtilitiesKit/JobRunner/JobQueue.swift b/SessionUtilitiesKit/JobRunner/JobQueue.swift new file mode 100644 index 0000000000..45a39d3801 --- /dev/null +++ b/SessionUtilitiesKit/JobRunner/JobQueue.swift @@ -0,0 +1,696 @@ +// Copyright © 2025 Rangeproof Pty Ltd. All rights reserved. + +import Foundation +import GRDB + +public actor JobQueue: Hashable { + private static let deferralLoopThreshold: Int = 3 + + private let dependencies: Dependencies + nonisolated private let id: UUID = UUID() + private let type: QueueType + private let executionType: ExecutionType + private let priority: TaskPriority + nonisolated public let jobVariants: [Job.Variant] + private let maxDeferralsPerSecond: Int + + private var executorMap: [Job.Variant: JobExecutor.Type] = [:] + private var pendingJobsQueue: [Job] = [] + private var currentlyRunningJobs: [Int64: (info: JobRunner.JobInfo, task: Task)] = [:] + private var deferLoopTracker: [Int64: (count: Int, times: [TimeInterval])] = [:] + + private var processingTask: Task? = nil + private var nextTriggerTask: Task? = nil + + // MARK: - Initialization + + public init( + type: QueueType, + executionType: ExecutionType, + priority: TaskPriority, + isTestingJobRunner: Bool, + jobVariants: [Job.Variant], + using dependencies: Dependencies + ) { + self.dependencies = dependencies + self.type = type + self.executionType = executionType + self.priority = priority + self.jobVariants = jobVariants + self.maxDeferralsPerSecond = (isTestingJobRunner ? 10 : 1) /// Allow for tripping the defer loop in tests + } + + // MARK: - Hashable + + nonisolated public func hash(into hasher: inout Hasher) { + id.hash(into: &hasher) + } + + public static func == (lhs: JobQueue, rhs: JobQueue) -> Bool { + return (lhs.id == rhs.id) + } + + // MARK: - Configuration + + func setExecutor(_ executor: JobExecutor.Type, for variant: Job.Variant) { + executorMap[variant] = executor + } + + // MARK: - State + + func infoForAllCurrentlyRunningJobs() -> [Int64: JobRunner.JobInfo] { + return currentlyRunningJobs.mapValues { $0.info } + } + + func infoForAllPendingJobs() -> [Int64: JobRunner.JobInfo] { + pendingJobsQueue.enumerated().reduce(into: [:]) { result, jobInfo in + guard let jobId: Int64 = jobInfo.element.id else { return } + + result[jobId] = JobRunner.JobInfo(job: jobInfo.element, queueIndex: jobInfo.offset) + } + } + + // MARK: - Scheduling + + @discardableResult func add(_ job: Job, canStart: Bool) -> Bool { + /// Check if the job should be added to the queue + guard + canStart, + job.behaviour != .runOnceNextLaunch, + job.nextRunTimestamp <= dependencies.dateNow.timeIntervalSince1970 + else { return false } + guard job.id != nil else { + Log.info(.jobRunner, "Prevented attempt to add \(job) without id to queue") + return false + } + + pendingJobsQueue.append(job) + start(canStart: canStart, drainOnly: false) + return true + } + + @discardableResult func upsert(_ job: Job, canStart: Bool) -> Bool { + guard let jobId: Int64 = job.id else { + Log.warn(.jobRunner, "Prevented attempt to upsert \(job) without id to queue") + return false + } + + /// If the job is currently running, we can't update it + guard currentlyRunningJobs[jobId] == nil else { + Log.warn(.jobRunner, "Prevented attempt to upsert a currently running job: \(job)") + return false + } + + /// If it's already in the queue then just update the existing job + if let index: Array.Index = pendingJobsQueue.firstIndex(where: { $0.id == jobId }) { + pendingJobsQueue[index] = job + start(canStart: canStart, drainOnly: false) + return true + } + + /// Otherwise add it to the queue + return add(job, canStart: canStart) + } + + @discardableResult func insert(_ job: Job, before otherJob: Job) -> Bool { + guard job.id != nil else { + Log.info(.jobRunner, "Prevented attempt to insert \(job) without id to queue") + return false + } + + /// Insert the job before the current job (re-adding the current job to the start of the `pendingJobsQueue` if it's not in + /// there) - this will mean the new job will run and then the `otherJob` will run (or run again) once it's done + if let otherJobIndex: Array.Index = pendingJobsQueue.firstIndex(of: otherJob) { + pendingJobsQueue.insert(job, at: otherJobIndex) + return true + } + + /// The `otherJob` wasn't already in the queue so just add them both at the start (we add at the start because generally + /// this function only gets called when dealing with dependencies at runtime - ie. in situations where we would want the jobs + /// to run immediately) + pendingJobsQueue.insert(contentsOf: [job, otherJob], at: 0) + return true + } + + func enqueueDependencies(_ jobs: [Job]) { + let jobIdsToMove: Set = Set(jobs.compactMap { $0.id }) + + /// Pull out any existing jobs that need to be prioritised + var existingJobs: [Job] = [] + pendingJobsQueue.removeAll { job in + if jobIdsToMove.contains(job.id ?? -1) { + existingJobs.append(job) + return true + } + + return false + } + + /// Use the instances of jobs that were already in the queue (in case they have state that is relevant) + let jobsToPrepend: [Job] = jobs.reduce(into: []) { result, next in + result.append(existingJobs.first(where: { $0.id == next.id }) ?? next) + } + + pendingJobsQueue.insert(contentsOf: jobsToPrepend, at: 0) + } + + func removePendingJob(_ jobId: Int64?) { + guard let jobId: Int64 = jobId else { return } + + pendingJobsQueue.removeAll { $0.id == jobId } + } + + func addJobsFromLifecycle(_ jobs: [Job], canStart: Bool) { + let currentJobIds: Set = Set(pendingJobsQueue.compactMap(\.id) + currentlyRunningJobs.keys) + let newJobs: [Job] = jobs.filter { !currentJobIds.contains($0.id ?? -1) } + pendingJobsQueue.append(contentsOf: newJobs) + start(canStart: canStart, drainOnly: false) + } + + func drainInBackground() -> Task? { + return start(canStart: true, drainOnly: true) + } + + // MARK: - Execution Management + + @discardableResult func start(canStart: Bool, drainOnly: Bool) -> Task? { + guard canStart, processingTask == nil else { return processingTask } + + /// Cancel any scheduled future work (since we are starting now) + nextTriggerTask?.cancel() + nextTriggerTask = nil + + Log.info(.jobRunner, "Starting JobQueue-\(type.name)... (Drain only: \(drainOnly))") + + processingTask = Task(priority: priority) { + await processQueue(drainOnly: drainOnly) + processingTask = nil + Log.info(.jobRunner, "JobQueue-\(type.name) has drained and is now idle") + } + + return processingTask + } + + func stopAndClear() { + nextTriggerTask?.cancel() + nextTriggerTask = nil + processingTask?.cancel() + processingTask = nil + + /// Cancel all individual jobs that are currently running + currentlyRunningJobs.values.forEach { _, task in task.cancel() } + + /// Clear the state + currentlyRunningJobs.removeAll() + pendingJobsQueue.removeAll() + deferLoopTracker.removeAll() + + Log.info(.jobRunner, "Stopped and cleared JobQueue-\(type.name)") + } + + func deferCount(for jobId: Int64) -> Int { + return (deferLoopTracker[jobId]?.count ?? 0) + } + + func matches(filters: JobRunner.Filters) -> Bool { + /// A queue matches if *any* of its variants match the filter + for variant in jobVariants { + let pseudoInfo: JobRunner.JobInfo = JobRunner.JobInfo( + id: nil, + variant: variant, + threadId: nil, + interactionId: nil, + queueIndex: nil, + detailsData: nil + ) + + if filters.matches(pseudoInfo) { + return true + } + } + + return false + } + + // MARK: - Processing Loop + + private func processQueue(drainOnly: Bool) async { + /// If we aren't just draining the queue then load and add any pending jobs from the database into the queue + if !drainOnly { + await loadPendingJobsFromDatabase() + } + + while let nextJob: Job = fetchNextJob() { + guard !Task.isCancelled else { break } + + switch executionType { + case .serial: + /// Wait for each task to be complete + await executeJob(nextJob) + + case .concurrent: + /// Spin up an individual task for each job + Task { await executeJob(nextJob) } + } + } + + /// If we aren't just draining the queue then when it's empty we should schedule the next job based on `nextRunTimestamp` + if !drainOnly { + await scheduleNextSoonestJob() + } + } + + private func fetchNextJob() -> Job? { + guard !pendingJobsQueue.isEmpty else { return nil } + + return pendingJobsQueue.removeFirst() + } + + private func executeJob(_ job: Job) async { + guard + let jobId: Int64 = job.id, + let executor: (any JobExecutor.Type) = executorMap[job.variant] + else { + await handleJobFailed(job, error: JobRunnerError.executorMissing, permanentFailure: true) + return + } + + /// Ensure the job has everything is needs before trying to start it + let precheckResult: JobExecutionPrecheckResult = await prepareToExecute(job) + + switch precheckResult { + case .permanentlyFail(let error): + await handleJobFailed(job, error: error, permanentFailure: true) + return + + case .deferUntilDependenciesMet: + await handleJobDeferred(job) + return + + case .ready: break + } + + /// Create a task to execute the job (this allows us to cancel if needed without impacting the queue) + let jobTask: Task = Task { + try await executor.run(job, using: dependencies) + } + + /// Track the running job + currentlyRunningJobs[jobId] = (info: JobRunner.JobInfo(job: job, queueIndex: 0), task: jobTask) + Log.info(.jobRunner, "JobQueue-\(type.name) started \(job)") + + /// Wait for the task to complete + let executionOutcome: Result = await jobTask.result + let finalResult: JobRunner.JobResult + + switch executionOutcome { + case .success(let result): + await handleJobResult(result) + finalResult = result.publicResult + + case .failure(let error): + let isPermanent: Bool = (error as? JobError)?.isPermanent ?? false + await handleJobFailed(job, error: error, permanentFailure: isPermanent) + finalResult = .failed(error, isPermanent) + } + + /// Cleanup after the job is finished + currentlyRunningJobs.removeValue(forKey: jobId) + Log.info(.jobRunner, "JobQueue-\(type.name) finished \(job)") + + await dependencies[singleton: .jobRunner].didCompleteJob(id: jobId, result: finalResult) + } + + // MARK: - Result Handling + + private func handleJobResult(_ result: JobExecutionResult) async { + switch result { + case .success(let updatedJob, let stop): await handleJobSucceeded(updatedJob, shouldStop: stop) + case .deferred(let updatedJob): await handleJobDeferred(updatedJob) + } + } + + private func handleJobSucceeded(_ job: Job, shouldStop: Bool) async { + do { + let dependantJobs: [Job] = try await dependencies[singleton: .storage].writeAsync { [dependencies] db in + /// Retrieve the dependant jobs first (the `JobDependecies` table has cascading deletion when the original `Job` is + /// removed so we need to retrieve these records before that happens) + let dependantJobIds: Set = try JobDependencies + .select(.jobId) + .filter(JobDependencies.Columns.dependantId == job.id) + .asRequest(of: Int64.self) + .fetchSet(db) + let dependantJobs: [Job] = try Job.fetchAll(db, ids: dependantJobIds) + // TODO: Need to test that the above is the same as this + let dependantJobs2: [Job] = try job.dependantJobs.fetchAll(db) + + switch job.behaviour { + /// Since this job has been completed we can update the dependencies so other job that were dependant + /// on this one can be run + case .runOnce, .runOnceNextLaunch, .runOnceAfterConfigSyncIgnoringPermanentFailure: + _ = try JobDependencies + .filter(JobDependencies.Columns.dependantId == job.id) + .deleteAll(db) + _ = try job.delete(db) + + /// Since this job has been completed we can update the dependencies so other job that were dependant + /// on this one can be run + case .recurring where shouldStop == true: + _ = try JobDependencies + .filter(JobDependencies.Columns.dependantId == job.id) + .deleteAll(db) + _ = try job.delete(db) + + /// For `recurring` jobs which have already run, they should automatically run again but we want at least 1 second + /// to pass before doing so - the job itself should really update it's own `nextRunTimestamp` (this is just a safety net) + case .recurring where job.nextRunTimestamp <= dependencies.dateNow.timeIntervalSince1970: + guard let jobId: Int64 = job.id else { break } + + _ = try Job + .filter(id: jobId) + .updateAll( + db, + Job.Columns.failureCount.set(to: 0), + Job.Columns.nextRunTimestamp.set(to: (dependencies.dateNow.timeIntervalSince1970 + 1)) + ) + + /// For `recurringOnLaunch/Active` jobs which have already run but failed once, we need to clear their + /// `failureCount` and `nextRunTimestamp` to prevent them from endlessly running over and over again + case .recurringOnLaunch, .recurringOnActive: + guard + let jobId: Int64 = job.id, + job.failureCount != 0 && + job.nextRunTimestamp > TimeInterval.leastNonzeroMagnitude + else { break } + + _ = try Job + .filter(id: jobId) + .updateAll( + db, + Job.Columns.failureCount.set(to: 0), + Job.Columns.nextRunTimestamp.set(to: 0) + ) + + /// Otherwise just save the updated job (just in case it was modified and hasn't been saved yet) + case .recurring: _ = try job.upserted(db) + } + + return dependantJobs + } + + /// This needs to call back to the JobRunner to enqueue these jobs on their correct queues + await dependencies[singleton: .jobRunner].enqueueDependenciesIfNeeded(dependantJobs) + } catch { + Log.error(.jobRunner, "Failed to process successful job \(job) in database: \(error)") + } + } + + private func handleJobFailed(_ job: Job, error: Error, permanentFailure: Bool) async { + let jobExists: Bool = ((try? await dependencies[singleton: .storage] + .readAsync { db in + try Job.exists(db, id: job.id ?? -1) + }) ?? false) + + guard jobExists else { + Log.info(.jobRunner, "JobQueue-\(type.name) \(job) canceled") + return + } + + // TODO: Should this be moved into the `JobRunner` instead???? + if self.type == .blocking && job.shouldBlock && (error as? JobRunnerError)?.wasPossibleDeferralLoop != true { + Log.info(.jobRunner, "JobQueue-\(type.name) \(job) failed due to error: \(error); retrying immediately") + pendingJobsQueue.insert(job, at: 0) + return + } + + /// Get the max failure count for the job (a value of '-1' means it will retry indefinitely) + let maxFailureCount: Int = (executorMap[job.variant]?.maxFailureCount ?? 0) + let tooManyRetries: Bool = (maxFailureCount >= 0 && (job.failureCount + 1) > maxFailureCount) + let isPermanent: Bool = (permanentFailure || tooManyRetries) + + do { + let dependantJobIds: Set = try await dependencies[singleton: .storage].writeAsync { [type, dependencies] db in + let dependantJobIds: Set = try JobDependencies + .select(.jobId) + .filter(JobDependencies.Columns.dependantId == job.id) + .asRequest(of: Int64.self) + .fetchSet(db) + + guard !isPermanent || job.behaviour == .runOnceAfterConfigSyncIgnoringPermanentFailure else { + Log.error(.jobRunner, "JobQueue-\(type.name) \(job) failed permanently due to error: \(error)\(tooManyRetries ? "; too many retries" : "")") + + /// If the job permanently failed or we have performed all of our retry attempts then delete the job and all of it's + /// dependant jobs (it'll probably never succeed) + _ = try job.dependantJobs.deleteAll(db) + _ = try job.delete(db) + return dependantJobIds + } + + let updatedFailureCount: UInt = (job.failureCount + 1) + let nextRunTimestamp: TimeInterval = (dependencies.dateNow.timeIntervalSince1970 + job.retryInterval) + Log.error(.jobRunner, "JobQueue-\(type.name) \(job) failed due to error: \(error); scheduling retry (failure count is \(updatedFailureCount))") + + let updatedJob: Job = job.with( + failureCount: updatedFailureCount, + nextRunTimestamp: nextRunTimestamp + ) + _ = try updatedJob.upserted(db) + + /// Update the `failureCount` and `nextRunTimestamp` on dependant jobs as well (update the + /// `nextRunTimestamp` value to be 1ms later so when the queue gets regenerated they'll come after the dependency) + // TODO: Need to confirm these match + let dependantJobs1: [Job] = try Job.fetchAll(db, ids: dependantJobIds) + // TODO: Need to test that the above is the same as this + let dependantJobs2: [Job] = try job.dependantJobs.fetchAll(db) + + try Job + .filter(dependantJobIds.contains(Job.Columns.id)) + .updateAll( + db, + Job.Columns.failureCount.set(to: updatedFailureCount), + Job.Columns.nextRunTimestamp.set(to: (nextRunTimestamp + (1 / 1000))) + ) + + return dependantJobIds + } + + if !dependantJobIds.isEmpty { + pendingJobsQueue.removeAll { dependantJobIds.contains($0.id ?? -1) } + } + } + catch { + Log.error(.jobRunner, "Failed to update database for failed job \(job): \(error)") + } + } + + private func handleJobDeferred(_ job: Job) async { + var stuckInDeferLoop: Bool = false + let jobId: Int64 = (job.id ?? -1) + + if let record: (count: Int, times: [TimeInterval]) = deferLoopTracker[jobId] { + let timeNow: TimeInterval = dependencies.dateNow.timeIntervalSince1970 + stuckInDeferLoop = ( + record.count >= JobQueue.deferralLoopThreshold && + (timeNow - record.times[0]) < CGFloat(record.count * maxDeferralsPerSecond) + ) + + /// Only store the last `deferralLoopThreshold` times to ensure we aren't running faster than one loop per second + deferLoopTracker[jobId] = ( + record.count + 1, + record.times.suffix(JobQueue.deferralLoopThreshold - 1) + [timeNow] + ) + } else { + deferLoopTracker[jobId] = (1, [dependencies.dateNow.timeIntervalSince1970]) + } + + /// It's possible (by introducing bugs) to create a loop where a `Job` tries to run and immediately defers itself but then attempts + /// to run again (resulting in an infinite loop); this won't block the app since it's on a background thread but can result in 100% of + /// a CPU being used (and a battery drain) + /// + /// This code will maintain an in-memory store for any jobs which are deferred too quickly (ie. more than `deferralLoopThreshold` + /// times within `deferralLoopThreshold` seconds) + if stuckInDeferLoop { + deferLoopTracker.removeValue(forKey: jobId) + await handleJobFailed(job, error: JobRunnerError.possibleDeferralLoop, permanentFailure: false) + } + + do { + try await dependencies[singleton: .storage].writeAsync { db in + _ = try job.upserted(db) + } + } catch { + Log.error(.jobRunner, "Failed to save deferred job \(job): \(error)") + } + } + + // MARK: - Conenience + + private func loadPendingJobsFromDatabase() async { + let currentJobIds: Set = Set(pendingJobsQueue.compactMap(\.id) + currentlyRunningJobs.keys) + let jobsToRun: [Job] = ((try? await dependencies[singleton: .storage].readAsync { db in + try Job.filterPendingJobs( + variants: self.jobVariants, + excludeFutureJobs: true, + includeJobsWithDependencies: false + ) + .filter(!currentJobIds.contains(Job.Columns.id)) /// Exclude jobs already running/queued + .fetchAll(db) + }) ?? []) + + guard !jobsToRun.isEmpty else { return } + + pendingJobsQueue.append(contentsOf: jobsToRun) + } + + private func scheduleNextSoonestJob() async { + /// Retrieve the soonest `nextRunTimestamp` for jobs that should be running on this queue from the database + let jobVariants: [Job.Variant] = self.jobVariants + let jobIdsAlreadyRunning: Set = Set(currentlyRunningJobs.keys) + let maybeNextTimestamp: TimeInterval? = try? await dependencies[singleton: .storage].readAsync { db in + try Job.filterPendingJobs( + variants: jobVariants, + excludeFutureJobs: false, + includeJobsWithDependencies: false + ) + .select(.nextRunTimestamp) + .filter(!jobIdsAlreadyRunning.contains(Job.Columns.id)) /// Exclude jobs already running + .asRequest(of: TimeInterval.self) + .fetchOne(db) + } + + guard let nextTimestamp: TimeInterval = maybeNextTimestamp else { return } + + let delay: TimeInterval = (nextTimestamp - dependencies.dateNow.timeIntervalSince1970) + + /// If the job isn't ready then schedule a future restart + guard delay <= 0 else { + Log.info(.jobRunner, "Stopping JobQueue-\(type.name) until next job in \(.seconds(delay), unit: .s)") + nextTriggerTask = Task { + try? await Task.sleep(for: .seconds(Int(floor(delay)))) + + guard !Task.isCancelled else { return } + + start(canStart: true, drainOnly: false) + } + return + } + + /// Job is ready now so process it immediately (only add a log if the queue is getting restarted) + if executionType != .concurrent || currentlyRunningJobs.isEmpty { + let timingString: String = (nextTimestamp == 0 ? + "that should be in the queue" : + "scheduled \(.seconds(delay), unit: .s) ago" + ) + Log.info(.jobRunner, "Restarting JobQueue-\(type.name) queue immediately for job \(timingString)") + } + + start(canStart: true, drainOnly: false) + } + + private func prepareToExecute(_ job: Job) async -> JobExecutionPrecheckResult { + guard let executor: JobExecutor.Type = executorMap[job.variant] else { + Log.info(.jobRunner, "JobQueue-\(type.name) Unable to run \(job) due to missing executor") + return .permanentlyFail(error: JobRunnerError.executorMissing) + } + guard !executor.requiresThreadId || job.threadId != nil else { + Log.info(.jobRunner, "JobQueue-\(type.name) Unable to run \(job) due to missing required threadId") + return .permanentlyFail(error: JobRunnerError.requiredThreadIdMissing) + } + guard !executor.requiresInteractionId || job.interactionId != nil else { + Log.info(.jobRunner, "JobQueue-\(type.name) Unable to run \(job) due to missing required interactionId") + return .permanentlyFail(error: JobRunnerError.requiredInteractionIdMissing) + } + + /// Check if the next job has any dependencies + let dependencyInfo: (expectedCount: Int, jobs: Set) = ((try? await dependencies[singleton: .storage].readAsync { db in + let expectedDependencies: Set = try JobDependencies + .filter(JobDependencies.Columns.jobId == job.id) + .fetchSet(db) + let jobDependencies: Set = try Job + .filter(ids: expectedDependencies.compactMap { $0.dependantId }) + .fetchSet(db) + + return (expectedDependencies.count, jobDependencies) + }) ?? (0, [])) + + guard dependencyInfo.jobs.count == dependencyInfo.expectedCount else { + Log.info(.jobRunner, "JobQueue-\(type.name) Removing \(job) due to missing dependencies") + return .permanentlyFail(error: JobRunnerError.missingDependencies) + } + guard dependencyInfo.jobs.isEmpty else { + Log.info(.jobRunner, "JobQueue-\(type.name) Deferring \(job) until \(dependencyInfo.jobs.count) dependencies are completed") + + /// Enqueue the dependencies then defer the current job + await dependencies[singleton: .jobRunner].enqueueDependenciesIfNeeded(Array(dependencyInfo.jobs)) + return .deferUntilDependenciesMet + } + + return .ready + } +} + +// MARK: - QueueType + +public extension JobQueue { + enum QueueType: Hashable { + case blocking + case general(number: Int) + case messageSend + case messageReceive + case attachmentDownload + case displayPictureDownload + case expirationUpdate + + var name: String { + switch self { + case .blocking: return "Blocking" + case .general(let number): return "General-\(number)" + case .messageSend: return "MessageSend" + case .messageReceive: return "MessageReceive" + case .attachmentDownload: return "AttachmentDownload" + case .displayPictureDownload: return "DisplayPictureDownload" + case .expirationUpdate: return "ExpirationUpdate" + } + } + } +} + +// MARK: - ExecutionType + +public extension JobQueue { + enum ExecutionType { + /// A serial queue will execute one job at a time until the queue is empty, then will load any new/deferred + /// jobs and run those one at a time + case serial + + /// A concurrent queue will execute as many jobs as the device supports at once until the queue is empty, + /// then will load any new/deferred jobs and try to start them all + case concurrent + } +} + +// MARK: - JobExecutionPrecheckResult + +private extension JobQueue { + enum JobExecutionPrecheckResult { + case ready + case permanentlyFail(error: Error) + case deferUntilDependenciesMet + } +} + +// MARK: - Convenience + +private extension Job { + var retryInterval: TimeInterval { + // Arbitrary backoff factor... + // try 1 delay: 0.5s + // try 2 delay: 1s + // ... + // try 5 delay: 16s + // ... + // try 11 delay: 512s + let maxBackoff: Double = 10 * 60 // 10 minutes + return 0.25 * min(maxBackoff, pow(2, Double(failureCount))) + } +} diff --git a/SessionUtilitiesKit/JobRunner/JobRunner.swift b/SessionUtilitiesKit/JobRunner/JobRunner.swift index e3549638e3..7bfdf90b16 100644 --- a/SessionUtilitiesKit/JobRunner/JobRunner.swift +++ b/SessionUtilitiesKit/JobRunner/JobRunner.swift @@ -23,231 +23,125 @@ public extension Log.Category { // MARK: - JobRunnerType -public protocol JobRunnerType: AnyObject { +public protocol JobRunnerType: Actor { // MARK: - Configuration - func setExecutor(_ executor: JobExecutor.Type, for variant: Job.Variant) - func canStart(queue: JobQueue?) -> Bool - func afterBlockingQueue(callback: @escaping () -> ()) - func queue(for variant: Job.Variant) -> DispatchQueue? - - // MARK: - State Management + func setExecutor(_ executor: JobExecutor.Type, for variant: Job.Variant) async - func jobInfoFor(jobs: [Job]?, state: JobRunner.JobState, variant: Job.Variant?) -> [Int64: JobRunner.JobInfo] - func deferCount(for jobId: Int64?, of variant: Job.Variant) -> Int + // MARK: - State Management - func appDidFinishLaunching() - func appDidBecomeActive() - func startNonBlockingQueues() + func appDidFinishLaunching() async + func appDidBecomeActive() async - /// Stops and clears any pending jobs except for the specified variant, the `onComplete` closure will be called once complete providing a flag indicating whether any additional - /// processing was needed before the closure was called (if not then the closure will be called synchronously) - func stopAndClearPendingJobs(exceptForVariant: Job.Variant?, onComplete: ((Bool) -> ())?) + func jobInfoFor(state: JobRunner.JobState, filters: JobRunner.Filters) async -> [Int64: JobRunner.JobInfo] + func deferCount(for jobId: Int64?, of variant: Job.Variant) async -> Int + func stopAndClearPendingJobs(filters: JobRunner.Filters) async // MARK: - Job Scheduling - @discardableResult func add(_ db: ObservingDatabase, job: Job?, dependantJob: Job?, canStartJob: Bool) -> Job? - @discardableResult func upsert(_ db: ObservingDatabase, job: Job?, canStartJob: Bool) -> Job? - @discardableResult func insert(_ db: ObservingDatabase, job: Job?, before otherJob: Job) -> (Int64, Job)? - func enqueueDependenciesIfNeeded(_ jobs: [Job]) - func manuallyTriggerResult(_ job: Job?, result: JobRunner.JobResult) - func afterJob(_ job: Job?, state: JobRunner.JobState) -> AnyPublisher - func removePendingJob(_ job: Job?) + @discardableResult nonisolated func add( + _ db: ObservingDatabase, + job: Job?, + dependantJob: Job?, + canStartJob: Bool + ) -> Job? + @discardableResult nonisolated func upsert( + _ db: ObservingDatabase, + job: Job?, + canStartJob: Bool + ) -> Job? + @discardableResult nonisolated func insert( + _ db: ObservingDatabase, + job: Job?, + before otherJob: Job + ) -> (Int64, Job)? + + func enqueueDependenciesIfNeeded(_ jobs: [Job]) async + func removePendingJob(_ job: Job?) async + + // MARK: - Awaiting Job Resules + + func awaitBlockingQueueCompletion() async + func didCompleteJob(id: Int64, result: JobRunner.JobResult) + func awaitResult(forFirstJobMatching filters: JobRunner.Filters, in state: JobRunner.JobState) async -> JobRunner.JobResult + + // MARK: - Recurring Jobs func registerRecurringJobs(scheduleInfo: [JobRunner.ScheduleInfo]) - func scheduleRecurringJobsIfNeeded() + func scheduleRecurringJobsIfNeeded() async } // MARK: - JobRunnerType Convenience public extension JobRunnerType { - func allJobInfo() -> [Int64: JobRunner.JobInfo] { return jobInfoFor(jobs: nil, state: .any, variant: nil) } - - func jobInfoFor(jobs: [Job]) -> [Int64: JobRunner.JobInfo] { - return jobInfoFor(jobs: jobs, state: .any, variant: nil) - } - - func jobInfoFor(jobs: [Job], state: JobRunner.JobState) -> [Int64: JobRunner.JobInfo] { - return jobInfoFor(jobs: jobs, state: state, variant: nil) + func allJobInfo() async -> [Int64: JobRunner.JobInfo] { + return await jobInfoFor(state: .any, filters: .matchingAll) } - func jobInfoFor(state: JobRunner.JobState) -> [Int64: JobRunner.JobInfo] { - return jobInfoFor(jobs: nil, state: state, variant: nil) + func jobInfoFor(filters: JobRunner.Filters) async -> [Int64: JobRunner.JobInfo] { + return await jobInfoFor(state: .any, filters: filters) } - func jobInfoFor(state: JobRunner.JobState, variant: Job.Variant) -> [Int64: JobRunner.JobInfo] { - return jobInfoFor(jobs: nil, state: state, variant: variant) - } - - func jobInfoFor(variant: Job.Variant) -> [Int64: JobRunner.JobInfo] { - return jobInfoFor(jobs: nil, state: .any, variant: variant) + func jobInfoFor(state: JobRunner.JobState) async -> [Int64: JobRunner.JobInfo] { + return await jobInfoFor(state: state, filters: .matchingAll) } - func isCurrentlyRunning(_ job: Job?) -> Bool { - guard let job: Job = job else { return false } + func isCurrentlyRunning(_ job: Job?) async -> Bool { + guard let jobId: Int64 = job?.id else { return false } - return !jobInfoFor(jobs: [job], state: .running, variant: nil).isEmpty - } - - func hasJob( - of variant: Job.Variant? = nil, - inState state: JobRunner.JobState = .any, - with jobDetails: T - ) -> Bool { - guard - let detailsData: Data = try? JSONEncoder() - .with(outputFormatting: .sortedKeys) // Needed for deterministic comparison - .encode(jobDetails) - else { return false } + let jobResults: [Int64: JobRunner.JobInfo] = await jobInfoFor( + state: .running, + filters: JobRunner.Filters( + include: [.jobId(jobId)], + exclude: [] + ) + ) - return jobInfoFor(jobs: nil, state: state, variant: variant) - .values - .contains(where: { $0.detailsData == detailsData }) + return !jobResults.isEmpty } - func stopAndClearPendingJobs() { - stopAndClearPendingJobs(exceptForVariant: nil, onComplete: nil) + func stopAndClearPendingJobs() async { + await stopAndClearPendingJobs(filters: .matchingAll) } // MARK: -- Job Scheduling - @discardableResult func add(_ db: ObservingDatabase, job: Job?, canStartJob: Bool) -> Job? { + @discardableResult nonisolated func add(_ db: ObservingDatabase, job: Job?, canStartJob: Bool) -> Job? { return add(db, job: job, dependantJob: nil, canStartJob: canStartJob) } - func afterJob(_ job: Job?) -> AnyPublisher { - return afterJob(job, state: .any) - } -} - -// MARK: - JobExecutor - -public protocol JobExecutor { - /// The maximum number of times the job can fail before it fails permanently - /// - /// **Note:** A value of `-1` means it will retry indefinitely - static var maxFailureCount: Int { get } - static var requiresThreadId: Bool { get } - static var requiresInteractionId: Bool { get } - - /// This method contains the logic needed to complete a job - /// - /// **Note:** The code in this method should run synchronously and the various - /// "result" blocks should not be called within a database closure - /// - /// - Parameters: - /// - job: The job which is being run - /// - success: The closure which is called when the job succeeds (with an - /// updated `job` and a flag indicating whether the job should forcibly stop running) - /// - failure: The closure which is called when the job fails (with an updated - /// `job`, an `Error` (if applicable) and a flag indicating whether it was a permanent - /// failure) - /// - deferred: The closure which is called when the job is deferred (with an - /// updated `job`) - static func run( - _ job: Job, - scheduler: S, - success: @escaping (Job, Bool) -> Void, - failure: @escaping (Job, Error, Bool) -> Void, - deferred: @escaping (Job) -> Void, - using dependencies: Dependencies - ) -} - -// MARK: - JobRunner - -public final class JobRunner: JobRunnerType { - public struct JobState: OptionSet, Hashable { - public let rawValue: UInt8 - - public init(rawValue: UInt8) { - self.rawValue = rawValue - } - - public static let pending: JobState = JobState(rawValue: 1 << 0) - public static let running: JobState = JobState(rawValue: 1 << 1) - - public static let any: JobState = [ .pending, .running ] + func awaitResult(forFirstJobMatching filters: JobRunner.Filters) async -> JobRunner.JobResult { + return await awaitResult(forFirstJobMatching: filters, in: .any) } - public enum JobResult: Equatable { - case succeeded - case failed(Error, Bool) - case deferred - case notFound + func awaitResult(for job: Job) async -> JobRunner.JobResult { + guard let jobId: Int64 = job.id else { return .notFound } - public static func == (lhs: JobRunner.JobResult, rhs: JobRunner.JobResult) -> Bool { - switch (lhs, rhs) { - case (.succeeded, .succeeded): return true - case (.failed(let lhsError, let lhsPermanent), .failed(let rhsError, let rhsPermanent)): - return ( - // Not a perfect solution but should be good enough - "\(lhsError)" == "\(rhsError)" && - lhsPermanent == rhsPermanent - ) - - case (.deferred, .deferred): return true - default: return false - } - } + return await awaitResult(forFirstJobMatching: JobRunner.Filters(include: [.jobId(jobId)]), in: .any) } +} - public struct JobInfo: Equatable, CustomDebugStringConvertible { - public let variant: Job.Variant - public let threadId: String? - public let interactionId: Int64? - public let detailsData: Data? - public let uniqueHashValue: Int? - - public var debugDescription: String { - let dataDescription: String = detailsData - .map { data in "Data(hex: \(data.toHexString()), \(data.bytes.count) bytes" } - .defaulting(to: "nil") - - return [ - "JobRunner.JobInfo(", - "variant: \(variant),", - " threadId: \(threadId ?? "nil"),", - " interactionId: \(interactionId.map { "\($0)" } ?? "nil"),", - " detailsData: \(dataDescription),", - " uniqueHashValue: \(uniqueHashValue.map { "\($0)" } ?? "nil")", - ")" - ].joined() - } - } - - public typealias ScheduleInfo = ( - variant: Job.Variant, - behaviour: Job.Behaviour, - shouldBlock: Bool, - shouldSkipLaunchBecomeActive: Bool - ) - - private enum Validation { - case enqueueOnly - case persist - } - +// MARK: - JobRunner + +public actor JobRunner: JobRunnerType { // MARK: - Variables private let dependencies: Dependencies private let allowToExecuteJobs: Bool - @ThreadSafeObject private var blockingQueue: JobQueue? - @ThreadSafeObject private var queues: [Job.Variant: JobQueue] - @ThreadSafeObject private var blockingQueueDrainCallback: [() -> ()] = [] - @ThreadSafeObject private var registeredRecurringJobs: [JobRunner.ScheduleInfo] = [] + private let blockingQueue: JobQueue + private var queues: [Job.Variant: JobQueue] = [:] + private var registeredRecurringJobs: [JobRunner.ScheduleInfo] = [] - @ThreadSafe internal var appReadyToStartQueues: Bool = false - @ThreadSafe internal var appHasBecomeActive: Bool = false - @ThreadSafeObject internal var perSessionJobsCompleted: Set = [] - @ThreadSafe internal var hasCompletedInitialBecomeActive: Bool = false - @ThreadSafeObject internal var shutdownBackgroundTask: SessionBackgroundTask? = nil + private var appReadyToStartQueues: Bool = false + private var appHasBecomeActive: Bool = false + private var hasCompletedInitialBecomeActive: Bool = false - private var canStartNonBlockingQueue: Bool { - _blockingQueue.performMap { - $0?.hasStartedAtLeastOnce == true && - $0?.isRunning != true - } && + private var blockingQueueTask: Task? + private var shutdownBackgroundTask: SessionBackgroundTask? = nil + private var resultStreams: [Int64: CancellationAwareAsyncStream] = [:] + + private var canStartNonBlockingQueues: Bool { + (blockingQueueTask == nil || blockingQueueTask?.isCancelled == true) && appHasBecomeActive } @@ -269,23 +163,22 @@ public final class JobRunner: JobRunnerType { !SNUtilitiesKit.isRunningTests ) ) - self._blockingQueue = ThreadSafeObject( - JobQueue( - type: .blocking, - executionType: .serial, - qos: .default, - isTestingJobRunner: isTestingJobRunner, - jobVariants: [], - using: dependencies - ) + self.blockingQueue = JobQueue( + type: .blocking, + executionType: .serial, + priority: .userInitiated, + isTestingJobRunner: isTestingJobRunner, + jobVariants: [], + using: dependencies ) - self._queues = ThreadSafeObject([ + + let queueList: [JobQueue] = [ // MARK: -- Message Send Queue JobQueue( type: .messageSend, - executionType: .concurrent, // Allow as many jobs to run at once as supported by the device - qos: .default, + executionType: .concurrent, /// Allow as many jobs to run at once as supported by the device + priority: .medium, isTestingJobRunner: isTestingJobRunner, jobVariants: [ jobVariants.remove(.attachmentUpload), @@ -304,13 +197,13 @@ public final class JobRunner: JobRunnerType { JobQueue( type: .messageReceive, - // Explicitly serial as executing concurrently means message receives getting processed at - // different speeds which can result in: - // • Small batches of messages appearing in the UI before larger batches - // • Closed group messages encrypted with updated keys could start parsing before it's key - // update message has been processed (ie. guaranteed to fail) + /// Explicitly serial as executing concurrently means message receives getting processed at different speeds which + /// can result in: + /// • Small batches of messages appearing in the UI before larger batches + /// • Closed group messages encrypted with updated keys could start parsing before it's key update message has + /// been processed (ie. guaranteed to fail) executionType: .serial, - qos: .default, + priority: .medium, isTestingJobRunner: isTestingJobRunner, jobVariants: [ jobVariants.remove(.messageReceive), @@ -324,7 +217,7 @@ public final class JobRunner: JobRunnerType { JobQueue( type: .attachmentDownload, executionType: .serial, - qos: .utility, + priority: .utility, isTestingJobRunner: isTestingJobRunner, jobVariants: [ jobVariants.remove(.attachmentDownload) @@ -336,14 +229,14 @@ public final class JobRunner: JobRunnerType { JobQueue( type: .expirationUpdate, - executionType: .concurrent, // Allow as many jobs to run at once as supported by the device - qos: .default, + executionType: .concurrent, /// Allow as many jobs to run at once as supported by the device + priority: .medium, isTestingJobRunner: isTestingJobRunner, jobVariants: [ jobVariants.remove(.expirationUpdate), jobVariants.remove(.getExpiration), jobVariants.remove(.disappearingMessages), - jobVariants.remove(.checkForAppUpdates) // Don't want this to block other jobs + jobVariants.remove(.checkForAppUpdates) /// Don't want this to block other jobs ].compactMap { $0 }, using: dependencies ), @@ -353,7 +246,7 @@ public final class JobRunner: JobRunnerType { JobQueue( type: .displayPictureDownload, executionType: .serial, - qos: .utility, + priority: .utility, isTestingJobRunner: isTestingJobRunner, jobVariants: [ jobVariants.remove(.displayPictureDownload) @@ -366,165 +259,40 @@ public final class JobRunner: JobRunnerType { JobQueue( type: .general(number: 0), executionType: .serial, - qos: .utility, + priority: .utility, isTestingJobRunner: isTestingJobRunner, jobVariants: Array(jobVariants), using: dependencies ) - ].reduce(into: [:]) { prev, next in + ] + + self.queues = queueList.reduce(into: [:]) { prev, next in next.jobVariants.forEach { variant in prev[variant] = next } - }) - - // Now that we've finished setting up the JobRunner, update the queue closures - self._blockingQueue.perform { - $0?.canStart = { [weak self] queue -> Bool in (self?.canStart(queue: queue) == true) } - $0?.onQueueDrained = { [weak self] in - // Once all blocking jobs have been completed we want to start running - // the remaining job queues - self?.startNonBlockingQueues() - - self?._blockingQueueDrainCallback.performUpdate { - $0.forEach { $0() } - return [] - } - } - } - - self._queues.perform { - $0.values.forEach { queue in - queue.canStart = { [weak self] targetQueue -> Bool in (self?.canStart(queue: targetQueue) == true) } - } } } // MARK: - Configuration public func setExecutor(_ executor: JobExecutor.Type, for variant: Job.Variant) { - _blockingQueue.perform { $0?.setExecutor(executor, for: variant) } // The blocking queue can run any job - queues[variant]?.setExecutor(executor, for: variant) - } - - public func canStart(queue: JobQueue?) -> Bool { - return ( - allowToExecuteJobs && - appReadyToStartQueues && ( - queue?.type == .blocking || - canStartNonBlockingQueue - ) - ) - } - - public func afterBlockingQueue(callback: @escaping () -> ()) { - guard - _blockingQueue.performMap({ - ($0?.hasStartedAtLeastOnce != true) || - ($0?.isRunning == true) - }) - else { return callback() } - - _blockingQueueDrainCallback.performUpdate { $0.appending(callback) } - } - - public func queue(for variant: Job.Variant) -> DispatchQueue? { - return queues[variant]?.targetQueue() - } - - // MARK: - State Management - - public func jobInfoFor( - jobs: [Job]?, - state: JobRunner.JobState, - variant: Job.Variant? - ) -> [Int64: JobRunner.JobInfo] { - var result: [(Int64, JobRunner.JobInfo)] = [] - let targetKeys: [JobQueue.JobKey] = (jobs?.compactMap { JobQueue.JobKey($0) } ?? []) - let targetVariants: [Job.Variant] = (variant.map { [$0] } ?? jobs?.map { $0.variant }) - .defaulting(to: []) - - // Insert the state of any pending jobs - if state.contains(.pending) { - func infoFor(queue: JobQueue?, variants: [Job.Variant]) -> [(Int64, JobRunner.JobInfo)] { - return (queue?.pendingJobsQueue - .filter { variants.isEmpty || variants.contains($0.variant) } - .compactMap { job -> (Int64, JobRunner.JobInfo)? in - guard let jobKey: JobQueue.JobKey = JobQueue.JobKey(job) else { return nil } - guard - targetKeys.isEmpty || - targetKeys.contains(jobKey) - else { return nil } - - return ( - jobKey.id, - JobRunner.JobInfo( - variant: job.variant, - threadId: job.threadId, - interactionId: job.interactionId, - detailsData: job.details, - uniqueHashValue: job.uniqueHashValue - ) - ) - }) - .defaulting(to: []) - } - - _blockingQueue.perform { - result.append(contentsOf: infoFor(queue: $0, variants: targetVariants)) - } - queues - .filter { key, _ -> Bool in targetVariants.isEmpty || targetVariants.contains(key) } - .map { _, queue in queue } - .asSet() - .forEach { queue in result.append(contentsOf: infoFor(queue: queue, variants: targetVariants)) } + Task { + /// The blocking queue can run any job + await blockingQueue.setExecutor(executor, for: variant) + await queues[variant]?.setExecutor(executor, for: variant) } - - // Insert the state of any running jobs - if state.contains(.running) { - func infoFor(queue: JobQueue?, variants: [Job.Variant]) -> [(Int64, JobRunner.JobInfo)] { - return (queue?.infoForAllCurrentlyRunningJobs() - .filter { variants.isEmpty || variants.contains($0.value.variant) } - .compactMap { jobId, info -> (Int64, JobRunner.JobInfo)? in - guard - targetKeys.isEmpty || - targetKeys.contains(JobQueue.JobKey(id: jobId, variant: info.variant)) - else { return nil } - - return (jobId, info) - }) - .defaulting(to: []) - } - - _blockingQueue.perform { - result.append(contentsOf: infoFor(queue: $0, variants: targetVariants)) - } - queues - .filter { key, _ -> Bool in targetVariants.isEmpty || targetVariants.contains(key) } - .map { _, queue in queue } - .asSet() - .forEach { queue in result.append(contentsOf: infoFor(queue: queue, variants: targetVariants)) } - } - - return result - .reduce(into: [:]) { result, next in - result[next.0] = next.1 - } } - public func deferCount(for jobId: Int64?, of variant: Job.Variant) -> Int { - guard let jobId: Int64 = jobId else { return 0 } - - return (queues[variant]?.deferLoopTracker[jobId]?.count ?? 0) - } + // MARK: - State Management - public func appDidFinishLaunching() { - // Flag that the JobRunner can start it's queues + public func appDidFinishLaunching() async { + /// Flag that the JobRunner can start it's queues appReadyToStartQueues = true - // Note: 'appDidBecomeActive' will run on first launch anyway so we can - // leave those jobs out and can wait until then to start the JobRunner - let jobsToRun: (blocking: [Job], nonBlocking: [Job]) = dependencies[singleton: .storage] - .read { db in + /// **Note:** `appDidBecomeActive` will run on first launch anyway so we can leave those jobs out and can wait until + /// then to start the JobRunner + let jobsToRun: (blocking: [Job], nonBlocking: [Job]) = ((try? await dependencies[singleton: .storage] + .readAsync { db in let blockingJobs: [Job] = try Job .filter( [ @@ -553,60 +321,64 @@ public final class JobRunner: JobRunnerType { .fetchAll(db) return (blockingJobs, nonblockingJobs) - } - .defaulting(to: ([], [])) + }) ?? ([], [])) - // Add and start any blocking jobs - _blockingQueue.perform { - $0?.appDidFinishLaunching( - with: jobsToRun.blocking.map { job -> Job in - guard job.behaviour == .recurringOnLaunch else { return job } - - // If the job is a `recurringOnLaunch` job then we reset the `nextRunTimestamp` - // value on the instance because the assumption is that `recurringOnLaunch` will - // run a job regardless of how many times it previously failed - return job.with(nextRunTimestamp: 0) - }, - canStart: true - ) - } + /// Add any blocking jobs + await blockingQueue.addJobsFromLifecycle( + jobsToRun.blocking.map { job -> Job in + guard job.behaviour == .recurringOnLaunch else { return job } + + /// If the job is a `recurringOnLaunch` job then we reset the `nextRunTimestamp` value on the instance + /// because the assumption is that `recurringOnLaunch` will run a job regardless of how many times it + /// previously failed + return job.with(nextRunTimestamp: 0) + }, + canStart: false + ) - // Add any non-blocking jobs (we don't start these incase there are blocking "on active" - // jobs as well) + /// Add any non-blocking jobs let jobsByVariant: [Job.Variant: [Job]] = jobsToRun.nonBlocking.grouped(by: \.variant) - let jobQueues: [Job.Variant: JobQueue] = queues - jobsByVariant.forEach { variant, jobs in - jobQueues[variant]?.appDidFinishLaunching( - with: jobs.map { job -> Job in - guard job.behaviour == .recurringOnLaunch else { return job } - - // If the job is a `recurringOnLaunch` job then we reset the `nextRunTimestamp` - // value on the instance because the assumption is that `recurringOnLaunch` will - // run a job regardless of how many times it previously failed - return job.with(nextRunTimestamp: 0) - }, - canStart: false - ) + for (variant, jobs) in jobsByVariant { + if let queue: JobQueue = queues[variant] { + await queue.addJobsFromLifecycle( + jobs.map { job -> Job in + guard job.behaviour == .recurringOnLaunch else { return job } + + /// If the job is a `recurringOnLaunch` job then we reset the `nextRunTimestamp` value on the instance + /// because the assumption is that `recurringOnLaunch` will run a job regardless of how many times it + /// previously failed + return job.with(nextRunTimestamp: 0) + }, + canStart: false + ) + } + } + + /// Create and store the task for the blocking queue, start the `blockingQueue` and, once it's complete, trigger the + /// `startNonBlockingQueues` function + blockingQueueTask = Task { + let canStart: Bool = (allowToExecuteJobs && appReadyToStartQueues) + + _ = await blockingQueue.start(canStart: canStart, drainOnly: false) + await self.startNonBlockingQueues() } } - public func appDidBecomeActive() { - // Flag that the JobRunner can start it's queues and start queueing non-launch jobs + public func appDidBecomeActive() async { + /// Flag that the JobRunner can start it's queues and start queueing non-launch jobs appReadyToStartQueues = true appHasBecomeActive = true - // If we have a running "sutdownBackgroundTask" then we want to cancel it as otherwise it - // can result in the database being suspended and us being unable to interact with it at all - _shutdownBackgroundTask.performUpdate { - $0?.cancel() - return nil - } + /// If we have a running `sutdownBackgroundTask` then we want to cancel it as otherwise it can result in the database + /// being suspended and us being unable to interact with it at all + shutdownBackgroundTask?.cancel() + shutdownBackgroundTask = nil - // Retrieve any jobs which should run when becoming active + /// Retrieve any jobs which should run when becoming active let hasCompletedInitialBecomeActive: Bool = self.hasCompletedInitialBecomeActive - let jobsToRun: [Job] = dependencies[singleton: .storage] - .read { db in + let jobsToRun: [Job] = ((try? await dependencies[singleton: .storage] + .readAsync { db in return try Job .filter(Job.Columns.behaviour == Job.Behaviour.recurringOnActive) .order( @@ -614,134 +386,131 @@ public final class JobRunner: JobRunnerType { Job.Columns.id ) .fetchAll(db) - } - .defaulting(to: []) + }) ?? [] ) .filter { hasCompletedInitialBecomeActive || !$0.shouldSkipLaunchBecomeActive } - // Store the current queue state locally to avoid multiple atomic retrievals - let jobQueues: [Job.Variant: JobQueue] = queues - let blockingQueueIsRunning: Bool = _blockingQueue.performMap { $0?.isRunning == true } - - // Reset the 'isRunningInBackgroundTask' flag just in case (since we aren't in the background anymore) - jobQueues.forEach { _, queue in - queue.setIsRunningBackgroundTask(false) + /// Add and start any non-blocking jobs (if there are no blocking jobs) + /// + /// We only want to trigger the queue to start once so we need to consolidate the queues to list of jobs (as queues can handle + /// multiple job variants), this means that `onActive` jobs will be queued before any standard jobs + let jobsByVariant: [Job.Variant: [Job]] = jobsToRun.grouped(by: \.variant) + for (variant, jobs) in jobsByVariant { + guard let queue: JobQueue = queues[variant] else { continue } + + Task { await queue.addJobsFromLifecycle(jobs, canStart: false) } } - guard !jobsToRun.isEmpty else { - if !blockingQueueIsRunning { - jobQueues.map { _, queue in queue }.asSet().forEach { $0.start() } - } - return + /// If the blocking queue isn't running, it's safe to start the non-blocking ones + if blockingQueueTask == nil || blockingQueueTask?.isCancelled == true { + Task { await self.startNonBlockingQueues() } } - // Add and start any non-blocking jobs (if there are no blocking jobs) - // - // We only want to trigger the queue to start once so we need to consolidate the - // queues to list of jobs (as queues can handle multiple job variants), this means - // that 'onActive' jobs will be queued before any standard jobs - let jobsByVariant: [Job.Variant: [Job]] = jobsToRun.grouped(by: \.variant) - jobQueues - .reduce(into: [:]) { result, variantAndQueue in - result[variantAndQueue.value] = (result[variantAndQueue.value] ?? []) - .appending(contentsOf: (jobsByVariant[variantAndQueue.key] ?? [])) - } - .forEach { queue, jobs in - queue.appDidBecomeActive( - with: jobs.map { job -> Job in - // We reset the `nextRunTimestamp` value on the instance because the - // assumption is that `recurringOnActive` will run a job regardless - // of how many times it previously failed - job.with(nextRunTimestamp: 0) - }, - canStart: !blockingQueueIsRunning - ) - } - self.hasCompletedInitialBecomeActive = true } - public func startNonBlockingQueues() { - queues.map { _, queue in queue }.asSet().forEach { queue in - queue.start() + private func startNonBlockingQueues() async { + guard canStartNonBlockingQueues else { return } + + let canStart: Bool = (allowToExecuteJobs && appReadyToStartQueues) + + /// Start all non-blocking queues concurrently + await withTaskGroup(of: Void.self) { group in + for queue in queues.values { + group.addTask { + _ = await queue.start(canStart: canStart, drainOnly: false) + } + } } } - public func stopAndClearPendingJobs( - exceptForVariant: Job.Variant?, - onComplete: ((Bool) -> ())? - ) { - // Inform the JobRunner that it can't start any queues (this is to prevent queues from - // rescheduling themselves while in the background, when the app restarts or becomes active - // the JobRunenr will update this flag) + public func stopAndClearPendingJobs(filters: JobRunner.Filters) async { + /// Inform the `JobRunner` that it can't start any queues (this is to prevent queues from rescheduling themselves while in the + /// background, when the app restarts or becomes active the `JobRunner` will update this flag) appReadyToStartQueues = false appHasBecomeActive = false - // Stop all queues except for the one containing the `exceptForVariant` - queues - .map { _, queue in queue } - .asSet() - .filter { queue -> Bool in - guard let exceptForVariant: Job.Variant = exceptForVariant else { return true } - - return !queue.jobVariants.contains(exceptForVariant) - } - .forEach { $0.stopAndClearPendingJobs() } + let uniqueQueues = Set(queues.values) - // Ensure the queue is actually running (if not the trigger the callback immediately) - guard - let exceptForVariant: Job.Variant = exceptForVariant, - let queue: JobQueue = queues[exceptForVariant], - queue.isRunning == true - else { - onComplete?(false) - return + await withTaskGroup(of: Void.self) { group in + /// Stop any queues which match the filters + for queue in uniqueQueues { + group.addTask { + if await queue.matches(filters: filters) { + await queue.stopAndClear() + } + } + } + + /// Also handle blocking queue + group.addTask { + if await self.blockingQueue.matches(filters: filters) { + await self.blockingQueue.stopAndClear() + } + } } - - let oldQueueDrained: (() -> ())? = queue.onQueueDrained - queue.setIsRunningBackgroundTask(true) - - // Create a backgroundTask to give the queue the chance to properly be drained - _shutdownBackgroundTask.performUpdate { _ in - SessionBackgroundTask(label: #function, using: dependencies) { [weak queue] state in - // If the background task didn't succeed then trigger the onComplete (and hope we have - // enough time to complete it's logic) - guard state != .cancelled else { - queue?.setIsRunningBackgroundTask(false) - queue?.onQueueDrained = oldQueueDrained - return + } + + private func cancelShutdown() { + shutdownBackgroundTask?.cancel() + shutdownBackgroundTask = nil + } + + public func jobInfoFor( + state: JobRunner.JobState, + filters: JobRunner.Filters + ) async -> [Int64: JobRunner.JobInfo] { + var allInfo: [Int64: JobRunner.JobInfo] = [:] + let uniqueQueues: Set = Set(queues.values).union([blockingQueue]) + + await withTaskGroup(of: [Int64: JobRunner.JobInfo].self) { group in + for queue in uniqueQueues { + if state.contains(.running) { + group.addTask { await queue.infoForAllCurrentlyRunningJobs() } } - guard state != .success else { return } - onComplete?(true) - queue?.setIsRunningBackgroundTask(false) - queue?.onQueueDrained = oldQueueDrained - queue?.stopAndClearPendingJobs() + if state.contains(.pending) { + group.addTask { await queue.infoForAllPendingJobs() } + } + } + + for await infoDict in group { + allInfo.merge(infoDict, uniquingKeysWith: { (current, _) in current }) } } - // Add a callback to be triggered once the queue is drained - queue.onQueueDrained = { [weak self, weak queue] in - oldQueueDrained?() - queue?.setIsRunningBackgroundTask(false) - queue?.onQueueDrained = oldQueueDrained - onComplete?(true) - - self?._shutdownBackgroundTask.performUpdate { _ in nil } + /// If the filter is `.matchingAll`, we can return early + if filters.include.isEmpty && filters.exclude.isEmpty { + return allInfo + } + + /// Apply the filters to the collected results + return allInfo.filter { _, jobInfo in + filters.matches(jobInfo) } } - // MARK: - Execution + public func deferCount(for jobId: Int64?, of variant: Job.Variant) async -> Int { + guard let jobId: Int64 = jobId else { return 0 } + + /// We should also check the `blockingQueue` just in case, so return the max value from both + return max( + await blockingQueue.deferCount(for: jobId), + await (queues[variant]?.deferCount(for: jobId) ?? 0) + ) + } - @discardableResult public func add( + // MARK: - Job Scheduling + + @discardableResult nonisolated public func add( _ db: ObservingDatabase, job: Job?, dependantJob: Job?, canStartJob: Bool ) -> Job? { - guard let updatedJob: Job = validatedJob(db, job: job, validation: .persist) else { return nil } + guard let savedJob: Job = validatedJob(db, job: job) else { return nil } - // If we are adding a job that's dependant on another job then create the dependency between them - if let jobId: Int64 = updatedJob.id, let dependantJobId: Int64 = dependantJob?.id { + /// If we are adding a job that's dependant on another job then create the dependency between them + if let jobId: Int64 = savedJob.id, let dependantJobId: Int64 = dependantJob?.id { try? JobDependencies( jobId: jobId, dependantId: dependantJobId @@ -749,73 +518,29 @@ public final class JobRunner: JobRunnerType { .insert(db) } - // Get the target queue - let jobQueue: JobQueue? = queues[updatedJob.variant] - - // Don't add to the queue if it should only run after the next config sync or the JobRunner - // isn't ready (it's been saved to the db so it'll be loaded once the queue actually get - // started later) - guard - job?.behaviour != .runOnceAfterConfigSyncIgnoringPermanentFailure && ( - canAddToQueue(updatedJob) || - jobQueue?.isRunningInBackgroundTask == true - ) - else { return updatedJob } - - // The queue is ready or running in a background task so we can add the job - jobQueue?.add(db, job: updatedJob, canStartJob: canStartJob) - - // Don't start the queue if the job can't be started - guard canStartJob else { return updatedJob } - - // Start the job runner if needed - db.afterCommit(dedupeId: "JobRunner-Start: \(jobQueue?.queueContext ?? "N/A")") { - jobQueue?.start() + /// Start the job runner if needed + db.afterCommit { [weak self] in + Task { [weak self] in await self?.addJobToQueue(savedJob, canStartJob: canStartJob) } } - return updatedJob + return savedJob } - public func upsert( + @discardableResult nonisolated public func upsert( _ db: ObservingDatabase, job: Job?, canStartJob: Bool ) -> Job? { - guard let job: Job = job else { return nil } // Ignore null jobs - guard job.id != nil else { - // When we upsert a job that should be unique we want to return the existing job (if it exists) - switch job.uniqueHashValue { - case .none: return add(db, job: job, canStartJob: canStartJob) - case .some: - let existingJob: Job? = try? Job - .filter(Job.Columns.variant == job.variant) - .filter(Job.Columns.uniqueHashValue == job.uniqueHashValue) - .fetchOne(db) - - return (existingJob ?? add(db, job: job, canStartJob: canStartJob)) - } - } - guard let updatedJob: Job = validatedJob(db, job: job, validation: .enqueueOnly) else { return nil } - - // Don't add to the queue if the JobRunner isn't ready (it's been saved to the db so it'll be loaded - // once the queue actually get started later) - guard canAddToQueue(updatedJob) else { return updatedJob } + guard let savedJob: Job = validatedJob(db, job: job) else { return nil } - let jobQueue: JobQueue? = queues[updatedJob.variant] - guard jobQueue?.upsert(db, job: updatedJob, canStartJob: canStartJob) == true else { return nil } - - // Don't start the queue if the job can't be started - guard canStartJob else { return updatedJob } - - // Start the job runner if needed - db.afterCommit(dedupeId: "JobRunner-Start: \(jobQueue?.queueContext ?? "N/A")") { - jobQueue?.start() + db.afterCommit { [weak self] in + Task { [weak self] in await self?.upsertJobInQueue(savedJob, canStartJob: canStartJob) } } - return updatedJob + return savedJob } - @discardableResult public func insert( + @discardableResult nonisolated public func insert( _ db: ObservingDatabase, job: Job?, before otherJob: Job @@ -830,83 +555,132 @@ public final class JobRunner: JobRunnerType { } guard - let updatedJob: Job = validatedJob(db, job: job, validation: .persist), - let jobId: Int64 = updatedJob.id + let savedJob: Job = validatedJob(db, job: job), + let savedJobId: Int64 = savedJob.id else { return nil } - queues[updatedJob.variant]?.insert(updatedJob, before: otherJob) + db.afterCommit { [weak self] in + Task { [weak self] in await self?.insertJobIntoQueue(savedJob, before: otherJob) } + } + + return (savedJobId, savedJob) + } + + private func addJobToQueue(_ job: Job, canStartJob: Bool) async { + guard canAddToQueue(job) else { return } + guard let queue: JobQueue = queues[job.variant] else { + Log.critical(.jobRunner, "Attempted to add job \(job) with variant \(job.variant) which has no assigned queue.") + return + } + + await queue.add(job, canStart: canStartJob && allowToExecuteJobs && appReadyToStartQueues) + } + + private func upsertJobInQueue(_ job: Job, canStartJob: Bool) async { + guard canAddToQueue(job) else { return } + guard let queue: JobQueue = queues[job.variant] else { + Log.critical(.jobRunner, "Attempted to upsert job \(job) with variant \(job.variant) which has no assigned queue.") + return + } + + await queue.upsert(job, canStart: canStartJob && allowToExecuteJobs && appReadyToStartQueues) + } + + private func insertJobIntoQueue(_ job: Job, before otherJob: Job) async { + guard let queue: JobQueue = queues[otherJob.variant] else { + Log.critical(.jobRunner, "Attempted to insert job before \(otherJob) with variant \(otherJob.variant) which has no assigned queue.") + return + } - return (jobId, updatedJob) + await queue.insert(job, before: otherJob) } /// Job dependencies can be quite messy as they might already be running or scheduled on different queues from the related job, this could result /// in some odd inter-dependencies between the JobQueues. Instead of this we want all jobs to run on their original assigned queues (so the /// concurrency rules remain consistent and easy to reason with), the only downside to this approach is serial queues could potentially be blocked /// waiting on unrelated dependencies to be run as this method will insert jobs at the start of the `pendingJobsQueue` - public func enqueueDependenciesIfNeeded(_ jobs: [Job]) { + public func enqueueDependenciesIfNeeded(_ jobs: [Job]) async { /// Do nothing if we weren't given any jobs guard !jobs.isEmpty else { return } - /// Ignore any dependencies which are already running or scheduled - let dependencyJobQueues: Set = jobs - .compactMap { queues[$0.variant] } - .asSet() - let allCurrentlyRunningJobIds: [Int64] = dependencyJobQueues - .flatMap { $0.currentlyRunningJobIds } - let jobsToEnqueue: [JobQueue: [Job]] = jobs - .compactMap { job in job.id.map { ($0, job) } } - .filter { jobId, _ in !allCurrentlyRunningJobIds.contains(jobId) } - .compactMap { _, job in queues[job.variant].map { (job, $0) } } - .grouped(by: { _, queue in queue }) - .mapValues { data in data.map { job, _ in job } } - - /// Regardless of whether the jobs are dependant jobs or dependencies we want them to be moved to the start of the - /// `pendingJobsQueue` because at least one job in the job chain has been triggered so we want to try to complete - /// the entire job chain rather than worry about deadlocks between different job chains - /// - /// **Note:** If any of these `dependantJobs` have other dependencies then when they attempt to start they will be - /// removed from the queue, replaced by their dependencies - jobsToEnqueue.forEach { queue, jobs in - queue.insertJobsIfNeeded(jobs, index: 0) + /// Group jobs by queue + let jobsByQueue: [JobQueue: [Job]] = jobs.reduce(into: [:]) { result, next in + guard let queue: JobQueue = queues[next.variant] else { + Log.critical(.jobRunner, "Attempted to add dependency \(next) with variant \(next.variant) which has no assigned queue.") + return + } - // Start the job queue if needed (might be a different queue from the currently executing one) - queue.start() + result[queue, default: []].append(next) + } + + await withTaskGroup(of: Void.self) { group in + for (queue, jobsForQueue) in jobsByQueue { + group.addTask { + await queue.enqueueDependencies(jobsForQueue) + } + } } } - public func manuallyTriggerResult(_ job: Job?, result: JobRunner.JobResult) { - guard let job: Job = job, let queue: JobQueue = queues[job.variant] else { return } + public func removePendingJob(_ job: Job?) async { + guard let job: Job = job, let jobId: Int64 = job.id else { return } - switch result { - case .notFound: return - case .succeeded: queue.handleJobSucceeded(job, shouldStop: false) - case .deferred: queue.handleJobDeferred(job) - case .failed(let error, let permanent): queue.handleJobFailed(job, error: error, permanentFailure: permanent) - } + await queues[job.variant]?.removePendingJob(jobId) + } + + // MARK: - Awaiting Job Results + + public func awaitBlockingQueueCompletion() async { + await blockingQueueTask?.value } - public func afterJob(_ job: Job?, state: JobRunner.JobState) -> AnyPublisher { - guard let job: Job = job, let jobId: Int64 = job.id, let queue: JobQueue = queues[job.variant] else { - return Just(.notFound).eraseToAnyPublisher() + public func didCompleteJob(id: Int64, result: JobRunner.JobResult) { + if let stream: CancellationAwareAsyncStream = resultStreams[id] { + Task { + await stream.send(result) + await stream.finishCurrentStreams() + resultStreams.removeValue(forKey: id) + } } - - return queue.afterJob(jobId, state: state) } - public func removePendingJob(_ job: Job?) { - guard let job: Job = job, let jobId: Int64 = job.id else { return } + public func awaitResult(forFirstJobMatching filters: JobRunner.Filters, in state: JobRunner.JobState) async -> JobRunner.JobResult { + /// Ensure we know about the job + let info: [Int64: JobInfo] = await jobInfoFor(state: state, filters: filters) + + guard + let targetJobId: Int64 = info + .sorted(by: { lhs, rhs in (lhs.value.queueIndex ?? 0) < (rhs.value.queueIndex ?? 0) }) + .first? + .key + else { return .notFound } + + /// Get or create a stream for the job + let stream: CancellationAwareAsyncStream = resultStreams[ + targetJobId, + default: CancellationAwareAsyncStream() + ] + resultStreams[targetJobId] = stream + + /// Await the first result from the stream + for await result in stream.stream { + return result + } - queues[job.variant]?.removePendingJob(jobId) + /// If the stream finishes without a result, something went wrong + return .notFound } + // MARK: - Recurring Jobs + public func registerRecurringJobs(scheduleInfo: [JobRunner.ScheduleInfo]) { - _registeredRecurringJobs.performUpdate { $0.appending(contentsOf: scheduleInfo) } + registeredRecurringJobs.append(contentsOf: scheduleInfo) } - public func scheduleRecurringJobsIfNeeded() { + public func scheduleRecurringJobsIfNeeded() async { let scheduleInfo: [ScheduleInfo] = registeredRecurringJobs let variants: Set = Set(scheduleInfo.map { $0.variant }) - let maybeExistingJobs: [Job]? = dependencies[singleton: .storage].read { db in + let maybeExistingJobs: [Job]? = try? await dependencies[singleton: .storage].readAsync { db in try Job .filter(variants.contains(Job.Columns.variant)) .fetchAll(db) @@ -929,41 +703,33 @@ public final class JobRunner: JobRunnerType { guard !missingScheduledJobs.isEmpty else { return } - var numScheduledJobs: Int = 0 - dependencies[singleton: .storage].write { db in - try missingScheduledJobs.forEach { variant, behaviour, shouldBlock, shouldSkipLaunchBecomeActive in - _ = try Job( - variant: variant, - behaviour: behaviour, - shouldBlock: shouldBlock, - shouldSkipLaunchBecomeActive: shouldSkipLaunchBecomeActive - ).inserted(db) - numScheduledJobs += 1 + do { + try await dependencies[singleton: .storage].writeAsync { db in + try missingScheduledJobs.forEach { variant, behaviour, shouldBlock, shouldSkipLaunchBecomeActive in + _ = try Job( + variant: variant, + behaviour: behaviour, + shouldBlock: shouldBlock, + shouldSkipLaunchBecomeActive: shouldSkipLaunchBecomeActive + ).inserted(db) + } } + Log.info(.jobRunner, "Scheduled \(missingScheduledJobs.count) missing recurring job(s)") } - - switch numScheduledJobs == missingScheduledJobs.count { - case true: Log.info(.jobRunner, "Scheduled \(numScheduledJobs) missing recurring job(s)") - case false: Log.error(.jobRunner, "Failed to schedule \(missingScheduledJobs.count - numScheduledJobs) recurring job(s)") + catch { + Log.error(.jobRunner, "Failed to schedule \(missingScheduledJobs.count) recurring job(s): \(error)") } } // MARK: - Convenience - - fileprivate static func getRetryInterval(for job: Job) -> TimeInterval { - // Arbitrary backoff factor... - // try 1 delay: 0.5s - // try 2 delay: 1s - // ... - // try 5 delay: 16s - // ... - // try 11 delay: 512s - let maxBackoff: Double = 10 * 60 // 10 minutes - return 0.25 * min(maxBackoff, pow(2, Double(job.failureCount))) - } fileprivate func canAddToQueue(_ job: Job) -> Bool { - // We can only start the job if it's an "on launch" job or the app has become active + /// A job should not be added to the in-memory queue if it's waiting for a config sync + guard job.behaviour != .runOnceAfterConfigSyncIgnoringPermanentFailure else { + return false + } + + /// We can only start the job if it's an "on launch" job or the app has become active return ( job.behaviour == .runOnceNextLaunch || job.behaviour == .recurringOnLaunch || @@ -971,996 +737,174 @@ public final class JobRunner: JobRunnerType { ) } - private func validatedJob(_ db: ObservingDatabase, job: Job?, validation: Validation) -> Job? { + nonisolated private func validatedJob(_ db: ObservingDatabase, job: Job?) -> Job? { guard let job: Job = job else { return nil } - switch (validation, job.uniqueHashValue) { - case (.enqueueOnly, .none): return job - case (.enqueueOnly, .some(let uniqueHashValue)): - // Nothing currently running or sitting in a JobQueue - guard !allJobInfo().contains(where: { _, info -> Bool in info.uniqueHashValue == uniqueHashValue }) else { - Log.info(.jobRunner, "Unable to add \(job) due to unique constraint") - return nil - } - - return job - - case (.persist, .some(let uniqueHashValue)): - guard - // Nothing currently running or sitting in a JobQueue - !allJobInfo().contains(where: { _, info -> Bool in info.uniqueHashValue == uniqueHashValue }) && - // Nothing in the database - !Job.filter(Job.Columns.uniqueHashValue == uniqueHashValue).isNotEmpty(db) - else { - Log.info(.jobRunner, "Unable to add \(job) due to unique constraint") - return nil - } - - fallthrough // Validation passed so try to persist the job - - case (.persist, .none): - guard let updatedJob: Job = try? job.inserted(db), updatedJob.id != nil else { - Log.info(.jobRunner, "Unable to add \(job)\(job.id == nil ? " due to missing id" : "")") - return nil - } - - return updatedJob - } - } -} - -// MARK: - JobQueue - -public final class JobQueue: Hashable { - fileprivate enum QueueType: Hashable { - case blocking - case general(number: Int) - case messageSend - case messageReceive - case attachmentDownload - case displayPictureDownload - case expirationUpdate - - var name: String { - switch self { - case .blocking: return "Blocking" - case .general(let number): return "General-\(number)" - case .messageSend: return "MessageSend" - case .messageReceive: return "MessageReceive" - case .attachmentDownload: return "AttachmentDownload" - case .displayPictureDownload: return "DisplayPictureDownload" - case .expirationUpdate: return "ExpirationUpdate" - } - } - } - - fileprivate enum ExecutionType { - /// A serial queue will execute one job at a time until the queue is empty, then will load any new/deferred - /// jobs and run those one at a time - case serial - - /// A concurrent queue will execute as many jobs as the device supports at once until the queue is empty, - /// then will load any new/deferred jobs and try to start them all - case concurrent - } - - private class Trigger { - private var timer: Timer? - fileprivate var fireTimestamp: TimeInterval = 0 - - static func create( - queue: JobQueue, - timestamp: TimeInterval, - using dependencies: Dependencies - ) -> Trigger? { - /// Setup the trigger (wait at least 1 second before triggering) - /// - /// **Note:** We use the `Timer.scheduledTimerOnMainThread` method because running a timer - /// on our random queue threads results in the timer never firing, the `start` method will redirect itself to - /// the correct thread - let trigger: Trigger = Trigger() - trigger.fireTimestamp = max(1, (timestamp - dependencies.dateNow.timeIntervalSince1970)) - trigger.timer = Timer.scheduledTimerOnMainThread( - withTimeInterval: trigger.fireTimestamp, - repeats: false, - using: dependencies, - block: { [weak queue] _ in - queue?.start(forceWhenAlreadyRunning: (queue?.executionType == .concurrent)) - } - ) - return trigger - } + /// Job already exists, no need to do anything + guard job.id == nil else { return job } - func invalidate() { - // Need to do this to prevent a strong reference cycle - timer?.invalidate() - timer = nil - } - } - - fileprivate struct JobKey: Equatable, Hashable { - fileprivate let id: Int64 - fileprivate let variant: Job.Variant - - fileprivate init(id: Int64, variant: Job.Variant) { - self.id = id - self.variant = variant - } - - fileprivate init?(_ job: Job?) { - guard let id: Int64 = job?.id, let variant: Job.Variant = job?.variant else { return nil } + do { + let insertedJob: Job = try job.inserted(db) - self.id = id - self.variant = variant - } - } - - private static let deferralLoopThreshold: Int = 3 - - private let dependencies: Dependencies - private let id: UUID = UUID() - fileprivate let type: QueueType - private let executionType: ExecutionType - private let qosClass: DispatchQoS - private let queueKey: DispatchSpecificKey = DispatchSpecificKey() - fileprivate let queueContext: String - fileprivate let jobVariants: [Job.Variant] - - private lazy var internalQueue: DispatchQueue = { - let result: DispatchQueue = DispatchQueue( - label: self.queueContext, - qos: self.qosClass, - attributes: (self.executionType == .concurrent ? [.concurrent] : []), - autoreleaseFrequency: .inherit, - target: nil - ) - result.setSpecific(key: queueKey, value: queueContext) - - return result - }() - - @ThreadSafeObject private var executorMap: [Job.Variant: JobExecutor.Type] = [:] - fileprivate var canStart: ((JobQueue?) -> Bool)? - fileprivate var onQueueDrained: (() -> ())? - @ThreadSafe fileprivate var hasStartedAtLeastOnce: Bool = false - @ThreadSafe fileprivate var isRunning: Bool = false - @ThreadSafeObject fileprivate var pendingJobsQueue: [Job] = [] - @ThreadSafe fileprivate var isRunningInBackgroundTask: Bool = false - - @ThreadSafeObject private var nextTrigger: Trigger? = nil - @ThreadSafeObject fileprivate var currentlyRunningJobIds: Set = [] - @ThreadSafeObject private var currentlyRunningJobInfo: [Int64: JobRunner.JobInfo] = [:] - @ThreadSafeObject fileprivate var deferLoopTracker: [Int64: (count: Int, times: [TimeInterval])] = [:] - private let maxDeferralsPerSecond: Int - private let jobCompletedSubject: PassthroughSubject<(Int64?, JobRunner.JobResult), Never> = PassthroughSubject() - - fileprivate var hasPendingJobs: Bool { !pendingJobsQueue.isEmpty } - - // MARK: - Initialization - - fileprivate init( - type: QueueType, - executionType: ExecutionType, - qos: DispatchQoS, - isTestingJobRunner: Bool, - jobVariants: [Job.Variant], - using dependencies: Dependencies - ) { - self.dependencies = dependencies - self.type = type - self.executionType = executionType - self.queueContext = "JobQueue-\(type.name)" - self.qosClass = qos - self.maxDeferralsPerSecond = (isTestingJobRunner ? 10 : 1) // Allow for tripping the defer loop in tests - self.jobVariants = jobVariants - } - - // MARK: - Hashable - - public func hash(into hasher: inout Hasher) { - id.hash(into: &hasher) - } - - public static func == (lhs: JobQueue, rhs: JobQueue) -> Bool { - return (lhs.id == rhs.id) - } - - // MARK: - Configuration - - fileprivate func setExecutor(_ executor: JobExecutor.Type, for variant: Job.Variant) { - _executorMap.performUpdate { $0.setting(variant, executor) } - } - - fileprivate func setIsRunningBackgroundTask(_ value: Bool) { - isRunningInBackgroundTask = value - } - - fileprivate func insertJobsIfNeeded(_ jobs: [Job], index: Int) { - _pendingJobsQueue.performUpdate { pendingJobs in - pendingJobs - .filter { !jobs.contains($0) } - .inserting(contentsOf: jobs, at: 0) - } - } - - // MARK: - Execution - - fileprivate func targetQueue() -> DispatchQueue { - /// As it turns out Combine doesn't play too nicely with concurrent Dispatch Queues, in Combine events are dispatched asynchronously to - /// the queue which means an odd situation can occasionally occur where the `finished` event can actually run before the `output` - /// event - this can result in unexpected behaviours (for more information see https://github.com/groue/GRDB.swift/issues/1334) - /// - /// Due to this if a job is meant to run on a concurrent queue then we actually want to create a temporary serial queue just for the execution - /// of that job - guard executionType == .concurrent else { return internalQueue } - - return DispatchQueue( - label: "\(self.queueContext)-serial", - qos: self.qosClass, - attributes: [], - autoreleaseFrequency: .inherit, - target: nil - ) - } - - fileprivate func add( - _ db: ObservingDatabase, - job: Job, - canStartJob: Bool - ) { - // Check if the job should be added to the queue - guard - canStartJob, - job.behaviour != .runOnceNextLaunch, - job.nextRunTimestamp <= dependencies.dateNow.timeIntervalSince1970 - else { return } - guard job.id != nil else { - Log.info(.jobRunner, "Prevented attempt to add \(job) without id to queue") - return - } - - _pendingJobsQueue.performUpdate { $0.appending(job) } - - // If this is a concurrent queue then we should immediately start the next job - guard executionType == .concurrent else { return } - - // Ensure that the database commit has completed and then trigger the next job to run (need - // to ensure any interactions have been correctly inserted first) - db.afterCommit(dedupeId: "JobRunner-Add: \(job.variant)") { [weak self] in - self?.runNextJob() - } - } - - /// Upsert a job onto the queue, if the queue isn't currently running and 'canStartJob' is true then this will start - /// the JobRunner - /// - /// **Note:** If the job has a `behaviour` of `runOnceNextLaunch` or the `nextRunTimestamp` - /// is in the future then the job won't be started - fileprivate func upsert( - _ db: ObservingDatabase, - job: Job, - canStartJob: Bool - ) -> Bool { - guard let jobId: Int64 = job.id else { - Log.warn(.jobRunner, "Prevented attempt to upsert \(job) without id to queue") - return false - } - - // Lock the pendingJobsQueue while checking the index and inserting to ensure we don't run into - // any multi-threading shenanigans - // - // Note: currently running jobs are removed from the pendingJobsQueue so we don't need to check - // the 'jobsCurrentlyRunning' set - var didUpdateExistingJob: Bool = false - - _pendingJobsQueue.performUpdate { queue in - if let jobIndex: Array.Index = queue.firstIndex(where: { $0.id == jobId }) { - didUpdateExistingJob = true - return queue.setting(jobIndex, job) + guard insertedJob.id != nil else { + Log.info(.jobRunner, "Unable to add \(job) due to DB insertion failure.") + return nil } - return queue - } - - // If we didn't update an existing job then we need to add it to the pendingJobsQueue - guard !didUpdateExistingJob else { return true } - - // Make sure the job isn't already running before we add it to the queue - guard !currentlyRunningJobIds.contains(jobId) else { - Log.warn(.jobRunner, "Prevented attempt to upsert \(job) which is currently running") - return false + return insertedJob + } catch { + Log.info(.jobRunner, "Unable to add \(job) due to error: \(error)") + return nil } - - add(db, job: job, canStartJob: canStartJob) - return true } - - fileprivate func insert(_ job: Job, before otherJob: Job) { - guard job.id != nil else { - Log.info(.jobRunner, "Prevented attempt to insert \(job) without id to queue") - return - } +} + +// MARK: - JobRunner.JobInfo + +public extension JobRunner { + struct JobInfo: Equatable, CustomDebugStringConvertible { + public let id: Int64? + public let variant: Job.Variant + public let threadId: String? + public let interactionId: Int64? + public let nextRunTimestamp: TimeInterval + public let queueIndex: Int? + public let detailsData: Data? - // Insert the job before the current job (re-adding the current job to - // the start of the pendingJobsQueue if it's not in there) - this will mean the new - // job will run and then the otherJob will run (or run again) once it's - // done - _pendingJobsQueue.performUpdate { - guard let otherJobIndex: Int = $0.firstIndex(of: otherJob) else { - return $0.inserting(contentsOf: [job, otherJob], at: 0) - } + public var debugDescription: String { + let dataDescription: String = detailsData + .map { data in "Data(hex: \(data.toHexString()), \(data.bytes.count) bytes" } + .defaulting(to: "nil") - return $0.inserting(job, at: otherJobIndex) + return """ + JobRunner.JobInfo( + id: \(id.map { "\($0)" } ?? "nil"), + variant: \(variant), + threadId: \(threadId ?? "nil"), + interactionId: \(interactionId.map { "\($0)" } ?? "nil"), + nextRunTimestamp: \(nextRunTimestamp), + queueIndex: \(queueIndex.map { "\($0)" } ?? "nil"), + detailsData: \(dataDescription) + ) + """ } } - - fileprivate func appDidFinishLaunching(with jobs: [Job], canStart: Bool) { - _pendingJobsQueue.performUpdate { $0.appending(contentsOf: jobs) } - - // Start the job runner if needed - if canStart && !isRunning { - start() - } +} + +public extension JobRunner.JobInfo { + init(job: Job, queueIndex: Int) { + self.id = job.id + self.variant = job.variant + self.threadId = job.threadId + self.interactionId = job.interactionId + self.nextRunTimestamp = job.nextRunTimestamp + self.queueIndex = queueIndex + self.detailsData = job.details } - - fileprivate func appDidBecomeActive(with jobs: [Job], canStart: Bool) { - let currentlyRunningJobIds: Set = currentlyRunningJobIds +} + +// MARK: - JobRunner.Filters + +public extension JobRunner { + struct Filters { + public static let matchingAll: Filters = Filters(include: [], exclude: []) + public static let matchingNone: Filters = Filters(include: [.never], exclude: []) - _pendingJobsQueue.performUpdate { queue in - // Avoid re-adding jobs to the queue that are already in it (this can - // happen if the user sends the app to the background before the 'onActive' - // jobs and then brings it back to the foreground) - let jobsNotAlreadyInQueue: [Job] = jobs - .filter { job in - !currentlyRunningJobIds.contains(job.id ?? -1) && - !queue.contains(where: { $0.id == job.id }) - } + public enum FilterType: Hashable { + case jobId(Int64) + case interactionId(Int64) + case threadId(String) + case variant(Job.Variant) - return queue.appending(contentsOf: jobsNotAlreadyInQueue) + case never } - // Start the job runner if needed - if canStart && !isRunning { - start() - } - } - - fileprivate func infoForAllCurrentlyRunningJobs() -> [Int64: JobRunner.JobInfo] { - return currentlyRunningJobInfo - } - - fileprivate func afterJob(_ jobId: Int64, state: JobRunner.JobState) -> AnyPublisher { - /// Check if the current job state matches the requested state (if not then the job in the requested state can't be found so stop here) - switch (state, currentlyRunningJobIds.contains(jobId)) { - case (.running, false): return Just(.notFound).eraseToAnyPublisher() - case (.pending, true): return Just(.notFound).eraseToAnyPublisher() - default: break - } + let include: Set + let exclude: Set - return jobCompletedSubject - .filter { $0.0 == jobId } - .map { $0.1 } - .eraseToAnyPublisher() - } - - fileprivate func hasPendingOrRunningJobWith( - threadId: String? = nil, - interactionId: Int64? = nil, - detailsData: Data? = nil - ) -> Bool { - let pendingJobs: [Job] = pendingJobsQueue - let currentlyRunningJobInfo: [Int64: JobRunner.JobInfo] = currentlyRunningJobInfo - var possibleJobIds: Set = Set(currentlyRunningJobInfo.keys) - .inserting(contentsOf: pendingJobs.compactMap { $0.id }.asSet()) + // MARK: - Initialization - // Remove any which don't have the matching threadId (if provided) - if let targetThreadId: String = threadId { - let pendingJobIdsWithWrongThreadId: Set = pendingJobs - .filter { $0.threadId != targetThreadId } - .compactMap { $0.id } - .asSet() - let runningJobIdsWithWrongThreadId: Set = currentlyRunningJobInfo - .filter { _, info -> Bool in info.threadId != targetThreadId } - .map { key, _ in key } - .asSet() - - possibleJobIds = possibleJobIds - .subtracting(pendingJobIdsWithWrongThreadId) - .subtracting(runningJobIdsWithWrongThreadId) + public init( + include: [FilterType] = [], + exclude: [FilterType] = [] + ) { + self.include = Set(include) + self.exclude = Set(exclude) } - // Remove any which don't have the matching interactionId (if provided) - if let targetInteractionId: Int64 = interactionId { - let pendingJobIdsWithWrongInteractionId: Set = pendingJobs - .filter { $0.interactionId != targetInteractionId } - .compactMap { $0.id } - .asSet() - let runningJobIdsWithWrongInteractionId: Set = currentlyRunningJobInfo - .filter { _, info -> Bool in info.interactionId != targetInteractionId } - .map { key, _ in key } - .asSet() - - possibleJobIds = possibleJobIds - .subtracting(pendingJobIdsWithWrongInteractionId) - .subtracting(runningJobIdsWithWrongInteractionId) - } + // MARK: - Functions - // Remove any which don't have the matching details (if provided) - if let targetDetailsData: Data = detailsData { - let pendingJobIdsWithWrongDetailsData: Set = pendingJobs - .filter { $0.details != targetDetailsData } - .compactMap { $0.id } - .asSet() - let runningJobIdsWithWrongDetailsData: Set = currentlyRunningJobInfo - .filter { _, info -> Bool in info.detailsData != detailsData } - .map { key, _ in key } - .asSet() + func matches(_ jobInfo: JobRunner.JobInfo) -> Bool { + let infoSet: Set = Set([ + jobInfo.id.map { .jobId($0) }, + .variant(jobInfo.variant), + jobInfo.threadId.map { .threadId($0) }, + jobInfo.interactionId.map { .interactionId($0) } + ].compactMap { $0 }) - possibleJobIds = possibleJobIds - .subtracting(pendingJobIdsWithWrongDetailsData) - .subtracting(runningJobIdsWithWrongDetailsData) - } - - return !possibleJobIds.isEmpty - } - - fileprivate func removePendingJob(_ jobId: Int64) { - _pendingJobsQueue.performUpdate { queue in - queue.filter { $0.id != jobId } - } - } - - // MARK: - Job Running - - fileprivate func start(forceWhenAlreadyRunning: Bool = false) { - // Only start if the JobRunner is allowed to start the queue or if this queue is running in - // a background task - let isRunningInBackgroundTask: Bool = self.isRunningInBackgroundTask - - guard canStart?(self) == true || isRunningInBackgroundTask else { return } - guard forceWhenAlreadyRunning || !isRunning || isRunningInBackgroundTask else { return } - - // The JobRunner runs synchronously so we need to ensure this doesn't start on the main - // thread and do so by creating a number of background queues to run the jobs on, if this - // function was called on the wrong queue then we need to dispatch to the correct one - guard DispatchQueue.with(key: queueKey, matches: queueContext, using: dependencies) else { - internalQueue.async(using: dependencies) { [weak self] in - self?.start(forceWhenAlreadyRunning: forceWhenAlreadyRunning) - } - return - } - - // Flag the JobRunner as running (to prevent something else from trying to start it - // and messing with the execution behaviour) - let wasAlreadyRunning: Bool = _isRunning.performUpdateAndMap { (true, $0) } - hasStartedAtLeastOnce = true - - // Get any pending jobs - - let jobVariants: [Job.Variant] = self.jobVariants - let jobIdsAlreadyRunning: Set = currentlyRunningJobIds - let jobsAlreadyInQueue: Set = pendingJobsQueue.compactMap { $0.id }.asSet() - let jobsToRun: [Job] - - switch isRunningInBackgroundTask { - case true: jobsToRun = [] // When running in a background task we don't want to schedule extra jobs - case false: - jobsToRun = dependencies[singleton: .storage].read { db in - try Job - .filterPendingJobs( - variants: jobVariants, - excludeFutureJobs: true, - includeJobsWithDependencies: false - ) - .filter(!jobIdsAlreadyRunning.contains(Job.Columns.id)) // Exclude jobs already running - .filter(!jobsAlreadyInQueue.contains(Job.Columns.id)) // Exclude jobs already in the queue - .fetchAll(db) - } - .defaulting(to: []) - } - - // Determine the number of jobs to run - let jobCount: Int = _pendingJobsQueue.performUpdateAndMap { queue in - let updatedQueue: [Job] = queue.appending(contentsOf: jobsToRun) - return (updatedQueue, updatedQueue.count) - } - - // If there are no pending jobs and nothing in the queue then schedule the JobRunner - // to start again when the next scheduled job should start - guard jobCount > 0 else { - if jobIdsAlreadyRunning.isEmpty { - isRunning = false - scheduleNextSoonestJob() - } - return - } - - // Run the first job in the pendingJobsQueue - if !wasAlreadyRunning { - Log.info(.jobRunner, "Starting \(queueContext) with \(jobCount) jobs") - } - runNextJob() - } - - fileprivate func stopAndClearPendingJobs() { - isRunning = false - _pendingJobsQueue.set(to: []) - _deferLoopTracker.set(to: [:]) - } - - private func runNextJob() { - // Ensure the queue is running (if we've stopped the queue then we shouldn't start the next job) - guard isRunning else { return } - - // Ensure this is running on the correct queue - guard DispatchQueue.with(key: queueKey, matches: queueContext, using: dependencies) else { - internalQueue.async(using: dependencies) { [weak self] in - self?.runNextJob() - } - return - } - guard executionType == .concurrent || currentlyRunningJobIds.isEmpty else { - return Log.info(.jobRunner, "\(queueContext) Ignoring 'runNextJob' due to currently running job in serial queue") - } - guard - let (nextJob, numJobsRemaining): (Job, Int) = _pendingJobsQueue.performUpdateAndMap({ queue in - var updatedQueue: [Job] = queue - let nextJob: Job? = updatedQueue.popFirst() - - return (updatedQueue, nextJob.map { ($0, updatedQueue.count) }) - }) - else { - // If it's a serial queue, or there are no more jobs running then update the 'isRunning' flag - if executionType != .concurrent || currentlyRunningJobIds.isEmpty { - isRunning = false - } - - // Always attempt to schedule the next soonest job (otherwise if enough jobs get started in rapid - // succession then pending/failed jobs in the database may never get re-started in a concurrent queue) - scheduleNextSoonestJob() - return - } - guard let jobExecutor: JobExecutor.Type = executorMap[nextJob.variant] else { - Log.info(.jobRunner, "\(queueContext) Unable to run \(nextJob) due to missing executor") - return handleJobFailed( - nextJob, - error: JobRunnerError.executorMissing, - permanentFailure: true - ) - } - guard !jobExecutor.requiresThreadId || nextJob.threadId != nil else { - Log.info(.jobRunner, "\(queueContext) Unable to run \(nextJob) due to missing required threadId") - return handleJobFailed( - nextJob, - error: JobRunnerError.requiredThreadIdMissing, - permanentFailure: true - ) - } - guard !jobExecutor.requiresInteractionId || nextJob.interactionId != nil else { - Log.info(.jobRunner, "\(queueContext) Unable to run \(nextJob) due to missing required interactionId") - return handleJobFailed( - nextJob, - error: JobRunnerError.requiredInteractionIdMissing, - permanentFailure: true - ) - } - guard nextJob.id != nil else { - Log.info(.jobRunner, "\(queueContext) Unable to run \(nextJob) due to missing id") - return handleJobFailed( - nextJob, - error: JobRunnerError.jobIdMissing, - permanentFailure: false - ) - } - - // If the 'nextRunTimestamp' for the job is in the future then don't run it yet - guard nextJob.nextRunTimestamp <= dependencies.dateNow.timeIntervalSince1970 else { - handleJobDeferred(nextJob) - return - } - - // Check if the next job has any dependencies - let dependencyInfo: (expectedCount: Int, jobs: Set) = dependencies[singleton: .storage].read { db in - let expectedDependencies: Set = try JobDependencies - .filter(JobDependencies.Columns.jobId == nextJob.id) - .fetchSet(db) - let jobDependencies: Set = try Job - .filter(ids: expectedDependencies.compactMap { $0.dependantId }) - .fetchSet(db) - - return (expectedDependencies.count, jobDependencies) - } - .defaulting(to: (0, [])) - - guard dependencyInfo.jobs.count == dependencyInfo.expectedCount else { - Log.info(.jobRunner, "\(queueContext) Removing \(nextJob) due to missing dependencies") - return handleJobFailed( - nextJob, - error: JobRunnerError.missingDependencies, - permanentFailure: true - ) - } - guard dependencyInfo.jobs.isEmpty else { - Log.info(.jobRunner, "\(queueContext) Deferring \(nextJob) until \(dependencyInfo.jobs.count) dependencies are completed") - - // Enqueue the dependencies then defer the current job - dependencies[singleton: .jobRunner].enqueueDependenciesIfNeeded(Array(dependencyInfo.jobs)) - handleJobDeferred(nextJob) - return - } - - // Update the state to indicate the particular job is running - // - // Note: We need to store 'numJobsRemaining' in it's own variable because - // the 'Log.info' seems to dispatch to it's own queue which ends up getting - // blocked by the JobRunner's queue becuase 'jobQueue' is Atomic - var numJobsRunning: Int = 0 - _nextTrigger.performUpdate { trigger in - trigger?.invalidate() // Need to invalidate to prevent a memory leak - return nil - } - _currentlyRunningJobIds.performUpdate { currentlyRunningJobIds in - let result: Set = currentlyRunningJobIds.inserting(nextJob.id) - numJobsRunning = currentlyRunningJobIds.count - return result - } - _currentlyRunningJobInfo.performUpdate { currentlyRunningJobInfo in - currentlyRunningJobInfo.setting( - nextJob.id, - JobRunner.JobInfo( - variant: nextJob.variant, - threadId: nextJob.threadId, - interactionId: nextJob.interactionId, - detailsData: nextJob.details, - uniqueHashValue: nextJob.uniqueHashValue - ) - ) - } - Log.info(.jobRunner, "\(queueContext) started \(nextJob) (\(executionType == .concurrent ? "\(numJobsRunning) currently running, " : "")\(numJobsRemaining) remaining)") - - jobExecutor.run( - nextJob, - scheduler: targetQueue(), - success: handleJobSucceeded, - failure: handleJobFailed, - deferred: handleJobDeferred, - using: dependencies - ) - - // If this queue executes concurrently and there are still jobs remaining then immediately attempt - // to start the next job - if executionType == .concurrent && numJobsRemaining > 0 { - internalQueue.async(using: dependencies) { [weak self] in - self?.runNextJob() - } - } - } - - private func scheduleNextSoonestJob() { - // Retrieve any pending jobs from the database - let jobVariants: [Job.Variant] = self.jobVariants - let jobIdsAlreadyRunning: Set = currentlyRunningJobIds - let nextJobTimestamp: TimeInterval? = dependencies[singleton: .storage].read { db in - try Job - .filterPendingJobs( - variants: jobVariants, - excludeFutureJobs: false, - includeJobsWithDependencies: false - ) - .select(.nextRunTimestamp) - .filter(!jobIdsAlreadyRunning.contains(Job.Columns.id)) // Exclude jobs already running - .asRequest(of: TimeInterval.self) - .fetchOne(db) - } - - // If there are no remaining jobs or the JobRunner isn't allowed to start any queues then trigger - // the 'onQueueDrained' callback and stop - guard let nextJobTimestamp: TimeInterval = nextJobTimestamp, canStart?(self) == true else { - if executionType != .concurrent || currentlyRunningJobIds.isEmpty { - self.onQueueDrained?() - } - return - } - - // If the next job isn't scheduled in the future then just restart the JobRunner immediately - let secondsUntilNextJob: TimeInterval = (nextJobTimestamp - dependencies.dateNow.timeIntervalSince1970) - - guard secondsUntilNextJob > 0 else { - // Only log that the queue is getting restarted if this queue had actually been about to stop - if executionType != .concurrent || currentlyRunningJobIds.isEmpty { - let timingString: String = (nextJobTimestamp == 0 ? - "that should be in the queue" : - "scheduled \(.seconds(secondsUntilNextJob), unit: .s) ago" - ) - Log.info(.jobRunner, "Restarting \(queueContext) immediately for job \(timingString)") + /// If the job is explicitly excluded then it doesn't match the filters + if !exclude.intersection(infoSet).isEmpty { + return false } - // Trigger the 'start' function to load in any pending jobs that aren't already in the - // queue (for concurrent queues we want to force them to load in pending jobs and add - // them to the queue regardless of whether the queue is already running) - internalQueue.async(using: dependencies) { [weak self] in - self?.start(forceWhenAlreadyRunning: (self?.executionType != .concurrent)) - } - return - } - - // Only schedule a trigger if the queue is concurrent, or it has actually completed - guard executionType == .concurrent || currentlyRunningJobIds.isEmpty else { return } - - // Setup a trigger - Log.info(.jobRunner, "Stopping \(queueContext) until next job in \(.seconds(secondsUntilNextJob), unit: .s)") - _nextTrigger.performUpdate { trigger in - trigger?.invalidate() // Need to invalidate the old trigger to prevent a memory leak - return Trigger.create(queue: self, timestamp: nextJobTimestamp, using: dependencies) + /// If `include` is empty, or the job is explicitly included then it does match the filters + return (include.isEmpty || !include.intersection(infoSet).isEmpty) } } - - // MARK: - Handling Results - - /// This function is called when a job succeeds - fileprivate func handleJobSucceeded(_ job: Job, shouldStop: Bool) { - dependencies[singleton: .storage].writeAsync( - updates: { [dependencies] db -> [Job] in - /// Retrieve the dependant jobs first (the `JobDependecies` table has cascading deletion when the original `Job` is - /// removed so we need to retrieve these records before that happens) - let dependantJobs: [Job] = try job.dependantJobs.fetchAll(db) - - switch job.behaviour { - case .runOnce, .runOnceNextLaunch, .runOnceAfterConfigSyncIgnoringPermanentFailure: - /// Since this job has been completed we can update the dependencies so other job that were dependant - /// on this one can be run - _ = try JobDependencies - .filter(JobDependencies.Columns.dependantId == job.id) - .deleteAll(db) - - _ = try job.delete(db) - - case .recurring where shouldStop == true: - /// Since this job has been completed we can update the dependencies so other job that were dependant - /// on this one can be run - _ = try JobDependencies - .filter(JobDependencies.Columns.dependantId == job.id) - .deleteAll(db) - - _ = try job.delete(db) - - /// For `recurring` jobs which have already run, they should automatically run again but we want at least 1 second - /// to pass before doing so - the job itself should really update it's own `nextRunTimestamp` (this is just a safety net) - case .recurring where job.nextRunTimestamp <= dependencies.dateNow.timeIntervalSince1970: - guard let jobId: Int64 = job.id else { break } - - _ = try Job - .filter(id: jobId) - .updateAll( - db, - Job.Columns.failureCount.set(to: 0), - Job.Columns.nextRunTimestamp.set(to: (dependencies.dateNow.timeIntervalSince1970 + 1)) - ) - - /// For `recurringOnLaunch/Active` jobs which have already run but failed once, we need to clear their - /// `failureCount` and `nextRunTimestamp` to prevent them from endlessly running over and over again - case .recurringOnLaunch, .recurringOnActive: - guard - let jobId: Int64 = job.id, - job.failureCount != 0 && - job.nextRunTimestamp > TimeInterval.leastNonzeroMagnitude - else { break } - - _ = try Job - .filter(id: jobId) - .updateAll( - db, - Job.Columns.failureCount.set(to: 0), - Job.Columns.nextRunTimestamp.set(to: 0) - ) - - default: break - } - - return dependantJobs - }, - completion: { [weak self, dependencies] result in - switch result { - case .failure: break - case .success(let dependantJobs): - /// Now that the job has been completed we want to enqueue any jobs that were dependant on it - dependencies[singleton: .jobRunner].enqueueDependenciesIfNeeded(dependantJobs) - } - - /// Perform job cleanup and start the next job - self?.performCleanUp(for: job, result: .succeeded) - self?.internalQueue.async(using: dependencies) { [weak self] in - self?.runNextJob() - } - } - ) - } +} - /// This function is called when a job fails, if it's wasn't a permanent failure then the 'failureCount' for the job will be incremented and it'll - /// be re-run after a retry interval has passed - fileprivate func handleJobFailed( - _ job: Job, - error: Error, - permanentFailure: Bool - ) { - guard dependencies[singleton: .storage].read({ db in try Job.exists(db, id: job.id ?? -1) }) == true else { - Log.info(.jobRunner, "\(queueContext) \(job) canceled") - performCleanUp(for: job, result: .failed(error, permanentFailure)) - - internalQueue.async(using: dependencies) { [weak self] in - self?.runNextJob() - } - return - } - - // If this is the blocking queue and a "blocking" job failed then rerun it - // immediately (in this case we don't trigger any job callbacks because the - // job isn't actually done, it's going to try again immediately) - if self.type == .blocking && job.shouldBlock { - Log.info(.jobRunner, "\(queueContext) \(job) failed due to error: \(error); retrying immediately") - - // If it was a possible deferral loop then we don't actually want to - // retry the job (even if it's a blocking one, this gives a small chance - // that the app could continue to function) - performCleanUp( - for: job, - result: .failed(error, permanentFailure), - shouldTriggerCallbacks: ((error as? JobRunnerError)?.wasPossibleDeferralLoop == true) - ) - - // Only add it back to the queue if it wasn't a deferral loop - if (error as? JobRunnerError)?.wasPossibleDeferralLoop != true { - _pendingJobsQueue.performUpdate { $0.inserting(job, at: 0) } - } - - internalQueue.async(using: dependencies) { [weak self] in - self?.runNextJob() - } - return - } - - // Get the max failure count for the job (a value of '-1' means it will retry indefinitely) - let maxFailureCount: Int = (executorMap[job.variant]?.maxFailureCount ?? 0) - let nextRunTimestamp: TimeInterval = (dependencies.dateNow.timeIntervalSince1970 + JobRunner.getRetryInterval(for: job)) - var dependantJobIds: [Int64] = [] - var failureText: String = "failed due to error: \(error)" - - dependencies[singleton: .storage].write { db in - /// Retrieve a list of dependant jobs so we can clear them from the queue - dependantJobIds = try job.dependantJobs - .select(.id) - .asRequest(of: Int64.self) - .fetchAll(db) +// MARK: - JobRunner.JobState - /// Delete/update the failed jobs and any dependencies - let updatedFailureCount: UInt = (job.failureCount + 1) +public extension JobRunner { + struct JobState: OptionSet, Hashable { + public let rawValue: UInt8 - guard - !permanentFailure && ( - maxFailureCount < 0 || - updatedFailureCount <= maxFailureCount || - job.behaviour == .runOnceAfterConfigSyncIgnoringPermanentFailure - ) - else { - failureText = (maxFailureCount >= 0 && updatedFailureCount > maxFailureCount ? - "failed permanently due to error: \(error); too many retries" : - "failed permanently due to error: \(error)" - ) - - // If the job permanently failed or we have performed all of our retry attempts - // then delete the job and all of it's dependant jobs (it'll probably never succeed) - _ = try job.dependantJobs - .deleteAll(db) - - _ = try job.delete(db) - return - } - - failureText = "failed due to error: \(error); scheduling retry (failure count is \(updatedFailureCount))" - - try job - .with( - failureCount: updatedFailureCount, - nextRunTimestamp: nextRunTimestamp - ) - .upserted(db) - - // Update the failureCount and nextRunTimestamp on dependant jobs as well (update the - // 'nextRunTimestamp' value to be 1ms later so when the queue gets regenerated they'll - // come after the dependency) - try job.dependantJobs - .updateAll( - db, - Job.Columns.failureCount.set(to: updatedFailureCount), - Job.Columns.nextRunTimestamp.set(to: (nextRunTimestamp + (1 / 1000))) - ) + public init(rawValue: UInt8) { + self.rawValue = rawValue } - /// Remove any dependant jobs from the queue (shouldn't be in there but filter the queue just in case so we don't try - /// to run a deleted job or get stuck in a loop of trying to run dependencies indefinitely) - if !dependantJobIds.isEmpty { - _pendingJobsQueue.performUpdate { queue in - queue.filter { !dependantJobIds.contains($0.id ?? -1) } - } - } + public static let pending: JobState = JobState(rawValue: 1 << 0) + public static let running: JobState = JobState(rawValue: 1 << 1) - Log.error(.jobRunner, "\(queueContext) \(job) \(failureText)") - performCleanUp(for: job, result: .failed(error, permanentFailure)) - internalQueue.async(using: dependencies) { [weak self] in - self?.runNextJob() - } + public static let any: JobState = [ .pending, .running ] } - - /// This function is called when a job neither succeeds or fails (this should only occur if the job has specific logic that makes it dependant - /// on other jobs, and it should automatically manage those dependencies) - fileprivate func handleJobDeferred(_ job: Job) { - var stuckInDeferLoop: Bool = false +} + +public extension JobRunner { + enum JobResult: Equatable { + case succeeded + case failed(Error, Bool) + case deferred + case notFound - _deferLoopTracker.performUpdate { - guard let lastRecord: (count: Int, times: [TimeInterval]) = $0[job.id] else { - return $0.setting( - job.id, - (1, [dependencies.dateNow.timeIntervalSince1970]) - ) + public static func == (lhs: JobRunner.JobResult, rhs: JobRunner.JobResult) -> Bool { + switch (lhs, rhs) { + case (.succeeded, .succeeded): return true + case (.failed(let lhsError, let lhsPermanent), .failed(let rhsError, let rhsPermanent)): + return ( + // Not a perfect solution but should be good enough + "\(lhsError)" == "\(rhsError)" && + lhsPermanent == rhsPermanent + ) + + case (.deferred, .deferred): return true + default: return false } - - let timeNow: TimeInterval = dependencies.dateNow.timeIntervalSince1970 - stuckInDeferLoop = ( - lastRecord.count >= JobQueue.deferralLoopThreshold && - (timeNow - lastRecord.times[0]) < CGFloat(lastRecord.count * maxDeferralsPerSecond) - ) - - return $0.setting( - job.id, - ( - lastRecord.count + 1, - // Only store the last 'deferralLoopThreshold' times to ensure we aren't running faster - // than one loop per second - lastRecord.times.suffix(JobQueue.deferralLoopThreshold - 1) + [timeNow] - ) - ) - } - - // It's possible (by introducing bugs) to create a loop where a Job tries to run and immediately - // defers itself but then attempts to run again (resulting in an infinite loop); this won't block - // the app since it's on a background thread but can result in 100% of a CPU being used (and a - // battery drain) - // - // This code will maintain an in-memory store for any jobs which are deferred too quickly (ie. - // more than 'deferralLoopThreshold' times within 'deferralLoopThreshold' seconds) - guard !stuckInDeferLoop else { - _deferLoopTracker.performUpdate { $0.removingValue(forKey: job.id) } - handleJobFailed( - job, - error: JobRunnerError.possibleDeferralLoop, - permanentFailure: false - ) - return } - - performCleanUp(for: job, result: .deferred) - internalQueue.async(using: dependencies) { [weak self] in - self?.runNextJob() - } - } - - fileprivate func performCleanUp( - for job: Job, - result: JobRunner.JobResult, - shouldTriggerCallbacks: Bool = true - ) { - // The job is removed from the queue before it runs so all we need to to is remove it - // from the 'currentlyRunning' set - _currentlyRunningJobIds.performUpdate { $0.removing(job.id) } - _currentlyRunningJobInfo.performUpdate { $0.removingValue(forKey: job.id) } - - guard shouldTriggerCallbacks else { return } - - // Notify any listeners of the job result - jobCompletedSubject.send((job.id, result)) } } +// MARK: - JobRunner.JobState + +public extension JobRunner { + typealias ScheduleInfo = ( + variant: Job.Variant, + behaviour: Job.Behaviour, + shouldBlock: Bool, + shouldSkipLaunchBecomeActive: Bool + ) +} + // MARK: - Formatting private extension String.StringInterpolation { diff --git a/SessionUtilitiesKit/JobRunner/JobRunnerError.swift b/SessionUtilitiesKit/JobRunner/JobRunnerError.swift index e9a69b5986..19ccaa958d 100644 --- a/SessionUtilitiesKit/JobRunner/JobRunnerError.swift +++ b/SessionUtilitiesKit/JobRunner/JobRunnerError.swift @@ -4,7 +4,7 @@ import Foundation -public enum JobRunnerError: Error, Equatable, CustomStringConvertible { +public enum JobRunnerError: Error, CustomStringConvertible { case executorMissing case jobIdMissing case requiredThreadIdMissing @@ -16,6 +16,8 @@ public enum JobRunnerError: Error, Equatable, CustomStringConvertible { case possibleDuplicateJob(permanentFailure: Bool) case possibleDeferralLoop + case permanentFailure(Error) + var wasPossibleDeferralLoop: Bool { switch self { case .possibleDeferralLoop: return true @@ -35,6 +37,24 @@ public enum JobRunnerError: Error, Equatable, CustomStringConvertible { case .possibleDuplicateJob: return "This job might be the duplicate of another running job." case .possibleDeferralLoop: return "The job might have been stuck in a deferral loop." + + case .permanentFailure(let underlyingError): return "A permanent failure occurred: \(underlyingError)" + } + } +} + +extension JobRunnerError: JobError { + public var isPermanent: Bool { + switch self { + case .executorMissing: return true + case .jobIdMissing: return true + case .requiredThreadIdMissing: return true + case .requiredInteractionIdMissing: return true + case .missingRequiredDetails: return true + case .missingDependencies: return true + case .possibleDuplicateJob(let permanentFailure): return permanentFailure + case .possibleDeferralLoop: return false + case .permanentFailure: return true } } } diff --git a/SessionUtilitiesKit/Types/CancellationAwareAsyncStream.swift b/SessionUtilitiesKit/Types/CancellationAwareAsyncStream.swift new file mode 100644 index 0000000000..42a56bace5 --- /dev/null +++ b/SessionUtilitiesKit/Types/CancellationAwareAsyncStream.swift @@ -0,0 +1,72 @@ +// Copyright © 2025 Rangeproof Pty Ltd. All rights reserved. + +import Foundation + +// MARK: - CancellationAwareAsyncStream + +public actor CancellationAwareAsyncStream: CancellationAwareStreamType { + private let lifecycleManager: StreamLifecycleManager = StreamLifecycleManager() + + // MARK: - Initialization + + public init() {} + + // MARK: - Functions + + public func send(_ newValue: Element) async { + lifecycleManager.send(newValue) + } + + public func finishCurrentStreams() async { + lifecycleManager.finishCurrentStreams() + } + + public func beforeYield(to continuation: AsyncStream.Continuation) async { + // No-op - no initial value + } + + public func makeTrackedStream() -> AsyncStream { + lifecycleManager.makeTrackedStream().stream + } +} + +// MARK: - CancellationAwareStreamType + +public protocol CancellationAwareStreamType: Actor { + associatedtype Element: Sendable + + func send(_ newValue: Element) async + func finishCurrentStreams() async + + /// This function gets called when a stream is initially created but before the inner stream is created, it shouldn't be called directly + func beforeYield(to continuation: AsyncStream.Continuation) async + + /// This is an internal function which shouldn't be called directly + func makeTrackedStream() async -> AsyncStream +} + +public extension CancellationAwareStreamType { + /// Every time `stream` is accessed it will create a **new** stream + /// + /// **Note:** This is non-isolated so it can be exposed via protocols without `async`, this is safe because `AsyncStream` is + /// thread-safe internally and `Element` is `Sendable` so it's verified to be safe to send concurrently + nonisolated var stream: AsyncStream { + AsyncStream { continuation in + let bridgingTask = Task { + await self.beforeYield(to: continuation) + + let internalStream: AsyncStream = await self.makeTrackedStream() + + for await element in internalStream { + continuation.yield(element) + } + + continuation.finish() + } + + continuation.onTermination = { @Sendable _ in + bridgingTask.cancel() + } + } + } +} diff --git a/SessionUtilitiesKit/Types/CurrentValueAsyncStream.swift b/SessionUtilitiesKit/Types/CurrentValueAsyncStream.swift index 26c6b9245e..b2f9f13e1a 100644 --- a/SessionUtilitiesKit/Types/CurrentValueAsyncStream.swift +++ b/SessionUtilitiesKit/Types/CurrentValueAsyncStream.swift @@ -2,34 +2,34 @@ import Foundation -public actor CurrentValueAsyncStream { - private var _currentValue: Element - private let continuation: AsyncStream.Continuation - public let stream: AsyncStream - - public var currentValue: Element { _currentValue } +public actor CurrentValueAsyncStream: CancellationAwareStreamType { + private let lifecycleManager: StreamLifecycleManager = StreamLifecycleManager() + + /// This is the most recently emitted value + public private(set) var currentValue: Element // MARK: - Initialization public init(_ initialValue: Element) { - self._currentValue = initialValue - - /// We use `.bufferingNewest(1)` to ensure that the stream always holds the most recent value. When a new iterator is - /// created for the stream, it will receive this buffered value first. - let (stream, continuation) = AsyncStream.makeStream(of: Element.self, bufferingPolicy: .bufferingNewest(1)) - self.stream = stream - self.continuation = continuation - self.continuation.yield(initialValue) + self.currentValue = initialValue } // MARK: - Functions - public func send(_ newValue: Element) { - _currentValue = newValue - continuation.yield(newValue) + public func send(_ newValue: Element) async { + currentValue = newValue + lifecycleManager.send(newValue) } - public func finish() { - continuation.finish() + public func finishCurrentStreams() async { + lifecycleManager.finishCurrentStreams() + } + + public func beforeYield(to continuation: AsyncStream.Continuation) async { + continuation.yield(currentValue) + } + + public func makeTrackedStream() -> AsyncStream { + lifecycleManager.makeTrackedStream().stream } } diff --git a/SessionUtilitiesKit/Types/StreamLifecycleManager.swift b/SessionUtilitiesKit/Types/StreamLifecycleManager.swift new file mode 100644 index 0000000000..444e431525 --- /dev/null +++ b/SessionUtilitiesKit/Types/StreamLifecycleManager.swift @@ -0,0 +1,61 @@ +// Copyright © 2025 Rangeproof Pty Ltd. All rights reserved. + +import Foundation + +public final class StreamLifecycleManager: @unchecked Sendable { + private let lock: NSLock = NSLock() + private var continuations: [UUID: AsyncStream.Continuation] = [:] + + // MARK: - Initialization + + public init() {} + + deinit { + finishCurrentStreams() + } + + // MARK: - Functions + + func makeTrackedStream() -> (stream: AsyncStream, id: UUID) { + let (stream, continuation) = AsyncStream.makeStream(of: Element.self) + let id: UUID = UUID() + + lock.withLock { continuations[id] = continuation } + + continuation.onTermination = { @Sendable [self] _ in + self.finishStream(id: id) + } + + return (stream, id) + } + + func send(_ value: Element) { + /// Capture current continuations before sending to avoid deadlocks where yielding could result in a new continuation being + /// added while the lock is held + let currentContinuations: [UUID: AsyncStream.Continuation] = lock.withLock { continuations } + + for continuation in currentContinuations.values { + continuation.yield(value) + } + } + + func finishStream(id: UUID) { + lock.withLock { + if let continuation: AsyncStream.Continuation = continuations.removeValue(forKey: id) { + continuation.finish() + } + } + } + + func finishCurrentStreams() { + let currentContinuations: [UUID: AsyncStream.Continuation] = lock.withLock { + let continuationsToFinish: [UUID: AsyncStream.Continuation] = continuations + continuations.removeAll() + return continuationsToFinish + } + + for continuation in currentContinuations.values { + continuation.finish() + } + } +}