diff --git a/Mixin.xcodeproj/project.pbxproj b/Mixin.xcodeproj/project.pbxproj index ddc346cb85..eadfb8167e 100644 --- a/Mixin.xcodeproj/project.pbxproj +++ b/Mixin.xcodeproj/project.pbxproj @@ -540,6 +540,7 @@ 7BFE47E32284394000FC4379 /* CheckmarkPeerCell.xib in Resources */ = {isa = PBXBuildFile; fileRef = 7BFE47E12284394000FC4379 /* CheckmarkPeerCell.xib */; }; 7BFE47E52284530200FC4379 /* PeerHeaderView.swift in Sources */ = {isa = PBXBuildFile; fileRef = 7BFE47E42284530200FC4379 /* PeerHeaderView.swift */; }; 7BFE47E722845DE100FC4379 /* MessageReceiver.swift in Sources */ = {isa = PBXBuildFile; fileRef = 7BFE47E622845DE100FC4379 /* MessageReceiver.swift */; }; + 7C07ED36287D549400685322 /* BackupJob.swift in Sources */ = {isa = PBXBuildFile; fileRef = 7C07ED35287D549400685322 /* BackupJob.swift */; }; 7C0D997F26CA62CD00356655 /* StaticAudioMessagePlayingManager.swift in Sources */ = {isa = PBXBuildFile; fileRef = 7C0D997E26CA62CD00356655 /* StaticAudioMessagePlayingManager.swift */; }; 7C0E15DF27005376002FC718 /* UnknownURLWindow.swift in Sources */ = {isa = PBXBuildFile; fileRef = 7C0E15DE27005376002FC718 /* UnknownURLWindow.swift */; }; 7C0E15E1270053AC002FC718 /* UnknownURLWindow.xib in Resources */ = {isa = PBXBuildFile; fileRef = 7C0E15E0270053AC002FC718 /* UnknownURLWindow.xib */; }; @@ -602,6 +603,7 @@ 7C8FA78F2768822800855AFD /* DeleteAccountTableHeaderView.xib in Resources */ = {isa = PBXBuildFile; fileRef = 7C8FA78E2768822800855AFD /* DeleteAccountTableHeaderView.xib */; }; 7C8FA8F42768909300855AFD /* AccountSettingViewController.swift in Sources */ = {isa = PBXBuildFile; fileRef = 7C8FA8F32768909300855AFD /* AccountSettingViewController.swift */; }; 7C8FA8F62768926D00855AFD /* SecuritySettingViewController.swift in Sources */ = {isa = PBXBuildFile; fileRef = 7C8FA8F52768926D00855AFD /* SecuritySettingViewController.swift */; }; + 7C9279AB28854F5C00321DFF /* RestoreJob.swift in Sources */ = {isa = PBXBuildFile; fileRef = 7C9279AA28854F5C00321DFF /* RestoreJob.swift */; }; 7C952CFE27E035080083F92B /* ExpiredMessageViewController.swift in Sources */ = {isa = PBXBuildFile; fileRef = 7C952CFD27E035080083F92B /* ExpiredMessageViewController.swift */; }; 7C952D0027E036240083F92B /* ExpiredMessageTableHeaderView.xib in Resources */ = {isa = PBXBuildFile; fileRef = 7C952CFF27E036240083F92B /* ExpiredMessageTableHeaderView.xib */; }; 7C9A734027392FAF00E0127A /* PinSettingTableHeaderView.swift in Sources */ = {isa = PBXBuildFile; fileRef = 7C9A733F27392FAF00E0127A /* PinSettingTableHeaderView.swift */; }; @@ -629,6 +631,7 @@ 7CF5929827979CCF00015495 /* DeleteAccountVerifyPinWindow.xib in Resources */ = {isa = PBXBuildFile; fileRef = 7CF5929727979CCF00015495 /* DeleteAccountVerifyPinWindow.xib */; }; 7CF7416E27DAD93000DA0004 /* SnapCenterFlowLayout.swift in Sources */ = {isa = PBXBuildFile; fileRef = 7CF7416D27DAD93000DA0004 /* SnapCenterFlowLayout.swift */; }; 7CF836F127E334B0002E2A98 /* ExpiredMessageDurationFormatter.swift in Sources */ = {isa = PBXBuildFile; fileRef = 7CF836F027E334B0002E2A98 /* ExpiredMessageDurationFormatter.swift */; }; + 7CFD746E2889228100D7A0EE /* CloudJob.swift in Sources */ = {isa = PBXBuildFile; fileRef = 7CFD746D2889228100D7A0EE /* CloudJob.swift */; }; 811C8154F03C8CBB72DBA1F4 /* Pods_MixinShare.framework in Frameworks */ = {isa = PBXBuildFile; fileRef = 67A8E0E9B65F16ADB27E6F25 /* Pods_MixinShare.framework */; }; 842347EE2695BA6400009A39 /* InitializeBotJob.swift in Sources */ = {isa = PBXBuildFile; fileRef = 842347ED2695BA6400009A39 /* InitializeBotJob.swift */; }; 94046B91272DC265007C1D4A /* GroupCallMembersDataSource.swift in Sources */ = {isa = PBXBuildFile; fileRef = 94046B90272DC265007C1D4A /* GroupCallMembersDataSource.swift */; }; @@ -762,7 +765,6 @@ DF1ED8DB20BBED24003E10E8 /* PickerViewController.swift in Sources */ = {isa = PBXBuildFile; fileRef = DF1ED8DA20BBED24003E10E8 /* PickerViewController.swift */; }; DF1ED8E220BC0794003E10E8 /* Photo.storyboard in Resources */ = {isa = PBXBuildFile; fileRef = DF1ED8E120BC0794003E10E8 /* Photo.storyboard */; }; DF1F277C21A53585009A74C6 /* BackupViewController.swift in Sources */ = {isa = PBXBuildFile; fileRef = DF1F277B21A53585009A74C6 /* BackupViewController.swift */; }; - DF1F278421A5637B009A74C6 /* BackupJob.swift in Sources */ = {isa = PBXBuildFile; fileRef = DF1F278321A5637B009A74C6 /* BackupJob.swift */; }; DF1F278621A59A60009A74C6 /* BackupJobQueue.swift in Sources */ = {isa = PBXBuildFile; fileRef = DF1F278521A59A60009A74C6 /* BackupJobQueue.swift */; }; DF2081B22005FF3500B87DB0 /* Camera.storyboard in Resources */ = {isa = PBXBuildFile; fileRef = DF2081B12005FF3500B87DB0 /* Camera.storyboard */; }; DF2819752014669E001EE5FA /* RefreshAccountJob.swift in Sources */ = {isa = PBXBuildFile; fileRef = DF2819742014669E001EE5FA /* RefreshAccountJob.swift */; }; @@ -843,7 +845,6 @@ DFB19002233219650021CAF3 /* LogViewController.swift in Sources */ = {isa = PBXBuildFile; fileRef = DFB19001233219650021CAF3 /* LogViewController.swift */; }; DFB19006233220290021CAF3 /* PINLogCell.swift in Sources */ = {isa = PBXBuildFile; fileRef = DFB19005233220290021CAF3 /* PINLogCell.swift */; }; DFB2062821ABC088006E4341 /* RestoreViewController.swift in Sources */ = {isa = PBXBuildFile; fileRef = DFB2062721ABC088006E4341 /* RestoreViewController.swift */; }; - DFB2062A21AC1771006E4341 /* RestoreJob.swift in Sources */ = {isa = PBXBuildFile; fileRef = DFB2062921AC1771006E4341 /* RestoreJob.swift */; }; DFB6CE1E23C4805B00FB6615 /* KeychainExtension.swift in Sources */ = {isa = PBXBuildFile; fileRef = DFB6CE1D23C4805B00FB6615 /* KeychainExtension.swift */; }; DFB6CE2123C485AB00FB6615 /* SendMessageService+Sending.swift in Sources */ = {isa = PBXBuildFile; fileRef = DFB6CE2023C485AB00FB6615 /* SendMessageService+Sending.swift */; }; DFB6CE2323C485D400FB6615 /* SystemConversationAction+Description.swift in Sources */ = {isa = PBXBuildFile; fileRef = DFB6CE2223C485D400FB6615 /* SystemConversationAction+Description.swift */; }; @@ -1530,6 +1531,7 @@ 7C07ED372880F31E00685322 /* ja */ = {isa = PBXFileReference; lastKnownFileType = text.plist.strings; name = ja; path = ja.lproj/InfoPlist.strings; sourceTree = ""; }; 7C07ED382880F31E00685322 /* ja */ = {isa = PBXFileReference; lastKnownFileType = text.plist.strings; name = ja; path = ja.lproj/Localizable.strings; sourceTree = ""; }; 7C07ED392880F31E00685322 /* ja */ = {isa = PBXFileReference; lastKnownFileType = text.plist.strings; name = ja; path = ja.lproj/Localizable.strings; sourceTree = ""; }; + 7C07ED35287D549400685322 /* BackupJob.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = BackupJob.swift; sourceTree = ""; }; 7C0D997E26CA62CD00356655 /* StaticAudioMessagePlayingManager.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = StaticAudioMessagePlayingManager.swift; sourceTree = ""; }; 7C0E15DE27005376002FC718 /* UnknownURLWindow.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = UnknownURLWindow.swift; sourceTree = ""; }; 7C0E15E0270053AC002FC718 /* UnknownURLWindow.xib */ = {isa = PBXFileReference; lastKnownFileType = file.xib; path = UnknownURLWindow.xib; sourceTree = ""; }; @@ -1595,6 +1597,7 @@ 7C8FA78E2768822800855AFD /* DeleteAccountTableHeaderView.xib */ = {isa = PBXFileReference; lastKnownFileType = file.xib; path = DeleteAccountTableHeaderView.xib; sourceTree = ""; }; 7C8FA8F32768909300855AFD /* AccountSettingViewController.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = AccountSettingViewController.swift; sourceTree = ""; }; 7C8FA8F52768926D00855AFD /* SecuritySettingViewController.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = SecuritySettingViewController.swift; sourceTree = ""; }; + 7C9279AA28854F5C00321DFF /* RestoreJob.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = RestoreJob.swift; sourceTree = ""; }; 7C952CFD27E035080083F92B /* ExpiredMessageViewController.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = ExpiredMessageViewController.swift; sourceTree = ""; }; 7C952CFF27E036240083F92B /* ExpiredMessageTableHeaderView.xib */ = {isa = PBXFileReference; lastKnownFileType = file.xib; path = ExpiredMessageTableHeaderView.xib; sourceTree = ""; }; 7C9A733F27392FAF00E0127A /* PinSettingTableHeaderView.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = PinSettingTableHeaderView.swift; sourceTree = ""; }; @@ -1624,6 +1627,7 @@ 7CFD7471288FC88900D7A0EE /* zh-Hant */ = {isa = PBXFileReference; lastKnownFileType = text.plist.strings; name = "zh-Hant"; path = "zh-Hant.lproj/InfoPlist.strings"; sourceTree = ""; }; 7CFD7472288FC88A00D7A0EE /* zh-Hant */ = {isa = PBXFileReference; lastKnownFileType = text.plist.strings; name = "zh-Hant"; path = "zh-Hant.lproj/Localizable.strings"; sourceTree = ""; }; 7CFD7473288FC88A00D7A0EE /* zh-Hant */ = {isa = PBXFileReference; lastKnownFileType = text.plist.strings; name = "zh-Hant"; path = "zh-Hant.lproj/Localizable.strings"; sourceTree = ""; }; + 7CFD746D2889228100D7A0EE /* CloudJob.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = CloudJob.swift; sourceTree = ""; }; 842347ED2695BA6400009A39 /* InitializeBotJob.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = InitializeBotJob.swift; sourceTree = ""; }; 8C43D9D96FCB101481DFD90F /* Pods-Mixin.release.xcconfig */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = text.xcconfig; name = "Pods-Mixin.release.xcconfig"; path = "Pods/Target Support Files/Pods-Mixin/Pods-Mixin.release.xcconfig"; sourceTree = ""; }; 94046B90272DC265007C1D4A /* GroupCallMembersDataSource.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = GroupCallMembersDataSource.swift; sourceTree = ""; }; @@ -1762,7 +1766,6 @@ DF1ED8DA20BBED24003E10E8 /* PickerViewController.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = PickerViewController.swift; sourceTree = ""; }; DF1ED8E120BC0794003E10E8 /* Photo.storyboard */ = {isa = PBXFileReference; lastKnownFileType = file.storyboard; path = Photo.storyboard; sourceTree = ""; }; DF1F277B21A53585009A74C6 /* BackupViewController.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = BackupViewController.swift; sourceTree = ""; }; - DF1F278321A5637B009A74C6 /* BackupJob.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = BackupJob.swift; sourceTree = ""; }; DF1F278521A59A60009A74C6 /* BackupJobQueue.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = BackupJobQueue.swift; sourceTree = ""; }; DF2081B12005FF3500B87DB0 /* Camera.storyboard */ = {isa = PBXFileReference; lastKnownFileType = file.storyboard; path = Camera.storyboard; sourceTree = ""; }; DF2819742014669E001EE5FA /* RefreshAccountJob.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = RefreshAccountJob.swift; sourceTree = ""; }; @@ -1847,7 +1850,6 @@ DFB19001233219650021CAF3 /* LogViewController.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = LogViewController.swift; sourceTree = ""; }; DFB19005233220290021CAF3 /* PINLogCell.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = PINLogCell.swift; sourceTree = ""; }; DFB2062721ABC088006E4341 /* RestoreViewController.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = RestoreViewController.swift; sourceTree = ""; }; - DFB2062921AC1771006E4341 /* RestoreJob.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = RestoreJob.swift; sourceTree = ""; }; DFB6CE1D23C4805B00FB6615 /* KeychainExtension.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = KeychainExtension.swift; sourceTree = ""; }; DFB6CE2023C485AB00FB6615 /* SendMessageService+Sending.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = "SendMessageService+Sending.swift"; sourceTree = ""; }; DFB6CE2223C485D400FB6615 /* SystemConversationAction+Description.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "SystemConversationAction+Description.swift"; sourceTree = ""; }; @@ -3600,8 +3602,9 @@ isa = PBXGroup; children = ( DF1F278521A59A60009A74C6 /* BackupJobQueue.swift */, - DF1F278321A5637B009A74C6 /* BackupJob.swift */, - DFB2062921AC1771006E4341 /* RestoreJob.swift */, + 7CFD746D2889228100D7A0EE /* CloudJob.swift */, + 7C07ED35287D549400685322 /* BackupJob.swift */, + 7C9279AA28854F5C00321DFF /* RestoreJob.swift */, DF03C88C1FF4D9A100C1ED6B /* RefreshGroupIconJob.swift */, DFD089E11FE4041400A7D815 /* AttachmentUploadJob.swift */, 7BF3F67B22AE64FA007B6C77 /* ImageUploadJob.swift */, @@ -4519,6 +4522,7 @@ 7B54F95422B23A5600908A9D /* EmergencyContactSelectorViewController.swift in Sources */, DF3FF0552011E9B8000A0C0A /* FileUploadJob.swift in Sources */, 7BBCEC382523A2B400F270DF /* MinimizedClipSwitcherViewController.swift in Sources */, + 7C9279AB28854F5C00321DFF /* RestoreJob.swift in Sources */, E0C7674D23CC9411003F9215 /* BackgroundedTrailingInfoViewModel.swift in Sources */, 7CA5EE58280EA06B00BF3CD0 /* ScreenLockTimeFormatter.swift in Sources */, 7B6A4046228400AF0037C7E5 /* MessageReceiverViewController.swift in Sources */, @@ -4595,6 +4599,7 @@ 7B1D7BA51FBE944E00FDA52C /* ConversationViewController.swift in Sources */, 7BFE47E722845DE100FC4379 /* MessageReceiver.swift in Sources */, E01BCE2823ACFA66005D3FF3 /* LoginManager+Provision.swift in Sources */, + 7CFD746E2889228100D7A0EE /* CloudJob.swift in Sources */, 7C4C039F28530C9E003DE0C0 /* ContactViewController.swift in Sources */, 7B63C49121A4334C0044C4BF /* DepositFieldView.swift in Sources */, 7BB6FD252011AD3400E84C5C /* CardMessageViewModel.swift in Sources */, @@ -4918,6 +4923,7 @@ 7BAD2E55207DE046006D7887 /* UnreadHintMessageCell.swift in Sources */, 7B86B1EF23F6A70300C80AD9 /* AudioMessageActionView.swift in Sources */, 7C4E2B0626A9BE50008190F5 /* StickersEditingCell.swift in Sources */, + 7C07ED36287D549400685322 /* BackupJob.swift in Sources */, 947B36A925DEB72A00146111 /* PlaylistManager.swift in Sources */, 7BA9D9C4226DCFFC00255943 /* SearchConversationViewController.swift in Sources */, DFB2062821ABC088006E4341 /* RestoreViewController.swift in Sources */, @@ -4950,7 +4956,6 @@ 7BD344072334CD5F005C26E3 /* UserHandleViewController.swift in Sources */, 7B28FA18201196F80023B28D /* DataMessageCell.swift in Sources */, DF121F311FA1C767000F701D /* ConversationCell.swift in Sources */, - DFB2062A21AC1771006E4341 /* RestoreJob.swift in Sources */, 7B9EDF101FB588D700D26989 /* UIImageExtension.swift in Sources */, 7B2636D5224E2A7E0057116D /* InfiniteTopView.swift in Sources */, 7B66AC072428D72000869DBD /* PostWebViewController.swift in Sources */, @@ -5045,7 +5050,6 @@ 94D9DF6125F89D6E00FC2F28 /* BulletinContent.swift in Sources */, 7B3CDA6824FFF2D8003A3E80 /* AnimatedStickerView.swift in Sources */, 7C8FA8F62768926D00855AFD /* SecuritySettingViewController.swift in Sources */, - DF1F278421A5637B009A74C6 /* BackupJob.swift in Sources */, 7B36919F233A1962007321A7 /* LocationPickerViewController.swift in Sources */, 7B7E7146217D89790052C7DD /* CallViewController.swift in Sources */, 7C2ACDAE27D73F7C00E9DDB3 /* LeftAlignedCollectionViewFlowLayout.swift in Sources */, diff --git a/Mixin/Service/Job/BackupJob.swift b/Mixin/Service/Job/BackupJob.swift index 3572b792c0..f7ac6f5aa7 100644 --- a/Mixin/Service/Job/BackupJob.swift +++ b/Mixin/Service/Job/BackupJob.swift @@ -1,291 +1,265 @@ import Foundation -import GRDB import MixinServices -class BackupJob: BaseJob { +class BackupJob: CloudJob { - static let sharedId = "backup" + var preparedProgress: Float64 { + totalFileCount == 0 ? 0 : Float64(preparedFileCount) / Float64(totalFileCount) + } + + private(set) var isPreparing = true - static let backupDidChangeNotification = Notification.Name("one.mixin.messenger.Application.backupDidChange") - - private let monitorQueue = DispatchQueue(label: "one.mixin.messenger.queue.backup") private let immediatelyBackup: Bool - private var monitors = SafeDictionary() - private var withoutUploadSize: Int64 = 0 - private var realUploadedSize: Int64 = 0 - private var isStoppedQuery = false - private var isContinueBackup: Bool { - return !isCancelled && ReachabilityManger.shared.isReachableOnEthernetOrWiFi - } - - private(set) var isBackingUp = true - private(set) var totalFileSize: Int64 = 0 - private(set) var backupTotalSize: Int64 = 0 - private(set) var backupSize: Int64 = 0 - - var uploadedSize: Int64 { - return realUploadedSize + withoutUploadSize - } - + private let maxConcurrentUploadCount = 10 + private let queue = DispatchQueue(label: "one.mixin.messenger.backup") + + private var totalFileCount = 0 + private var preparedFileCount = 0 + init(immediatelyBackup: Bool = false) { self.immediatelyBackup = immediatelyBackup super.init() + NotificationCenter.default.addObserver(self, selector: #selector(networkChanged), name: ReachabilityManger.reachabilityDidChangeNotification, object: nil) } - - override func getJobId() -> String { - return BackupJob.sharedId + + override class var jobId: String { + "backup" } - - override func run() throws { + + override func execute() -> Bool { guard FileManager.default.ubiquityIdentityToken != nil else { - return + return false } guard let backupUrl = backupUrl else { + return false + } + guard isBackupNow() else { + return false + } + AppGroupUserDefaults.Account.hasUnfinishedBackup = true + guard prepare(backupUrl: backupUrl) else { + return false + } + guard pendingFiles.count > 0 else { + backupFinished() + return true + } + guard isContinueProcessing else { + return false + } + setupQuery(backupUrl: backupUrl) + startQuery() + queue.async(execute: backupNextFile) + return true + } + + override func setupQuery(backupUrl: URL) { + super.setupQuery(backupUrl: backupUrl) + query.valueListAttributes = [NSMetadataUbiquitousItemPercentUploadedKey, + NSMetadataUbiquitousItemIsUploadedKey, + NSMetadataUbiquitousItemUploadingErrorKey] + } + + override func queryDidUpdate(notification: Notification) { + guard let metadataItems = notification.userInfo?[NSMetadataQueryUpdateChangedItemsKey] as? [NSMetadataItem] else { return } + queue.async { [weak self] in + guard let self = self else { + return + } + for item in metadataItems { + guard + let filename = item.value(forAttribute: NSMetadataItemFSNameKey) as? String, + let file = self.processingFiles[filename] + else { + continue + } + let isUploaded = item.value(forAttribute: NSMetadataUbiquitousItemIsUploadedKey) as? Bool ?? false + if isUploaded { + self.processingFiles.removeValue(forKey: filename) + self.processedFileSize += file.size + if ReachabilityManger.shared.isReachableOnEthernetOrWiFi { + self.backupNextFile() + } + } else { + let percent = item.value(forAttribute: NSMetadataUbiquitousItemPercentUploadedKey) as? Double ?? 0 + self.processingFiles[filename]?.processedSize = Int64(Double(file.size) * percent / 100) + } + if let error = item.value(forAttribute: NSMetadataUbiquitousItemUploadingErrorKey) as? NSError { + Logger.general.error(category: "Backup", message: "Upload item at \(file.srcURL) failed, error: \(error)") + } + } + if self.totalProcessedSize >= self.totalFileSize { + self.backupFinished() + } + } + } + +} +extension BackupJob { + + private func isBackupNow() -> Bool { if !immediatelyBackup && !AppGroupUserDefaults.Account.hasUnfinishedBackup, let lastBackupDate = AppGroupUserDefaults.User.lastBackupDate { switch AppGroupUserDefaults.User.autoBackup { case .off: - return + return false case .daily: if -lastBackupDate.timeIntervalSinceNow < 86400 { - return + return false } case .weekly: if -lastBackupDate.timeIntervalSinceNow < 86400 * 7 { - return + return false } case .monthly: if -lastBackupDate.timeIntervalSinceNow < 86400 * 30 { - return + return false } } } - - AppGroupUserDefaults.Account.hasUnfinishedBackup = true - - try FileManager.default.createDirectory(at: backupUrl, withIntermediateDirectories: true, attributes: nil) - - var categories: [AttachmentContainer.Category] = [.photos, .audios] - if AppGroupUserDefaults.User.backupFiles { - categories.append(.files) - } else { - let url = backupUrl.appendingPathComponent(AttachmentContainer.Category.files.pathComponent, isDirectory: true) - try? FileManager.default.removeItem(at: url) - } - if AppGroupUserDefaults.User.backupVideos { - categories.append(.videos) - } else { - let url = backupUrl.appendingPathComponent(AttachmentContainer.Category.videos.pathComponent, isDirectory: true) - try? FileManager.default.removeItem(at: url) + return true + } + + private func prepare(backupUrl: URL) -> Bool { + isPreparing = true + defer { + isPreparing = false } - - var localPaths = Set() - var cloudPaths = Set() - - for category in categories { - let localUrl = AttachmentContainer.url(for: category, filename: nil) - let cloudUrl = backupUrl.appendingPathComponent(category.pathComponent) - - if localUrl.fileExists { - localPaths.formUnion(try FileManager.default.contentsOfDirectory(atPath: localUrl.path).map { "\(category.pathComponent)/\($0)" }) - } - if cloudUrl.fileExists { - cloudPaths.formUnion(try FileManager.default.contentsOfDirectory(atPath: cloudUrl.path).map { "\(category.pathComponent)/\($0)" }) + do { + try FileManager.default.createDirectory(at: backupUrl, withIntermediateDirectories: true, attributes: nil) + + var categories: [AttachmentContainer.Category] = [.photos, .audios] + if AppGroupUserDefaults.User.backupFiles { + categories.append(.files) } else { - try FileManager.default.createDirectory(at: cloudUrl, withIntermediateDirectories: true, attributes: nil) + let url = backupUrl.appendingPathComponent(AttachmentContainer.Category.files.pathComponent, isDirectory: true) + try? FileManager.default.removeItem(at: url) } - } - - for path in cloudPaths { - if !localPaths.contains(path) { - try? FileManager.default.removeItem(at: backupUrl.appendingPathComponent(path)) + if AppGroupUserDefaults.User.backupVideos { + categories.append(.videos) + } else { + let url = backupUrl.appendingPathComponent(AttachmentContainer.Category.videos.pathComponent, isDirectory: true) + try? FileManager.default.removeItem(at: url) } - } - - guard isContinueBackup else { - return - } - - var uploadPaths: [String] = [] - var backupPaths: [String] = [] - monitors = SafeDictionary() - totalFileSize = 0 - withoutUploadSize = 0 - realUploadedSize = 0 - backupSize = 0 - backupTotalSize = 0 - isStoppedQuery = false - isBackingUp = true - - for filename in localPaths { - let localURL = AttachmentContainer.url.appendingPathComponent(filename) - let cloudURL = backupUrl.appendingPathComponent(filename) - let localFileSize = FileManager.default.fileSize(localURL.path) - let cloudExists = FileManager.default.fileExists(atPath: cloudURL.path) - - if !cloudExists || FileManager.default.fileSize(cloudURL.path) != localFileSize { - backupPaths.append(filename) - backupTotalSize += localFileSize - - uploadPaths.append(filename) - } else if cloudExists { - if cloudURL.isUploaded { - withoutUploadSize += localFileSize + + var localPaths = Set() + var cloudPaths = Set() + for category in categories { + let localURL = AttachmentContainer.url(for: category, filename: nil) + if localURL.fileExists { + localPaths.formUnion(try FileManager.default.contentsOfDirectory(atPath: localURL.path).map { "\(category.pathComponent)/\($0)" }) + } + let cloudURL = backupUrl.appendingPathComponent(category.pathComponent) + if cloudURL.fileExists { + cloudPaths.formUnion(try FileManager.default.contentsOfDirectory(atPath: cloudURL.path).map { "\(category.pathComponent)/\($0)" }) } else { - uploadPaths.append(filename) + try FileManager.default.createDirectory(at: cloudURL, withIntermediateDirectories: true, attributes: nil) } } - totalFileSize += localFileSize - } - - let databaseFileSize = getDatabaseFileSize() - let databaseCloudURL = backupUrl.appendingPathComponent(backupDatabaseName) - let isBackupDatabase = !FileManager.default.fileExists(atPath: databaseCloudURL.path) || FileManager.default.fileSize(databaseCloudURL.path) != databaseFileSize - - if !isBackupDatabase { - withoutUploadSize += databaseFileSize - totalFileSize += databaseFileSize - } - - guard isContinueBackup else { - return - } - - let semaphore = DispatchSemaphore(value: 0) - let query = NSMetadataQuery() - - let observer = NotificationCenter.default.addObserver(forName: .NSMetadataQueryDidUpdate, object: nil, queue: nil) { [weak self](notification) in - guard let metadataItems = (notification.userInfo?[NSMetadataQueryUpdateChangedItemsKey] as? [NSMetadataItem]) else { - return + for path in cloudPaths where !localPaths.contains(path) { + try? FileManager.default.removeItem(at: backupUrl.appendingPathComponent(path)) } - self?.monitorQueue.async { - guard let weakSelf = self else { - return - } - guard weakSelf.isContinueBackup else { - weakSelf.stopQuery(query: query, semaphore: semaphore) - return - } - - for metadataItem in metadataItems { - let name = metadataItem.value(forAttribute: NSMetadataItemFSNameKey) as? String - let fileSize = (metadataItem.value(forAttribute: NSMetadataItemFSSizeKey) as? NSNumber)?.int64Value ?? 0 - let percent = (metadataItem.value(forAttribute: NSMetadataUbiquitousItemPercentUploadedKey) as? NSNumber)?.floatValue ?? 0 - let isUploaded = (metadataItem.value(forAttribute: NSMetadataUbiquitousItemIsUploadedKey) as? NSNumber)?.boolValue ?? false - - if let fileName = name, fileSize > 0, percent > 0, weakSelf.monitors[fileName] != nil { - weakSelf.monitors[fileName] = isUploaded ? fileSize : Int64(Float(fileSize) * percent / 100) - } - } - weakSelf.realUploadedSize = weakSelf.monitors.values.map { $0 }.reduce(0, +) - if weakSelf.uploadedSize >= weakSelf.totalFileSize { - weakSelf.stopQuery(query: query, semaphore: semaphore) + + totalFileCount = localPaths.count + 1 + + func process(localURL: URL, cloudURL: URL, fileSize: Int64) { + let isUploaded = FileManager.default.fileExists(atPath: cloudURL.path) && FileManager.default.fileSize(cloudURL.path) == fileSize + if isUploaded { + processedFileSize += fileSize + } else { + let file = File(srcURL: localURL, dstURL: cloudURL, size: fileSize) + pendingFiles[file.name] = file } + totalFileSize += fileSize + preparedFileCount += 1 } - } - - query.searchScopes = [NSMetadataQueryUbiquitousDataScope] - query.valueListAttributes = [ NSMetadataUbiquitousItemPercentUploadedKey, - NSMetadataUbiquitousItemIsUploadingKey, - NSMetadataUbiquitousItemUploadingErrorKey, - NSMetadataUbiquitousItemIsUploadedKey] - query.predicate = NSPredicate(format: "%K BEGINSWITH[c] %@ && kMDItemContentType != 'public.folder'", NSMetadataItemPathKey, backupUrl.path) - DispatchQueue.main.async { - query.start() - } - - if isBackupDatabase { - copyToCloud(from: AppGroupContainer.userDatabaseUrl, destination: databaseCloudURL, isDatabase: true) - } - for path in backupPaths { - guard isContinueBackup else { - return - } - copyToCloud(from: AttachmentContainer.url.appendingPathComponent(path), destination: backupUrl.appendingPathComponent(path)) - } - - isBackingUp = false - - if uploadedSize >= totalFileSize || !isContinueBackup { - DispatchQueue.main.async { - query.stop() + + let fileSize = databaseSizeAfterCompression() + let localURL = AppGroupContainer.userDatabaseUrl + let cloudURL = backupUrl.appendingPathComponent(backupDatabaseName) + process(localURL: localURL, cloudURL: cloudURL, fileSize: fileSize) + + for path in localPaths { + let localURL = AttachmentContainer.url.appendingPathComponent(path) + let cloudURL = backupUrl.appendingPathComponent(path) + let fileSize = FileManager.default.fileSize(localURL.path) + process(localURL: localURL, cloudURL: cloudURL, fileSize: fileSize) } - } else { - semaphore.wait() - } - NotificationCenter.default.removeObserver(observer) - - if uploadedSize >= totalFileSize { - removeOldFiles(backupDir: backupUrl) - AppGroupUserDefaults.User.lastBackupDate = Date() - AppGroupUserDefaults.User.lastBackupSize = totalFileSize - AppGroupUserDefaults.Account.hasUnfinishedBackup = false + return true + } catch { + Logger.general.error(category: "BackupJob", message: "Prepare failed: \(error)") + return false } - - NotificationCenter.default.post(onMainThread: BackupJob.backupDidChangeNotification, object: self) } - - private func getDatabaseFileSize() -> Int64 { + + private func databaseSizeAfterCompression() -> Int64 { try? UserDatabase.current.writeAndReturnError { (db) -> Void in try db.checkpoint(.full, on: nil) } - if AppGroupUserDefaults.Database.isFTSInitialized && -AppGroupUserDefaults.Database.vacuumDate.timeIntervalSinceNow >= 86400 * 14 { AppGroupUserDefaults.Database.vacuumDate = Date() try? UserDatabase.current.vacuum() } return FileManager.default.fileSize(AppGroupContainer.userDatabaseUrl.path) } - - private func stopQuery(query: NSMetadataQuery, semaphore: DispatchSemaphore) { - guard !isStoppedQuery else { - return - } - isStoppedQuery = true - query.stop() - semaphore.signal() - } - - private func copyToCloud(from: URL, destination: URL, isDatabase: Bool = false) { + + private func copyFileToiCloud(_ file: File) { let tmpFile = FileManager.default.temporaryDirectory.appendingPathComponent(UUID().uuidString) - monitors[destination.lastPathComponent] = 0 do { - try FileManager.default.copyItem(at: from, to: tmpFile) - if FileManager.default.fileExists(atPath: destination.path) { - try FileManager.default.removeItem(at: destination) - } - try FileManager.default.setUbiquitous(true, itemAt: tmpFile, destinationURL: destination) - backupSize += from.fileSize - if isDatabase { - backupTotalSize += destination.fileSize - totalFileSize += destination.fileSize + try FileManager.default.copyItem(at: file.srcURL, to: tmpFile) + if FileManager.default.fileExists(atPath: file.dstURL.path) { + try FileManager.default.removeItem(at: file.dstURL) } + try FileManager.default.setUbiquitous(true, itemAt: tmpFile, destinationURL: file.dstURL) } catch { - if !isDatabase { - backupTotalSize -= from.fileSize - } - monitors.removeValue(forKey: destination.lastPathComponent) + processingFiles.removeValue(forKey: file.name) + totalFileSize -= file.size if tmpFile.fileExists { try? FileManager.default.removeItem(at: tmpFile) } + Logger.general.error(category: "BackupJob", message: "Failed to copy \(file.name) to iCloud, error: \(error)") reporter.report(error: error) } } - - private func removeOldFiles(backupDir: URL) { - let files = ["mixin.backup.db", - "mixin.photos.zip", - "mixin.audios.zip"] - + + private func backupNextFile() { + guard let files = pendingFiles.values as? [File] else { + return + } + for file in files { + guard processingFiles.count < maxConcurrentUploadCount else { + return + } + let name = file.name + pendingFiles.removeValue(forKey: name) + processingFiles[name] = file + copyFileToiCloud(file) + } + } + + private func backupFinished() { + stopQuery() + deleteLegacyBackup() + AppGroupUserDefaults.User.lastBackupDate = Date() + AppGroupUserDefaults.User.lastBackupSize = totalFileSize + AppGroupUserDefaults.Account.hasUnfinishedBackup = false + NotificationCenter.default.post(onMainThread: BackupJob.backupDidChangeNotification, object: self) + finishJob() + } + + private func deleteLegacyBackup() { + guard let backupUrl = backupUrl else { + return + } let baseDir = try! FileManager.default.url(for: .documentDirectory, in: .userDomainMask, appropriateFor: nil, create: false) + let files = ["mixin.backup.db", "mixin.photos.zip", "mixin.audios.zip"] for file in files { - let cloudURL = backupDir.appendingPathComponent(file) + let cloudURL = backupUrl.appendingPathComponent(file) if cloudURL.isStoredCloud { try? FileManager.default.removeItem(at: cloudURL) } - if file.hasSuffix(".zip") { let localURL = baseDir.appendingPathComponent(file) if localURL.fileExists { @@ -294,4 +268,23 @@ class BackupJob: BaseJob { } } } + + @objc private func networkChanged() { + guard ReachabilityManger.shared.isReachableOnEthernetOrWiFi else { + return + } + queue.async(execute: backupNextFile) + } + +} + +extension BackupJob { + + struct File: CloudJobFile { + var srcURL: URL + var dstURL: URL + var size: Int64 + var processedSize: Int64 = 0 + } + } diff --git a/Mixin/Service/Job/BackupJobQueue.swift b/Mixin/Service/Job/BackupJobQueue.swift index 362a191f4c..f8cb3d74aa 100644 --- a/Mixin/Service/Job/BackupJobQueue.swift +++ b/Mixin/Service/Job/BackupJobQueue.swift @@ -14,11 +14,11 @@ class BackupJobQueue: JobQueue { } var backupJob: BackupJob? { - return findJobById(jodId: BackupJob.sharedId) as? BackupJob + return findJobById(jodId: BackupJob.jobId) as? BackupJob } var restoreJob: RestoreJob? { - return findJobById(jodId: RestoreJob.sharedId) as? RestoreJob + return findJobById(jodId: RestoreJob.jobId) as? RestoreJob } init() { diff --git a/Mixin/Service/Job/CloudJob.swift b/Mixin/Service/Job/CloudJob.swift new file mode 100644 index 0000000000..603c2327a7 --- /dev/null +++ b/Mixin/Service/Job/CloudJob.swift @@ -0,0 +1,73 @@ +import Foundation +import MixinServices + +protocol CloudJobFile { + var srcURL: URL { get set } + var dstURL: URL { get set } + var size: Int64 { get set } + var processedSize: Int64 { get set } +} + +extension CloudJobFile { + var name: String { srcURL.lastPathComponent } + var isProcessingCompleted: Bool { processedSize >= size } +} + +class CloudJob: AsynchronousJob { + + static let backupDidChangeNotification = Notification.Name("one.mixin.messenger.CloudJob.backupDidChange") + + var progress: Float { + Float(Float64(totalProcessedSize) / Float64(totalFileSize)) + } + + var totalProcessedSize: Int64 { + processedFileSize + processingFiles.values.map { $0.processedSize }.reduce(0, +) + } + + var isContinueProcessing: Bool { + !isCancelled && ReachabilityManger.shared.isReachableOnEthernetOrWiFi + } + + var pendingFiles = SafeDictionary() + var processingFiles = SafeDictionary() + var totalFileSize: Int64 = 0 + var processedFileSize: Int64 = 0 + + let query = NSMetadataQuery() + + class var jobId: String { + "" + } + + override func getJobId() -> String { + Self.jobId + } + + override init() { + super.init() + NotificationCenter.default.addObserver(self, selector: #selector(queryDidUpdate(notification:)), name: .NSMetadataQueryDidUpdate, object: nil) + } + + func setupQuery(backupUrl: URL) { + query.searchScopes = [NSMetadataQueryUbiquitousDataScope] + query.predicate = NSPredicate(format: "%K BEGINSWITH[c] %@ && kMDItemContentType != 'public.folder'", NSMetadataItemPathKey, backupUrl.path) + } + + func startQuery() { + DispatchQueue.main.async { [weak self] in + _ = self?.query.start() + } + } + + func stopQuery() { + DispatchQueue.main.async { [weak self] in + self?.query.stop() + } + } + + @objc func queryDidUpdate(notification: Notification) { + + } + +} diff --git a/Mixin/Service/Job/RestoreJob.swift b/Mixin/Service/Job/RestoreJob.swift index 9c28ca3d38..038b708743 100644 --- a/Mixin/Service/Job/RestoreJob.swift +++ b/Mixin/Service/Job/RestoreJob.swift @@ -2,277 +2,195 @@ import Foundation import Zip import MixinServices -class RestoreJob: BaseJob { - - private let monitorQueue = DispatchQueue(label: "one.mixin.messenger.queue.restore.download") - private let restoreQueue = DispatchQueue(label: "one.mixin.messenger.queue.restore") - private var monitors = SafeDictionary() - private var totalFileSize: Int64 = 0 - private var downloadedSize: Int64 = 0 - private var isStoppedQuery = false - private var isContinueRestore: Bool { - return !isCancelled && ReachabilityManger.shared.isReachableOnEthernetOrWiFi - } - private var isRestoredAllFiles: Bool { - return monitors.values.first(where: { !$0.isRestored }) == nil - } - - var progress: Float { - return Float(Float64(downloadedSize) / Float64(totalFileSize)) +class RestoreJob: CloudJob { + + private let queue = DispatchQueue(label: "one.mixin.messenger.restore") + + override class var jobId: String { + "restore" } - - static let sharedId = "restore" - - override func getJobId() -> String { - return RestoreJob.sharedId - } - - override func run() throws { - guard AppGroupUserDefaults.Account.canRestoreMedia else { - return - } + + override func execute() -> Bool { guard FileManager.default.ubiquityIdentityToken != nil else { - return + return false } guard let backupUrl = backupUrl else { - return + return false } - - let categories: [AttachmentContainer.Category] = [.photos, .audios, .files, .videos] - - monitors = SafeDictionary() - totalFileSize = 0 - downloadedSize = 0 - isStoppedQuery = false - - for category in categories { - try FileManager.default.createDirectory(at: AttachmentContainer.url(for: category, filename: nil), withIntermediateDirectories: true, attributes: nil) - - if category == .photos, backupUrl.appendingPathComponent("mixin.photos.zip").isStoredCloud { - let filename = "mixin.photos.zip" - monitorURL(cloudURL: backupUrl.appendingPathComponent(filename), - localURL: AttachmentContainer.url.appendingPathComponent(filename), - category: category, - isZipFile: true) - } - if category == .audios, backupUrl.appendingPathComponent("mixin.audios.zip").isStoredCloud { - let filename = "mixin.audios.zip" - monitorURL(cloudURL: backupUrl.appendingPathComponent(filename), - localURL: AttachmentContainer.url.appendingPathComponent(filename), - category: category, - isZipFile: true) - } - - let cloudDir = backupUrl.appendingPathComponent(category.pathComponent) - guard FileManager.default.directoryExists(atPath: cloudDir.path) else { - continue - } - - let contents = try FileManager.default.contentsOfDirectory(atPath: cloudDir.path) - guard contents.count > 0 else { - continue - } - - let localDir = AttachmentContainer.url.appendingPathComponent(category.pathComponent) - for content in contents { - var filename = content - if filename.hasSuffix(".icloud") { - filename = String(filename[filename.index(filename.startIndex, offsetBy: 1).. 0 else { + restoreFinished() + return true + } + guard isContinueProcessing else { + return false + } + setupQuery(backupUrl: backupUrl) + startQuery() + queue.async(execute: downloadFiles) + return true + } + + override func setupQuery(backupUrl: URL) { + super.setupQuery(backupUrl: backupUrl) + query.valueListAttributes = [NSMetadataUbiquitousItemPercentDownloadedKey, + NSMetadataUbiquitousItemDownloadingErrorKey, + NSMetadataUbiquitousItemDownloadingStatusKey] + } + + override func queryDidUpdate(notification: Notification) { + guard let metadataItems = notification.userInfo?[NSMetadataQueryUpdateChangedItemsKey] as? [NSMetadataItem] else { return } - - let semaphore = DispatchSemaphore(value: 0) - let query = NSMetadataQuery() - - let observer = NotificationCenter.default.addObserver(forName: .NSMetadataQueryDidUpdate, object: nil, queue: nil) { [weak self](notification) in - guard let metadataItems = (notification.userInfo?[NSMetadataQueryUpdateChangedItemsKey] as? [NSMetadataItem]) else { + queue.async { [weak self] in + guard let self = self else { return } - self?.monitorQueue.async { - guard let weakSelf = self else { - return + for item in metadataItems { + guard + let fileName = item.value(forAttribute: NSMetadataItemFSNameKey) as? String, + let file = self.processingFiles[fileName] as? File + else { + continue } - guard weakSelf.isContinueRestore else { - weakSelf.stopQuery(query: query, semaphore: semaphore) - return + let status = item.value(forAttribute: NSMetadataUbiquitousItemDownloadingStatusKey) as? String + let isDownloaded = status == NSMetadataUbiquitousItemDownloadingStatusCurrent + if isDownloaded { + self.processingFiles.removeValue(forKey: fileName) + self.processedFileSize += file.size + self.restore(file) + } else { + let percent = (item.value(forAttribute: NSMetadataUbiquitousItemPercentDownloadedKey) as? NSNumber)?.floatValue ?? 0 + self.processingFiles[fileName]?.processedSize = Int64(Float(file.size) * percent / 100) } + if let error = item.value(forAttribute: NSMetadataUbiquitousItemDownloadingErrorKey) as? NSError { + Logger.general.error(category: "RestoreJob", message: "Download item at \(file.srcURL) failed, error: \(error)") + } + } + } + } + +} - for metadataItem in metadataItems { - let name = metadataItem.value(forAttribute: NSMetadataItemFSNameKey) as? String - let fileSize = (metadataItem.value(forAttribute: NSMetadataItemFSSizeKey) as? NSNumber)?.int64Value ?? 0 - let percent = (metadataItem.value(forAttribute: NSMetadataUbiquitousItemPercentDownloadedKey) as? NSNumber)?.floatValue ?? 0 - let status = (metadataItem.value(forAttribute: NSMetadataUbiquitousItemDownloadingStatusKey) as? String) - let isDownloaded = status == NSMetadataUbiquitousItemDownloadingStatusCurrent - - if let error = metadataItem.value(forAttribute: NSMetadataUbiquitousItemIsDownloadingKey) as? NSError { - reporter.report(error: error) +extension RestoreJob { + + private func prepare(backupUrl: URL) -> Bool { + do { + func process(cloudURL: URL, localURL: URL, category: AttachmentContainer.Category, isZipFile: Bool) { + let fileSize = cloudURL.fileSize + let downloadedSize: Int64 = cloudURL.isDownloaded ? fileSize : 0 + let file = File(srcURL: cloudURL, dstURL: localURL, size: fileSize, processedSize: downloadedSize, isZipFile: isZipFile, category: category) + pendingFiles[file.name] = file + totalFileSize += fileSize + } + let categories: [AttachmentContainer.Category] = [.photos, .audios, .files, .videos] + for category in categories { + try FileManager.default.createDirectory(at: AttachmentContainer.url(for: category, filename: nil), withIntermediateDirectories: true, attributes: nil) + + if category == .photos || category == .audios { + let name = category == .photos ? "mixin.photos.zip" : "mixin.audios.zip" + if backupUrl.appendingPathComponent(name).isStoredCloud { + let cloudURL = backupUrl.appendingPathComponent(name) + let localURL = AttachmentContainer.url.appendingPathComponent(name) + process(cloudURL: cloudURL, localURL: localURL, category: category, isZipFile: true) } - - if let fileName = name, fileSize > 0, percent > 0, var monitorFile = weakSelf.monitors[fileName] { - monitorFile.downloadedSize = isDownloaded ? fileSize : Int64(Float(fileSize) * percent / 100) - monitorFile.isDownloaded = isDownloaded - monitorFile.fileSize = fileSize - weakSelf.monitors[fileName] = monitorFile - if isDownloaded { - weakSelf.restoreFromCloud(fileName: fileName, chatDir: AttachmentContainer.url, semaphore: semaphore, query: query) - } + } + + let cloudDir = backupUrl.appendingPathComponent(category.pathComponent) + guard FileManager.default.directoryExists(atPath: cloudDir.path) else { + Logger.general.info(category: "RestoreJob", message: "Directory not exists at: \(cloudDir.path)") + continue + } + let contents = try FileManager.default.contentsOfDirectory(atPath: cloudDir.path) + guard contents.count > 0 else { + continue + } + let localDir = AttachmentContainer.url.appendingPathComponent(category.pathComponent) + for content in contents { + let name: String + if content.hasSuffix(".icloud") { + name = String(content[content.index(content.startIndex, offsetBy: 1)..