From 57205facbc1ee00b48286ed25362cc5a19fdb473 Mon Sep 17 00:00:00 2001 From: Matthew Chen Date: Fri, 30 Nov 2018 16:46:23 -0500 Subject: [PATCH] Batch backup exports. --- .../ViewControllers/DebugUI/DebugUIBackup.m | 8 +- Signal/src/util/Backup/OWSBackupAPI.swift | 235 +---------------- Signal/src/util/Backup/OWSBackupExportJob.m | 247 ++++++++++-------- 3 files changed, 144 insertions(+), 346 deletions(-) diff --git a/Signal/src/ViewControllers/DebugUI/DebugUIBackup.m b/Signal/src/ViewControllers/DebugUI/DebugUIBackup.m index 0f291c8fa..be3d5c5f6 100644 --- a/Signal/src/ViewControllers/DebugUI/DebugUIBackup.m +++ b/Signal/src/ViewControllers/DebugUI/DebugUIBackup.m @@ -96,9 +96,11 @@ NS_ASSUME_NONNULL_BEGIN OWSAssertDebug(success); NSString *recipientId = self.tsAccountManager.localNumber; - [[self.backup ensureCloudKitAccess].then(^{ - return - [OWSBackupAPI saveTestFileToCloudObjcWithRecipientId:recipientId fileUrl:[NSURL fileURLWithPath:filePath]]; + NSString *recordName = [OWSBackupAPI recordNameForTestFileWithRecipientId:recipientId]; + CKRecord *record = [OWSBackupAPI recordForFileUrl:[NSURL fileURLWithPath:filePath] recordName:recordName]; + + [[self.backup ensureCloudKitAccess].thenInBackground(^{ + return [OWSBackupAPI saveRecordsToCloudObjcWithRecords:@[ record ]]; }) retainUntilComplete]; } diff --git a/Signal/src/util/Backup/OWSBackupAPI.swift b/Signal/src/util/Backup/OWSBackupAPI.swift index a4a8489c4..f2acd8dbc 100644 --- a/Signal/src/util/Backup/OWSBackupAPI.swift +++ b/Signal/src/util/Backup/OWSBackupAPI.swift @@ -24,10 +24,6 @@ import PromiseKit static let payloadKey = "payload" static let maxRetries = 5 - private class func recordIdForTest() -> String { - return "test-\(NSUUID().uuidString)" - } - private class func database() -> CKDatabase { let myContainer = CKContainer.default() let privateDatabase = myContainer.privateCloudDatabase @@ -43,18 +39,8 @@ import PromiseKit // MARK: - Upload @objc - public class func saveTestFileToCloudObjc(recipientId: String, - fileUrl: URL) -> AnyPromise { - return AnyPromise(saveTestFileToCloud(recipientId: recipientId, - fileUrl: fileUrl)) - } - - public class func saveTestFileToCloud(recipientId: String, - fileUrl: URL) -> Promise { - let recordName = "\(recordNamePrefix(forRecipientId: recipientId))test-\(NSUUID().uuidString)" - return saveFileToCloud(fileUrl: fileUrl, - recordName: recordName, - recordType: signalBackupRecordType) + public class func recordNameForTestFile(recipientId: String) -> String { + return "\(recordNamePrefix(forRecipientId: recipientId))test-\(NSUUID().uuidString)" } // "Ephemeral" files are specific to this backup export and will always need to @@ -67,28 +53,6 @@ import PromiseKit return "\(recordNamePrefix(forRecipientId: recipientId))ephemeral-\(label)-\(NSUUID().uuidString)" } - // "Ephemeral" files are specific to this backup export and will always need to - // be saved. For example, a complete image of the database is exported each time. - // We wouldn't want to overwrite previous images until the entire backup export is - // complete. - @objc - public class func saveEphemeralFileToCloudObjc(recipientId: String, - label: String, - fileUrl: URL) -> AnyPromise { - return AnyPromise(saveEphemeralFileToCloud(recipientId: recipientId, - label: label, - fileUrl: fileUrl)) - } - - public class func saveEphemeralFileToCloud(recipientId: String, - label: String, - fileUrl: URL) -> Promise { - let recordName = recordNameForEphemeralFile(recipientId: recipientId, label: label) - return saveFileToCloud(fileUrl: fileUrl, - recordName: recordName, - recordType: signalBackupRecordType) - } - // "Persistent" files may be shared between backup export; they should only be saved // once. For example, attachment files should only be uploaded once. Subsequent // backups can reuse the same record. @@ -152,127 +116,6 @@ import PromiseKit return recipientIds } - // "Persistent" files may be shared between backup export; they should only be saved - // once. For example, attachment files should only be uploaded once. Subsequent - // backups can reuse the same record. - @objc - public class func savePersistentFileOnceToCloudObjc(recipientId: String, - fileId: String, - fileUrlBlock: @escaping () -> URL?) -> AnyPromise { - return AnyPromise(savePersistentFileOnceToCloud(recipientId: recipientId, - fileId: fileId, - fileUrlBlock: fileUrlBlock)) - } - - public class func savePersistentFileOnceToCloud(recipientId: String, - fileId: String, - fileUrlBlock: @escaping () -> URL?) -> Promise { - let recordName = recordNameForPersistentFile(recipientId: recipientId, fileId: fileId) - return saveFileOnceToCloud(recordName: recordName, - recordType: signalBackupRecordType, - fileUrlBlock: fileUrlBlock) - } - - @objc - public class func upsertManifestFileToCloudObjc(recipientId: String, - fileUrl: URL) -> AnyPromise { - return AnyPromise(upsertManifestFileToCloud(recipientId: recipientId, - fileUrl: fileUrl)) - } - - public class func upsertManifestFileToCloud(recipientId: String, - fileUrl: URL) -> Promise { - // We want to use a well-known record id and type for manifest files. - let recordName = recordNameForManifest(recipientId: recipientId) - return upsertFileToCloud(fileUrl: fileUrl, - recordName: recordName, - recordType: signalBackupRecordType) - } - - @objc - public class func saveFileToCloudObjc(fileUrl: URL, - recordName: String) -> AnyPromise { - return AnyPromise(saveFileToCloud(fileUrl: fileUrl, - recordName: recordName, - recordType: signalBackupRecordType)) - } - - public class func saveFileToCloud(fileUrl: URL, - recordName: String, - recordType: String) -> Promise { - let recordID = CKRecordID(recordName: recordName) - let record = CKRecord(recordType: recordType, recordID: recordID) - let asset = CKAsset(fileURL: fileUrl) - record[payloadKey] = asset - - return saveRecordToCloud(record: record) - } - - @objc - public class func saveRecordToCloudObjc(record: CKRecord) -> AnyPromise { - return AnyPromise(saveRecordToCloud(record: record)) - } - - public class func saveRecordToCloud(record: CKRecord) -> Promise { - return saveRecordToCloud(record: record, - remainingRetries: maxRetries) - } - - private class func saveRecordToCloud(record: CKRecord, - remainingRetries: Int) -> Promise { - - Logger.verbose("saveRecordToCloud \(record.recordID.recordName)") - - return Promise { resolver in - let saveOperation = CKModifyRecordsOperation(recordsToSave: [record ], recordIDsToDelete: nil) - saveOperation.modifyRecordsCompletionBlock = { (_, _, error) in - - let outcome = outcomeForCloudKitError(error: error, - remainingRetries: remainingRetries, - label: "Save Record") - switch outcome { - case .success: - let recordName = record.recordID.recordName - resolver.fulfill(recordName) - case .failureDoNotRetry(let outcomeError): - resolver.reject(outcomeError) - case .failureRetryAfterDelay(let retryDelay): - DispatchQueue.global().asyncAfter(deadline: DispatchTime.now() + retryDelay, execute: { - saveRecordToCloud(record: record, - remainingRetries: remainingRetries - 1) - .done { (recordName) in - resolver.fulfill(recordName) - }.catch { (error) in - resolver.reject(error) - }.retainUntilComplete() - }) - case .failureRetryWithoutDelay: - DispatchQueue.global().async { - saveRecordToCloud(record: record, - remainingRetries: remainingRetries - 1) - .done { (recordName) in - resolver.fulfill(recordName) - }.catch { (error) in - resolver.reject(error) - }.retainUntilComplete() - } - case .unknownItem: - owsFailDebug("unexpected CloudKit response.") - resolver.reject(invalidServiceResponseError()) - } - } - saveOperation.isAtomic = false - - // These APIs are only available in iOS 9.3 and later. - if #available(iOS 9.3, *) { - saveOperation.isLongLived = true - saveOperation.qualityOfService = .background - } - - database().add(saveOperation) - } - } - @objc public class func record(forFileUrl fileUrl: URL, recordName: String) -> CKRecord { @@ -366,80 +209,6 @@ import PromiseKit } } - // Compare: - // * An "upsert" creates a new record if none exists and - // or updates if there is an existing record. - // * A "save once" creates a new record if none exists and - // does nothing if there is an existing record. - @objc - public class func upsertFileToCloudObjc(fileUrl: URL, - recordName: String, - recordType: String) -> AnyPromise { - return AnyPromise(upsertFileToCloud(fileUrl: fileUrl, - recordName: recordName, - recordType: recordType)) - } - - public class func upsertFileToCloud(fileUrl: URL, - recordName: String, - recordType: String) -> Promise { - - return checkForFileInCloud(recordName: recordName, - remainingRetries: maxRetries) - .then { (record: CKRecord?) -> Promise in - if let record = record { - // Record found, updating existing record. - let asset = CKAsset(fileURL: fileUrl) - record[payloadKey] = asset - return saveRecordToCloud(record: record) - } - - // No record found, saving new record. - return saveFileToCloud(fileUrl: fileUrl, - recordName: recordName, - recordType: recordType) - } - } - - // Compare: - // * An "upsert" creates a new record if none exists and - // or updates if there is an existing record. - // * A "save once" creates a new record if none exists and - // does nothing if there is an existing record. - @objc - public class func saveFileOnceToCloudObjc(recordName: String, - recordType: String, - fileUrlBlock: @escaping () -> URL?) -> AnyPromise { - return AnyPromise(saveFileOnceToCloud(recordName: recordName, - recordType: recordType, - fileUrlBlock: fileUrlBlock)) - } - - public class func saveFileOnceToCloud(recordName: String, - recordType: String, - fileUrlBlock: @escaping () -> URL?) -> Promise { - - return checkForFileInCloud(recordName: recordName, - remainingRetries: maxRetries) - .then { (record: CKRecord?) -> Promise in - if record != nil { - // Record found, skipping save. - return Promise.value(recordName) - } - // No record found, saving new record. - guard let fileUrl = fileUrlBlock() else { - Logger.error("error preparing file for upload.") - return Promise(error: OWSErrorWithCodeDescription(.exportBackupError, - NSLocalizedString("BACKUP_EXPORT_ERROR_SAVE_FILE_TO_CLOUD_FAILED", - comment: "Error indicating the backup export failed to save a file to the cloud."))) - } - - return saveFileToCloud(fileUrl: fileUrl, - recordName: recordName, - recordType: recordType) - } - } - // MARK: - Delete @objc diff --git a/Signal/src/util/Backup/OWSBackupExportJob.m b/Signal/src/util/Backup/OWSBackupExportJob.m index 5a6dafcf1..e22c462e2 100644 --- a/Signal/src/util/Backup/OWSBackupExportJob.m +++ b/Signal/src/util/Backup/OWSBackupExportJob.m @@ -740,8 +740,6 @@ NS_ASSUME_NONNULL_BEGIN // This method returns YES IFF "work was done and there might be more work to do". - (AnyPromise *)saveDatabaseFilesToCloud { - // AnyPromise *promise = [AnyPromise promiseWithValue:@(1)]; - if (self.isComplete) { return [AnyPromise promiseWithValue:OWSBackupErrorWithDescription(@"Backup export no longer active.")]; } @@ -759,7 +757,7 @@ NS_ASSUME_NONNULL_BEGIN } // TODO: Expose progress. - return [OWSBackupAPI saveRecordsToCloudObjcWithRecords:records].thenInBackground(^(NSString *recordName) { + return [OWSBackupAPI saveRecordsToCloudObjcWithRecords:records].thenInBackground(^{ OWSAssertDebug(items.count == records.count); NSUInteger count = MIN(items.count, records.count); for (NSUInteger i = 0; i < count; i++) { @@ -778,121 +776,152 @@ NS_ASSUME_NONNULL_BEGIN // This method returns YES IFF "work was done and there might be more work to do". - (AnyPromise *)saveAttachmentFilesToCloud { + if (self.isComplete) { + return [AnyPromise promiseWithValue:OWSBackupErrorWithDescription(@"Backup export no longer active.")]; + } + AnyPromise *promise = [AnyPromise promiseWithValue:@(1)]; + NSMutableArray *items = [NSMutableArray new]; + NSMutableArray *records = [NSMutableArray new]; for (OWSAttachmentExport *attachmentExport in self.unsavedAttachmentExports) { + if ([self tryToSkipAttachmentUpload:attachmentExport]) { + continue; + } + promise = promise.thenInBackground(^{ - if (self.isComplete) { - return [AnyPromise promiseWithValue:OWSBackupErrorWithDescription(@"Backup export no longer active.")]; + @autoreleasepool { + // OWSAttachmentExport is used to lazily write an encrypted copy of the + // attachment to disk. + if (![attachmentExport prepareForUpload]) { + // Attachment files are non-critical so any error uploading them is recoverable. + return @(1); + } + OWSAssertDebug(attachmentExport.relativeFilePath.length > 0); + OWSAssertDebug(attachmentExport.encryptedItem); + } + + NSURL *_Nullable fileUrl = ^{ + if (attachmentExport.encryptedItem.filePath.length < 1) { + OWSLogError(@"attachment export missing temp file path"); + return (NSURL *)nil; + } + if (attachmentExport.relativeFilePath.length < 1) { + OWSLogError(@"attachment export missing relative file path"); + return (NSURL *)nil; + } + return [NSURL fileURLWithPath:attachmentExport.encryptedItem.filePath]; + }(); + + if (!fileUrl) { + // Attachment files are non-critical so any error uploading them is recoverable. + return @(1); } - return [self saveAttachmentFileToCloud:attachmentExport]; + + NSString *recordName = + [OWSBackupAPI recordNameForPersistentFileWithRecipientId:self.recipientId + fileId:attachmentExport.attachmentId]; + CKRecord *record = [OWSBackupAPI recordForFileUrl:fileUrl recordName:recordName]; + [records addObject:record]; + [items addObject:attachmentExport]; + return @(1); }); } - [self.unsavedAttachmentExports removeAllObjects]; - return promise; + + // TODO: Expose progress. + dispatch_queue_t backgroundQueue = dispatch_get_global_queue(0, 0); + return promise + .thenInBackground(^{ + return [OWSBackupAPI saveRecordsToCloudObjcWithRecords:records]; + }) + .ensureOn(backgroundQueue, + ^{ + for (OWSAttachmentExport *attachmentExport in items) { + if (![attachmentExport cleanUp]) { + OWSLogError(@"couldn't clean up attachment export."); + // Attachment files are non-critical so any error uploading them is recoverable. + } + } + }) + .thenInBackground(^{ + OWSAssertDebug(items.count == records.count); + NSUInteger count = MIN(items.count, records.count); + for (NSUInteger i = 0; i < count; i++) { + OWSAttachmentExport *attachmentExport = items[i]; + CKRecord *record = records[i]; + NSString *recordName = record.recordID.recordName; + OWSAssertDebug(recordName.length > 0); + + OWSBackupExportItem *exportItem = [OWSBackupExportItem new]; + exportItem.encryptedItem = attachmentExport.encryptedItem; + exportItem.recordName = recordName; + exportItem.attachmentExport = attachmentExport; + [self.savedAttachmentItems addObject:exportItem]; + + // Immediately save the record metadata to facilitate export resume. + OWSBackupFragment *backupFragment = [[OWSBackupFragment alloc] initWithUniqueId:recordName]; + backupFragment.recordName = recordName; + backupFragment.encryptionKey = exportItem.encryptedItem.encryptionKey; + backupFragment.relativeFilePath = attachmentExport.relativeFilePath; + backupFragment.attachmentId = attachmentExport.attachmentId; + backupFragment.uncompressedDataLength = exportItem.uncompressedDataLength; + [self.dbConnection readWriteWithBlock:^(YapDatabaseReadWriteTransaction *transaction) { + [backupFragment saveWithTransaction:transaction]; + }]; + + OWSLogVerbose(@"saved attachment: %@ as %@", + attachmentExport.attachmentFilePath, + attachmentExport.relativeFilePath); + } + }) + .catchInBackground(^{ + // Attachment files are non-critical so any error uploading them is recoverable. + return [AnyPromise promiseWithValue:@(1)]; + }); } -- (AnyPromise *)saveAttachmentFileToCloud:(OWSAttachmentExport *)attachmentExport +- (BOOL)tryToSkipAttachmentUpload:(OWSAttachmentExport *)attachmentExport { - if (self.lastValidRecordNames) { - // Wherever possible, we do incremental backups and re-use fragments of the last - // backup and/or restore. - // Recycling fragments doesn't just reduce redundant network activity, - // it allows us to skip the local export work, i.e. encryption. - // To do so, we must preserve the metadata for these fragments. - // - // We check two things: - // - // * That we already know the metadata for this fragment (from a previous backup - // or restore). - // * That this record does in fact exist in our CloudKit database. - NSString *lastRecordName = - [OWSBackupAPI recordNameForPersistentFileWithRecipientId:self.recipientId - fileId:attachmentExport.attachmentId]; - OWSBackupFragment *_Nullable lastBackupFragment = [OWSBackupFragment fetchObjectWithUniqueID:lastRecordName]; - if (lastBackupFragment && [self.lastValidRecordNames containsObject:lastRecordName]) { - OWSAssertDebug(lastBackupFragment.encryptionKey.length > 0); - OWSAssertDebug(lastBackupFragment.relativeFilePath.length > 0); - - // Recycle the metadata from the last backup's manifest. - OWSBackupEncryptedItem *encryptedItem = [OWSBackupEncryptedItem new]; - encryptedItem.encryptionKey = lastBackupFragment.encryptionKey; - attachmentExport.encryptedItem = encryptedItem; - attachmentExport.relativeFilePath = lastBackupFragment.relativeFilePath; - - OWSBackupExportItem *exportItem = [OWSBackupExportItem new]; - exportItem.encryptedItem = attachmentExport.encryptedItem; - exportItem.recordName = lastRecordName; - exportItem.attachmentExport = attachmentExport; - [self.savedAttachmentItems addObject:exportItem]; - - OWSLogVerbose(@"recycled attachment: %@ as %@", - attachmentExport.attachmentFilePath, - attachmentExport.relativeFilePath); - return [AnyPromise promiseWithValue:@(1)]; - } + if (!self.lastValidRecordNames) { + return NO; } - @autoreleasepool { - // OWSAttachmentExport is used to lazily write an encrypted copy of the - // attachment to disk. - if (![attachmentExport prepareForUpload]) { - // Attachment files are non-critical so any error uploading them is recoverable. - return [AnyPromise promiseWithValue:@(1)]; - } - OWSAssertDebug(attachmentExport.relativeFilePath.length > 0); - OWSAssertDebug(attachmentExport.encryptedItem); + // Wherever possible, we do incremental backups and re-use fragments of the last + // backup and/or restore. + // Recycling fragments doesn't just reduce redundant network activity, + // it allows us to skip the local export work, i.e. encryption. + // To do so, we must preserve the metadata for these fragments. + // + // We check two things: + // + // * That we already know the metadata for this fragment (from a previous backup + // or restore). + // * That this record does in fact exist in our CloudKit database. + NSString *recordName = + [OWSBackupAPI recordNameForPersistentFileWithRecipientId:self.recipientId fileId:attachmentExport.attachmentId]; + OWSBackupFragment *_Nullable lastBackupFragment = [OWSBackupFragment fetchObjectWithUniqueID:recordName]; + if (!lastBackupFragment || ![self.lastValidRecordNames containsObject:recordName]) { + return NO; } - return [OWSBackupAPI - savePersistentFileOnceToCloudObjcWithRecipientId:self.recipientId - fileId:attachmentExport.attachmentId - fileUrlBlock:^{ - if (attachmentExport.encryptedItem.filePath.length < 1) { - OWSLogError(@"attachment export missing temp file path"); - return (NSURL *)nil; - } - if (attachmentExport.relativeFilePath.length < 1) { - OWSLogError(@"attachment export missing relative file path"); - return (NSURL *)nil; - } - return [NSURL fileURLWithPath:attachmentExport.encryptedItem.filePath]; - }] - .thenInBackground(^(NSString *recordName) { - if (![attachmentExport cleanUp]) { - OWSLogError(@"couldn't clean up attachment export."); - // Attachment files are non-critical so any error uploading them is recoverable. - } + OWSAssertDebug(lastBackupFragment.encryptionKey.length > 0); + OWSAssertDebug(lastBackupFragment.relativeFilePath.length > 0); - OWSBackupExportItem *exportItem = [OWSBackupExportItem new]; - exportItem.encryptedItem = attachmentExport.encryptedItem; - exportItem.recordName = recordName; - exportItem.attachmentExport = attachmentExport; - [self.savedAttachmentItems addObject:exportItem]; - - // Immediately save the record metadata to facilitate export resume. - OWSBackupFragment *backupFragment = [[OWSBackupFragment alloc] initWithUniqueId:recordName]; - backupFragment.recordName = recordName; - backupFragment.encryptionKey = exportItem.encryptedItem.encryptionKey; - backupFragment.relativeFilePath = attachmentExport.relativeFilePath; - backupFragment.attachmentId = attachmentExport.attachmentId; - backupFragment.uncompressedDataLength = exportItem.uncompressedDataLength; - [self.dbConnection readWriteWithBlock:^(YapDatabaseReadWriteTransaction *transaction) { - [backupFragment saveWithTransaction:transaction]; - }]; + // Recycle the metadata from the last backup's manifest. + OWSBackupEncryptedItem *encryptedItem = [OWSBackupEncryptedItem new]; + encryptedItem.encryptionKey = lastBackupFragment.encryptionKey; + attachmentExport.encryptedItem = encryptedItem; + attachmentExport.relativeFilePath = lastBackupFragment.relativeFilePath; - OWSLogVerbose( - @"saved attachment: %@ as %@", attachmentExport.attachmentFilePath, attachmentExport.relativeFilePath); - }) - .catchInBackground(^{ - if (![attachmentExport cleanUp]) { - OWSLogError(@"couldn't clean up attachment export."); - // Attachment files are non-critical so any error uploading them is recoverable. - } + OWSBackupExportItem *exportItem = [OWSBackupExportItem new]; + exportItem.encryptedItem = attachmentExport.encryptedItem; + exportItem.recordName = recordName; + exportItem.attachmentExport = attachmentExport; + [self.savedAttachmentItems addObject:exportItem]; - // Attachment files are non-critical so any error uploading them is recoverable. - return [AnyPromise promiseWithValue:@(1)]; - }); + OWSLogVerbose( + @"recycled attachment: %@ as %@", attachmentExport.attachmentFilePath, attachmentExport.relativeFilePath); + return YES; } - (AnyPromise *)saveLocalProfileAvatarToCloud @@ -922,8 +951,6 @@ NS_ASSUME_NONNULL_BEGIN return [OWSBackupAPI saveRecordsToCloudObjcWithRecords:@[ record ]].thenInBackground(^{ exportItem.recordName = recordName; self.localProfileAvatarItem = exportItem; - - return [AnyPromise promiseWithValue:@(1)]; }); } @@ -941,14 +968,14 @@ NS_ASSUME_NONNULL_BEGIN OWSBackupExportItem *exportItem = [OWSBackupExportItem new]; exportItem.encryptedItem = encryptedItem; - return [OWSBackupAPI upsertManifestFileToCloudObjcWithRecipientId:self.recipientId - fileUrl:[NSURL fileURLWithPath:encryptedItem.filePath]] - .thenInBackground(^(NSString *recordName) { - exportItem.recordName = recordName; - self.manifestItem = exportItem; - // All files have been saved to the cloud. - }); + NSString *recordName = [OWSBackupAPI recordNameForManifestWithRecipientId:self.recipientId]; + CKRecord *record = + [OWSBackupAPI recordForFileUrl:[NSURL fileURLWithPath:encryptedItem.filePath] recordName:recordName]; + return [OWSBackupAPI saveRecordsToCloudObjcWithRecords:@[ record ]].thenInBackground(^{ + exportItem.recordName = recordName; + self.manifestItem = exportItem; + }); } - (nullable OWSBackupEncryptedItem *)writeManifestFile