From c2751665c346726785220925a93d1788a5b44687 Mon Sep 17 00:00:00 2001 From: Matthew Chen Date: Tue, 6 Mar 2018 15:58:06 -0300 Subject: [PATCH] Only backup attachments once. --- Signal/src/util/OWSBackupAPI.swift | 95 ++++++++++++++++--- Signal/src/util/OWSBackupExport.m | 142 +++++++++++++++++++++-------- 2 files changed, 186 insertions(+), 51 deletions(-) diff --git a/Signal/src/util/OWSBackupAPI.swift b/Signal/src/util/OWSBackupAPI.swift index 68fa85116..631e488a2 100644 --- a/Signal/src/util/OWSBackupAPI.swift +++ b/Signal/src/util/OWSBackupAPI.swift @@ -32,13 +32,32 @@ import CloudKit failure: failure) } + // "Ephemeral" files are specific to this backup export and will always need to + // be saved. For example, a complete image of the database is exported each time. + // We wouldn't want to overwrite previous images until the entire backup export is + // complete. @objc - public class func saveBackupFileToCloud(fileUrl: URL, + public class func saveEphemeralDatabaseFileToCloud(fileUrl: URL, success: @escaping (String) -> Swift.Void, failure: @escaping (Error) -> Swift.Void) { saveFileToCloud(fileUrl: fileUrl, recordName: NSUUID().uuidString, - recordType: "backupFile", + recordType: "ephemeralFile", + success: success, + failure: failure) + } + + // "Persistent" files may be shared between backup export; they should only be saved + // once. For example, attachment files should only be uploaded once. Subsequent + // + @objc + public class func savePersistentFileOnceToCloud(fileId: String, + fileUrlBlock: @escaping (Swift.Void) -> URL?, + success: @escaping (String) -> Swift.Void, + failure: @escaping (Error) -> Swift.Void) { + saveFileOnceToCloud(recordName: "persistentFile-\(fileId)", + recordType: "persistentFile", + fileUrlBlock: fileUrlBlock, success: success, failure: failure) } @@ -48,16 +67,28 @@ import CloudKit static let manifestRecordType = "manifest" static let payloadKey = "payload" + @objc + public class func upsertAttachmentToCloud(fileUrl: URL, + success: @escaping (String) -> Swift.Void, + failure: @escaping (Error) -> Swift.Void) { + // We want to use a well-known record id and type for manifest files. + upsertFileToCloud(fileUrl: fileUrl, + recordName: manifestRecordName, + recordType: manifestRecordType, + success: success, + failure: failure) + } + @objc public class func upsertManifestFileToCloud(fileUrl: URL, - success: @escaping (String) -> Swift.Void, - failure: @escaping (Error) -> Swift.Void) { + success: @escaping (String) -> Swift.Void, + failure: @escaping (Error) -> Swift.Void) { // We want to use a well-known record id and type for manifest files. upsertFileToCloud(fileUrl: fileUrl, - recordName: manifestRecordName, - recordType: manifestRecordType, - success: success, - failure: failure) + recordName: manifestRecordName, + recordType: manifestRecordType, + success: success, + failure: failure) } @objc @@ -117,10 +148,10 @@ import CloudKit if ckerror.code == .unknownItem { // No record found to update, saving new record. saveFileToCloud(fileUrl: fileUrl, - recordName: recordName, - recordType: recordType, - success: success, - failure: failure) + recordName: recordName, + recordType: recordType, + success: success, + failure: failure) return } Logger.error("\(self.logTag) error fetching record: \(error) \(ckerror.code).") @@ -144,7 +175,47 @@ import CloudKit saveRecordToCloud(record: record, success: success, failure: failure) + } + let myContainer = CKContainer.default() + let privateDatabase = myContainer.privateCloudDatabase + privateDatabase.add(fetchOperation) + } + + @objc + public class func saveFileOnceToCloud(recordName: String, + recordType: String, + fileUrlBlock: @escaping (Swift.Void) -> URL?, + success: @escaping (String) -> Swift.Void, + failure: @escaping (Error) -> Swift.Void) { + let recordId = CKRecordID(recordName: recordName) + let fetchOperation = CKFetchRecordsOperation(recordIDs: [recordId ]) + fetchOperation.perRecordCompletionBlock = { (record, recordId, error) in + if let error = error { + if let ckerror = error as? CKError { + if ckerror.code == .unknownItem { + // No record found to update, saving new record. + + guard let fileUrl = fileUrlBlock() else { + Logger.error("\(self.logTag) error preparing file for upload: \(error).") + return + } + saveFileToCloud(fileUrl: fileUrl, + recordName: recordName, + recordType: recordType, + success: success, + failure: failure) + return + } + Logger.error("\(self.logTag) error fetching record: \(error) \(ckerror.code).") + } else { + Logger.error("\(self.logTag) error fetching record: \(error).") + } + failure(error) + return + } + Logger.info("\(self.logTag) record already exists; skipping save.") + success(recordName) } let myContainer = CKContainer.default() let privateDatabase = myContainer.privateCloudDatabase diff --git a/Signal/src/util/OWSBackupExport.m b/Signal/src/util/OWSBackupExport.m index 0bde672ac..50cced4f5 100644 --- a/Signal/src/util/OWSBackupExport.m +++ b/Signal/src/util/OWSBackupExport.m @@ -25,6 +25,82 @@ NS_ASSUME_NONNULL_BEGIN typedef void (^OWSBackupExportBoolCompletion)(BOOL success); typedef void (^OWSBackupExportCompletion)(NSError *_Nullable error); +#pragma mark - + +@interface OWSAttachmentExport : NSObject + +@property (nonatomic) NSString *exportDirPath; +@property (nonatomic) NSString *attachmentId; +@property (nonatomic) NSString *attachmentFilePath; +@property (nonatomic, nullable) NSString *tempFilePath; +@property (nonatomic, nullable) NSString *relativeFilePath; + +@end + +#pragma mark - + +@implementation OWSAttachmentExport + +- (void)dealloc +{ + // Surface memory leaks by logging the deallocation. + DDLogVerbose(@"Dealloc: %@", self.class); + + // Delete temporary file ASAP. + if (self.tempFilePath) { + [OWSFileSystem deleteFileIfExists:self.tempFilePath]; + } +} + +// On success, tempFilePath will be non-nil. +- (void)prepareForUpload +{ + OWSAssert(self.exportDirPath.length > 0); + OWSAssert(self.attachmentId.length > 0); + OWSAssert(self.attachmentFilePath.length > 0); + + NSString *attachmentsDirPath = [TSAttachmentStream attachmentsFolder]; + if (![self.attachmentFilePath hasPrefix:attachmentsDirPath]) { + DDLogError(@"%@ attachment has unexpected path.", self.logTag); + OWSFail(@"%@ attachment has unexpected path: %@", self.logTag, self.attachmentFilePath); + return; + } + NSString *relativeFilePath = [self.attachmentFilePath substringFromIndex:attachmentsDirPath.length]; + NSString *pathSeparator = @"/"; + if ([relativeFilePath hasPrefix:pathSeparator]) { + relativeFilePath = [relativeFilePath substringFromIndex:pathSeparator.length]; + } + self.relativeFilePath = relativeFilePath; + + NSString *_Nullable tempFilePath = [self encryptAsTempFile:self.attachmentFilePath]; + if (!tempFilePath) { + DDLogError(@"%@ attachment could not be encrypted.", self.logTag); + OWSFail(@"%@ attachment could not be encrypted: %@", self.logTag, self.attachmentFilePath); + return; + } +} + +- (nullable NSString *)encryptAsTempFile:(NSString *)srcFilePath +{ + OWSAssert(self.exportDirPath.length > 0); + + // TODO: Encrypt the file using self.delegate.backupKey; + + NSString *dstFilePath = [self.exportDirPath stringByAppendingPathComponent:[NSUUID UUID].UUIDString]; + NSFileManager *fileManager = [NSFileManager defaultManager]; + NSError *error; + BOOL success = [fileManager copyItemAtPath:srcFilePath toPath:dstFilePath error:&error]; + if (!success || error) { + OWSProdLogAndFail(@"%@ error writing encrypted file: %@", self.logTag, error); + return nil; + } + return dstFilePath; +} + +@end + +#pragma mark - + @interface OWSBackupExport () @property (nonatomic, weak) id delegate; @@ -46,7 +122,8 @@ typedef void (^OWSBackupExportCompletion)(NSError *_Nullable error); // A map of "record name"-to-"file name". @property (nonatomic) NSMutableDictionary *databaseRecordMap; -@property (nonatomic) NSMutableArray *attachmentFilePaths; +// A map of "attachment id"-to-"local file path". +@property (nonatomic) NSMutableDictionary *attachmentFilePathMap; // A map of "record name"-to-"file relative path". @property (nonatomic) NSMutableDictionary *attachmentRecordMap; @@ -210,7 +287,7 @@ typedef void (^OWSBackupExportCompletion)(NSError *_Nullable error); __block unsigned long long copiedEntities = 0; __block unsigned long long copiedAttachments = 0; - self.attachmentFilePaths = [NSMutableArray new]; + self.attachmentFilePathMap = [NSMutableDictionary new]; [self.srcDBConnection readWithBlock:^(YapDatabaseReadTransaction *srcTransaction) { [self.dstDBConnection readWriteWithBlock:^(YapDatabaseReadWriteTransaction *dstTransaction) { @@ -280,7 +357,8 @@ typedef void (^OWSBackupExportCompletion)(NSError *_Nullable error); TSAttachmentStream *attachmentStream = object; NSString *_Nullable filePath = attachmentStream.filePath; if (filePath) { - [self.attachmentFilePaths addObject:filePath]; + OWSAssert(attachmentStream.uniqueId.length > 0); + self.attachmentFilePathMap[attachmentStream.uniqueId] = filePath; } } TSAttachment *attachment = object; @@ -337,7 +415,7 @@ typedef void (^OWSBackupExportCompletion)(NSError *_Nullable error); [self.databaseFilePaths removeLastObject]; // Database files are encrypted and can be safely stored unencrypted in the cloud. // TODO: Security review. - [OWSBackupAPI saveBackupFileToCloudWithFileUrl:[NSURL fileURLWithPath:filePath] + [OWSBackupAPI saveEphemeralDatabaseFileToCloudWithFileUrl:[NSURL fileURLWithPath:filePath] success:^(NSString *recordName) { // Ensure that we continue to perform the backup export // off the main thread. @@ -357,46 +435,35 @@ typedef void (^OWSBackupExportCompletion)(NSError *_Nullable error); return; } - if (self.attachmentFilePaths.count > 0) { - NSString *attachmentFilePath = self.attachmentFilePaths.lastObject; - [self.attachmentFilePaths removeLastObject]; - - NSString *attachmentsDirPath = [TSAttachmentStream attachmentsFolder]; - if (![attachmentFilePath hasPrefix:attachmentsDirPath]) { - DDLogError(@"%@ attachment has unexpected path.", self.logTag); - OWSFail(@"%@ attachment has unexpected path: %@", self.logTag, attachmentFilePath); - // Attachment files are non-critical so any error uploading them is recoverable. - [weakSelf saveNextFileToCloud:completion]; - return; - } - NSString *relativeFilePath = [attachmentFilePath substringFromIndex:attachmentsDirPath.length]; - NSString *pathSeparator = @"/"; - if ([relativeFilePath hasPrefix:pathSeparator]) { - relativeFilePath = [relativeFilePath substringFromIndex:pathSeparator.length]; - } - - // TODO: Make this incremental. - NSString *_Nullable tempFilePath = [self encryptAsTempFile:attachmentFilePath]; - if (!tempFilePath) { - DDLogError(@"%@ attachment could not be encrypted.", self.logTag); - OWSFail(@"%@ attachment could not be encrypted: %@", self.logTag, attachmentFilePath); - // Attachment files are non-critical so any error uploading them is recoverable. - [weakSelf saveNextFileToCloud:completion]; - return; - } - [OWSBackupAPI saveBackupFileToCloudWithFileUrl:[NSURL fileURLWithPath:tempFilePath] + if (self.attachmentFilePathMap.count > 0) { + NSString *attachmentId = self.attachmentFilePathMap.allKeys.lastObject; + NSString *attachmentFilePath = self.attachmentFilePathMap[attachmentId]; + [self.attachmentFilePathMap removeObjectForKey:attachmentId]; + + // OWSAttachmentExport is used to lazily write an encrypted copy of the + // attachment to disk. + OWSAttachmentExport *attachmentExport = [OWSAttachmentExport new]; + attachmentExport.exportDirPath = self.exportDirPath; + attachmentExport.attachmentId = attachmentId; + attachmentExport.attachmentFilePath = attachmentFilePath; + + [OWSBackupAPI savePersistentFileOnceToCloudWithFileId:attachmentId + fileUrlBlock:^{ + [attachmentExport prepareForUpload]; + if (attachmentExport.tempFilePath.length < 1 || attachmentExport.relativeFilePath.length < 1) { + return (NSURL *)nil; + } + return [NSURL fileURLWithPath:attachmentExport.tempFilePath]; + } success:^(NSString *recordName) { // Ensure that we continue to perform the backup export // off the main thread. dispatch_async(dispatch_get_global_queue(DISPATCH_QUEUE_PRIORITY_DEFAULT, 0), ^{ - // Delete temporary file ASAP. - [OWSFileSystem deleteFileIfExists:tempFilePath]; - OWSBackupExport *strongSelf = weakSelf; if (!strongSelf) { return; } - strongSelf.attachmentRecordMap[recordName] = relativeFilePath; + strongSelf.attachmentRecordMap[recordName] = attachmentExport.relativeFilePath; [strongSelf saveNextFileToCloud:completion]; }); } @@ -404,9 +471,6 @@ typedef void (^OWSBackupExportCompletion)(NSError *_Nullable error); // Ensure that we continue to perform the backup export // off the main thread. dispatch_async(dispatch_get_global_queue(DISPATCH_QUEUE_PRIORITY_DEFAULT, 0), ^{ - // Delete temporary file ASAP. - [OWSFileSystem deleteFileIfExists:tempFilePath]; - // Attachment files are non-critical so any error uploading them is recoverable. [weakSelf saveNextFileToCloud:completion]; });