Only backup attachments once.

pull/1/head
Matthew Chen 7 years ago
parent 20587ba377
commit c2751665c3

@ -32,13 +32,32 @@ import CloudKit
failure: failure) failure: failure)
} }
// "Ephemeral" files are specific to this backup export and will always need to
// be saved. For example, a complete image of the database is exported each time.
// We wouldn't want to overwrite previous images until the entire backup export is
// complete.
@objc @objc
public class func saveBackupFileToCloud(fileUrl: URL, public class func saveEphemeralDatabaseFileToCloud(fileUrl: URL,
success: @escaping (String) -> Swift.Void, success: @escaping (String) -> Swift.Void,
failure: @escaping (Error) -> Swift.Void) { failure: @escaping (Error) -> Swift.Void) {
saveFileToCloud(fileUrl: fileUrl, saveFileToCloud(fileUrl: fileUrl,
recordName: NSUUID().uuidString, recordName: NSUUID().uuidString,
recordType: "backupFile", recordType: "ephemeralFile",
success: success,
failure: failure)
}
// "Persistent" files may be shared between backup export; they should only be saved
// once. For example, attachment files should only be uploaded once. Subsequent
//
@objc
public class func savePersistentFileOnceToCloud(fileId: String,
fileUrlBlock: @escaping (Swift.Void) -> URL?,
success: @escaping (String) -> Swift.Void,
failure: @escaping (Error) -> Swift.Void) {
saveFileOnceToCloud(recordName: "persistentFile-\(fileId)",
recordType: "persistentFile",
fileUrlBlock: fileUrlBlock,
success: success, success: success,
failure: failure) failure: failure)
} }
@ -48,16 +67,28 @@ import CloudKit
static let manifestRecordType = "manifest" static let manifestRecordType = "manifest"
static let payloadKey = "payload" static let payloadKey = "payload"
@objc
public class func upsertAttachmentToCloud(fileUrl: URL,
success: @escaping (String) -> Swift.Void,
failure: @escaping (Error) -> Swift.Void) {
// We want to use a well-known record id and type for manifest files.
upsertFileToCloud(fileUrl: fileUrl,
recordName: manifestRecordName,
recordType: manifestRecordType,
success: success,
failure: failure)
}
@objc @objc
public class func upsertManifestFileToCloud(fileUrl: URL, public class func upsertManifestFileToCloud(fileUrl: URL,
success: @escaping (String) -> Swift.Void, success: @escaping (String) -> Swift.Void,
failure: @escaping (Error) -> Swift.Void) { failure: @escaping (Error) -> Swift.Void) {
// We want to use a well-known record id and type for manifest files. // We want to use a well-known record id and type for manifest files.
upsertFileToCloud(fileUrl: fileUrl, upsertFileToCloud(fileUrl: fileUrl,
recordName: manifestRecordName, recordName: manifestRecordName,
recordType: manifestRecordType, recordType: manifestRecordType,
success: success, success: success,
failure: failure) failure: failure)
} }
@objc @objc
@ -117,10 +148,10 @@ import CloudKit
if ckerror.code == .unknownItem { if ckerror.code == .unknownItem {
// No record found to update, saving new record. // No record found to update, saving new record.
saveFileToCloud(fileUrl: fileUrl, saveFileToCloud(fileUrl: fileUrl,
recordName: recordName, recordName: recordName,
recordType: recordType, recordType: recordType,
success: success, success: success,
failure: failure) failure: failure)
return return
} }
Logger.error("\(self.logTag) error fetching record: \(error) \(ckerror.code).") Logger.error("\(self.logTag) error fetching record: \(error) \(ckerror.code).")
@ -144,7 +175,47 @@ import CloudKit
saveRecordToCloud(record: record, saveRecordToCloud(record: record,
success: success, success: success,
failure: failure) failure: failure)
}
let myContainer = CKContainer.default()
let privateDatabase = myContainer.privateCloudDatabase
privateDatabase.add(fetchOperation)
}
@objc
public class func saveFileOnceToCloud(recordName: String,
recordType: String,
fileUrlBlock: @escaping (Swift.Void) -> URL?,
success: @escaping (String) -> Swift.Void,
failure: @escaping (Error) -> Swift.Void) {
let recordId = CKRecordID(recordName: recordName)
let fetchOperation = CKFetchRecordsOperation(recordIDs: [recordId ])
fetchOperation.perRecordCompletionBlock = { (record, recordId, error) in
if let error = error {
if let ckerror = error as? CKError {
if ckerror.code == .unknownItem {
// No record found to update, saving new record.
guard let fileUrl = fileUrlBlock() else {
Logger.error("\(self.logTag) error preparing file for upload: \(error).")
return
}
saveFileToCloud(fileUrl: fileUrl,
recordName: recordName,
recordType: recordType,
success: success,
failure: failure)
return
}
Logger.error("\(self.logTag) error fetching record: \(error) \(ckerror.code).")
} else {
Logger.error("\(self.logTag) error fetching record: \(error).")
}
failure(error)
return
}
Logger.info("\(self.logTag) record already exists; skipping save.")
success(recordName)
} }
let myContainer = CKContainer.default() let myContainer = CKContainer.default()
let privateDatabase = myContainer.privateCloudDatabase let privateDatabase = myContainer.privateCloudDatabase

@ -25,6 +25,82 @@ NS_ASSUME_NONNULL_BEGIN
typedef void (^OWSBackupExportBoolCompletion)(BOOL success); typedef void (^OWSBackupExportBoolCompletion)(BOOL success);
typedef void (^OWSBackupExportCompletion)(NSError *_Nullable error); typedef void (^OWSBackupExportCompletion)(NSError *_Nullable error);
#pragma mark -
@interface OWSAttachmentExport : NSObject
@property (nonatomic) NSString *exportDirPath;
@property (nonatomic) NSString *attachmentId;
@property (nonatomic) NSString *attachmentFilePath;
@property (nonatomic, nullable) NSString *tempFilePath;
@property (nonatomic, nullable) NSString *relativeFilePath;
@end
#pragma mark -
@implementation OWSAttachmentExport
- (void)dealloc
{
// Surface memory leaks by logging the deallocation.
DDLogVerbose(@"Dealloc: %@", self.class);
// Delete temporary file ASAP.
if (self.tempFilePath) {
[OWSFileSystem deleteFileIfExists:self.tempFilePath];
}
}
// On success, tempFilePath will be non-nil.
- (void)prepareForUpload
{
OWSAssert(self.exportDirPath.length > 0);
OWSAssert(self.attachmentId.length > 0);
OWSAssert(self.attachmentFilePath.length > 0);
NSString *attachmentsDirPath = [TSAttachmentStream attachmentsFolder];
if (![self.attachmentFilePath hasPrefix:attachmentsDirPath]) {
DDLogError(@"%@ attachment has unexpected path.", self.logTag);
OWSFail(@"%@ attachment has unexpected path: %@", self.logTag, self.attachmentFilePath);
return;
}
NSString *relativeFilePath = [self.attachmentFilePath substringFromIndex:attachmentsDirPath.length];
NSString *pathSeparator = @"/";
if ([relativeFilePath hasPrefix:pathSeparator]) {
relativeFilePath = [relativeFilePath substringFromIndex:pathSeparator.length];
}
self.relativeFilePath = relativeFilePath;
NSString *_Nullable tempFilePath = [self encryptAsTempFile:self.attachmentFilePath];
if (!tempFilePath) {
DDLogError(@"%@ attachment could not be encrypted.", self.logTag);
OWSFail(@"%@ attachment could not be encrypted: %@", self.logTag, self.attachmentFilePath);
return;
}
}
- (nullable NSString *)encryptAsTempFile:(NSString *)srcFilePath
{
OWSAssert(self.exportDirPath.length > 0);
// TODO: Encrypt the file using self.delegate.backupKey;
NSString *dstFilePath = [self.exportDirPath stringByAppendingPathComponent:[NSUUID UUID].UUIDString];
NSFileManager *fileManager = [NSFileManager defaultManager];
NSError *error;
BOOL success = [fileManager copyItemAtPath:srcFilePath toPath:dstFilePath error:&error];
if (!success || error) {
OWSProdLogAndFail(@"%@ error writing encrypted file: %@", self.logTag, error);
return nil;
}
return dstFilePath;
}
@end
#pragma mark -
@interface OWSBackupExport () <SSZipArchiveDelegate> @interface OWSBackupExport () <SSZipArchiveDelegate>
@property (nonatomic, weak) id<OWSBackupExportDelegate> delegate; @property (nonatomic, weak) id<OWSBackupExportDelegate> delegate;
@ -46,7 +122,8 @@ typedef void (^OWSBackupExportCompletion)(NSError *_Nullable error);
// A map of "record name"-to-"file name". // A map of "record name"-to-"file name".
@property (nonatomic) NSMutableDictionary<NSString *, NSString *> *databaseRecordMap; @property (nonatomic) NSMutableDictionary<NSString *, NSString *> *databaseRecordMap;
@property (nonatomic) NSMutableArray<NSString *> *attachmentFilePaths; // A map of "attachment id"-to-"local file path".
@property (nonatomic) NSMutableDictionary<NSString *, NSString *> *attachmentFilePathMap;
// A map of "record name"-to-"file relative path". // A map of "record name"-to-"file relative path".
@property (nonatomic) NSMutableDictionary<NSString *, NSString *> *attachmentRecordMap; @property (nonatomic) NSMutableDictionary<NSString *, NSString *> *attachmentRecordMap;
@ -210,7 +287,7 @@ typedef void (^OWSBackupExportCompletion)(NSError *_Nullable error);
__block unsigned long long copiedEntities = 0; __block unsigned long long copiedEntities = 0;
__block unsigned long long copiedAttachments = 0; __block unsigned long long copiedAttachments = 0;
self.attachmentFilePaths = [NSMutableArray new]; self.attachmentFilePathMap = [NSMutableDictionary new];
[self.srcDBConnection readWithBlock:^(YapDatabaseReadTransaction *srcTransaction) { [self.srcDBConnection readWithBlock:^(YapDatabaseReadTransaction *srcTransaction) {
[self.dstDBConnection readWriteWithBlock:^(YapDatabaseReadWriteTransaction *dstTransaction) { [self.dstDBConnection readWriteWithBlock:^(YapDatabaseReadWriteTransaction *dstTransaction) {
@ -280,7 +357,8 @@ typedef void (^OWSBackupExportCompletion)(NSError *_Nullable error);
TSAttachmentStream *attachmentStream = object; TSAttachmentStream *attachmentStream = object;
NSString *_Nullable filePath = attachmentStream.filePath; NSString *_Nullable filePath = attachmentStream.filePath;
if (filePath) { if (filePath) {
[self.attachmentFilePaths addObject:filePath]; OWSAssert(attachmentStream.uniqueId.length > 0);
self.attachmentFilePathMap[attachmentStream.uniqueId] = filePath;
} }
} }
TSAttachment *attachment = object; TSAttachment *attachment = object;
@ -337,7 +415,7 @@ typedef void (^OWSBackupExportCompletion)(NSError *_Nullable error);
[self.databaseFilePaths removeLastObject]; [self.databaseFilePaths removeLastObject];
// Database files are encrypted and can be safely stored unencrypted in the cloud. // Database files are encrypted and can be safely stored unencrypted in the cloud.
// TODO: Security review. // TODO: Security review.
[OWSBackupAPI saveBackupFileToCloudWithFileUrl:[NSURL fileURLWithPath:filePath] [OWSBackupAPI saveEphemeralDatabaseFileToCloudWithFileUrl:[NSURL fileURLWithPath:filePath]
success:^(NSString *recordName) { success:^(NSString *recordName) {
// Ensure that we continue to perform the backup export // Ensure that we continue to perform the backup export
// off the main thread. // off the main thread.
@ -357,46 +435,35 @@ typedef void (^OWSBackupExportCompletion)(NSError *_Nullable error);
return; return;
} }
if (self.attachmentFilePaths.count > 0) { if (self.attachmentFilePathMap.count > 0) {
NSString *attachmentFilePath = self.attachmentFilePaths.lastObject; NSString *attachmentId = self.attachmentFilePathMap.allKeys.lastObject;
[self.attachmentFilePaths removeLastObject]; NSString *attachmentFilePath = self.attachmentFilePathMap[attachmentId];
[self.attachmentFilePathMap removeObjectForKey:attachmentId];
NSString *attachmentsDirPath = [TSAttachmentStream attachmentsFolder];
if (![attachmentFilePath hasPrefix:attachmentsDirPath]) { // OWSAttachmentExport is used to lazily write an encrypted copy of the
DDLogError(@"%@ attachment has unexpected path.", self.logTag); // attachment to disk.
OWSFail(@"%@ attachment has unexpected path: %@", self.logTag, attachmentFilePath); OWSAttachmentExport *attachmentExport = [OWSAttachmentExport new];
// Attachment files are non-critical so any error uploading them is recoverable. attachmentExport.exportDirPath = self.exportDirPath;
[weakSelf saveNextFileToCloud:completion]; attachmentExport.attachmentId = attachmentId;
return; attachmentExport.attachmentFilePath = attachmentFilePath;
}
NSString *relativeFilePath = [attachmentFilePath substringFromIndex:attachmentsDirPath.length]; [OWSBackupAPI savePersistentFileOnceToCloudWithFileId:attachmentId
NSString *pathSeparator = @"/"; fileUrlBlock:^{
if ([relativeFilePath hasPrefix:pathSeparator]) { [attachmentExport prepareForUpload];
relativeFilePath = [relativeFilePath substringFromIndex:pathSeparator.length]; if (attachmentExport.tempFilePath.length < 1 || attachmentExport.relativeFilePath.length < 1) {
} return (NSURL *)nil;
}
// TODO: Make this incremental. return [NSURL fileURLWithPath:attachmentExport.tempFilePath];
NSString *_Nullable tempFilePath = [self encryptAsTempFile:attachmentFilePath]; }
if (!tempFilePath) {
DDLogError(@"%@ attachment could not be encrypted.", self.logTag);
OWSFail(@"%@ attachment could not be encrypted: %@", self.logTag, attachmentFilePath);
// Attachment files are non-critical so any error uploading them is recoverable.
[weakSelf saveNextFileToCloud:completion];
return;
}
[OWSBackupAPI saveBackupFileToCloudWithFileUrl:[NSURL fileURLWithPath:tempFilePath]
success:^(NSString *recordName) { success:^(NSString *recordName) {
// Ensure that we continue to perform the backup export // Ensure that we continue to perform the backup export
// off the main thread. // off the main thread.
dispatch_async(dispatch_get_global_queue(DISPATCH_QUEUE_PRIORITY_DEFAULT, 0), ^{ dispatch_async(dispatch_get_global_queue(DISPATCH_QUEUE_PRIORITY_DEFAULT, 0), ^{
// Delete temporary file ASAP.
[OWSFileSystem deleteFileIfExists:tempFilePath];
OWSBackupExport *strongSelf = weakSelf; OWSBackupExport *strongSelf = weakSelf;
if (!strongSelf) { if (!strongSelf) {
return; return;
} }
strongSelf.attachmentRecordMap[recordName] = relativeFilePath; strongSelf.attachmentRecordMap[recordName] = attachmentExport.relativeFilePath;
[strongSelf saveNextFileToCloud:completion]; [strongSelf saveNextFileToCloud:completion];
}); });
} }
@ -404,9 +471,6 @@ typedef void (^OWSBackupExportCompletion)(NSError *_Nullable error);
// Ensure that we continue to perform the backup export // Ensure that we continue to perform the backup export
// off the main thread. // off the main thread.
dispatch_async(dispatch_get_global_queue(DISPATCH_QUEUE_PRIORITY_DEFAULT, 0), ^{ dispatch_async(dispatch_get_global_queue(DISPATCH_QUEUE_PRIORITY_DEFAULT, 0), ^{
// Delete temporary file ASAP.
[OWSFileSystem deleteFileIfExists:tempFilePath];
// Attachment files are non-critical so any error uploading them is recoverable. // Attachment files are non-critical so any error uploading them is recoverable.
[weakSelf saveNextFileToCloud:completion]; [weakSelf saveNextFileToCloud:completion];
}); });

Loading…
Cancel
Save