Modify YapDatabase to read converted database, part 1.

Signal/test/util/OWSDatabaseConverterTest.m

@@ -35,8 +35,13 @@ NS_ASSUME_NONNULL_BEGIN
     return [Randomness generateRandomBytes:30];
 }
 
+// * Open a YapDatabase.
+// * Do some work with a block.
+// * Close the database.
+// * Verify that the database is closed.
 - (void)openYapDatabase:(NSString *)databaseFilePath
        databasePassword:(NSData *)databasePassword
+           databaseSalt:(NSData *_Nullable)databaseSalt
           databaseBlock:(void (^_Nonnull)(YapDatabase *))databaseBlock
 {
     OWSAssert(databaseFilePath.length > 0);
@@ -57,6 +62,14 @@ NS_ASSUME_NONNULL_BEGIN
         };
         options.enableMultiProcessSupport = YES;
 
+        if (databaseSalt) {
+            DDLogInfo(@"%@ Using salt & unencrypted header.", self.logTag);
+            options.cipherSaltBlock = ^{
+                return databaseSalt;
+            };
+            options.cipherUnencryptedHeaderLength = kSqliteHeaderLength;
+        }
+
         OWSAssert(options.cipherDefaultkdfIterNumber == 0);
         OWSAssert(options.kdfIterNumber == 0);
         OWSAssert(options.cipherPageSize == 0);
@@ -112,16 +125,16 @@ NS_ASSUME_NONNULL_BEGIN
     OWSAssert(!strongDatabase);
 }
 
-- (nullable NSString *)createUnconvertedDatabase:(NSData *)databasePassword
+- (void)createTestDatabase:(NSString *)databaseFilePath databasePassword:(NSData *)databasePassword
 {
-    NSString *temporaryDirectory = NSTemporaryDirectory();
+    OWSAssert(databaseFilePath.length > 0);
-    NSString *filename = [NSUUID UUID].UUIDString;
+    OWSAssert(databasePassword.length > 0);
-    NSString *databaseFilePath = [temporaryDirectory stringByAppendingPathComponent:filename];
 
-    DDLogInfo(@"%@ databaseFilePath: %@", self.logTag, databaseFilePath);
+    OWSAssert(![[NSFileManager defaultManager] fileExistsAtPath:databaseFilePath]);
 
     [self openYapDatabase:databaseFilePath
          databasePassword:databasePassword
+             databaseSalt:nil
             databaseBlock:^(YapDatabase *database) {
                 YapDatabaseConnection *dbConnection = database.newConnection;
                 [dbConnection setObject:@(YES) forKey:@"test_key_name" inCollection:@"test_collection_name"];
@@ -130,21 +143,49 @@ NS_ASSUME_NONNULL_BEGIN
 
     OWSAssert([[NSFileManager defaultManager] fileExistsAtPath:databaseFilePath]);
 
+    NSError *_Nullable error = nil;
+    NSDictionary *fileAttributes =
+        [[NSFileManager defaultManager] attributesOfItemAtPath:databaseFilePath error:&error];
+    OWSAssert(fileAttributes && !error);
+    DDLogVerbose(@"%@ test database file size: %@", self.logTag, fileAttributes[NSFileSize]);
+}
+
+- (BOOL)verifyTestDatabase:(NSString *)databaseFilePath
+          databasePassword:(NSData *)databasePassword
+              databaseSalt:(NSData *_Nullable)databaseSalt
+{
+    OWSAssert(databaseFilePath.length > 0);
+    OWSAssert(databasePassword.length > 0);
+
+    OWSAssert([[NSFileManager defaultManager] fileExistsAtPath:databaseFilePath]);
+
+    __block BOOL isValid = NO;
     [self openYapDatabase:databaseFilePath
          databasePassword:databasePassword
+             databaseSalt:databaseSalt
             databaseBlock:^(YapDatabase *database) {
                 YapDatabaseConnection *dbConnection = database.newConnection;
                 id _Nullable value = [dbConnection objectForKey:@"test_key_name" inCollection:@"test_collection_name"];
-                OWSAssert([@(YES) isEqual:value]);
+                isValid = [@(YES) isEqual:value];
             }];
 
     OWSAssert([[NSFileManager defaultManager] fileExistsAtPath:databaseFilePath]);
 
-    NSError *_Nullable error = nil;
+    return isValid;
-    NSDictionary *fileAttributes =
+}
-        [[NSFileManager defaultManager] attributesOfItemAtPath:databaseFilePath error:&error];
+
-    OWSAssert(fileAttributes && !error);
+- (nullable NSString *)createUnconvertedDatabase:(NSData *)databasePassword
-    DDLogVerbose(@"%@ test database file size: %@", self.logTag, fileAttributes[NSFileSize]);
+{
+    NSString *temporaryDirectory = NSTemporaryDirectory();
+    NSString *filename = [[NSUUID UUID].UUIDString stringByAppendingString:@".sqlite"];
+    NSString *databaseFilePath = [temporaryDirectory stringByAppendingPathComponent:filename];
+
+    DDLogInfo(@"%@ databaseFilePath: %@", self.logTag, databaseFilePath);
+
+    [self createTestDatabase:databaseFilePath databasePassword:databasePassword];
+
+    BOOL isValid = [self verifyTestDatabase:databaseFilePath databasePassword:databasePassword databaseSalt:nil];
+    OWSAssert(isValid);
 
     return databaseFilePath;
 }
@@ -166,13 +207,26 @@ NS_ASSUME_NONNULL_BEGIN
     NSData *databasePassword = [self randomDatabasePassword];
     NSString *_Nullable databaseFilePath = [self createUnconvertedDatabase:databasePassword];
     XCTAssertTrue([OWSDatabaseConverter doesDatabaseNeedToBeConverted:databaseFilePath]);
-    NSError *_Nullable error =
+
-        [OWSDatabaseConverter convertDatabaseIfNecessary:databaseFilePath databasePassword:databasePassword];
+    __block NSData *_Nullable databaseSalt = nil;
+    OWSDatabaseSaltBlock saltBlock = ^(NSData *saltData) {
+        OWSAssert(!databaseSalt);
+        OWSAssert(saltData);
+
+        databaseSalt = saltData;
+    };
+    NSError *_Nullable error = [OWSDatabaseConverter convertDatabaseIfNecessary:databaseFilePath
+                                                               databasePassword:databasePassword
+                                                                      saltBlock:saltBlock];
     if (error) {
         DDLogError(@"%s error: %@", __PRETTY_FUNCTION__, error);
     }
     XCTAssertNil(error);
     XCTAssertFalse([OWSDatabaseConverter doesDatabaseNeedToBeConverted:databaseFilePath]);
+
+    BOOL isValid =
+        [self verifyTestDatabase:databaseFilePath databasePassword:databasePassword databaseSalt:databaseSalt];
+    XCTAssertTrue(isValid);
 }
 
 @end

SignalMessaging/utils/OWSDatabaseConverter.h

@@ -4,6 +4,10 @@
 
 NS_ASSUME_NONNULL_BEGIN
 
+extern const NSUInteger kSqliteHeaderLength;
+
+typedef void (^OWSDatabaseSaltBlock)(NSData *saltData);
+
 // Used to convert YapDatabase/SQLCipher databases whose header is encrypted
 // to databases whose first 32 bytes are unencrypted so that iOS can determine
 // that this is a SQLite database using WAL and therefore not terminate the app
@@ -14,7 +18,8 @@ NS_ASSUME_NONNULL_BEGIN
 
 + (nullable NSError *)convertDatabaseIfNecessary;
 + (nullable NSError *)convertDatabaseIfNecessary:(NSString *)databaseFilePath
-                                databasePassword:(NSData *)databasePassword;
+                                databasePassword:(NSData *)databasePassword
+                                       saltBlock:(OWSDatabaseSaltBlock)saltBlock;
 
 @end
 

SignalMessaging/utils/OWSDatabaseConverter.m

@@ -87,25 +87,33 @@ const NSUInteger kSqliteHeaderLength = 32;
                        OWSErrorCodeDatabaseConversionFatalError, @"Failed to load database password"));
     }
 
-    return [self convertDatabaseIfNecessary:databaseFilePath databasePassword:databasePassword];
+    OWSDatabaseSaltBlock saltBlock = ^(NSData *saltData) {
+        [OWSStorage storeDatabaseSalt:saltData];
+    };
+
+    return [self convertDatabaseIfNecessary:databaseFilePath databasePassword:databasePassword saltBlock:saltBlock];
 }
 
 // TODO upon failure show user error UI
 // TODO upon failure anything we need to do "back out" partial migration
 + (nullable NSError *)convertDatabaseIfNecessary:(NSString *)databaseFilePath
                                 databasePassword:(NSData *)databasePassword
+                                       saltBlock:(OWSDatabaseSaltBlock)saltBlock
 {
     if (![self doesDatabaseNeedToBeConverted:databaseFilePath]) {
         return nil;
     }
 
-    return [self convertDatabase:(NSString *)databaseFilePath databasePassword:databasePassword];
+    return [self convertDatabase:(NSString *)databaseFilePath databasePassword:databasePassword saltBlock:saltBlock];
 }
 
-+ (nullable NSError *)convertDatabase:(NSString *)databaseFilePath databasePassword:(NSData *)databasePassword
++ (nullable NSError *)convertDatabase:(NSString *)databaseFilePath
+                     databasePassword:(NSData *)databasePassword
+                            saltBlock:(OWSDatabaseSaltBlock)saltBlock
 {
     OWSAssert(databaseFilePath.length > 0);
     OWSAssert(databasePassword.length > 0);
+    OWSAssert(saltBlock);
 
     NSData *sqlCipherSaltData;
     {
@@ -115,6 +123,11 @@ const NSUInteger kSqliteHeaderLength = 32;
         const NSUInteger kSQLCipherSaltLength = 16;
         OWSAssert(headerData.length >= kSQLCipherSaltLength);
         sqlCipherSaltData = [headerData subdataWithRange:NSMakeRange(0, kSQLCipherSaltLength)];
+
+        // Make sure we successfully persist the salt (persumably in the keychain) before
+        // proceeding with the database conversion or we could leave the app in an
+        // unrecoverable state.
+        saltBlock(sqlCipherSaltData);
     }
 
     // TODO: Write salt to keychain.

SignalServiceKit/src/Storage/OWSStorage.h

@@ -57,6 +57,9 @@ extern NSString *const StorageIsReadyNotification;
 
 + (nullable NSData *)tryToLoadDatabasePassword:(NSError **)errorHandle;
 
++ (nullable NSData *)tryToLoadDatabaseSalt:(NSError **)errorHandle;
++ (void)storeDatabaseSalt:(NSData *)saltData;
+
 @end
 
 NS_ASSUME_NONNULL_END

SignalServiceKit/src/Storage/OWSStorage.m

@@ -27,6 +27,7 @@ NSString *const OWSResetStorageNotification = @"OWSResetStorageNotification";
 
 static NSString *keychainService = @"TSKeyChainService";
 static NSString *keychainDBPassAccount = @"TSDatabasePass";
+static NSString *keychainDBSalt = @"OWSDatabaseSalt";
 
 #pragma mark -
 
@@ -500,7 +501,7 @@ static NSString *keychainDBPassAccount = @"TSDatabasePass";
     return @"";
 }
 
-#pragma mark - Password
+#pragma mark - Keychain
 
 + (BOOL)isDatabasePasswordAccessible
 {
@@ -519,15 +520,24 @@ static NSString *keychainDBPassAccount = @"TSDatabasePass";
     return NO;
 }
 
-+ (nullable NSData *)tryToLoadDatabasePassword:(NSError **)errorHandle
++ (nullable NSData *)tryToLoadKeyChainValue:(NSString *)keychainKey errorHandle:(NSError **)errorHandle
 {
+    OWSAssert(keychainKey.length > 0);
     OWSAssert(errorHandle);
 
     [SAMKeychain setAccessibilityType:kSecAttrAccessibleAfterFirstUnlockThisDeviceOnly];
 
-    NSData *_Nullable passwordData =
+    return [SAMKeychain passwordDataForService:keychainService account:keychainKey error:errorHandle];
-        [SAMKeychain passwordDataForService:keychainService account:keychainDBPassAccount error:errorHandle];
+}
-    return passwordData;
+
++ (nullable NSData *)tryToLoadDatabasePassword:(NSError **)errorHandle
+{
+    return [self tryToLoadKeyChainValue:keychainDBPassAccount errorHandle:errorHandle];
+}
+
++ (nullable NSData *)tryToLoadDatabaseSalt:(NSError **)errorHandle
+{
+    return [self tryToLoadKeyChainValue:keychainDBSalt errorHandle:errorHandle];
 }
 
 - (NSData *)databasePassword
@@ -604,6 +614,7 @@ static NSString *keychainDBPassAccount = @"TSDatabasePass";
 + (void)deletePasswordFromKeychain
 {
     [SAMKeychain deletePasswordForService:keychainService account:keychainDBPassAccount];
+    [SAMKeychain deletePasswordForService:keychainService account:keychainDBSalt];
 }
 
 - (unsigned long long)databaseFileSize
@@ -620,16 +631,16 @@ static NSString *keychainDBPassAccount = @"TSDatabasePass";
     return fileSize;
 }
 
-+ (void)storeDatabasePassword:(NSData *)passwordData
++ (void)storeKeyChainValue:(NSData *)data keychainKey:(NSString *)keychainKey
 {
+    OWSAssert(keychainKey.length > 0);
+    OWSAssert(data.length > 0);
+
     NSError *error;
     [SAMKeychain setAccessibilityType:kSecAttrAccessibleAfterFirstUnlockThisDeviceOnly];
-    BOOL success = [SAMKeychain setPasswordData:passwordData
+    BOOL success = [SAMKeychain setPasswordData:data forService:keychainService account:keychainKey error:&error];
-                                     forService:keychainService
-                                        account:keychainDBPassAccount
-                                          error:&error];
     if (!success || error) {
-        OWSProdCritical([OWSAnalyticsEvents storageErrorCouldNotStoreDatabasePassword]);
+        OWSProdCritical([OWSAnalyticsEvents storageErrorCouldNotStoreKeychainValue]);
 
         [OWSStorage deletePasswordFromKeychain];
 
@@ -637,12 +648,22 @@ static NSString *keychainDBPassAccount = @"TSDatabasePass";
         [NSThread sleepForTimeInterval:15.0f];
 
         [NSException raise:OWSStorageExceptionName_DatabasePasswordUnwritable
-                    format:@"Setting DB password failed with error: %@", error];
+                    format:@"Setting keychain value failed with error: %@", error];
     } else {
-        DDLogWarn(@"Succesfully set new DB password.");
+        DDLogWarn(@"Succesfully set new keychain value.");
     }
 }
 
++ (void)storeDatabasePassword:(NSData *)passwordData
+{
+    [self storeKeyChainValue:passwordData keychainKey:keychainDBPassAccount];
+}
+
++ (void)storeDatabaseSalt:(NSData *)saltData
+{
+    [self storeKeyChainValue:saltData keychainKey:keychainDBSalt];
+}
+
 @end
 
 NS_ASSUME_NONNULL_END

SignalServiceKit/src/Util/OWSAnalyticsEvents.h

@@ -1,5 +1,5 @@
 //
-//  Copyright (c) 2017 Open Whisper Systems. All rights reserved.
+//  Copyright (c) 2018 Open Whisper Systems. All rights reserved.
 //
 
 NS_ASSUME_NONNULL_BEGIN
@@ -220,7 +220,7 @@ NS_ASSUME_NONNULL_BEGIN
 
 + (NSString *)storageErrorCouldNotLoadDatabaseSecondAttempt;
 
-+ (NSString *)storageErrorCouldNotStoreDatabasePassword;
++ (NSString *)storageErrorCouldNotStoreKeychainValue;
 
 + (NSString *)storageErrorDeserialization;
 

SignalServiceKit/src/Util/OWSAnalyticsEvents.m

@@ -1,5 +1,5 @@
 //
-//  Copyright (c) 2017 Open Whisper Systems. All rights reserved.
+//  Copyright (c) 2018 Open Whisper Systems. All rights reserved.
 //
 
 #import "OWSAnalyticsEvents.h"
@@ -517,9 +517,9 @@ NS_ASSUME_NONNULL_BEGIN
     return @"storage_error_could_not_load_database_second_attempt";
 }
 
-+ (NSString *)storageErrorCouldNotStoreDatabasePassword
++ (NSString *)storageErrorCouldNotStoreKeychainValue
 {
-    return @"storage_error_could_not_store_database_password";
+    return @"storage_error_could_not_store_keychain_value";
 }
 
 + (NSString *)storageErrorDeserialization