Add tests around database conversion.

Signal.xcodeproj/project.pbxproj

@@ -186,6 +186,7 @@
 		34D8C0281ED3673300188D7C /* DebugUITableViewController.m in Sources */ = {isa = PBXBuildFile; fileRef = 34D8C0261ED3673300188D7C /* DebugUITableViewController.m */; };
 		34D8C02B1ED3685800188D7C /* DebugUIContacts.m in Sources */ = {isa = PBXBuildFile; fileRef = 34D8C02A1ED3685800188D7C /* DebugUIContacts.m */; };
 		34D99C931F2937CC00D284D6 /* OWSAnalytics.swift in Sources */ = {isa = PBXBuildFile; fileRef = 34D99C911F2937CC00D284D6 /* OWSAnalytics.swift */; };
+		34DB0BED2011548B007B313F /* OWSDatabaseConverterTest.m in Sources */ = {isa = PBXBuildFile; fileRef = 34DB0BEC2011548B007B313F /* OWSDatabaseConverterTest.m */; };
 		34DFCB851E8E04B500053165 /* AddToBlockListViewController.m in Sources */ = {isa = PBXBuildFile; fileRef = 34DFCB841E8E04B500053165 /* AddToBlockListViewController.m */; };
 		34E3E5681EC4B19400495BAC /* AudioProgressView.swift in Sources */ = {isa = PBXBuildFile; fileRef = 34E3E5671EC4B19400495BAC /* AudioProgressView.swift */; };
 		34E3EF0D1EFC235B007F6822 /* DebugUIDiskUsage.m in Sources */ = {isa = PBXBuildFile; fileRef = 34E3EF0C1EFC235B007F6822 /* DebugUIDiskUsage.m */; };
@@ -740,6 +741,8 @@
 		34D99C8A1F27B13B00D284D6 /* OWSViewController.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = OWSViewController.h; sourceTree = "<group>"; };
 		34D99C8B1F27B13B00D284D6 /* OWSViewController.m */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.objc; path = OWSViewController.m; sourceTree = "<group>"; };
 		34D99C911F2937CC00D284D6 /* OWSAnalytics.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = OWSAnalytics.swift; sourceTree = "<group>"; };
+		34DB0BEB2011548A007B313F /* OWSDatabaseConverterTest.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = OWSDatabaseConverterTest.h; sourceTree = "<group>"; };
+		34DB0BEC2011548B007B313F /* OWSDatabaseConverterTest.m */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.objc; path = OWSDatabaseConverterTest.m; sourceTree = "<group>"; };
 		34DFCB831E8E04B400053165 /* AddToBlockListViewController.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = AddToBlockListViewController.h; sourceTree = "<group>"; };
 		34DFCB841E8E04B500053165 /* AddToBlockListViewController.m */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.objc; path = AddToBlockListViewController.m; sourceTree = "<group>"; };
 		34E3E5671EC4B19400495BAC /* AudioProgressView.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = AudioProgressView.swift; sourceTree = "<group>"; };
@@ -1895,11 +1898,13 @@
 				B660F6AB1C29868000687D6E /* ExceptionsTest.m */,
 				B660F6AC1C29868000687D6E /* FunctionalUtilTest.h */,
 				B660F6AD1C29868000687D6E /* FunctionalUtilTest.m */,
-				B660F6B31C29868000687D6E /* UtilTest.h */,
-				B660F6B41C29868000687D6E /* UtilTest.m */,
-				45666F571D9B2880008FE134 /* OWSScrubbingLogFormatterTest.m */,
 				455AC69D1F4F8B0300134004 /* ImageCacheTest.swift */,
+				34DB0BEB2011548A007B313F /* OWSDatabaseConverterTest.h */,
+				34DB0BEC2011548B007B313F /* OWSDatabaseConverterTest.m */,
+				45666F571D9B2880008FE134 /* OWSScrubbingLogFormatterTest.m */,
 				45360B8F1F9527DA00FA666C /* SearcherTest.swift */,
+				B660F6B31C29868000687D6E /* UtilTest.h */,
+				B660F6B41C29868000687D6E /* UtilTest.m */,
 			);
 			path = util;
 			sourceTree = "<group>";
@@ -2998,6 +3003,7 @@
 				452D1EE81DCA90D100A57EC4 /* MesssagesBubblesSizeCalculatorTest.swift in Sources */,
 				45360B901F9527DA00FA666C /* SearcherTest.swift in Sources */,
 				B660F7561C29988E00687D6E /* PushManager.m in Sources */,
+				34DB0BED2011548B007B313F /* OWSDatabaseConverterTest.m in Sources */,
 				45360B911F952AA900FA666C /* MarqueeLabel.swift in Sources */,
 				454EBAB41F2BE14C00ACE0BB /* OWSAnalytics.swift in Sources */,
 				B660F7721C29988E00687D6E /* AppStoreRating.m in Sources */,

Signal/test/util/OWSDatabaseConverterTest.h

@@ -0,0 +1,13 @@
+//
+//  Copyright (c) 2018 Open Whisper Systems. All rights reserved.
+//
+
+#import <XCTest/XCTest.h>
+
+NS_ASSUME_NONNULL_BEGIN
+
+@interface OWSDatabaseConverterTest : XCTestCase
+
+@end
+
+NS_ASSUME_NONNULL_END

Signal/test/util/OWSDatabaseConverterTest.m

@@ -0,0 +1,80 @@
+//
+//  Copyright (c) 2018 Open Whisper Systems. All rights reserved.
+//
+
+#import "OWSDatabaseConverterTest.h"
+#import "OWSDatabaseConverter.h"
+#import <Curve25519Kit/Randomness.h>
+#import <SignalServiceKit/OWSStorage.h>
+#import <YapDatabase/YapDatabase.h>
+
+NS_ASSUME_NONNULL_BEGIN
+
+@interface OWSStorage (OWSDatabaseConverterTest)
+
++ (YapDatabaseDeserializer)logOnFailureDeserializer;
+
+@end
+
+#pragma mark -
+
+@interface OWSDatabaseConverter (OWSDatabaseConverterTest)
+
++ (BOOL)doesDatabaseNeedToBeConverted:(NSString *)databaseFilePath;
+
+@end
+
+#pragma mark -
+
+@implementation OWSDatabaseConverterTest
+
+- (NSData *)randomDatabasePassword
+{
+    return [Randomness generateRandomBytes:30];
+}
+
+- (nullable NSString *)createUnconvertedDatabase:(NSData *)passwordData
+{
+    NSString *temporaryDirectory = NSTemporaryDirectory();
+    NSString *filename = [NSUUID UUID].UUIDString;
+    NSString *databaseFilePath = [temporaryDirectory stringByAppendingPathComponent:filename];
+
+    YapDatabaseOptions *options = [[YapDatabaseOptions alloc] init];
+    options.corruptAction = YapDatabaseCorruptAction_Fail;
+    options.cipherKeyBlock = ^{
+        return passwordData;
+    };
+    options.enableMultiProcessSupport = YES;
+
+    YapDatabase *database = [[YapDatabase alloc] initWithPath:databaseFilePath
+                                                   serializer:nil
+                                                 deserializer:[OWSStorage logOnFailureDeserializer]
+                                                      options:options];
+    OWSAssert(database);
+    return database ? databaseFilePath : nil;
+}
+
+- (void)testDoesDatabaseNeedToBeConverted_Unconverted
+{
+    NSData *passwordData = [self randomDatabasePassword];
+    NSString *_Nullable databaseFilePath = [self createUnconvertedDatabase:passwordData];
+    XCTAssertTrue([OWSDatabaseConverter doesDatabaseNeedToBeConverted:databaseFilePath]);
+}
+
+- (void)testDoesDatabaseNeedToBeConverted_Converted
+{
+    // TODO: When we can create converted databases.
+}
+
+- (void)testDatabaseConversion
+{
+    NSData *passwordData = [self randomDatabasePassword];
+    NSString *_Nullable databaseFilePath = [self createUnconvertedDatabase:passwordData];
+    XCTAssertTrue([OWSDatabaseConverter doesDatabaseNeedToBeConverted:databaseFilePath]);
+    [OWSDatabaseConverter convertDatabaseIfNecessary];
+    XCTAssertFalse([OWSDatabaseConverter doesDatabaseNeedToBeConverted:databaseFilePath]);
+}
+
+@end
+
+NS_ASSUME_NONNULL_END

SignalMessaging/utils/OWSDatabaseConverter.h

@@ -13,6 +13,7 @@ NS_ASSUME_NONNULL_BEGIN
 - (instancetype)init NS_UNAVAILABLE;
 
 + (void)convertDatabaseIfNecessary;
++ (void)convertDatabaseIfNecessary:(NSString *)databaseFilePath;
 
 @end
 

SignalMessaging/utils/OWSDatabaseConverter.m

@@ -10,9 +10,8 @@ NS_ASSUME_NONNULL_BEGIN
 
 @implementation OWSDatabaseConverter
 
-+ (BOOL)doesDatabaseNeedToBeConverted
++ (BOOL)doesDatabaseNeedToBeConverted:(NSString *)databaseFilePath
 {
-    NSString *databaseFilePath = [TSStorageManager legacyDatabaseFilePath];
     if (![[NSFileManager defaultManager] fileExistsAtPath:databaseFilePath]) {
         DDLogVerbose(@"%@ Skipping database conversion; no legacy database found.", self.logTag);
         return NO;
@@ -50,7 +49,13 @@ NS_ASSUME_NONNULL_BEGIN
 
 + (void)convertDatabaseIfNecessary
 {
-    if (![self doesDatabaseNeedToBeConverted]) {
+    NSString *databaseFilePath = [TSStorageManager legacyDatabaseFilePath];
+    [self convertDatabaseIfNecessary:databaseFilePath];
+}
+
++ (void)convertDatabaseIfNecessary:(NSString *)databaseFilePath
+{
+    if (![self doesDatabaseNeedToBeConverted:databaseFilePath]) {
         return;
     }
 
@@ -60,6 +65,162 @@ NS_ASSUME_NONNULL_BEGIN
 + (void)convertDatabase
 {
     // TODO:
+
+    //    Hello Matthew,
+    //
+    //    I hope you're doing well. We've just pushed some changes out to the SQLCipher prerelease branch on GitHub that
+    //    implement the functionality we talked about that add a few new options:
+    //
+    //    1. PRAGMA cipher_plaintext_header_size - set or query the number of bytes to be left unencrypted on the start
+    //    of the first page. This pragma would be called after keying the database, but before use. In our testing 32
+    //    works for iOS
+    //        2. PRAGMA cipher_default_plaintext_header_size - set the "global" default to be used when opening database
+    //        connections
+    //        3. PRAGMA cipher_salt - set or query the salt for the database
+    //
+    //            When working with the SQLCipherVsSharedData application, there are two changes required. First, modify
+    //            the Podfile to reference SQLCipher with these changes:
+    //
+    //            pod 'SQLCipher', :git => 'https://github.com/sqlcipher/sqlcipher.git', :commit => 'd5c2bec'
+    //
+    //            Next, set the plaintext header size immediately after the key is provided:
+    //
+    //            int status = sqlite3_exec(db, "PRAGMA cipher_plaintext_header_size = 32;", NULL, NULL, NULL);
+    //
+    //
+    //    This should allow the demo app to background correctly.
+    //
+    //    In practice, for a real application, the other changes we talked about on the phone need occur, i.e. to
+    //    provide the salt to the application explicitly. The application can use a raw key spec, where the 96 hex are
+    //    provide (i.e. 64 hex for the 256 bit key, followed by 32 hex for the 128 bit salt) using explicit BLOB syntax,
+    //    e.g.
+    //
+    //        x'98483C6EB40B6C31A448C22A66DED3B5E5E8D5119CAC8327B655C8B5C483648101010101010101010101010101010101'
+    //
+    //        Alternately, the application can use the new cipher_salt PRAGMA to provide 32 hex to use as salt in
+    //        conjunction with a standard derived key, e.g.
+    //
+    //        PRAGMA cipher_salt = "x'01010101010101010101010101010101'";
+    //
+    //    Since you mentioned the Signal application is using a derived key, the second option might be easiest. You
+    //    could load the first 16 bytes of the existing file, or query the database using cipher_salt, and then store
+    //    that along side the key in the keychain. Then following migration you can provide both the key and the salt
+    //    explicitly.
+    //
+    //    With respect to migrating existing databases, it is possible to open a database, set the pragma, modify the
+    //    first page, then checkpoint to ensure that all WAL frames are written back to the main database. This allows
+    //    you to "decrypt" the first part of the header almost instantaneously, without having to re-encrypt all of the
+    //    content. Keep in mind that you'll need to record the salt separately in this case. There are a few examples of
+    //    this in the test cases we wrote up for this new functionality, starting here:
+    //
+    // https://github.com/sqlcipher/sqlcipher/blob/d5c2bec7688cef298292906c029d26b2c043219d/test/crypto.test#L2669
+    //
+    //    I was hoping you could take a look at this new functionality, provide feedback, and perform some initial
+    //    testing on your side. Please let us know if you have any questions, or would like to discuss the specifics of
+    //    implementation further. Thanks!
+    //
+    //        Cheers,
+    //        Stephen
+
+    //    - (BOOL)openDatabase
+    //    {
+    //        // Open the database connection.
+    //        //
+    //        // We use SQLITE_OPEN_NOMUTEX to use the multi-thread threading mode,
+    //        // as we will be serializing access to the connection externally.
+    //
+    //        int flags = SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE | SQLITE_OPEN_NOMUTEX | SQLITE_OPEN_PRIVATECACHE;
+    //
+    //        int status = sqlite3_open_v2([databasePath UTF8String], &db, flags, NULL);
+    //        if (status != SQLITE_OK)
+    //        {
+    //            // There are a few reasons why the database might not open.
+    //            // One possibility is if the database file has become corrupt.
+    //
+    //            // Sometimes the open function returns a db to allow us to query it for the error message.
+    //            // The openConfigCreate block will close it for us.
+    //            if (db) {
+    //                YDBLogError(@"Error opening database: %d %s", status, sqlite3_errmsg(db));
+    //            }
+    //            else {
+    //                YDBLogError(@"Error opening database: %d", status);
+    //            }
+    //
+    //            return NO;
+    //        }
+    //        // Add a busy handler if we are in multiprocess mode
+    //        if (options.enableMultiProcessSupport) {
+    //            sqlite3_busy_handler(db, connectionBusyHandler, (__bridge void *)(self));
+    //        }
+    //
+    //        return YES;
+    //    }
+    //
+    //
+    //
+    //#ifdef SQLITE_HAS_CODEC
+    //    /**
+    //     * Configures database encryption via SQLCipher.
+    //     **/
+    //    - (BOOL)configureEncryptionForDatabase:(sqlite3 *)sqlite
+    //    {
+    //        if (options.cipherKeyBlock)
+    //        {
+    //            NSData *keyData = options.cipherKeyBlock();
+    //
+    //            if (keyData == nil)
+    //            {
+    //                NSAssert(NO, @"YapDatabaseOptions.cipherKeyBlock cannot return nil!");
+    //                return NO;
+    //            }
+    //
+    //            //Setting the PBKDF2 default iteration number (this will have effect next time database is opened)
+    //            if (options.cipherDefaultkdfIterNumber > 0) {
+    //                char *errorMsg;
+    //                NSString *pragmaCommand = [NSString stringWithFormat:@"PRAGMA cipher_default_kdf_iter = %lu",
+    //                (unsigned long)options.cipherDefaultkdfIterNumber]; if (sqlite3_exec(sqlite, [pragmaCommand
+    //                UTF8String], NULL, NULL, &errorMsg) != SQLITE_OK)
+    //                {
+    //                    YDBLogError(@"failed to set database cipher_default_kdf_iter: %s", errorMsg);
+    //                    return NO;
+    //                }
+    //            }
+    //
+    //            //Setting the PBKDF2 iteration number
+    //            if (options.kdfIterNumber > 0) {
+    //                char *errorMsg;
+    //                NSString *pragmaCommand = [NSString stringWithFormat:@"PRAGMA kdf_iter = %lu", (unsigned
+    //                long)options.kdfIterNumber]; if (sqlite3_exec(sqlite, [pragmaCommand UTF8String], NULL, NULL,
+    //                &errorMsg) != SQLITE_OK)
+    //                {
+    //                    YDBLogError(@"failed to set database kdf_iter: %s", errorMsg);
+    //                    return NO;
+    //                }
+    //            }
+    //
+    //            //Setting the encrypted database page size
+    //            if (options.cipherPageSize > 0) {
+    //                char *errorMsg;
+    //                NSString *pragmaCommand = [NSString stringWithFormat:@"PRAGMA cipher_page_size = %lu", (unsigned
+    //                long)options.cipherPageSize]; if (sqlite3_exec(sqlite, [pragmaCommand UTF8String], NULL, NULL,
+    //                &errorMsg) != SQLITE_OK)
+    //                {
+    //                    YDBLogError(@"failed to set database cipher_page_size: %s", errorMsg);
+    //                    return NO;
+    //                }
+    //            }
+    //
+    //            int status = sqlite3_key(sqlite, [keyData bytes], (int)[keyData length]);
+    //            if (status != SQLITE_OK)
+    //            {
+    //                YDBLogError(@"Error setting SQLCipher key: %d %s", status, sqlite3_errmsg(sqlite));
+    //                return NO;
+    //            }
+    //        }
+    //
+    //        return YES;
+    //    }
+    //#endif
 }
 
 @end