Update to latest YapDB (with Signal patches applied)

Applied branches: - mkirk/invalidFinalIndices - mkirk/upstream-unencrypted-headers Also fixed up test suite.
7 years ago · df01c7e63e
parent 2ba5f65d89
commit df01c7e63e
6 changed files with 101 additions and 84 deletions
--- a/2
+++ b/2
@ -8,7 +8,7 @@ def shared_pods
  # pod 'SQLCipher', path: '../sqlcipher2'
  pod 'SQLCipher', :git => 'https://github.com/sqlcipher/sqlcipher.git', :commit => 'd5c2bec'
  # pod 'YapDatabase/SQLCipher', path: '../YapDatabase'
-  pod 'YapDatabase/SQLCipher', :git => 'https://github.com/signalapp/YapDatabase.git', branch: 'release/unencryptedHeaders'
+  pod 'YapDatabase/SQLCipher', :git => 'https://github.com/signalapp/YapDatabase.git', branch: 'signal-release'
  # pod 'AxolotlKit',   path: '../SignalProtocolKit'
  pod 'SignalServiceKit', path: '.'
  pod 'AxolotlKit', git: 'https://github.com/signalapp/SignalProtocolKit.git'
--- a/Podfile.lock
+++ b/Podfile.lock
@ -70,64 +70,67 @@ PODS:
    - SQLCipher/common
  - SSZipArchive (2.1.2)
  - SwiftProtobuf (1.0.3)
-  - YapDatabase/SQLCipher (3.0.2):
-    - YapDatabase/SQLCipher/Core (= 3.0.2)
-    - YapDatabase/SQLCipher/Extensions (= 3.0.2)
-  - YapDatabase/SQLCipher/Core (3.0.2):
+  - YapDatabase/SQLCipher (3.1.1):
+    - YapDatabase/SQLCipher/Core (= 3.1.1)
+    - YapDatabase/SQLCipher/Extensions (= 3.1.1)
+  - YapDatabase/SQLCipher/Core (3.1.1):
    - CocoaLumberjack
    - SQLCipher (>= 3.4.0)
-  - YapDatabase/SQLCipher/Extensions (3.0.2):
-    - YapDatabase/SQLCipher/Core
-    - YapDatabase/SQLCipher/Extensions/ActionManager (= 3.0.2)
-    - YapDatabase/SQLCipher/Extensions/AutoView (= 3.0.2)
-    - YapDatabase/SQLCipher/Extensions/CloudCore (= 3.0.2)
-    - YapDatabase/SQLCipher/Extensions/CloudKit (= 3.0.2)
-    - YapDatabase/SQLCipher/Extensions/ConnectionProxy (= 3.0.2)
-    - YapDatabase/SQLCipher/Extensions/CrossProcessNotification (= 3.0.2)
-    - YapDatabase/SQLCipher/Extensions/FilteredView (= 3.0.2)
-    - YapDatabase/SQLCipher/Extensions/FullTextSearch (= 3.0.2)
-    - YapDatabase/SQLCipher/Extensions/Hooks (= 3.0.2)
-    - YapDatabase/SQLCipher/Extensions/ManualView (= 3.0.2)
-    - YapDatabase/SQLCipher/Extensions/Relationships (= 3.0.2)
-    - YapDatabase/SQLCipher/Extensions/RTreeIndex (= 3.0.2)
-    - YapDatabase/SQLCipher/Extensions/SearchResultsView (= 3.0.2)
-    - YapDatabase/SQLCipher/Extensions/SecondaryIndex (= 3.0.2)
-    - YapDatabase/SQLCipher/Extensions/View (= 3.0.2)
-  - YapDatabase/SQLCipher/Extensions/ActionManager (3.0.2):
+  - YapDatabase/SQLCipher/Extensions (3.1.1):
+    - YapDatabase/SQLCipher/Core
+    - YapDatabase/SQLCipher/Extensions/ActionManager (= 3.1.1)
+    - YapDatabase/SQLCipher/Extensions/AutoView (= 3.1.1)
+    - YapDatabase/SQLCipher/Extensions/CloudCore (= 3.1.1)
+    - YapDatabase/SQLCipher/Extensions/CloudKit (= 3.1.1)
+    - YapDatabase/SQLCipher/Extensions/ConnectionPool (= 3.1.1)
+    - YapDatabase/SQLCipher/Extensions/ConnectionProxy (= 3.1.1)
+    - YapDatabase/SQLCipher/Extensions/CrossProcessNotification (= 3.1.1)
+    - YapDatabase/SQLCipher/Extensions/FilteredView (= 3.1.1)
+    - YapDatabase/SQLCipher/Extensions/FullTextSearch (= 3.1.1)
+    - YapDatabase/SQLCipher/Extensions/Hooks (= 3.1.1)
+    - YapDatabase/SQLCipher/Extensions/ManualView (= 3.1.1)
+    - YapDatabase/SQLCipher/Extensions/Relationships (= 3.1.1)
+    - YapDatabase/SQLCipher/Extensions/RTreeIndex (= 3.1.1)
+    - YapDatabase/SQLCipher/Extensions/SearchResultsView (= 3.1.1)
+    - YapDatabase/SQLCipher/Extensions/SecondaryIndex (= 3.1.1)
+    - YapDatabase/SQLCipher/Extensions/View (= 3.1.1)
+  - YapDatabase/SQLCipher/Extensions/ActionManager (3.1.1):
    - YapDatabase/SQLCipher/Core
    - YapDatabase/SQLCipher/Extensions/AutoView
-  - YapDatabase/SQLCipher/Extensions/AutoView (3.0.2):
+  - YapDatabase/SQLCipher/Extensions/AutoView (3.1.1):
    - YapDatabase/SQLCipher/Core
    - YapDatabase/SQLCipher/Extensions/View
-  - YapDatabase/SQLCipher/Extensions/CloudCore (3.0.2):
+  - YapDatabase/SQLCipher/Extensions/CloudCore (3.1.1):
    - YapDatabase/SQLCipher/Core
-  - YapDatabase/SQLCipher/Extensions/CloudKit (3.0.2):
+  - YapDatabase/SQLCipher/Extensions/CloudKit (3.1.1):
    - YapDatabase/SQLCipher/Core
-  - YapDatabase/SQLCipher/Extensions/ConnectionProxy (3.0.2):
+  - YapDatabase/SQLCipher/Extensions/ConnectionPool (3.1.1):
    - YapDatabase/SQLCipher/Core
-  - YapDatabase/SQLCipher/Extensions/CrossProcessNotification (3.0.2):
+  - YapDatabase/SQLCipher/Extensions/ConnectionProxy (3.1.1):
    - YapDatabase/SQLCipher/Core
-  - YapDatabase/SQLCipher/Extensions/FilteredView (3.0.2):
+  - YapDatabase/SQLCipher/Extensions/CrossProcessNotification (3.1.1):
+    - YapDatabase/SQLCipher/Core
+  - YapDatabase/SQLCipher/Extensions/FilteredView (3.1.1):
    - YapDatabase/SQLCipher/Core
    - YapDatabase/SQLCipher/Extensions/View
-  - YapDatabase/SQLCipher/Extensions/FullTextSearch (3.0.2):
+  - YapDatabase/SQLCipher/Extensions/FullTextSearch (3.1.1):
    - YapDatabase/SQLCipher/Core
-  - YapDatabase/SQLCipher/Extensions/Hooks (3.0.2):
+  - YapDatabase/SQLCipher/Extensions/Hooks (3.1.1):
    - YapDatabase/SQLCipher/Core
-  - YapDatabase/SQLCipher/Extensions/ManualView (3.0.2):
+  - YapDatabase/SQLCipher/Extensions/ManualView (3.1.1):
    - YapDatabase/SQLCipher/Core
    - YapDatabase/SQLCipher/Extensions/View
-  - YapDatabase/SQLCipher/Extensions/Relationships (3.0.2):
+  - YapDatabase/SQLCipher/Extensions/Relationships (3.1.1):
    - YapDatabase/SQLCipher/Core
-  - YapDatabase/SQLCipher/Extensions/RTreeIndex (3.0.2):
+  - YapDatabase/SQLCipher/Extensions/RTreeIndex (3.1.1):
    - YapDatabase/SQLCipher/Core
-  - YapDatabase/SQLCipher/Extensions/SearchResultsView (3.0.2):
+  - YapDatabase/SQLCipher/Extensions/SearchResultsView (3.1.1):
    - YapDatabase/SQLCipher/Core
    - YapDatabase/SQLCipher/Extensions/AutoView
    - YapDatabase/SQLCipher/Extensions/FullTextSearch
-  - YapDatabase/SQLCipher/Extensions/SecondaryIndex (3.0.2):
+  - YapDatabase/SQLCipher/Extensions/SecondaryIndex (3.1.1):
    - YapDatabase/SQLCipher/Core
-  - YapDatabase/SQLCipher/Extensions/View (3.0.2):
+  - YapDatabase/SQLCipher/Extensions/View (3.1.1):
    - YapDatabase/SQLCipher/Core
  - YYImage (1.0.4):
    - YYImage/Core (= 1.0.4)
@ -147,7 +150,7 @@ DEPENDENCIES:
  - SocketRocket (from `https://github.com/facebook/SocketRocket.git`)
  - SQLCipher (from `https://github.com/sqlcipher/sqlcipher.git`, commit `d5c2bec`)
  - SSZipArchive
-  - YapDatabase/SQLCipher (from `https://github.com/signalapp/YapDatabase.git`, branch `release/unencryptedHeaders`)
+  - YapDatabase/SQLCipher (from `https://github.com/signalapp/YapDatabase.git`, branch `signal-release`)
  - YYImage

 SPEC REPOS:
@ -185,7 +188,7 @@ EXTERNAL SOURCES:
    :commit: d5c2bec
    :git: https://github.com/sqlcipher/sqlcipher.git
  YapDatabase:
-    :branch: release/unencryptedHeaders
+    :branch: signal-release
    :git: https://github.com/signalapp/YapDatabase.git

 CHECKOUT OPTIONS:
@ -208,7 +211,7 @@ CHECKOUT OPTIONS:
    :commit: d5c2bec
    :git: https://github.com/sqlcipher/sqlcipher.git
  YapDatabase:
-    :commit: 764e949142ba1bada99aeedeeaccfb68047a6e79
+    :commit: f1fa4545e1e1594fb80065ffca52a682f5a2e71c
    :git: https://github.com/signalapp/YapDatabase.git

 SPEC CHECKSUMS:
@ -231,9 +234,9 @@ SPEC CHECKSUMS:
  SQLCipher: f9fcf29b2e59ced7defc2a2bdd0ebe79b40d4990
  SSZipArchive: d4009d2ce5520a421f231fd97028cc0e2667eed8
  SwiftProtobuf: 5ccc0e4054e37c75800e5744acb2aa80bb72b39c
-  YapDatabase: 299a32de9d350d37a9ac5b0532609d87d5d2a5de
+  YapDatabase: b418a4baa6906e8028748938f9159807fd039af4
  YYImage: 1e1b62a9997399593e4b9c4ecfbbabbf1d3f3b54

-PODFILE CHECKSUM: db797890d3df475827063ba956cb42afa148ff19
+PODFILE CHECKSUM: b378e1ac40edbfc9d4f66410d26ea18a8a0a32dc

 COCOAPODS: 1.5.3
--- a/2
+++ b/2
@ -1 +1 @@
-Subproject commit 74a0f3857847cf9658b10b4bdce7b19f53e891ec
+Subproject commit 9f36102cbbd5555045626b87fc2dde4dde05ab61
--- a/Signal/test/util/DisplayableTextFilterTest.swift
+++ b/Signal/test/util/DisplayableTextFilterTest.swift
@ -64,7 +64,14 @@ class DisplayableTextTest: XCTestCase {
        XCTAssertEqual("Příliš žluťoučký kůň úpěl ďábelské ódy.".glyphCount, 39)

        // Excessive diacritics
-        XCTAssertEqual("H҉̸̧͘͠A͢͞V̛̛I̴̸N͏̕͏G҉̵͜͏͢ ̧̧́T̶̛͘͡R̸̵̨̢̀O̷̡U͡҉B̶̛͢͞L̸̸͘͢͟É̸ ̸̛͘͏R͟È͠͞A̸͝Ḑ̕͘͜I̵͘҉͜͞N̷̡̢͠G̴͘͠ ͟͞T͏̢́͡È̀X̕҉̢̀T̢͠?̕͏̢͘͢".glyphCount, 115)
+
+        // some insignificant discrepencies across iOS versions        
+        if #available(iOS 11, *) {
+            XCTAssertEqual("H҉̸̧͘͠A͢͞V̛̛I̴̸N͏̕͏G҉̵͜͏͢ ̧̧́T̶̛͘͡R̸̵̨̢̀O̷̡U͡҉B̶̛͢͞L̸̸͘͢͟É̸ ̸̛͘͏R͟È͠͞A̸͝Ḑ̕͘͜I̵͘҉͜͞N̷̡̢͠G̴͘͠ ͟͞T͏̢́͡È̀X̕҉̢̀T̢͠?̕͏̢͘͢".glyphCount, 115)
+        } else {
+            XCTAssertEqual("H҉̸̧͘͠A͢͞V̛̛I̴̸N͏̕͏G҉̵͜͏͢ ̧̧́T̶̛͘͡R̸̵̨̢̀O̷̡U͡҉B̶̛͢͞L̸̸͘͢͟É̸ ̸̛͘͏R͟È͠͞A̸͝Ḑ̕͘͜I̵͘҉͜͞N̷̡̢͠G̴͘͠ ͟͞T͏̢́͡È̀X̕҉̢̀T̢͠?̕͏̢͘͢".glyphCount, 109)
+        }
+
        XCTAssertEqual("L̷̳͔̲͝Ģ̵̮̯̤̩̙͍̬̟͉̹̘̹͍͈̮̦̰̣͟͝O̶̴̮̻̮̗͘͡!̴̷̟͓͓".glyphCount, 43)
    }

--- a/Signal/test/util/OWSDatabaseConverterTest.m
+++ b/Signal/test/util/OWSDatabaseConverterTest.m
@ -14,6 +14,13 @@

 NS_ASSUME_NONNULL_BEGIN

+@interface YapDatabase (OWSDatabaseConverterTest)
+
+- (void)flushInternalQueue;
+- (void)flushCheckpointQueue;
+
+@end
+
@interface OWSStorage (OWSDatabaseConverterTest)

 + (YapDatabaseDeserializer)logOnFailureDeserializer;
--- a/SignalServiceKit/src/Storage/FullTextSearchFinder.swift
+++ b/SignalServiceKit/src/Storage/FullTextSearchFinder.swift
@ -7,14 +7,14 @@ import Foundation
 // Create a searchable index for objects of type T
 public class SearchIndexer<T> {

-    private let indexBlock: (T) -> String
+    private let indexBlock: (T, YapDatabaseReadTransaction) -> String

-    public init(indexBlock: @escaping (T) -> String) {
+    public init(indexBlock: @escaping (T, YapDatabaseReadTransaction) -> String) {
        self.indexBlock = indexBlock
    }

-    public func index(_ item: T) -> String {
-        return normalize(indexingText: indexBlock(item))
+    public func index(_ item: T, transaction: YapDatabaseReadTransaction) -> String {
+        return normalize(indexingText: indexBlock(item, transaction))
    }

    private func normalize(indexingText: String) -> String {
@ -157,22 +157,22 @@ public class FullTextSearchFinder: NSObject {
        return TextSecureKitEnv.shared().contactsManager
    }

-    private static let groupThreadIndexer: SearchIndexer<TSGroupThread> = SearchIndexer { (groupThread: TSGroupThread) in
+    private static let groupThreadIndexer: SearchIndexer<TSGroupThread> = SearchIndexer { (groupThread: TSGroupThread, transaction: YapDatabaseReadTransaction) in
        let groupName = groupThread.groupModel.groupName ?? ""

        let memberStrings = groupThread.groupModel.groupMemberIds.map { recipientId in
-            recipientIndexer.index(recipientId)
+            recipientIndexer.index(recipientId, transaction: transaction)
        }.joined(separator: " ")

        return "\(groupName) \(memberStrings)"
    }

-    private static let contactThreadIndexer: SearchIndexer<TSContactThread> = SearchIndexer { (contactThread: TSContactThread) in
+    private static let contactThreadIndexer: SearchIndexer<TSContactThread> = SearchIndexer { (contactThread: TSContactThread, transaction: YapDatabaseReadTransaction) in
        let recipientId =  contactThread.contactIdentifier()
-        return recipientIndexer.index(recipientId)
+        return recipientIndexer.index(recipientId, transaction: transaction)
    }

-    private static let recipientIndexer: SearchIndexer<String> = SearchIndexer { (recipientId: String) in
+    private static let recipientIndexer: SearchIndexer<String> = SearchIndexer { (recipientId: String, _: YapDatabaseReadTransaction) in
        let displayName = contactsManager.displayName(forPhoneIdentifier: recipientId)

        let nationalNumber: String = { (recipientId: String) -> String in
@ -193,46 +193,46 @@ public class FullTextSearchFinder: NSObject {
        return "\(recipientId) \(nationalNumber) \(displayName)"
    }

-    private static let messageIndexer: SearchIndexer<TSMessage> = SearchIndexer { (message: TSMessage) in
+    private static let messageIndexer: SearchIndexer<TSMessage> = SearchIndexer { (message: TSMessage, transaction: YapDatabaseReadTransaction) in
        if let body = message.body, body.count > 0 {
            return body
        }
-        if let oversizeText = oversizeText(forMessage: message) {
+        if let oversizeText = oversizeText(forMessage: message, transaction: transaction) {
            return oversizeText
        }
        return ""
    }

-    private static func oversizeText(forMessage message: TSMessage) -> String? {
+    private static func oversizeText(forMessage message: TSMessage, transaction: YapDatabaseReadTransaction) -> String? {
        guard message.hasAttachments() else {
            return nil
        }
-        let dbConnection = OWSPrimaryStorage.shared().dbReadConnection
-        var oversizeText: String?
-        dbConnection.read({ (transaction) in
-            guard let attachment = message.attachment(with: transaction) else {
-                // This can happen during the initial save of incoming messages.
-                Logger.warn("Could not load attachment for search indexing.")
-                return
-            }
-            guard let attachmentStream = attachment as? TSAttachmentStream else {
-                return
-            }
-            guard attachmentStream.isOversizeText() else {
-                return
-            }
-            guard let text = attachmentStream.readOversizeText() else {
-                owsFail("Could not load oversize text attachment")
-                return
-            }
-            oversizeText = text
-        })
-        return oversizeText
+
+        guard let attachment = message.attachment(with: transaction) else {
+            // This can happen during the initial save of incoming messages.
+            Logger.warn("Could not load attachment for search indexing.")
+            return nil
+        }
+
+        guard let attachmentStream = attachment as? TSAttachmentStream else {
+            return nil
+        }
+
+        guard attachmentStream.isOversizeText() else {
+            return nil
+        }
+
+        guard let text = attachmentStream.readOversizeText() else {
+            owsFail("Could not load oversize text attachment")
+            return nil
+        }
+
+        return text
    }

-    private class func indexContent(object: Any) -> String? {
+    private class func indexContent(object: Any, transaction: YapDatabaseReadTransaction) -> String? {
        if let groupThread = object as? TSGroupThread {
-            return self.groupThreadIndexer.index(groupThread)
+            return self.groupThreadIndexer.index(groupThread, transaction: transaction)
        } else if let contactThread = object as? TSContactThread {
            guard contactThread.hasEverHadMessage else {
                // If we've never sent/received a message in a TSContactThread,
@ -240,11 +240,11 @@ public class FullTextSearchFinder: NSObject {
                // than in the "Conversations" section.
                return nil
            }
-            return self.contactThreadIndexer.index(contactThread)
+            return self.contactThreadIndexer.index(contactThread, transaction: transaction)
        } else if let message = object as? TSMessage {
-            return self.messageIndexer.index(message)
+            return self.messageIndexer.index(message, transaction: transaction)
        } else if let signalAccount = object as? SignalAccount {
-            return self.recipientIndexer.index(signalAccount.recipientId)
+            return self.recipientIndexer.index(signalAccount.recipientId, transaction: transaction)
        } else {
            return nil
        }
@ -277,8 +277,8 @@ public class FullTextSearchFinder: NSObject {

        let contentColumnName = "content"

-        let handler = YapDatabaseFullTextSearchHandler.withObjectBlock { (dict: NSMutableDictionary, _: String, _: String, object: Any) in
-            if let content: String = indexContent(object: object) {
+        let handler = YapDatabaseFullTextSearchHandler.withObjectBlock { (transaction: YapDatabaseReadTransaction, dict: NSMutableDictionary, _: String, _: String, object: Any) in
+            if let content: String = indexContent(object: object, transaction: transaction) {
                dict[contentColumnName] = content
            }
        }