Gallery performance

- [x] share uiDatabaseConnection to share cache
- [x] increase cache size
- [x] load less initially
- [x] lazy loading
  - [x] slider view
  - [x] tile view

// FREEBIE

Signal.xcodeproj/project.pbxproj

@@ -272,6 +272,7 @@
 		452D1EE81DCA90D100A57EC4 /* MesssagesBubblesSizeCalculatorTest.swift in Sources */ = {isa = PBXBuildFile; fileRef = 452D1EE71DCA90D100A57EC4 /* MesssagesBubblesSizeCalculatorTest.swift */; };
 		452EA09E1EA7ABE00078744B /* AttachmentPointerView.swift in Sources */ = {isa = PBXBuildFile; fileRef = 452EA09D1EA7ABE00078744B /* AttachmentPointerView.swift */; };
 		452EC6DF205E9E30000E787C /* MediaGalleryViewController.swift in Sources */ = {isa = PBXBuildFile; fileRef = 452EC6DE205E9E30000E787C /* MediaGalleryViewController.swift */; };
+		452EC6E1205FF5DC000E787C /* Bench.swift in Sources */ = {isa = PBXBuildFile; fileRef = 452EC6E0205FF5DC000E787C /* Bench.swift */; };
 		452ECA4D1E087E7200E2F016 /* MessageFetcherJob.swift in Sources */ = {isa = PBXBuildFile; fileRef = 452ECA4C1E087E7200E2F016 /* MessageFetcherJob.swift */; };
 		4535186B1FC635DD00210559 /* ShareViewController.swift in Sources */ = {isa = PBXBuildFile; fileRef = 4535186A1FC635DD00210559 /* ShareViewController.swift */; };
 		4535186E1FC635DD00210559 /* MainInterface.storyboard in Resources */ = {isa = PBXBuildFile; fileRef = 4535186C1FC635DD00210559 /* MainInterface.storyboard */; };
@@ -856,6 +857,7 @@
 		452D1EE71DCA90D100A57EC4 /* MesssagesBubblesSizeCalculatorTest.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; name = MesssagesBubblesSizeCalculatorTest.swift; path = Models/MesssagesBubblesSizeCalculatorTest.swift; sourceTree = "<group>"; };
 		452EA09D1EA7ABE00078744B /* AttachmentPointerView.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = AttachmentPointerView.swift; sourceTree = "<group>"; };
 		452EC6DE205E9E30000E787C /* MediaGalleryViewController.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = MediaGalleryViewController.swift; sourceTree = "<group>"; };
+		452EC6E0205FF5DC000E787C /* Bench.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = Bench.swift; sourceTree = "<group>"; };
 		452ECA4C1E087E7200E2F016 /* MessageFetcherJob.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = MessageFetcherJob.swift; sourceTree = "<group>"; };
 		453034AA200289F50018945D /* VideoPlayerView.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = VideoPlayerView.swift; sourceTree = "<group>"; };
 		453518681FC635DD00210559 /* SignalShareExtension.appex */ = {isa = PBXFileReference; explicitFileType = "wrapper.app-extension"; includeInIndex = 0; path = SignalShareExtension.appex; sourceTree = BUILT_PRODUCTS_DIR; };
@@ -1326,6 +1328,7 @@
 		34480B471FD0A60200BC14EF /* utils */ = {
 			isa = PBXGroup;
 			children = (
+				452EC6E0205FF5DC000E787C /* Bench.swift */,
 				343D3D991E9283F100165CA4 /* BlockListUIUtils.h */,
 				343D3D9A1E9283F100165CA4 /* BlockListUIUtils.m */,
 				451777C71FD61554001225FF /* ConversationSearcher.swift */,
@@ -3012,6 +3015,7 @@
 				34074F61203D0CBE004596AE /* OWSSounds.m in Sources */,
 				346129B51FD1F7E800532771 /* OWSProfileManager.m in Sources */,
 				346129701FD1D74C00532771 /* Release.m in Sources */,
+				452EC6E1205FF5DC000E787C /* Bench.swift in Sources */,
 				3478506C1FD9B78A007B8332 /* NoopNotificationsManager.swift in Sources */,
 				34480B621FD0A98800BC14EF /* UIColor+OWS.m in Sources */,
 				34480B531FD0A7A400BC14EF /* OWSLogger.m in Sources */,

Signal/src/ViewControllers/ConversationView/ConversationViewController.m

@@ -2033,8 +2033,9 @@ typedef enum : NSUInteger {
     }
     TSMessage *mediaMessage = (TSMessage *)viewItem.interaction;
 
-    MediaGalleryViewController *vc =
+    MediaGalleryViewController *vc = [[MediaGalleryViewController alloc] initWithThread:self.thread
-        [[MediaGalleryViewController alloc] initWithThread:self.thread mediaMessage:mediaMessage];
+                                                                           mediaMessage:mediaMessage
+                                                                   uiDatabaseConnection:self.uiDatabaseConnection];
 
     [vc presentDetailViewFromViewController:self replacingView:imageView];
 }
@@ -2055,8 +2056,9 @@ typedef enum : NSUInteger {
     }
     TSMessage *mediaMessage = (TSMessage *)viewItem.interaction;
 
-    MediaGalleryViewController *vc =
+    MediaGalleryViewController *vc = [[MediaGalleryViewController alloc] initWithThread:self.thread
-        [[MediaGalleryViewController alloc] initWithThread:self.thread mediaMessage:mediaMessage];
+                                                                           mediaMessage:mediaMessage
+                                                                   uiDatabaseConnection:self.uiDatabaseConnection];
 
     [vc presentDetailViewFromViewController:self replacingView:imageView];
 }
@@ -2807,6 +2809,8 @@ typedef enum : NSUInteger {
     NSAssert([NSThread isMainThread], @"Must access uiDatabaseConnection on main thread!");
     if (!_uiDatabaseConnection) {
         _uiDatabaseConnection = [self.primaryStorage newDatabaseConnection];
+        // Increase object cache limit. Default is 250.
+        _uiDatabaseConnection.objectCacheLimit = 500;
         [_uiDatabaseConnection beginLongLivedReadTransaction];
     }
     return _uiDatabaseConnection;

Signal/src/ViewControllers/MediaGalleryViewController.swift

@@ -4,10 +4,22 @@
 
 import Foundation
 
+public enum GalleryDirection {
+    case before, after, around
+}
+
 public struct MediaGalleryItem: Equatable {
+    let logTag = "[MediaGalleryItem]"
+
     let message: TSMessage
     let attachmentStream: TSAttachmentStream
-    let logTag = "[MediaGalleryItem]"
+    let galleryDate: GalleryDate
+
+    init(message: TSMessage, attachmentStream: TSAttachmentStream) {
+        self.message = message
+        self.attachmentStream = attachmentStream
+        self.galleryDate = GalleryDate(message: message)
+    }
 
     var isVideo: Bool {
         return attachmentStream.isVideo()
@@ -29,7 +41,7 @@ public struct MediaGalleryItem: Equatable {
     }
 }
 
-public struct GalleryDate: Hashable {
+public struct GalleryDate: Hashable, Comparable, Equatable {
     let year: Int
     let month: Int
 
@@ -41,6 +53,8 @@ public struct GalleryDate: Hashable {
     }
 
     init(year: Int, month: Int) {
+        assert(month >= 1 && month <= 12)
+
         self.year = year
         self.month = month
     }
@@ -101,20 +115,71 @@ public struct GalleryDate: Hashable {
         return month.hashValue ^ year.hashValue
     }
 
+    // Mark: Comparable
+
+    public static func < (lhs: GalleryDate, rhs: GalleryDate) -> Bool {
+        if lhs.year != rhs.year {
+            return lhs.year < rhs.year
+        } else if lhs.month != rhs.month {
+            return lhs.month < rhs.month
+        } else {
+            return false
+        }
+    }
+
     // MARK: Equatable
 
     public static func == (lhs: GalleryDate, rhs: GalleryDate) -> Bool {
         return lhs.month == rhs.month && lhs.year == rhs.year
     }
+
+//    // MARK: Sequence / IteratorProtocol
+//    public func until(_ toDate: GalleryDate) -> GalleryDateSequence {
+//        return GalleryDateSequence(from: self, to: toDate)
+//    }
+//
+//    public class GalleryDateSequence: Sequence, IteratorProtocol {
+//        public typealias Element = GalleryDate
+//
+//        var currentDate: GalleryDate
+//        let toDate: GalleryDate
+//
+//        init(from: GalleryDate, to: GalleryDate) {
+//            self.currentDate = from
+//            self.toDate = to
+//        }
+//
+//        public func next() -> GalleryDate? {
+//            guard currentDate < toDate else {
+//                return nil
+//            }
+//
+//            let nextDate: GalleryDate = {
+//                if currentDate.month == 12 {
+//                    return GalleryDate(year: currentDate.year + 1, month: 1)
+//                } else {
+//                    return GalleryDate(year: currentDate.year, month: currentDate.month + 1)
+//                }
+//            }()
+//            currentDate = nextDate
+//            return nextDate
+//        }
+//    }
+
 }
 
 protocol MediaGalleryDataSource: class {
+    var hasFetchedOldest: Bool { get }
+    var hasFetchedMostRecent: Bool { get }
+
     var galleryItems: [MediaGalleryItem] { get }
     var galleryItemCount: Int { get }
 
     var sections: [GalleryDate: [MediaGalleryItem]] { get }
     var sectionDates: [GalleryDate] { get }
 
+    func ensureGalleryItemsLoaded(_ direction: GalleryDirection, item: MediaGalleryItem, amount: UInt, completion: ((IndexSet, [IndexPath]) -> Void)?)
+
     func galleryItem(before currentItem: MediaGalleryItem) -> MediaGalleryItem?
     func galleryItem(after currentItem: MediaGalleryItem) -> MediaGalleryItem?
 
@@ -137,26 +202,29 @@ class MediaGalleryViewController: UINavigationController, MediaGalleryDataSource
     private let thread: TSThread
     private let includeGallery: Bool
 
-    convenience init(thread: TSThread, mediaMessage: TSMessage) {
+    // we start with a small range size for quick loading.
-        self.init(thread: thread, mediaMessage: mediaMessage, includeGallery: true)
+    private let fetchRangeSize: UInt = 10
+
+    convenience init(thread: TSThread, mediaMessage: TSMessage, uiDatabaseConnection: YapDatabaseConnection) {
+        self.init(thread: thread, mediaMessage: mediaMessage, uiDatabaseConnection: uiDatabaseConnection, includeGallery: true)
     }
 
-    init(thread: TSThread, mediaMessage: TSMessage, includeGallery: Bool) {
+    init(thread: TSThread, mediaMessage: TSMessage, uiDatabaseConnection: YapDatabaseConnection, includeGallery: Bool) {
         self.thread = thread
+        assert(uiDatabaseConnection.isInLongLivedReadTransaction())
+        self.uiDatabaseConnection = uiDatabaseConnection
         self.includeGallery = includeGallery
-
+        self.mediaGalleryFinder = OWSMediaGalleryFinder(thread: thread)
-        self.uiDatabaseConnection = OWSPrimaryStorage.shared().newDatabaseConnection()
-        self.mediaGalleryFinder = OWSMediaGalleryFinder()
 
         super.init(nibName: nil, bundle: nil)
 
-        uiDatabaseConnection.beginLongLivedReadTransaction()
-
         uiDatabaseConnection.read { transaction in
             self.initialGalleryItem = self.buildGalleryItem(message: mediaMessage, transaction: transaction)!
         }
 
-        updateGalleryItems(thread: thread)
+        // For a speedy load, we only fetch a few items on either side of
+        // the initial message
+        ensureGalleryItemsLoaded(.around, item: self.initialGalleryItem, amount: 10)
     }
 
     required init?(coder aDecoder: NSCoder) {
@@ -417,6 +485,9 @@ class MediaGalleryViewController: UINavigationController, MediaGalleryDataSource
     // MARK: MediaGalleryDataSource
 
     func showAllMedia() {
+
+        ensureGalleryItemsLoaded(.around, item: self.initialGalleryItem, amount: 100)
+
         // TODO fancy animation - zoom media item into it's tile in the all media grid
         let allMediaController = MediaTileViewController(mediaGalleryDataSource: self, uiDatabaseConnection: self.uiDatabaseConnection)
         allMediaController.delegate = self
@@ -427,6 +498,8 @@ class MediaGalleryViewController: UINavigationController, MediaGalleryDataSource
     var galleryItems: [MediaGalleryItem] = []
     var sections: [GalleryDate: [MediaGalleryItem]] = [:]
     var sectionDates: [GalleryDate] = []
+    var hasFetchedOldest = false
+    var hasFetchedMostRecent = false
 
     func buildGalleryItem(message: TSMessage, transaction: YapDatabaseReadTransaction) -> MediaGalleryItem? {
         guard let attachmentStream = message.attachment(with: transaction) as? TSAttachmentStream else {
@@ -437,42 +510,149 @@ class MediaGalleryViewController: UINavigationController, MediaGalleryDataSource
         return MediaGalleryItem(message: message, attachmentStream: attachmentStream)
     }
 
-    func updateGalleryItems(thread: TSThread) {
+    // Range instead of indexSet since it's contiguous?
-        var galleryItems: [MediaGalleryItem] = []
+    var fetchedIndexSet = IndexSet() {
-        var sections: [GalleryDate: [MediaGalleryItem]] = [:]
+        didSet {
-        var sectionDates: [GalleryDate] = []
+            Logger.debug("\(logTag) in \(#function) \(oldValue) -> \(fetchedIndexSet)")
+        }
+    }
+
+    func ensureGalleryItemsLoaded(_ direction: GalleryDirection, item: MediaGalleryItem, amount: UInt, completion: ((IndexSet, [IndexPath]) -> Void)? = nil ) {
+
+        // TODO avoid copy?
+        // TODO read off main thread?
+        var galleryItems: [MediaGalleryItem] = self.galleryItems
+        var sections: [GalleryDate: [MediaGalleryItem]] = self.sections
+        var sectionDates: [GalleryDate] = self.sectionDates
+
+        var newGalleryItems: [MediaGalleryItem] = []
+        var newDates: [GalleryDate] = []
+
+        Bench(title: "fetching gallery items") {
+            self.uiDatabaseConnection.read { transaction in
+
+                let initialIndex: Int = Int(self.mediaGalleryFinder.mediaIndex(message: item.message, transaction: transaction))
+                let mediaCount: Int = Int(self.mediaGalleryFinder.mediaCount(transaction: transaction))
+
+                let requestRange: Range<Int> = { () -> Range<Int> in
+                    let range: Range<Int> = { () -> Range<Int> in
+                        switch direction {
+                        case .around:
+                            // To keep it simple, this isn't exactly *amount* sized if `message` window overlaps the end or
+                            // beginning of the view. Still, we have sufficient buffer to fetch more as the user swipes.
+                            let start: Int = initialIndex - Int(amount) / 2
+                            let end: Int = initialIndex + Int(amount) / 2
+
+                            return start..<end
+                        case .before:
+                            let start: Int = initialIndex - Int(amount)
+                            let end: Int = initialIndex
+
+                            return start..<end
+                        case  .after:
+                            let start: Int = initialIndex
+                            let end: Int = initialIndex  + Int(amount)
 
-        self.uiDatabaseConnection.read { transaction in
+                            return start..<end
-            self.mediaGalleryFinder.enumerateMediaMessages(with: thread, transaction: transaction) { (message: TSMessage) in
+                        }
+                    }()
 
-                guard let item: MediaGalleryItem = self.buildGalleryItem(message: message, transaction: transaction) else {
+                    return range.clamped(to: 0..<mediaCount)
-                    owsFail("\(self.logTag) in \(#function) unexpectedly failed to buildGalleryItem")
+                }()
+
+                let requestSet = IndexSet(integersIn: requestRange)
+                guard !self.fetchedIndexSet.contains(integersIn: requestSet) else {
+                    Logger.debug("\(self.logTag) in \(#function) all requested messages have already been loaded.")
                     return
                 }
 
-                let date = GalleryDate(message: message)
+                let unfetchedSet = requestSet.subtracting(self.fetchedIndexSet)
+                guard unfetchedSet.count > (requestSet.count / 2) else {
+                    // For perf we only want to fetch a relatively full batch, unless the requestSet is very small.
+                    Logger.debug("\(self.logTag) in \(#function) ignoring small fetch request: \(unfetchedSet.count)")
+                    return
+                }
 
-                // TODO do we need to box this for reasonable perf?
+                Logger.debug("\(self.logTag) in \(#function) fetching set: \(unfetchedSet)")
-                galleryItems.append(item)
+                let nsRange: NSRange = NSRange(location: unfetchedSet.min()!, length: unfetchedSet.count)
-                if sections[date] != nil {
+                self.mediaGalleryFinder.enumerateMediaMessages(range: nsRange, transaction: transaction) { (message: TSMessage) in
-                    // TODO do we need to box this for reasonable perf?
+                    guard let item: MediaGalleryItem = self.buildGalleryItem(message: message, transaction: transaction) else {
-                    sections[date]!.append(item)
+                        owsFail("\(self.logTag) in \(#function) unexpectedly failed to buildGalleryItem")
-                } else {
+                        return
-                    sectionDates.append(date)
+                    }
-                    sections[date] = [item]
+
+                    let date = item.galleryDate
+
+                    galleryItems.append(item)
+                    if sections[date] != nil {
+                        sections[date]!.append(item)
+
+                        // so we can update collectionView
+                        newGalleryItems.append(item)
+                    } else {
+                        sectionDates.append(date)
+                        sections[date] = [item]
+
+                        // so we can update collectionView
+                        newDates.append(date)
+                        newGalleryItems.append(item)
+                    }
+                }
+
+                self.fetchedIndexSet = self.fetchedIndexSet.union(unfetchedSet)
+                self.hasFetchedOldest = self.fetchedIndexSet.min() == 0
+                self.hasFetchedMostRecent = self.fetchedIndexSet.max() == mediaCount - 1
+            }
+        }
+
+        // TODO only sort if changed
+        var sortedSections: [GalleryDate: [MediaGalleryItem]] = [:]
+
+        Bench(title: "sorting gallery items") {
+            galleryItems.sort { lhs, rhs -> Bool in
+                return lhs.message.timestampForSorting() < rhs.message.timestampForSorting()
+            }
+            sectionDates.sort()
+
+            for (date, galleryItems) in sections {
+                sortedSections[date] = galleryItems.sorted { lhs, rhs -> Bool in
+                    return lhs.message.timestampForSorting() < rhs.message.timestampForSorting()
                 }
             }
         }
 
         self.galleryItems = galleryItems
-        self.sections = sections
+        self.sections = sortedSections
         self.sectionDates = sectionDates
+
+        if let completionBlock = completion {
+            Bench(title: "calculating changes for collectionView") {
+                // FIXME can we avoid this index offset?
+                let dateIndices = newDates.map { sectionDates.index(of: $0)! + 1 }
+                let addedSections: IndexSet = IndexSet(dateIndices)
+
+                let addedItems: [IndexPath] = newGalleryItems.map { galleryItem in
+                    let sectionIdx = sectionDates.index(of: galleryItem.galleryDate)!
+                    let section = sections[galleryItem.galleryDate]!
+                    let itemIdx = section.index(of: galleryItem)!
+
+                    // FIXME can we avoid this index offset?
+                    return IndexPath(item: itemIdx, section: sectionIdx + 1)
+                }
+
+                completionBlock(addedSections, addedItems)
+            }
+        }
     }
 
+    let kGallerySwipeLoadBatchSize: UInt = 5
+
     // TODO extract to public extension?
     internal func galleryItem(after currentItem: MediaGalleryItem) -> MediaGalleryItem? {
         Logger.debug("\(logTag) in \(#function)")
 
+        self.ensureGalleryItemsLoaded(.after, item: currentItem, amount: kGallerySwipeLoadBatchSize)
+
         guard let currentIndex = galleryItems.index(of: currentItem) else {
             owsFail("currentIndex was unexpectedly nil in \(#function)")
             return nil
@@ -485,6 +665,8 @@ class MediaGalleryViewController: UINavigationController, MediaGalleryDataSource
     internal func galleryItem(before currentItem: MediaGalleryItem) -> MediaGalleryItem? {
         Logger.debug("\(logTag) in \(#function)")
 
+        self.ensureGalleryItemsLoaded(.before, item: currentItem, amount: kGallerySwipeLoadBatchSize)
+
         guard let currentIndex = galleryItems.index(of: currentItem) else {
             owsFail("currentIndex was unexpectedly nil in \(#function)")
             return nil
@@ -497,7 +679,7 @@ class MediaGalleryViewController: UINavigationController, MediaGalleryDataSource
     var galleryItemCount: Int {
         var count: UInt = 0
         self.uiDatabaseConnection.read { (transaction: YapDatabaseReadTransaction) in
-            count = self.mediaGalleryFinder.mediaCount(thread: self.thread, transaction: transaction)
+            count = self.mediaGalleryFinder.mediaCount(transaction: transaction)
         }
         return Int(count)
     }

Signal/src/ViewControllers/MediaPageViewController.swift

@@ -72,7 +72,7 @@ class MediaPageViewController: UIPageViewController, UIPageViewControllerDataSou
     }
 
     init(initialItem: MediaGalleryItem, mediaGalleryDataSource: MediaGalleryDataSource, uiDatabaseConnection: YapDatabaseConnection, includeGallery: Bool) {
-        // TODO move responsibility to MediaGalleryVC?
+        assert(uiDatabaseConnection.isInLongLivedReadTransaction())
         self.uiDatabaseConnection = uiDatabaseConnection
         self.includeGallery = includeGallery
         self.mediaGalleryDataSource = mediaGalleryDataSource

Signal/src/ViewControllers/MediaTileViewController.swift

@@ -13,10 +13,10 @@ public class MediaTileViewController: UICollectionViewController, MediaGalleryCe
     // TODO weak?
     private var mediaGalleryDataSource: MediaGalleryDataSource
 
-    private var sections: [GalleryDate: [MediaGalleryItem]] {
+    private var galleryItems: [GalleryDate: [MediaGalleryItem]] {
         return mediaGalleryDataSource.sections
     }
-    private var sectionDates: [GalleryDate] {
+    private var galleryDates: [GalleryDate] {
         return mediaGalleryDataSource.sectionDates
     }
 
@@ -24,12 +24,10 @@ public class MediaTileViewController: UICollectionViewController, MediaGalleryCe
 
     public weak var delegate: MediaTileViewControllerDelegate?
 
-    let kSectionHeaderReuseIdentifier = "kSectionHeaderReuseIdentifier"
-    let kCellReuseIdentifier = "kCellReuseIdentifier"
-
     init(mediaGalleryDataSource: MediaGalleryDataSource, uiDatabaseConnection: YapDatabaseConnection) {
 
         self.mediaGalleryDataSource = mediaGalleryDataSource
+        assert(uiDatabaseConnection.isInLongLivedReadTransaction())
         self.uiDatabaseConnection = uiDatabaseConnection
 
         // Layout Setup
@@ -48,12 +46,7 @@ public class MediaTileViewController: UICollectionViewController, MediaGalleryCe
         layout.minimumLineSpacing = kInterItemSpacing
         layout.sectionHeadersPinToVisibleBounds = true
 
-        let kHeaderHeight: CGFloat = 50
-        layout.headerReferenceSize = CGSize(width: 0, height: kHeaderHeight)
-
         super.init(collectionViewLayout: layout)
-
-        updateSections()
     }
 
     required public init?(coder aDecoder: NSCoder) {
@@ -71,39 +64,70 @@ public class MediaTileViewController: UICollectionViewController, MediaGalleryCe
             owsFail("\(logTag) in \(#function) collectionView was unexpectedly nil")
             return
         }
+
         collectionView.backgroundColor = UIColor.white
-        collectionView.register(MediaGalleryCell.self, forCellWithReuseIdentifier: kCellReuseIdentifier)
-        collectionView.register(MediaGallerySectionHeader.self, forSupplementaryViewOfKind: UICollectionElementKindSectionHeader, withReuseIdentifier: kSectionHeaderReuseIdentifier)
 
+        collectionView.register(MediaGalleryCell.self, forCellWithReuseIdentifier: MediaGalleryCell.reuseIdentifier)
+        collectionView.register(MediaGallerySectionHeader.self, forSupplementaryViewOfKind: UICollectionElementKindSectionHeader, withReuseIdentifier: MediaGallerySectionHeader.reuseIdentifier)
+        collectionView.register(MediaGalleryLoadingHeader.self, forSupplementaryViewOfKind: UICollectionElementKindSectionHeader, withReuseIdentifier: MediaGalleryLoadingHeader.reuseIdentifier)
+
+        collectionView.delegate = self
+
+        // TODO iPhoneX
         // feels a bit weird to have content smashed all the way to the bottom edge.
         collectionView.contentInset = UIEdgeInsets(top: 0, left: 0, bottom: 20, right: 0)
 
-        // FIXME: For some reason this is scrolling not *quite* to the bottom in viewDidLoad.
-        // It does work in viewDidAppear. What changes?
         self.view.layoutIfNeeded()
         scrollToBottom(animated: false)
     }
 
+    // MARK: UIColletionViewDelegate
+
+    override public func scrollViewDidScroll(_ scrollView: UIScrollView) {
+        self.autoLoadMoreIfNecessary()
+    }
+
+    override public func scrollViewWillBeginDragging(_ scrollView: UIScrollView) {
+        self.isUserScrolling = true
+    }
+
+    override public func scrollViewDidEndDragging(_ scrollView: UIScrollView, willDecelerate decelerate: Bool) {
+        self.isUserScrolling = false
+    }
+
+    private var isUserScrolling: Bool = false {
+        didSet {
+            autoLoadMoreIfNecessary()
+        }
+    }
+
     // MARK: UIColletionViewDataSource
 
     override public func numberOfSections(in collectionView: UICollectionView) -> Int {
-        return sections.keys.count
+        return galleryItems.keys.count + 2
     }
 
     override public func collectionView(_ collectionView: UICollectionView, numberOfItemsInSection sectionIdx: Int) -> Int {
-        guard let sectionDate = self.sectionDates[safe: sectionIdx] else {
+        if sectionIdx == kLoadOlderSectionIdx {
+            // load older
+            return 0
+        }
+
+        if sectionIdx == kLoadNewerSectionIdx {
+            // load more recent
+            return 0
+        }
+
+        guard let sectionDate = self.galleryDates[safe: sectionIdx - 1] else {
             owsFail("\(logTag) in \(#function) unknown section: \(sectionIdx)")
             return 0
         }
 
-        guard let section = self.sections[sectionDate] else {
+        guard let section = self.galleryItems[sectionDate] else {
             owsFail("\(logTag) in \(#function) no section for date: \(sectionDate)")
             return 0
         }
 
-        // We shouldn't show empty sections
-        assert(section.count > 0)
-
         return section.count
     }
 
@@ -111,51 +135,101 @@ public class MediaTileViewController: UICollectionViewController, MediaGalleryCe
 
         let defaultView = UICollectionReusableView()
         if (kind == UICollectionElementKindSectionHeader) {
-            guard let sectionHeader = collectionView.dequeueReusableSupplementaryView(ofKind: kind, withReuseIdentifier: kSectionHeaderReuseIdentifier, for: indexPath) as? MediaGallerySectionHeader else {
+            switch indexPath.section {
-                owsFail("\(logTag) in \(#function) unable to build section header for indexPath: \(indexPath)")
+            case kLoadOlderSectionIdx:
-                return defaultView
+                guard let sectionHeader = collectionView.dequeueReusableSupplementaryView(ofKind: kind, withReuseIdentifier: MediaGalleryLoadingHeader.reuseIdentifier, for: indexPath) as? MediaGalleryLoadingHeader else {
+
+                    owsFail("\(logTag) in \(#function) unable to build section header for kLoadOlderSectionIdx")
+                    return defaultView
+                }
+                // TODO localize
+                sectionHeader.configure(title: "Loading older...")
+                return sectionHeader
+            case kLoadNewerSectionIdx:
+                guard let sectionHeader = collectionView.dequeueReusableSupplementaryView(ofKind: kind, withReuseIdentifier: MediaGalleryLoadingHeader.reuseIdentifier, for: indexPath) as? MediaGalleryLoadingHeader else {
+
+                    owsFail("\(logTag) in \(#function) unable to build section header for kLoadOlderSectionIdx")
+                    return defaultView
+                }
+                // TODO localize
+                sectionHeader.configure(title: "Loading newer...")
+                return sectionHeader
+            default:
+                guard let sectionHeader = collectionView.dequeueReusableSupplementaryView(ofKind: kind, withReuseIdentifier: MediaGallerySectionHeader.reuseIdentifier, for: indexPath) as? MediaGallerySectionHeader else {
+                    owsFail("\(logTag) in \(#function) unable to build section header for indexPath: \(indexPath)")
+                    return defaultView
+                }
+                guard let date = self.galleryDates[safe: indexPath.section - 1] else {
+                    owsFail("\(logTag) in \(#function) unknown section for indexPath: \(indexPath)")
+                    return defaultView
+                }
+
+                sectionHeader.configure(title: date.localizedString)
+                return sectionHeader
             }
-            guard let date = self.sectionDates[safe: indexPath.section] else {
-                owsFail("\(logTag) in \(#function) unknown section for indexPath: \(indexPath)")
-                return defaultView
-            }
-
-            sectionHeader.configure(title: date.localizedString)
-            return sectionHeader
         }
 
         return defaultView
     }
 
     override public func collectionView(_ collectionView: UICollectionView, cellForItemAt indexPath: IndexPath) -> UICollectionViewCell {
+        Logger.debug("\(logTag) in \(#function) indexPath: \(indexPath)")
 
         let defaultCell = UICollectionViewCell()
 
-        guard let sectionDate = self.sectionDates[safe: indexPath.section] else {
+        switch indexPath.section {
-            owsFail("\(logTag) in \(#function) unknown section: \(indexPath.section)")
+        case kLoadOlderSectionIdx:
+            owsFail("\(logTag) in \(#function) unexpected cell for kLoadOlderSectionIdx")
             return defaultCell
-        }
+        case kLoadNewerSectionIdx:
-
+            owsFail("\(logTag) in \(#function) unexpected cell for kLoadNewerSectionIdx")
-        guard let section = self.sections[sectionDate] else {
-            owsFail("\(logTag) in \(#function) no section for date: \(sectionDate)")
             return defaultCell
-        }
+        default:
+            guard let sectionDate = self.galleryDates[safe: indexPath.section - 1] else {
+                owsFail("\(logTag) in \(#function) unknown section: \(indexPath.section)")
+                return defaultCell
+            }
 
-        guard let galleryItem = section[safe: indexPath.row] else {
+            guard let sectionItems = self.galleryItems[sectionDate] else {
-            owsFail("\(logTag) in \(#function) no message for row: \(indexPath.row)")
+                owsFail("\(logTag) in \(#function) no section for date: \(sectionDate)")
-            return defaultCell
+                return defaultCell
-        }
+            }
 
-        guard let cell = self.collectionView?.dequeueReusableCell(withReuseIdentifier: kCellReuseIdentifier, for: indexPath) as? MediaGalleryCell else {
+            guard let galleryItem = sectionItems[safe: indexPath.row] else {
-            owsFail("\(logTag) in \(#function) unexptected cell for indexPath: \(indexPath)")
+                owsFail("\(logTag) in \(#function) no message for row: \(indexPath.row)")
-            return defaultCell
+                return defaultCell
-        }
+            }
+
+            guard let cell = self.collectionView?.dequeueReusableCell(withReuseIdentifier: MediaGalleryCell.reuseIdentifier, for: indexPath) as? MediaGalleryCell else {
+                owsFail("\(logTag) in \(#function) unexpected cell for indexPath: \(indexPath)")
+                return defaultCell
+            }
 
-        cell.configure(item: galleryItem, delegate: self)
+            cell.configure(item: galleryItem, delegate: self)
 
-        return cell
+            return cell
+        }
     }
 
+    // MARK: UICollectionViewDelegateFlowLayout
+
+    public func collectionView(_ collectionView: UICollectionView,
+                               layout collectionViewLayout: UICollectionViewLayout,
+                               referenceSizeForHeaderInSection section: Int) -> CGSize {
+
+        let kHeaderHeight: CGFloat = 50
+
+        switch section {
+        case kLoadOlderSectionIdx:
+            // Show "loading older..." iff there is still older data to be fetched
+            return self.mediaGalleryDataSource.hasFetchedOldest ? CGSize.zero : CGSize(width: 0, height: 100)
+        case kLoadNewerSectionIdx:
+            // Show "loading newer..." iff there is still more recent data to be fetched
+            return self.mediaGalleryDataSource.hasFetchedMostRecent ? CGSize.zero : CGSize(width: 0, height: 100)
+        default:
+            return CGSize(width: 0, height: kHeaderHeight)
+        }
+    }
     // MARK: MediaGalleryDelegate
 
     public func didTapCell(_ cell: MediaGalleryCell, item: MediaGalleryItem) {
@@ -163,6 +237,115 @@ public class MediaTileViewController: UICollectionViewController, MediaGalleryCe
         self.delegate?.mediaTileViewController(self, didTapMediaGalleryItem: item)
     }
 
+    // MARK: Lazy Loading
+
+    // This should be substantially larger than one screen size so we don't have to call it
+    // multiple times in a rapid succession.
+    let kMediaTileViewLoadBatchSize: UInt = 200
+    var oldestLoadedItem: MediaGalleryItem? {
+        guard let oldestDate = galleryDates.first else {
+            return nil
+        }
+
+        return galleryItems[oldestDate]?.first
+    }
+
+    var mostRecentLoadedItem: MediaGalleryItem? {
+        guard let mostRecentDate = galleryDates.last else {
+            return nil
+        }
+
+        return galleryItems[mostRecentDate]?.last
+    }
+
+    var isFetchingMoreData: Bool = false
+
+    let kLoadOlderSectionIdx = 0
+    var kLoadNewerSectionIdx: Int {
+        return galleryDates.count + 1
+    }
+
+    public func autoLoadMoreIfNecessary() {
+        let kEdgeThreshold: CGFloat = 800
+
+        if (self.isUserScrolling) {
+            return
+        }
+
+        guard let collectionView = self.collectionView else {
+            owsFail("\(logTag) in \(#function) collectionView was unexpectedly nil")
+            return
+        }
+
+        let contentOffsetY = collectionView.contentOffset.y
+        let oldContentHeight = collectionView.contentSize.height
+
+        if contentOffsetY < kEdgeThreshold {
+            // Near the top, load older content
+
+            guard let oldestLoadedItem = self.oldestLoadedItem else {
+                Logger.debug("\(logTag) in \(#function) no oldest item")
+                return
+            }
+
+            guard !isFetchingMoreData else {
+                Logger.debug("\(logTag) in \(#function) already fetching more data")
+                return
+            }
+
+            isFetchingMoreData = true
+
+            let scrollDistanceToBottom = oldContentHeight - contentOffsetY
+
+            collectionView.performBatchUpdates({
+                self.mediaGalleryDataSource.ensureGalleryItemsLoaded(.before, item: oldestLoadedItem, amount: self.kMediaTileViewLoadBatchSize) { addedSections, addedItems in
+                    Logger.debug("\(self.logTag) in \(#function) insertingSections: \(addedSections) items: \(addedItems)")
+
+                    collectionView.insertSections(addedSections)
+                    collectionView.insertItems(at: addedItems)
+                }
+            }, completion: { finished in
+
+                // Adjust content offset to affect change in content height so that the same content is visible after
+                // the update.
+                let newContentOffset = CGPoint(x: 0, y: collectionView.contentSize.height - scrollDistanceToBottom)
+                collectionView.setContentOffset(newContentOffset, animated: false)
+
+                Logger.debug("\(self.logTag) in \(#function) performBatchUpdates finished: \(finished)")
+                self.isFetchingMoreData = false
+            })
+        } else if oldContentHeight - contentOffsetY < kEdgeThreshold {
+            // Near the bottom, load newer content
+
+            guard let mostRecentLoadedItem = self.mostRecentLoadedItem else {
+                Logger.debug("\(logTag) in \(#function) no mostRecent item")
+                return
+            }
+
+            guard !isFetchingMoreData else {
+                Logger.debug("\(logTag) in \(#function) already fetching more data")
+                return
+            }
+
+            isFetchingMoreData = true
+            collectionView.performBatchUpdates({
+                self.mediaGalleryDataSource.ensureGalleryItemsLoaded(.after, item: mostRecentLoadedItem, amount: self.kMediaTileViewLoadBatchSize) { addedSections, addedItems in
+                    guard let collectionView = self.collectionView else {
+                        Logger.debug("\(self.logTag) in \(#function) collectionView was unexpectedly nil")
+                        return
+                    }
+                    Logger.debug("\(self.logTag) in \(#function) insertingSections: \(addedSections), items: \(addedItems)")
+
+                    collectionView.insertSections(addedSections)
+                    collectionView.insertItems(at: addedItems)
+                }
+            }, completion: { finished in
+                Logger.debug("\(self.logTag) in \(#function) performBatchUpdates finished: \(finished)")
+                self.isFetchingMoreData = false
+            })
+        }
+    }
+
     // MARK: Util
 
     private func scrollToBottom(animated isAnimated: Bool) {
@@ -176,16 +359,12 @@ public class MediaTileViewController: UICollectionViewController, MediaGalleryCe
 
         collectionView.setContentOffset(offset, animated: isAnimated)
     }
-
-    // TODO? dbModified? Is this even  necessary?
-    private func updateSections() {
-        self.collectionView?.reloadData()
-    }
-
 }
 
 class MediaGallerySectionHeader: UICollectionReusableView {
 
+    static let reuseIdentifier = "MediaGallerySectionHeader"
+
     // HACK: scrollbar incorrectly appears *behind* section headers
     // in collection view on iOS11 =(
     private class AlwaysOnTopLayer: CALayer {
@@ -248,8 +427,40 @@ public protocol MediaGalleryCellDelegate: class {
     func didTapCell(_ cell: MediaGalleryCell, item: MediaGalleryItem)
 }
 
+public class MediaGalleryLoadingHeader: UICollectionViewCell {
+
+    static let reuseIdentifier = "MediaGalleryLoadingHeader"
+
+    let label = UILabel()
+
+    override init(frame: CGRect) {
+        super.init(frame: frame)
+
+        // TODO add spinnner, start/stop animating on will/end display
+        self.backgroundColor = UIColor.green
+        addSubview(label)
+
+        label.autoCenterInSuperview()
+    }
+
+    @available(*, unavailable, message: "Unimplemented")
+    required public init?(coder aDecoder: NSCoder) {
+        fatalError("init(coder:) has not been implemented")
+    }
+
+    public func configure(title: String) {
+        self.label.text = title
+    }
+
+    public override func prepareForReuse() {
+        self.label.text = nil
+    }
+}
+
 public class MediaGalleryCell: UICollectionViewCell {
 
+    static let reuseIdentifier = "MediaGalleryCell"
+
     private let imageView: UIImageView
     private var tapGesture: UITapGestureRecognizer!
 

Signal/src/ViewControllers/MessageDetailViewController.swift

@@ -762,7 +762,7 @@ class MessageDetailViewController: OWSViewController, UIScrollViewDelegate, Medi
             return
         }
 
-        let mediaGalleryViewController = MediaGalleryViewController(thread: self.thread, mediaMessage: self.message, includeGallery: false)
+        let mediaGalleryViewController = MediaGalleryViewController(thread: self.thread, mediaMessage: self.message, uiDatabaseConnection: self.uiDatabaseConnection, includeGallery: false)
         mediaGalleryViewController.presentDetailView(fromViewController: self, replacingView: fromView)
     }
 }

SignalMessaging/utils/Bench.swift

@@ -0,0 +1,21 @@
+//
+//  Copyright (c) 2018 Open Whisper Systems. All rights reserved.
+//
+
+import Foundation
+
+public func BenchAsync(title: String, block: (() -> Void) -> Void) {
+    let startTime = CFAbsoluteTimeGetCurrent()
+
+    block {
+        let timeElapsed = CFAbsoluteTimeGetCurrent() - startTime
+        print("[Bench] title: \(title), duration: \(timeElapsed)")
+    }
+}
+
+public func Bench(title: String, block: () -> Void) {
+    BenchAsync(title: title) { finish in
+        block()
+        finish()
+    }
+}

SignalServiceKit/src/Storage/OWSMediaGalleryFinder.h

@@ -11,15 +11,22 @@ NS_ASSUME_NONNULL_BEGIN
 
 @interface OWSMediaGalleryFinder : NSObject
 
+- (instancetype)initWithThread:(TSThread *)thread NS_DESIGNATED_INITIALIZER;
+
 // How many media items a thread has
-- (NSUInteger)mediaCountForThread:(TSThread *)thread transaction:(YapDatabaseReadTransaction *)transaction NS_SWIFT_NAME(mediaCount(thread:transaction:));
+- (NSUInteger)mediaCountWithTransaction:(YapDatabaseReadTransaction *)transaction NS_SWIFT_NAME(mediaCount(transaction:));
 
 // The ordinal position of a message within a thread's media gallery
 - (NSUInteger)mediaIndexForMessage:(TSMessage *)message transaction:(YapDatabaseReadTransaction *)transaction NS_SWIFT_NAME(mediaIndex(message:transaction:));
 
-- (void)enumerateMediaMessagesWithThread:(TSThread *)thread
+- (nullable TSMessage *)oldestMediaMessageWithTransaction:(YapDatabaseReadTransaction *)transaction NS_SWIFT_NAME(oldestMediaMessage(transaction:));
-                             transaction:(YapDatabaseReadTransaction *)transaction
+- (nullable TSMessage *)mostRecentMediaMessageWithTransaction:(YapDatabaseReadTransaction *)transaction NS_SWIFT_NAME(mostRecentMediaMessage(transaction:));
-                                   block:(void (^)(TSMessage *))messageBlock;
+
+- (void)enumerateMediaMessagesWithRange:(NSRange)range
+                            transaction:(YapDatabaseReadTransaction *)transaction
+                                  block:(void (^)(TSMessage *))messageBlock NS_SWIFT_NAME(enumerateMediaMessages(range:transaction:block:));
+
+#pragma mark - Extension registration
 
 + (void)asyncRegisterDatabaseExtensionsWithPrimaryStorage:(OWSStorage *)storage;
 

SignalServiceKit/src/Storage/OWSMediaGalleryFinder.m

@@ -16,14 +16,31 @@ NS_ASSUME_NONNULL_BEGIN
 
 static NSString *const OWSMediaGalleryFinderExtensionName = @"OWSMediaGalleryFinderExtensionName";
 
+@interface OWSMediaGalleryFinder ()
+
+@property (nonatomic, readonly) TSThread *thread;
+
+@end
+
 @implementation OWSMediaGalleryFinder
 
+- (instancetype)initWithThread:(TSThread *)thread
+{
+    self = [super init];
+    if (!self) {
+        return self;
+    }
+
+    _thread = thread;
+
+    return self;
+}
+
 #pragma mark - Public Finder Methods
 
--  (NSUInteger)mediaCountForThread:(TSThread *)thread transaction:(YapDatabaseReadTransaction *)transaction
+- (NSUInteger)mediaCountWithTransaction:(YapDatabaseReadTransaction *)transaction
 {
-    NSString *group = [self mediaGroupWithThreadId:thread.uniqueId];
+    return [[self galleryExtensionWithTransaction:transaction] numberOfItemsInGroup:self.mediaGroup];
-    return [[self galleryExtensionWithTransaction:transaction] numberOfItemsInGroup:group];
 }
 
 - (NSUInteger)mediaIndexForMessage:(TSMessage *)message transaction:(YapDatabaseReadTransaction *)transaction
@@ -37,22 +54,36 @@ static NSString *const OWSMediaGalleryFinderExtensionName = @"OWSMediaGalleryFin
                                                                     inCollection:[TSMessage collection]];
 
     OWSAssert(wasFound);
+    OWSAssert([self.mediaGroup isEqual:groupId]);
 
     return index;
 }
 
-- (void)enumerateMediaMessagesWithThread:(TSThread *)thread
+- (nullable TSMessage *)oldestMediaMessageWithTransaction:(YapDatabaseReadTransaction *)transaction
-                             transaction:(YapDatabaseReadTransaction *)transaction
-                                   block:(void (^)(TSMessage *))messageBlock
 {
-    NSString *group = [self mediaGroupWithThreadId:thread.uniqueId];
+    return [[self galleryExtensionWithTransaction:transaction] firstObjectInGroup:self.mediaGroup];
+}
+
+- (nullable TSMessage *)mostRecentMediaMessageWithTransaction:(YapDatabaseReadTransaction *)transaction
+{
+    return [[self galleryExtensionWithTransaction:transaction] lastObjectInGroup:self.mediaGroup];
+}
+
+- (void)enumerateMediaMessagesWithRange:(NSRange)range
+                            transaction:(YapDatabaseReadTransaction *)transaction
+                                  block:(void (^)(TSMessage *))messageBlock
+{
+
     [[self galleryExtensionWithTransaction:transaction]
-        enumerateKeysAndObjectsInGroup:group
+        enumerateKeysAndObjectsInGroup:self.mediaGroup
+                           withOptions:0
+                                 range:range
                             usingBlock:^(NSString *_Nonnull collection,
                                 NSString *_Nonnull key,
                                 id _Nonnull object,
                                 NSUInteger index,
                                 BOOL *_Nonnull stop) {
+
                                 OWSAssert([object isKindOfClass:[TSMessage class]]);
                                 messageBlock((TSMessage *)object);
                             }];
@@ -73,9 +104,9 @@ static NSString *const OWSMediaGalleryFinderExtensionName = @"OWSMediaGalleryFin
     return [NSString stringWithFormat:@"%@-media", threadId];
 }
 
-- (NSString *)mediaGroupWithThreadId:(NSString *)threadId
+- (NSString *)mediaGroup
 {
-    return [[self class] mediaGroupWithThreadId:threadId];
+    return [[self class] mediaGroupWithThreadId:self.thread.uniqueId];
 }
 
 #pragma mark - Extension registration