From 7e39e58fc65476aed143adb4ea541f6db993f85d Mon Sep 17 00:00:00 2001 From: Michael Kirk Date: Tue, 3 Oct 2017 11:37:51 -0400 Subject: [PATCH 01/15] WebRTC M61 // FREEBIE --- Carthage | 2 +- Libraries/WebRTC/README.md | 7 - Libraries/WebRTC/RTCAudioSession.h | 224 ----------------------------- 3 files changed, 1 insertion(+), 232 deletions(-) delete mode 100644 Libraries/WebRTC/README.md delete mode 100644 Libraries/WebRTC/RTCAudioSession.h diff --git a/Carthage b/Carthage index fc8eebb92..31b7ea5a3 160000 --- a/Carthage +++ b/Carthage @@ -1 +1 @@ -Subproject commit fc8eebb92367031f60ea337c897551cc54baa4b2 +Subproject commit 31b7ea5a35345833d06d61b7da2333827de3a084 diff --git a/Libraries/WebRTC/README.md b/Libraries/WebRTC/README.md deleted file mode 100644 index 119fded51..000000000 --- a/Libraries/WebRTC/README.md +++ /dev/null @@ -1,7 +0,0 @@ -The RTCAudioSession.h header isn't included in the standard build of -WebRTC, so we've vendored it here. Otherwise we're using the vanilla -framework. - -We use the RTCAudioSession header to manually manage the RTC audio -session, so as to not start recording until the call is connected. - diff --git a/Libraries/WebRTC/RTCAudioSession.h b/Libraries/WebRTC/RTCAudioSession.h deleted file mode 100644 index 199fd17b2..000000000 --- a/Libraries/WebRTC/RTCAudioSession.h +++ /dev/null @@ -1,224 +0,0 @@ -/* - * Copyright 2016 The WebRTC Project Authors. All rights reserved. - * - * Use of this source code is governed by a BSD-style license - * that can be found in the LICENSE file in the root of the source - * tree. An additional intellectual property rights grant can be found - * in the file PATENTS. All contributing project authors may - * be found in the AUTHORS file in the root of the source tree. - */ - -#import -#import - -#import "WebRTC/RTCMacros.h" - -NS_ASSUME_NONNULL_BEGIN - -extern NSString * const kRTCAudioSessionErrorDomain; -/** Method that requires lock was called without lock. */ -extern NSInteger const kRTCAudioSessionErrorLockRequired; -/** Unknown configuration error occurred. */ -extern NSInteger const kRTCAudioSessionErrorConfiguration; - -@class RTCAudioSession; -@class RTCAudioSessionConfiguration; - -// Surfaces AVAudioSession events. WebRTC will listen directly for notifications -// from AVAudioSession and handle them before calling these delegate methods, -// at which point applications can perform additional processing if required. -RTC_EXPORT -@protocol RTCAudioSessionDelegate - -@optional -/** Called on a system notification thread when AVAudioSession starts an - * interruption event. - */ -- (void)audioSessionDidBeginInterruption:(RTCAudioSession *)session; - -/** Called on a system notification thread when AVAudioSession ends an - * interruption event. - */ -- (void)audioSessionDidEndInterruption:(RTCAudioSession *)session - shouldResumeSession:(BOOL)shouldResumeSession; - -/** Called on a system notification thread when AVAudioSession changes the - * route. - */ -- (void)audioSessionDidChangeRoute:(RTCAudioSession *)session - reason:(AVAudioSessionRouteChangeReason)reason - previousRoute:(AVAudioSessionRouteDescription *)previousRoute; - -/** Called on a system notification thread when AVAudioSession media server - * terminates. - */ -- (void)audioSessionMediaServerTerminated:(RTCAudioSession *)session; - -/** Called on a system notification thread when AVAudioSession media server - * restarts. - */ -- (void)audioSessionMediaServerReset:(RTCAudioSession *)session; - -// TODO(tkchin): Maybe handle SilenceSecondaryAudioHintNotification. - -- (void)audioSession:(RTCAudioSession *)session - didChangeCanPlayOrRecord:(BOOL)canPlayOrRecord; - -/** Called on a WebRTC thread when the audio device is notified to begin - * playback or recording. - */ -- (void)audioSessionDidStartPlayOrRecord:(RTCAudioSession *)session; - -/** Called on a WebRTC thread when the audio device is notified to stop - * playback or recording. - */ -- (void)audioSessionDidStopPlayOrRecord:(RTCAudioSession *)session; - -@end - -/** Proxy class for AVAudioSession that adds a locking mechanism similar to - * AVCaptureDevice. This is used to that interleaving configurations between - * WebRTC and the application layer are avoided. - * - * RTCAudioSession also coordinates activation so that the audio session is - * activated only once. See |setActive:error:|. - */ -RTC_EXPORT -@interface RTCAudioSession : NSObject - -/** Convenience property to access the AVAudioSession singleton. Callers should - * not call setters on AVAudioSession directly, but other method invocations - * are fine. - */ -@property(nonatomic, readonly) AVAudioSession *session; - -/** Our best guess at whether the session is active based on results of calls to - * AVAudioSession. - */ -@property(nonatomic, readonly) BOOL isActive; -/** Whether RTCAudioSession is currently locked for configuration. */ -@property(nonatomic, readonly) BOOL isLocked; - -/** If YES, WebRTC will not initialize the audio unit automatically when an - * audio track is ready for playout or recording. Instead, applications should - * call setIsAudioEnabled. If NO, WebRTC will initialize the audio unit - * as soon as an audio track is ready for playout or recording. - */ -@property(nonatomic, assign) BOOL useManualAudio; - -/** This property is only effective if useManualAudio is YES. - * Represents permission for WebRTC to initialize the VoIP audio unit. - * When set to NO, if the VoIP audio unit used by WebRTC is active, it will be - * stopped and uninitialized. This will stop incoming and outgoing audio. - * When set to YES, WebRTC will initialize and start the audio unit when it is - * needed (e.g. due to establishing an audio connection). - * This property was introduced to work around an issue where if an AVPlayer is - * playing audio while the VoIP audio unit is initialized, its audio would be - * either cut off completely or played at a reduced volume. By preventing - * the audio unit from being initialized until after the audio has completed, - * we are able to prevent the abrupt cutoff. - */ -@property(nonatomic, assign) BOOL isAudioEnabled; - -// Proxy properties. -@property(readonly) NSString *category; -@property(readonly) AVAudioSessionCategoryOptions categoryOptions; -@property(readonly) NSString *mode; -@property(readonly) BOOL secondaryAudioShouldBeSilencedHint; -@property(readonly) AVAudioSessionRouteDescription *currentRoute; -@property(readonly) NSInteger maximumInputNumberOfChannels; -@property(readonly) NSInteger maximumOutputNumberOfChannels; -@property(readonly) float inputGain; -@property(readonly) BOOL inputGainSettable; -@property(readonly) BOOL inputAvailable; -@property(readonly, nullable) - NSArray * inputDataSources; -@property(readonly, nullable) - AVAudioSessionDataSourceDescription *inputDataSource; -@property(readonly, nullable) - NSArray * outputDataSources; -@property(readonly, nullable) - AVAudioSessionDataSourceDescription *outputDataSource; -@property(readonly) double sampleRate; -@property(readonly) double preferredSampleRate; -@property(readonly) NSInteger inputNumberOfChannels; -@property(readonly) NSInteger outputNumberOfChannels; -@property(readonly) float outputVolume; -@property(readonly) NSTimeInterval inputLatency; -@property(readonly) NSTimeInterval outputLatency; -@property(readonly) NSTimeInterval IOBufferDuration; -@property(readonly) NSTimeInterval preferredIOBufferDuration; - -/** Default constructor. */ -+ (instancetype)sharedInstance; -- (instancetype)init NS_UNAVAILABLE; - -/** Adds a delegate, which is held weakly. */ -- (void)addDelegate:(id)delegate; -/** Removes an added delegate. */ -- (void)removeDelegate:(id)delegate; - -/** Request exclusive access to the audio session for configuration. This call - * will block if the lock is held by another object. - */ -- (void)lockForConfiguration; -/** Relinquishes exclusive access to the audio session. */ -- (void)unlockForConfiguration; - -/** If |active|, activates the audio session if it isn't already active. - * Successful calls must be balanced with a setActive:NO when activation is no - * longer required. If not |active|, deactivates the audio session if one is - * active and this is the last balanced call. When deactivating, the - * AVAudioSessionSetActiveOptionNotifyOthersOnDeactivation option is passed to - * AVAudioSession. - */ -- (BOOL)setActive:(BOOL)active - error:(NSError **)outError; - -// The following methods are proxies for the associated methods on -// AVAudioSession. |lockForConfiguration| must be called before using them -// otherwise they will fail with kRTCAudioSessionErrorLockRequired. - -- (BOOL)setCategory:(NSString *)category - withOptions:(AVAudioSessionCategoryOptions)options - error:(NSError **)outError; -- (BOOL)setMode:(NSString *)mode error:(NSError **)outError; -- (BOOL)setInputGain:(float)gain error:(NSError **)outError; -- (BOOL)setPreferredSampleRate:(double)sampleRate error:(NSError **)outError; -- (BOOL)setPreferredIOBufferDuration:(NSTimeInterval)duration - error:(NSError **)outError; -- (BOOL)setPreferredInputNumberOfChannels:(NSInteger)count - error:(NSError **)outError; -- (BOOL)setPreferredOutputNumberOfChannels:(NSInteger)count - error:(NSError **)outError; -- (BOOL)overrideOutputAudioPort:(AVAudioSessionPortOverride)portOverride - error:(NSError **)outError; -- (BOOL)setPreferredInput:(AVAudioSessionPortDescription *)inPort - error:(NSError **)outError; -- (BOOL)setInputDataSource:(AVAudioSessionDataSourceDescription *)dataSource - error:(NSError **)outError; -- (BOOL)setOutputDataSource:(AVAudioSessionDataSourceDescription *)dataSource - error:(NSError **)outError; - -@end - -@interface RTCAudioSession (Configuration) - -/** Applies the configuration to the current session. Attempts to set all - * properties even if previous ones fail. Only the last error will be - * returned. - * |lockForConfiguration| must be called first. - */ -- (BOOL)setConfiguration:(RTCAudioSessionConfiguration *)configuration - error:(NSError **)outError; - -/** Convenience method that calls both setConfiguration and setActive. - * |lockForConfiguration| must be called first. - */ -- (BOOL)setConfiguration:(RTCAudioSessionConfiguration *)configuration - active:(BOOL)active - error:(NSError **)outError; - -@end - -NS_ASSUME_NONNULL_END From f171c564886f709f718c1561664c3c4849c952ca Mon Sep 17 00:00:00 2001 From: Michael Kirk Date: Tue, 3 Oct 2017 17:26:14 -0400 Subject: [PATCH 02/15] Video calls use MetalKit when available Taking advantage of the new RTCMTLVideoView where available (64bit && >=iOS9), should help performance (battery and rendering quality). // FREEBIE --- Signal/src/Signal-Bridging-Header.h | 6 + .../ViewControllers/CallViewController.swift | 112 +++++++++++++----- Signal/src/call/CallService.swift | 2 +- 3 files changed, 87 insertions(+), 33 deletions(-) diff --git a/Signal/src/Signal-Bridging-Header.h b/Signal/src/Signal-Bridging-Header.h index 786e7205d..3f1222bcd 100644 --- a/Signal/src/Signal-Bridging-Header.h +++ b/Signal/src/Signal-Bridging-Header.h @@ -111,4 +111,10 @@ #import #import #import +#import +#import #import + +@interface RTCMTLVideoView (MakePrivatePublic) ++ (BOOL)isMetalAvailable; +@end diff --git a/Signal/src/ViewControllers/CallViewController.swift b/Signal/src/ViewControllers/CallViewController.swift index c3d91ca61..fde36bfc0 100644 --- a/Signal/src/ViewControllers/CallViewController.swift +++ b/Signal/src/ViewControllers/CallViewController.swift @@ -60,7 +60,10 @@ class CallViewController: OWSViewController, CallObserver, CallServiceObserver, // MARK: Video Views - var remoteVideoView: RTCEAGLVideoView! +// var remoteVideoView: RTCEAGLVideoView! + var remoteVideoView: (UIView & RTCVideoRenderer)! +// var remoteVideoView = UIView() +// var remoteVideoRenderView: (UIView & RTCVideoRenderer)! var localVideoView: RTCCameraPreviewView! weak var localVideoTrack: RTCVideoTrack? weak var remoteVideoTrack: RTCVideoTrack? @@ -232,14 +235,38 @@ class CallViewController: OWSViewController, CallObserver, CallServiceObserver, } func createVideoViews() { - remoteVideoView = RTCEAGLVideoView() - remoteVideoView.delegate = self - remoteVideoView.isUserInteractionEnabled = false localVideoView = RTCCameraPreviewView() + + if #available(iOS 9, *) { + if RTCMTLVideoView.isMetalAvailable() { + remoteVideoView = RTCMTLVideoView(frame: CGRect(x: 50, y: 50, width: 200, height: 300)) + remoteVideoView.setSize(self.view.frame.size) + } + } + + // Fallback on earlier versions + if remoteVideoView == nil { + let eaglVideoView = RTCEAGLVideoView() + eaglVideoView.delegate = self + remoteVideoView = eaglVideoView + } + + guard let remoteVideoView = self.remoteVideoView else { + owsFail("Failed to make remoteVideoView") + return + } + + remoteVideoView.backgroundColor = UIColor.yellow + remoteVideoView.isUserInteractionEnabled = false + remoteVideoView.layoutMargins = UIEdgeInsets.zero + remoteVideoView.isHidden = true +// remoteVideoView.isHidden = false localVideoView.isHidden = true self.view.addSubview(remoteVideoView) self.view.addSubview(localVideoView) + + remoteVideoView.autoPinEdgesToSuperviewEdges() } func createContactViews() { @@ -588,41 +615,56 @@ class CallViewController: OWSViewController, CallObserver, CallServiceObserver, } internal func updateRemoteVideoLayout() { +// return +// +// switch self.remoteVideoView { +// case let videoView as RTCMTLVideoView: +// videoView.frame = self.remoteVideView.bounds +// default: + if #available(iOS 9, *) { + if let mtlView = remoteVideoView as? RTCMTLVideoView { + mtlView.setSize(self.view.frame.size) + } + } + NSLayoutConstraint.deactivate(self.remoteVideoConstraints) - + var constraints: [NSLayoutConstraint] = [] + if (self.hasRemoteVideoTrack) { + // We fill the screen with the remote video. The remote video's + // aspect ratio may not (and in fact will very rarely) match the + // aspect ratio of the current device, so parts of the remote + // video will be hidden offscreen. + // + // It's better to trim the remote video than to adopt a letterboxed + // layout. + if remoteVideoSize.width > 0 && remoteVideoSize.height > 0 && + self.view.bounds.size.width > 0 && self.view.bounds.size.height > 0 { + + var remoteVideoWidth = self.view.bounds.size.width + var remoteVideoHeight = self.view.bounds.size.height + if remoteVideoSize.width / self.view.bounds.size.width > remoteVideoSize.height / self.view.bounds.size.height { + remoteVideoWidth = round(self.view.bounds.size.height * remoteVideoSize.width / remoteVideoSize.height) + } else { + remoteVideoHeight = round(self.view.bounds.size.width * remoteVideoSize.height / remoteVideoSize.width) + } + constraints.append(remoteVideoView.autoSetDimension(.width, toSize:remoteVideoWidth)) + constraints.append(remoteVideoView.autoSetDimension(.height, toSize:remoteVideoHeight)) + constraints += remoteVideoView.autoCenterInSuperview() - // We fill the screen with the remote video. The remote video's - // aspect ratio may not (and in fact will very rarely) match the - // aspect ratio of the current device, so parts of the remote - // video will be hidden offscreen. - // - // It's better to trim the remote video than to adopt a letterboxed - // layout. - if remoteVideoSize.width > 0 && remoteVideoSize.height > 0 && - self.view.bounds.size.width > 0 && self.view.bounds.size.height > 0 { - - var remoteVideoWidth = self.view.bounds.size.width - var remoteVideoHeight = self.view.bounds.size.height - if remoteVideoSize.width / self.view.bounds.size.width > remoteVideoSize.height / self.view.bounds.size.height { - remoteVideoWidth = round(self.view.bounds.size.height * remoteVideoSize.width / remoteVideoSize.height) + remoteVideoView.frame = CGRect(origin:CGPoint.zero, + size:CGSize(width:remoteVideoWidth, + height:remoteVideoHeight)) + + remoteVideoView.isHidden = false } else { - remoteVideoHeight = round(self.view.bounds.size.width * remoteVideoSize.height / remoteVideoSize.width) + remoteVideoView.frame = self.view.bounds + remoteVideoView.isHidden = false } - constraints.append(remoteVideoView.autoSetDimension(.width, toSize:remoteVideoWidth)) - constraints.append(remoteVideoView.autoSetDimension(.height, toSize:remoteVideoHeight)) - constraints += remoteVideoView.autoCenterInSuperview() - - remoteVideoView.frame = CGRect(origin:CGPoint.zero, - size:CGSize(width:remoteVideoWidth, - height:remoteVideoHeight)) - - remoteVideoView.isHidden = false } else { constraints += remoteVideoView.autoPinEdgesToSuperviewEdges() remoteVideoView.isHidden = true } - self.remoteVideoConstraints = constraints // We need to force relayout to occur immediately (and not @@ -988,6 +1030,9 @@ class CallViewController: OWSViewController, CallObserver, CallServiceObserver, updateLocalVideoLayout() } + var hasRemoteVideoTrack: Bool { + return self.remoteVideoTrack != nil + } internal func updateRemoteVideoTrack(remoteVideoTrack: RTCVideoTrack?) { AssertIsOnMainThread() guard self.remoteVideoTrack != remoteVideoTrack else { @@ -1064,14 +1109,17 @@ class CallViewController: OWSViewController, CallObserver, CallServiceObserver, remoteVideoTrack: RTCVideoTrack?) { AssertIsOnMainThread() - updateLocalVideoTrack(localVideoTrack:localVideoTrack) - updateRemoteVideoTrack(remoteVideoTrack:remoteVideoTrack) + updateLocalVideoTrack(localVideoTrack: localVideoTrack) + updateRemoteVideoTrack(remoteVideoTrack: remoteVideoTrack) } // MARK: - RTCEAGLVideoViewDelegate internal func videoView(_ videoView: RTCEAGLVideoView, didChangeVideoSize size: CGSize) { AssertIsOnMainThread() + if #available(iOS 9, *) { + assert(!RTCMTLVideoView.isMetalAvailable()) + } if videoView != remoteVideoView { return diff --git a/Signal/src/call/CallService.swift b/Signal/src/call/CallService.swift index 4d82d99e0..742d617eb 100644 --- a/Signal/src/call/CallService.swift +++ b/Signal/src/call/CallService.swift @@ -209,7 +209,7 @@ protocol CallServiceObserver: class { didSet { AssertIsOnMainThread() - Logger.info("\(self.TAG) \(#function)") + Logger.info("\(self.TAG) \(#function): \(isRemoteVideoEnabled)") fireDidUpdateVideoTracks() } From ff2f9ebaf00f0faa1c80fa80a7b29b83080311b0 Mon Sep 17 00:00:00 2001 From: Michael Kirk Date: Wed, 4 Oct 2017 11:00:59 -0400 Subject: [PATCH 03/15] fix compiling on 32bit. // FREEBIE --- Signal.xcodeproj/project.pbxproj | 6 + Signal/src/Signal-Bridging-Header.h | 8 +- .../ViewControllers/CallViewController.swift | 182 ++++++-------- Signal/src/call/CallAudioService.swift | 3 +- Signal/src/views/RemoteVideoView.h | 15 ++ Signal/src/views/RemoteVideoView.m | 224 ++++++++++++++++++ 6 files changed, 326 insertions(+), 112 deletions(-) create mode 100644 Signal/src/views/RemoteVideoView.h create mode 100644 Signal/src/views/RemoteVideoView.m diff --git a/Signal.xcodeproj/project.pbxproj b/Signal.xcodeproj/project.pbxproj index 387675b5a..943d0799d 100644 --- a/Signal.xcodeproj/project.pbxproj +++ b/Signal.xcodeproj/project.pbxproj @@ -124,6 +124,7 @@ 450873C71D9D867B006B54F2 /* OWSIncomingMessageCollectionViewCell.m in Sources */ = {isa = PBXBuildFile; fileRef = 450873C61D9D867B006B54F2 /* OWSIncomingMessageCollectionViewCell.m */; }; 450873C81D9D867B006B54F2 /* OWSIncomingMessageCollectionViewCell.m in Sources */ = {isa = PBXBuildFile; fileRef = 450873C61D9D867B006B54F2 /* OWSIncomingMessageCollectionViewCell.m */; }; 4509E79A1DD653700025A59F /* WebRTC.framework in Frameworks */ = {isa = PBXBuildFile; fileRef = 4509E7991DD653700025A59F /* WebRTC.framework */; }; + 450D19131F85236600970622 /* RemoteVideoView.m in Sources */ = {isa = PBXBuildFile; fileRef = 450D19121F85236600970622 /* RemoteVideoView.m */; }; 450DF2051E0D74AC003D14BE /* Platform.swift in Sources */ = {isa = PBXBuildFile; fileRef = 450DF2041E0D74AC003D14BE /* Platform.swift */; }; 450DF2091E0DD2C6003D14BE /* UserNotificationsAdaptee.swift in Sources */ = {isa = PBXBuildFile; fileRef = 450DF2081E0DD2C6003D14BE /* UserNotificationsAdaptee.swift */; }; 451686AB1F520CDA00AC3D4B /* MultiDeviceProfileKeyUpdateJob.swift in Sources */ = {isa = PBXBuildFile; fileRef = 451686AA1F520CDA00AC3D4B /* MultiDeviceProfileKeyUpdateJob.swift */; }; @@ -601,6 +602,8 @@ 450873C61D9D867B006B54F2 /* OWSIncomingMessageCollectionViewCell.m */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.objc; path = OWSIncomingMessageCollectionViewCell.m; sourceTree = ""; }; 450873C91D9D86F4006B54F2 /* OWSExpirableMessageView.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = OWSExpirableMessageView.h; sourceTree = ""; }; 4509E7991DD653700025A59F /* WebRTC.framework */ = {isa = PBXFileReference; lastKnownFileType = wrapper.framework; name = WebRTC.framework; path = Carthage/Build/iOS/WebRTC.framework; sourceTree = ""; }; + 450D19111F85236600970622 /* RemoteVideoView.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = RemoteVideoView.h; sourceTree = ""; }; + 450D19121F85236600970622 /* RemoteVideoView.m */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.objc; path = RemoteVideoView.m; sourceTree = ""; }; 450DF2041E0D74AC003D14BE /* Platform.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = Platform.swift; sourceTree = ""; }; 450DF2081E0DD2C6003D14BE /* UserNotificationsAdaptee.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; lineEnding = 0; name = UserNotificationsAdaptee.swift; path = UserInterface/Notifications/UserNotificationsAdaptee.swift; sourceTree = ""; xcLanguageSpecificationIdentifier = xcode.lang.swift; }; 451686AA1F520CDA00AC3D4B /* MultiDeviceProfileKeyUpdateJob.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = MultiDeviceProfileKeyUpdateJob.swift; sourceTree = ""; }; @@ -1526,6 +1529,8 @@ 34F3089D1ECA580B00BB7697 /* OWSUnreadIndicatorCell.h */, 34F3089E1ECA580B00BB7697 /* OWSUnreadIndicatorCell.m */, 45A6DAD51EBBF85500893231 /* ReminderView.swift */, + 450D19111F85236600970622 /* RemoteVideoView.h */, + 450D19121F85236600970622 /* RemoteVideoView.m */, ); name = Views; path = views; @@ -2259,6 +2264,7 @@ 45CD81EF1DC030E7004C9430 /* AccountManager.swift in Sources */, 45794E861E00620000066731 /* CallUIAdapter.swift in Sources */, 4585C4681ED8F8D200896AEA /* SafetyNumberConfirmationAlert.swift in Sources */, + 450D19131F85236600970622 /* RemoteVideoView.m in Sources */, FCFA64B71A24F6730007FB87 /* UIFont+OWS.m in Sources */, B6B9ECFC198B31BA00C620D3 /* PushManager.m in Sources */, 45DF5DF21DDB843F00C936C7 /* CompareSafetyNumbersActivity.swift in Sources */, diff --git a/Signal/src/Signal-Bridging-Header.h b/Signal/src/Signal-Bridging-Header.h index 3f1222bcd..62b761453 100644 --- a/Signal/src/Signal-Bridging-Header.h +++ b/Signal/src/Signal-Bridging-Header.h @@ -32,6 +32,7 @@ #import "ProfileViewController.h" #import "PushManager.h" #import "Release.h" +#import "RemoteVideoView.h" #import "TSMessageAdapter.h" #import "ThreadUtil.h" #import "UIColor+OWS.h" @@ -110,11 +111,4 @@ #import #import #import -#import -#import -#import #import - -@interface RTCMTLVideoView (MakePrivatePublic) -+ (BOOL)isMetalAvailable; -@end diff --git a/Signal/src/ViewControllers/CallViewController.swift b/Signal/src/ViewControllers/CallViewController.swift index fde36bfc0..e5f45ff8e 100644 --- a/Signal/src/ViewControllers/CallViewController.swift +++ b/Signal/src/ViewControllers/CallViewController.swift @@ -8,7 +8,8 @@ import PromiseKit // TODO: Add category so that button handlers can be defined where button is created. // TODO: Ensure buttons enabled & disabled as necessary. -class CallViewController: OWSViewController, CallObserver, CallServiceObserver, RTCEAGLVideoViewDelegate { +//class CallViewController: OWSViewController, CallObserver, CallServiceObserver, RTCEAGLVideoViewDelegate { +class CallViewController: OWSViewController, CallObserver, CallServiceObserver { let TAG = "[CallViewController]" @@ -60,14 +61,14 @@ class CallViewController: OWSViewController, CallObserver, CallServiceObserver, // MARK: Video Views -// var remoteVideoView: RTCEAGLVideoView! - var remoteVideoView: (UIView & RTCVideoRenderer)! -// var remoteVideoView = UIView() -// var remoteVideoRenderView: (UIView & RTCVideoRenderer)! + var remoteVideoView: RemoteVideoView! var localVideoView: RTCCameraPreviewView! weak var localVideoTrack: RTCVideoTrack? weak var remoteVideoTrack: RTCVideoTrack? + + // TODO delete? var remoteVideoSize: CGSize! = CGSize.zero + // TODO delete? var remoteVideoConstraints: [NSLayoutConstraint] = [] var localVideoConstraints: [NSLayoutConstraint] = [] @@ -236,37 +237,35 @@ class CallViewController: OWSViewController, CallObserver, CallServiceObserver, func createVideoViews() { localVideoView = RTCCameraPreviewView() - - if #available(iOS 9, *) { - if RTCMTLVideoView.isMetalAvailable() { - remoteVideoView = RTCMTLVideoView(frame: CGRect(x: 50, y: 50, width: 200, height: 300)) - remoteVideoView.setSize(self.view.frame.size) - } - } - // Fallback on earlier versions - if remoteVideoView == nil { - let eaglVideoView = RTCEAGLVideoView() - eaglVideoView.delegate = self - remoteVideoView = eaglVideoView - } +// if isMetalAvailable { +// if #available(iOS 9, *) { +// remoteVideoView = RTCMTLVideoView(frame: CGRect(x: 50, y: 50, width: 200, height: 300)) +//// remoteVideoView.setSize(self.view.frame.size) +// } else { +// owsFail("metal should only be available on ios9+") +// } +// } else { +// let eaglVideoView = RTCEAGLVideoView() +// eaglVideoView.delegate = self +// remoteVideoView = eaglVideoView +// } +// +// guard let remoteVideoView = self.remoteVideoView else { +// owsFail("Failed to make remoteVideoView") +// return +// } - guard let remoteVideoView = self.remoteVideoView else { - owsFail("Failed to make remoteVideoView") - return - } + remoteVideoView = RemoteVideoView() remoteVideoView.backgroundColor = UIColor.yellow remoteVideoView.isUserInteractionEnabled = false remoteVideoView.layoutMargins = UIEdgeInsets.zero remoteVideoView.isHidden = true -// remoteVideoView.isHidden = false localVideoView.isHidden = true self.view.addSubview(remoteVideoView) self.view.addSubview(localVideoView) - - remoteVideoView.autoPinEdgesToSuperviewEdges() } func createContactViews() { @@ -575,6 +574,8 @@ class CallViewController: OWSViewController, CallObserver, CallServiceObserver, localVideoView.autoSetDimension(.width, toSize:localVideoSize) localVideoView.autoSetDimension(.height, toSize:localVideoSize) + remoteVideoView.autoPinEdgesToSuperviewEdges() + contactNameLabel.autoPinEdge(toSuperviewEdge:.top, withInset:topMargin) contactNameLabel.autoPinLeadingToSuperview(withMargin: contactHMargin) contactNameLabel.setContentHuggingVerticalHigh() @@ -615,67 +616,8 @@ class CallViewController: OWSViewController, CallObserver, CallServiceObserver, } internal func updateRemoteVideoLayout() { -// return -// -// switch self.remoteVideoView { -// case let videoView as RTCMTLVideoView: -// videoView.frame = self.remoteVideView.bounds -// default: - if #available(iOS 9, *) { - if let mtlView = remoteVideoView as? RTCMTLVideoView { - mtlView.setSize(self.view.frame.size) - } - } - - NSLayoutConstraint.deactivate(self.remoteVideoConstraints) - - var constraints: [NSLayoutConstraint] = [] - if (self.hasRemoteVideoTrack) { - // We fill the screen with the remote video. The remote video's - // aspect ratio may not (and in fact will very rarely) match the - // aspect ratio of the current device, so parts of the remote - // video will be hidden offscreen. - // - // It's better to trim the remote video than to adopt a letterboxed - // layout. - if remoteVideoSize.width > 0 && remoteVideoSize.height > 0 && - self.view.bounds.size.width > 0 && self.view.bounds.size.height > 0 { - - var remoteVideoWidth = self.view.bounds.size.width - var remoteVideoHeight = self.view.bounds.size.height - if remoteVideoSize.width / self.view.bounds.size.width > remoteVideoSize.height / self.view.bounds.size.height { - remoteVideoWidth = round(self.view.bounds.size.height * remoteVideoSize.width / remoteVideoSize.height) - } else { - remoteVideoHeight = round(self.view.bounds.size.width * remoteVideoSize.height / remoteVideoSize.width) - } - constraints.append(remoteVideoView.autoSetDimension(.width, toSize:remoteVideoWidth)) - constraints.append(remoteVideoView.autoSetDimension(.height, toSize:remoteVideoHeight)) - constraints += remoteVideoView.autoCenterInSuperview() - - remoteVideoView.frame = CGRect(origin:CGPoint.zero, - size:CGSize(width:remoteVideoWidth, - height:remoteVideoHeight)) - - remoteVideoView.isHidden = false - } else { - remoteVideoView.frame = self.view.bounds - remoteVideoView.isHidden = false - } - } else { - constraints += remoteVideoView.autoPinEdgesToSuperviewEdges() - remoteVideoView.isHidden = true - } - self.remoteVideoConstraints = constraints - - // We need to force relayout to occur immediately (and not - // wait for a UIKit layout/render pass) or the remoteVideoView - // (which presumably is updating its CALayer directly) will - // ocassionally appear to have bad frames. - remoteVideoView.setNeedsLayout() - remoteVideoView.superview?.setNeedsLayout() - remoteVideoView.layoutIfNeeded() - remoteVideoView.superview?.layoutIfNeeded() - + remoteVideoView.isHidden = !self.hasRemoteVideoTrack + self.remoteVideoView.updateRemoteVideoLayout() updateCallUI(callState: call.state) } @@ -1033,6 +975,7 @@ class CallViewController: OWSViewController, CallObserver, CallServiceObserver, var hasRemoteVideoTrack: Bool { return self.remoteVideoTrack != nil } + internal func updateRemoteVideoTrack(remoteVideoTrack: RTCVideoTrack?) { AssertIsOnMainThread() guard self.remoteVideoTrack != remoteVideoTrack else { @@ -1112,22 +1055,53 @@ class CallViewController: OWSViewController, CallObserver, CallServiceObserver, updateLocalVideoTrack(localVideoTrack: localVideoTrack) updateRemoteVideoTrack(remoteVideoTrack: remoteVideoTrack) } - - // MARK: - RTCEAGLVideoViewDelegate - - internal func videoView(_ videoView: RTCEAGLVideoView, didChangeVideoSize size: CGSize) { - AssertIsOnMainThread() - if #available(iOS 9, *) { - assert(!RTCMTLVideoView.isMetalAvailable()) - } - - if videoView != remoteVideoView { - return - } - - Logger.info("\(TAG) \(#function): \(size)") - - remoteVideoSize = size - updateRemoteVideoLayout() - } +// +// // MARK: - RTCEAGLVideoViewDelegate +// +// internal func videoView(_ videoView: RTCEAGLVideoView, didChangeVideoSize size: CGSize) { +// AssertIsOnMainThread() +// assert(!isMetalAvailable) +// +// if videoView != remoteVideoView { +// return +// } +// +// Logger.info("\(TAG) \(#function): \(size)") +// +// remoteVideoSize = size +// updateRemoteVideoLayout() +// } } + +//protocol RemoteVideoViewAdaptee: RTCVideoRenderer { +// +//} +// +//class RemoteVideoViewAdapter: UIView, RTCVideoRenderer { +// let adaptee: RemoteVideoViewAdaptee +// +// var isMetalAvailable: Bool { +// // MetalKit is available on 64bit devices running iOS9+ +// if #available(iOS 9, *) { +// return objc_getClass("RTCMTLVideoView") != nil && RTCMTLVideoView.isMetalAvailable() +// } else { +// return false +// } +// } +// +// init() { +// #if defined(RTC_SUPPORTS_METAL) +// if #available(iOS 9, *) { +// adaptee = RTCMTLVideoView(frame: CGRect.zero) +// } +// #endif +// +// if adaptee == nil { +// let eaglVideoView = RTCEAGLVideoView() +// eaglVideoView.delegate = self +// adaptee = eaglVideoView +// } +// +// super.init() +// } +//} diff --git a/Signal/src/call/CallAudioService.swift b/Signal/src/call/CallAudioService.swift index 4f93b457b..d791606e8 100644 --- a/Signal/src/call/CallAudioService.swift +++ b/Signal/src/call/CallAudioService.swift @@ -418,7 +418,8 @@ struct AudioSource: Hashable { guard let availableInputs = session.availableInputs else { // I'm not sure when this would happen. - owsFail("No available inputs or inputs not ready") + // TODO this happens when the call ends (at leas ton iOS8) +// owsFail("No available inputs or inputs not ready") return [AudioSource.builtInSpeaker] } diff --git a/Signal/src/views/RemoteVideoView.h b/Signal/src/views/RemoteVideoView.h new file mode 100644 index 000000000..93da1cb2a --- /dev/null +++ b/Signal/src/views/RemoteVideoView.h @@ -0,0 +1,15 @@ +// +// Copyright (c) 2017 Open Whisper Systems. All rights reserved. +// + +#import + +/** + * Drives the full screen remote video, this class is backed by either the modern MetalKit backed view on supported + * systems or the leagacy EAGL view. MetalKit is supported on 64bit systems running iOS8 or newer. + */ +@interface RemoteVideoView : UIView + +- (void)updateRemoteVideoLayout; + +@end diff --git a/Signal/src/views/RemoteVideoView.m b/Signal/src/views/RemoteVideoView.m new file mode 100644 index 000000000..32c84b6fc --- /dev/null +++ b/Signal/src/views/RemoteVideoView.m @@ -0,0 +1,224 @@ +// +// Copyright (c) 2017 Open Whisper Systems. All rights reserved. +// + +#import "RemoteVideoView.h" +#import +#import +#import +#import + +@interface RTCMTLVideoView (MakePrivatePublic) ++ (BOOL)isMetalAvailable; +@end + +@interface RemoteVideoView () + +@property (nonatomic, readonly) __kindof UIView *adaptee; + +// Used for legacy EAGLVideoView +@property (nonatomic) CGSize remoteVideoSize; +@property (nullable, nonatomic) NSMutableArray *remoteVideoConstraints; + +@end + +@implementation RemoteVideoView +//@implementation RemoteVideoViewAdapter + +//protocol RemoteVideoViewAdaptee: RTCVideoRenderer { +// +//} +// +// +//#if defined(RTC_SUPPORTS_METAL) +// if #available(iOS 9, *) { +// +// } +//#endif +// +// if adaptee == nil { +// let eaglVideoView = RTCEAGLVideoView() +// eaglVideoView.delegate = self +// adaptee = eaglVideoView +// } +// +// super.init() +// } +//} +- (instancetype)init +{ + self = [super init]; + if (!self) { + return self; + } + +// This class is defined in objc in order to access this compile time macro +#if defined(RTC_SUPPORTS_METAL) + if ([RTCMTLVideoView isMetalAvailable]) { + _adaptee = [[RTCMTLVideoView alloc] initWithFrame:CGRectZero]; + } +#endif + if (_adaptee == nil) { + RTCEAGLVideoView *eaglVideoView = [RTCEAGLVideoView new]; + eaglVideoView.delegate = self; + _adaptee = eaglVideoView; + } + + [self addSubview:_adaptee]; + [_adaptee autoPinEdgesToSuperviewEdges]; + + return self; +} + +- (void)updateRemoteVideoLayout +{ +// if isMetalAvailable { +// if #available(iOS 9, *) { +// assert(remoteVideoView as? RTCMTLVideoView != nil) +// } else { +// owsFail("metal should only be available on iOS9+") +// } +// Logger.debug("no additional layout needed for RTCMTLVideoView") +// } else if let videoView = remoteVideoView as? RTCEAGLVideoView { +// NSLayoutConstraint.deactivate(self.remoteVideoConstraints) +// +// var constraints: [NSLayoutConstraint] = [] +// // We fill the screen with the remote video. The remote video's +// // aspect ratio may not (and in fact will very rarely) match the +// // aspect ratio of the current device, so parts of the remote +// // video will be hidden offscreen. +// // +// // It's better to trim the remote video than to adopt a letterboxed +// // layout. +// if remoteVideoSize.width > 0 && remoteVideoSize.height > 0 && +// self.view.bounds.size.width > 0 && self.view.bounds.size.height > 0 { +// +// var remoteVideoWidth = self.view.bounds.size.width +// var remoteVideoHeight = self.view.bounds.size.height +// if remoteVideoSize.width / self.view.bounds.size.width > remoteVideoSize.height / self.view.bounds.size.height { +// remoteVideoWidth = round(self.view.bounds.size.height * remoteVideoSize.width / remoteVideoSize.height) +// } else { +// remoteVideoHeight = round(self.view.bounds.size.width * remoteVideoSize.height / remoteVideoSize.width) +// } +// constraints.append(videoView.autoSetDimension(.width, toSize:remoteVideoWidth)) +// constraints.append(videoView.autoSetDimension(.height, toSize:remoteVideoHeight)) +// constraints += videoView.autoCenterInSuperview() +// +// videoView.frame = CGRect(origin:CGPoint.zero, +// size:CGSize(width:remoteVideoWidth, +// height:remoteVideoHeight)) +// +// } else { +// constraints += videoView.autoPinEdgesToSuperviewEdges() +// } +// self.remoteVideoConstraints = constraints +// // We need to force relayout to occur immediately (and not +// // wait for a UIKit layout/render pass) or the remoteVideoView +// // (which presumably is updating its CALayer directly) will +// // ocassionally appear to have bad frames. +// videoView.setNeedsLayout() +// videoView.superview?.setNeedsLayout() +// videoView.layoutIfNeeded() +// videoView.superview?.layoutIfNeeded() +// } else { +// owsFail("in \(#function) with unhandled remoteVideoView type: \(remoteVideoView)") +// } + + UIView *containingView = [self superview]; + if (containingView == nil) { + DDLogDebug(@"%@ Cannot layout video view without superview", self.logTag); + return; + } + + // We fill the screen with the remote video. The remote video's + // aspect ratio may not (and in fact will very rarely) match the + // aspect ratio of the current device, so parts of the remote + // video will be hidden offscreen. + // + // It's better to trim the remote video than to adopt a letterboxed + // layout. + // This is only required on the legacy EAGL view. The modern MetalKit + // backed view can scale using the AspectFill content mode + if ([self.adaptee isKindOfClass:[RTCEAGLVideoView class]]) { + RTCEAGLVideoView *videoView = (RTCEAGLVideoView *)self.adaptee; + [NSLayoutConstraint deactivateConstraints:self.remoteVideoConstraints]; + + CGSize remoteVideoSize = self.remoteVideoSize; + + NSMutableArray *constraints = [NSMutableArray new]; + if (remoteVideoSize.width > 0 && remoteVideoSize.height > 0 && + containingView.bounds.size.width > 0 && containingView.bounds.size.height > 0) { + + CGFloat remoteVideoWidth = containingView.bounds.size.width; + CGFloat remoteVideoHeight = containingView.bounds.size.height; + + if (remoteVideoSize.width / containingView.bounds.size.width > remoteVideoSize.height / containingView.bounds.size.height) { + remoteVideoWidth = round(containingView.bounds.size.height * remoteVideoSize.width / remoteVideoSize.height); + } else { + remoteVideoHeight = round(containingView.bounds.size.width * remoteVideoSize.height / remoteVideoSize.width); + } + [constraints addObject:[videoView autoSetDimension:ALDimensionWidth toSize:remoteVideoWidth]]; + [constraints addObject:[videoView autoSetDimension:ALDimensionHeight toSize:remoteVideoHeight]]; + [constraints addObjectsFromArray:[videoView autoCenterInSuperview]]; + + videoView.frame = CGRectMake(0, 0, remoteVideoWidth, remoteVideoHeight); + + } else { + [constraints addObjectsFromArray:[videoView autoPinEdgesToSuperviewEdges]]; + } + + self.remoteVideoConstraints = constraints; + // We need to force relayout to occur immediately (and not + // wait for a UIKit layout/render pass) or the remoteVideoView + // (which presumably is updating its CALayer directly) will + // ocassionally appear to have bad frames. + [videoView setNeedsLayout]; + [[videoView superview] setNeedsLayout]; + [videoView layoutIfNeeded]; + [[videoView superview] layoutIfNeeded]; + } +} + +// MARK: - RTCEAGLVideoViewDelegate + +- (void)videoView:(RTCEAGLVideoView *)videoView didChangeVideoSize:(CGSize)size +{ + AssertIsOnMainThread(); + + // TODO +// if videoView != remoteVideoView { +// return +// } + DDLogInfo(@"%s called", __PRETTY_FUNCTION__); + + self.remoteVideoSize = size; + [self updateRemoteVideoLayout]; +} + +#pragma mark - RTCVideoRenderer + +/** The size of the frame. */ +- (void)setSize:(CGSize)size +{ + [self.adaptee setSize:size]; +} + +/** The frame to be displayed. */ +- (void)renderFrame:(nullable RTCVideoFrame *)frame +{ + [self.adaptee renderFrame:frame]; +} + +#pragma mark - Logging + ++ (NSString *)logTag +{ + return [NSString stringWithFormat:@"[%@]", self.class]; +} + +- (NSString *)logTag +{ + return self.class.logTag; +} + +@end From 9b33bb0b6dc334e03295bbef0d3743ab74372fb6 Mon Sep 17 00:00:00 2001 From: Michael Kirk Date: Wed, 4 Oct 2017 11:19:45 -0400 Subject: [PATCH 04/15] fix layout on MetalKit view --- Signal/src/ViewControllers/CallViewController.swift | 1 + Signal/src/views/RemoteVideoView.m | 3 +++ 2 files changed, 4 insertions(+) diff --git a/Signal/src/ViewControllers/CallViewController.swift b/Signal/src/ViewControllers/CallViewController.swift index e5f45ff8e..f3103cb0c 100644 --- a/Signal/src/ViewControllers/CallViewController.swift +++ b/Signal/src/ViewControllers/CallViewController.swift @@ -617,6 +617,7 @@ class CallViewController: OWSViewController, CallObserver, CallServiceObserver { internal func updateRemoteVideoLayout() { remoteVideoView.isHidden = !self.hasRemoteVideoTrack + self.remoteVideoView.setSize(self.view.frame.size) self.remoteVideoView.updateRemoteVideoLayout() updateCallUI(callState: call.state) } diff --git a/Signal/src/views/RemoteVideoView.m b/Signal/src/views/RemoteVideoView.m index 32c84b6fc..ddba95794 100644 --- a/Signal/src/views/RemoteVideoView.m +++ b/Signal/src/views/RemoteVideoView.m @@ -45,6 +45,7 @@ // super.init() // } //} + - (instancetype)init { self = [super init]; @@ -65,6 +66,8 @@ } [self addSubview:_adaptee]; + + _adaptee.layoutMargins = UIEdgeInsetsZero; [_adaptee autoPinEdgesToSuperviewEdges]; return self; From 2a4e113c82bac4764af251aba153b5400d7d931c Mon Sep 17 00:00:00 2001 From: Michael Kirk Date: Wed, 4 Oct 2017 11:44:33 -0400 Subject: [PATCH 05/15] Cleanup // FREEBIE --- .../ViewControllers/CallViewController.swift | 80 -------- Signal/src/views/RemoteVideoView.h | 2 - Signal/src/views/RemoteVideoView.m | 179 ++---------------- 3 files changed, 20 insertions(+), 241 deletions(-) diff --git a/Signal/src/ViewControllers/CallViewController.swift b/Signal/src/ViewControllers/CallViewController.swift index f3103cb0c..354315fb9 100644 --- a/Signal/src/ViewControllers/CallViewController.swift +++ b/Signal/src/ViewControllers/CallViewController.swift @@ -65,11 +65,6 @@ class CallViewController: OWSViewController, CallObserver, CallServiceObserver { var localVideoView: RTCCameraPreviewView! weak var localVideoTrack: RTCVideoTrack? weak var remoteVideoTrack: RTCVideoTrack? - - // TODO delete? - var remoteVideoSize: CGSize! = CGSize.zero - // TODO delete? - var remoteVideoConstraints: [NSLayoutConstraint] = [] var localVideoConstraints: [NSLayoutConstraint] = [] var shouldRemoteVideoControlsBeHidden = false { @@ -237,30 +232,9 @@ class CallViewController: OWSViewController, CallObserver, CallServiceObserver { func createVideoViews() { localVideoView = RTCCameraPreviewView() - -// if isMetalAvailable { -// if #available(iOS 9, *) { -// remoteVideoView = RTCMTLVideoView(frame: CGRect(x: 50, y: 50, width: 200, height: 300)) -//// remoteVideoView.setSize(self.view.frame.size) -// } else { -// owsFail("metal should only be available on ios9+") -// } -// } else { -// let eaglVideoView = RTCEAGLVideoView() -// eaglVideoView.delegate = self -// remoteVideoView = eaglVideoView -// } -// -// guard let remoteVideoView = self.remoteVideoView else { -// owsFail("Failed to make remoteVideoView") -// return -// } - remoteVideoView = RemoteVideoView() - remoteVideoView.backgroundColor = UIColor.yellow remoteVideoView.isUserInteractionEnabled = false - remoteVideoView.layoutMargins = UIEdgeInsets.zero remoteVideoView.isHidden = true localVideoView.isHidden = true @@ -618,7 +592,6 @@ class CallViewController: OWSViewController, CallObserver, CallServiceObserver { internal func updateRemoteVideoLayout() { remoteVideoView.isHidden = !self.hasRemoteVideoTrack self.remoteVideoView.setSize(self.view.frame.size) - self.remoteVideoView.updateRemoteVideoLayout() updateCallUI(callState: call.state) } @@ -990,10 +963,6 @@ class CallViewController: OWSViewController, CallObserver, CallServiceObserver { self.remoteVideoTrack?.add(remoteVideoView) shouldRemoteVideoControlsBeHidden = false - if remoteVideoTrack == nil { - remoteVideoSize = CGSize.zero - } - updateRemoteVideoLayout() } @@ -1056,53 +1025,4 @@ class CallViewController: OWSViewController, CallObserver, CallServiceObserver { updateLocalVideoTrack(localVideoTrack: localVideoTrack) updateRemoteVideoTrack(remoteVideoTrack: remoteVideoTrack) } -// -// // MARK: - RTCEAGLVideoViewDelegate -// -// internal func videoView(_ videoView: RTCEAGLVideoView, didChangeVideoSize size: CGSize) { -// AssertIsOnMainThread() -// assert(!isMetalAvailable) -// -// if videoView != remoteVideoView { -// return -// } -// -// Logger.info("\(TAG) \(#function): \(size)") -// -// remoteVideoSize = size -// updateRemoteVideoLayout() -// } } - -//protocol RemoteVideoViewAdaptee: RTCVideoRenderer { -// -//} -// -//class RemoteVideoViewAdapter: UIView, RTCVideoRenderer { -// let adaptee: RemoteVideoViewAdaptee -// -// var isMetalAvailable: Bool { -// // MetalKit is available on 64bit devices running iOS9+ -// if #available(iOS 9, *) { -// return objc_getClass("RTCMTLVideoView") != nil && RTCMTLVideoView.isMetalAvailable() -// } else { -// return false -// } -// } -// -// init() { -// #if defined(RTC_SUPPORTS_METAL) -// if #available(iOS 9, *) { -// adaptee = RTCMTLVideoView(frame: CGRect.zero) -// } -// #endif -// -// if adaptee == nil { -// let eaglVideoView = RTCEAGLVideoView() -// eaglVideoView.delegate = self -// adaptee = eaglVideoView -// } -// -// super.init() -// } -//} diff --git a/Signal/src/views/RemoteVideoView.h b/Signal/src/views/RemoteVideoView.h index 93da1cb2a..6e84de450 100644 --- a/Signal/src/views/RemoteVideoView.h +++ b/Signal/src/views/RemoteVideoView.h @@ -10,6 +10,4 @@ */ @interface RemoteVideoView : UIView -- (void)updateRemoteVideoLayout; - @end diff --git a/Signal/src/views/RemoteVideoView.m b/Signal/src/views/RemoteVideoView.m index ddba95794..0133ab96b 100644 --- a/Signal/src/views/RemoteVideoView.m +++ b/Signal/src/views/RemoteVideoView.m @@ -3,10 +3,11 @@ // #import "RemoteVideoView.h" +#import +#import #import #import #import -#import @interface RTCMTLVideoView (MakePrivatePublic) + (BOOL)isMetalAvailable; @@ -14,37 +15,11 @@ @interface RemoteVideoView () -@property (nonatomic, readonly) __kindof UIView *adaptee; - -// Used for legacy EAGLVideoView -@property (nonatomic) CGSize remoteVideoSize; -@property (nullable, nonatomic) NSMutableArray *remoteVideoConstraints; +@property (nonatomic, readonly) __kindof UIView *videoRenderer; @end @implementation RemoteVideoView -//@implementation RemoteVideoViewAdapter - -//protocol RemoteVideoViewAdaptee: RTCVideoRenderer { -// -//} -// -// -//#if defined(RTC_SUPPORTS_METAL) -// if #available(iOS 9, *) { -// -// } -//#endif -// -// if adaptee == nil { -// let eaglVideoView = RTCEAGLVideoView() -// eaglVideoView.delegate = self -// adaptee = eaglVideoView -// } -// -// super.init() -// } -//} - (instancetype)init { @@ -54,148 +29,34 @@ } // This class is defined in objc in order to access this compile time macro +// Currently RTC only supports metal on 64bit machines #if defined(RTC_SUPPORTS_METAL) - if ([RTCMTLVideoView isMetalAvailable]) { - _adaptee = [[RTCMTLVideoView alloc] initWithFrame:CGRectZero]; + // RTCMTLVideoView requires the MTKView class, available in the iOS9+ MetalKit framework + if ([MTKView class]) { + _videoRenderer = [[RTCMTLVideoView alloc] initWithFrame:CGRectZero]; } #endif - if (_adaptee == nil) { - RTCEAGLVideoView *eaglVideoView = [RTCEAGLVideoView new]; + if (_videoRenderer == nil) { + RTCEAGLVideoView *eaglVideoView = [[RTCEAGLVideoView alloc] initWithFrame:CGRectZero]; eaglVideoView.delegate = self; - _adaptee = eaglVideoView; + _videoRenderer = eaglVideoView; } - - [self addSubview:_adaptee]; - _adaptee.layoutMargins = UIEdgeInsetsZero; - [_adaptee autoPinEdgesToSuperviewEdges]; + [self addSubview:_videoRenderer]; - return self; -} - -- (void)updateRemoteVideoLayout -{ -// if isMetalAvailable { -// if #available(iOS 9, *) { -// assert(remoteVideoView as? RTCMTLVideoView != nil) -// } else { -// owsFail("metal should only be available on iOS9+") -// } -// Logger.debug("no additional layout needed for RTCMTLVideoView") -// } else if let videoView = remoteVideoView as? RTCEAGLVideoView { -// NSLayoutConstraint.deactivate(self.remoteVideoConstraints) -// -// var constraints: [NSLayoutConstraint] = [] -// // We fill the screen with the remote video. The remote video's -// // aspect ratio may not (and in fact will very rarely) match the -// // aspect ratio of the current device, so parts of the remote -// // video will be hidden offscreen. -// // -// // It's better to trim the remote video than to adopt a letterboxed -// // layout. -// if remoteVideoSize.width > 0 && remoteVideoSize.height > 0 && -// self.view.bounds.size.width > 0 && self.view.bounds.size.height > 0 { -// -// var remoteVideoWidth = self.view.bounds.size.width -// var remoteVideoHeight = self.view.bounds.size.height -// if remoteVideoSize.width / self.view.bounds.size.width > remoteVideoSize.height / self.view.bounds.size.height { -// remoteVideoWidth = round(self.view.bounds.size.height * remoteVideoSize.width / remoteVideoSize.height) -// } else { -// remoteVideoHeight = round(self.view.bounds.size.width * remoteVideoSize.height / remoteVideoSize.width) -// } -// constraints.append(videoView.autoSetDimension(.width, toSize:remoteVideoWidth)) -// constraints.append(videoView.autoSetDimension(.height, toSize:remoteVideoHeight)) -// constraints += videoView.autoCenterInSuperview() -// -// videoView.frame = CGRect(origin:CGPoint.zero, -// size:CGSize(width:remoteVideoWidth, -// height:remoteVideoHeight)) -// -// } else { -// constraints += videoView.autoPinEdgesToSuperviewEdges() -// } -// self.remoteVideoConstraints = constraints -// // We need to force relayout to occur immediately (and not -// // wait for a UIKit layout/render pass) or the remoteVideoView -// // (which presumably is updating its CALayer directly) will -// // ocassionally appear to have bad frames. -// videoView.setNeedsLayout() -// videoView.superview?.setNeedsLayout() -// videoView.layoutIfNeeded() -// videoView.superview?.layoutIfNeeded() -// } else { -// owsFail("in \(#function) with unhandled remoteVideoView type: \(remoteVideoView)") -// } - - UIView *containingView = [self superview]; - if (containingView == nil) { - DDLogDebug(@"%@ Cannot layout video view without superview", self.logTag); - return; - } + _videoRenderer.layoutMargins = UIEdgeInsetsZero; + [_videoRenderer autoPinEdgesToSuperviewEdges]; - // We fill the screen with the remote video. The remote video's - // aspect ratio may not (and in fact will very rarely) match the - // aspect ratio of the current device, so parts of the remote - // video will be hidden offscreen. - // - // It's better to trim the remote video than to adopt a letterboxed - // layout. - // This is only required on the legacy EAGL view. The modern MetalKit - // backed view can scale using the AspectFill content mode - if ([self.adaptee isKindOfClass:[RTCEAGLVideoView class]]) { - RTCEAGLVideoView *videoView = (RTCEAGLVideoView *)self.adaptee; - [NSLayoutConstraint deactivateConstraints:self.remoteVideoConstraints]; - - CGSize remoteVideoSize = self.remoteVideoSize; - - NSMutableArray *constraints = [NSMutableArray new]; - if (remoteVideoSize.width > 0 && remoteVideoSize.height > 0 && - containingView.bounds.size.width > 0 && containingView.bounds.size.height > 0) { - - CGFloat remoteVideoWidth = containingView.bounds.size.width; - CGFloat remoteVideoHeight = containingView.bounds.size.height; - - if (remoteVideoSize.width / containingView.bounds.size.width > remoteVideoSize.height / containingView.bounds.size.height) { - remoteVideoWidth = round(containingView.bounds.size.height * remoteVideoSize.width / remoteVideoSize.height); - } else { - remoteVideoHeight = round(containingView.bounds.size.width * remoteVideoSize.height / remoteVideoSize.width); - } - [constraints addObject:[videoView autoSetDimension:ALDimensionWidth toSize:remoteVideoWidth]]; - [constraints addObject:[videoView autoSetDimension:ALDimensionHeight toSize:remoteVideoHeight]]; - [constraints addObjectsFromArray:[videoView autoCenterInSuperview]]; - - videoView.frame = CGRectMake(0, 0, remoteVideoWidth, remoteVideoHeight); - - } else { - [constraints addObjectsFromArray:[videoView autoPinEdgesToSuperviewEdges]]; - } - - self.remoteVideoConstraints = constraints; - // We need to force relayout to occur immediately (and not - // wait for a UIKit layout/render pass) or the remoteVideoView - // (which presumably is updating its CALayer directly) will - // ocassionally appear to have bad frames. - [videoView setNeedsLayout]; - [[videoView superview] setNeedsLayout]; - [videoView layoutIfNeeded]; - [[videoView superview] layoutIfNeeded]; - } + return self; } -// MARK: - RTCEAGLVideoViewDelegate +#pragma mark - RTCEAGLVideoViewDelegate - (void)videoView:(RTCEAGLVideoView *)videoView didChangeVideoSize:(CGSize)size { - AssertIsOnMainThread(); - - // TODO -// if videoView != remoteVideoView { -// return -// } - DDLogInfo(@"%s called", __PRETTY_FUNCTION__); - - self.remoteVideoSize = size; - [self updateRemoteVideoLayout]; + // Do nothing. In older versions of WebRTC we used this to fit the video to fullscreen, + // but now we use the RTCVideoRenderer.setSize. + // However setting a delegate is *required* when using EAGL view. } #pragma mark - RTCVideoRenderer @@ -203,13 +64,13 @@ /** The size of the frame. */ - (void)setSize:(CGSize)size { - [self.adaptee setSize:size]; + [self.videoRenderer setSize:size]; } /** The frame to be displayed. */ - (void)renderFrame:(nullable RTCVideoFrame *)frame { - [self.adaptee renderFrame:frame]; + [self.videoRenderer renderFrame:frame]; } #pragma mark - Logging From 39e5875a36ffa3ab2078fd20cfa2af0cab14ddfc Mon Sep 17 00:00:00 2001 From: Michael Kirk Date: Wed, 4 Oct 2017 12:54:05 -0400 Subject: [PATCH 06/15] remove overzealous assert --- Signal/src/call/CallService.swift | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/Signal/src/call/CallService.swift b/Signal/src/call/CallService.swift index 742d617eb..40cc4c886 100644 --- a/Signal/src/call/CallService.swift +++ b/Signal/src/call/CallService.swift @@ -1043,10 +1043,8 @@ protocol CallServiceObserver: class { AssertIsOnMainThread() guard let call = self.call else { - // This should never happen; return to a known good state. - owsFail("\(TAG) call was unexpectedly nil in \(#function)") - OWSProdError(OWSAnalyticsEvents.callServiceCallMissing(), file:#file, function:#function, line:#line) - handleFailedCurrentCall(error: CallError.assertionError(description:"\(TAG) call unexpectedly nil in \(#function)")) + // This can happen after a call has ended. Reproducible on iOS11, when the other party ends the call. + Logger.info("\(TAG) ignoring mute request for obsolete call") return } From f837a462459137be4f834529ac516ffbfb7c6557 Mon Sep 17 00:00:00 2001 From: Michael Kirk Date: Wed, 4 Oct 2017 15:05:10 -0400 Subject: [PATCH 07/15] Fix post call crash on iOS8 // FREEBIE --- Signal/src/views/RemoteVideoView.m | 64 ++++++++++++++++++- .../translations/en.lproj/Localizable.strings | 5 +- 2 files changed, 66 insertions(+), 3 deletions(-) diff --git a/Signal/src/views/RemoteVideoView.m b/Signal/src/views/RemoteVideoView.m index 0133ab96b..7f5cb8f07 100644 --- a/Signal/src/views/RemoteVideoView.m +++ b/Signal/src/views/RemoteVideoView.m @@ -3,16 +3,67 @@ // #import "RemoteVideoView.h" +#import "UIFont+OWS.h" +#import "UIView+OWS.h" #import #import #import #import #import -@interface RTCMTLVideoView (MakePrivatePublic) -+ (BOOL)isMetalAvailable; +NS_ASSUME_NONNULL_BEGIN + +// As of RTC M61, iOS8 crashes when ending call while de-alloc'ing the EAGLVideoView +// WebRTC doesn't seem to support iOS8 - e.g. their Podfile requires iOS9+ +// Until WebRTC supports iOS8, we show a "upgrade iOS to see remote video" view +// to our few remaining iOS8 users +@interface NullVideoRenderer : UIView + @end +@implementation NullVideoRenderer + +- (instancetype)initWithFrame:(CGRect)frame +{ + self = [super initWithFrame:frame]; + if (!self) { + return self; + } + + self.backgroundColor = UIColor.blackColor; + + UILabel *label = [UILabel new]; + label.numberOfLines = 0; + label.text + = NSLocalizedString(@"CALL_REMOTE_VIDEO_DISABLED", @"Text shown on call screen in place of remote video"); + label.textAlignment = NSTextAlignmentCenter; + label.font = [UIFont ows_boldFontWithSize:ScaleFromIPhone5(20)]; + label.textColor = UIColor.whiteColor; + + [self addSubview:label]; + [label autoVCenterInSuperview]; + [label autoPinWidthToSuperviewWithMargin:ScaleFromIPhone5(16)]; + + return self; +} + +#pragma mark - RTCVideoRenderer + +/** The size of the frame. */ +- (void)setSize:(CGSize)size +{ + // Do nothing. +} + +/** The frame to be displayed. */ +- (void)renderFrame:(nullable RTCVideoFrame *)frame +{ + // Do nothing. +} + +@end + + @interface RemoteVideoView () @property (nonatomic, readonly) __kindof UIView *videoRenderer; @@ -28,6 +79,11 @@ return self; } + if (!SYSTEM_VERSION_GREATER_THAN_OR_EQUAL_TO(9, 0)) { + _videoRenderer = [NullVideoRenderer new]; + } + + if (_videoRenderer == nil) { // This class is defined in objc in order to access this compile time macro // Currently RTC only supports metal on 64bit machines #if defined(RTC_SUPPORTS_METAL) @@ -36,6 +92,8 @@ _videoRenderer = [[RTCMTLVideoView alloc] initWithFrame:CGRectZero]; } #endif + } + if (_videoRenderer == nil) { RTCEAGLVideoView *eaglVideoView = [[RTCEAGLVideoView alloc] initWithFrame:CGRectZero]; eaglVideoView.delegate = self; @@ -86,3 +144,5 @@ } @end + +NS_ASSUME_NONNULL_END diff --git a/Signal/translations/en.lproj/Localizable.strings b/Signal/translations/en.lproj/Localizable.strings index 59137c75e..0a2dc6f9d 100644 --- a/Signal/translations/en.lproj/Localizable.strings +++ b/Signal/translations/en.lproj/Localizable.strings @@ -208,6 +208,9 @@ /* Accessibilty label for placing call button */ "CALL_LABEL" = "Call"; +/* Text shown on call screen in place of remote video */ +"CALL_REMOTE_VIDEO_DISABLED" = "Please upgrade to iOS 9 or newer to see remote video."; + /* Call setup status label after outgoing call times out */ "CALL_SCREEN_STATUS_NO_ANSWER" = "No Answer."; @@ -601,7 +604,7 @@ /* A label for generic attachments. */ "GENERIC_ATTACHMENT_LABEL" = "Attachment"; -/* Please enter your search. */ +/* Alert message shown when user tries to search for GIFs without entering any search terms. */ "GIF_PICKER_VIEW_MISSING_QUERY" = "Please enter your search."; /* Title for the 'gif picker' dialog. */ From c3dc8508abc037ff5e7d12efe193b1bfe85cb972 Mon Sep 17 00:00:00 2001 From: Michael Kirk Date: Wed, 4 Oct 2017 15:19:36 -0400 Subject: [PATCH 08/15] pre-PR cleanup // FREEBIE --- Signal/src/ViewControllers/CallViewController.swift | 4 +--- Signal/src/call/CallAudioService.swift | 7 +++++-- Signal/src/views/RemoteVideoView.h | 4 ++-- Signal/src/views/RemoteVideoView.m | 5 ++++- 4 files changed, 12 insertions(+), 8 deletions(-) diff --git a/Signal/src/ViewControllers/CallViewController.swift b/Signal/src/ViewControllers/CallViewController.swift index 354315fb9..750186916 100644 --- a/Signal/src/ViewControllers/CallViewController.swift +++ b/Signal/src/ViewControllers/CallViewController.swift @@ -8,7 +8,6 @@ import PromiseKit // TODO: Add category so that button handlers can be defined where button is created. // TODO: Ensure buttons enabled & disabled as necessary. -//class CallViewController: OWSViewController, CallObserver, CallServiceObserver, RTCEAGLVideoViewDelegate { class CallViewController: OWSViewController, CallObserver, CallServiceObserver { let TAG = "[CallViewController]" @@ -231,10 +230,9 @@ class CallViewController: OWSViewController, CallObserver, CallServiceObserver { } func createVideoViews() { - localVideoView = RTCCameraPreviewView() remoteVideoView = RemoteVideoView() - remoteVideoView.isUserInteractionEnabled = false + localVideoView = RTCCameraPreviewView() remoteVideoView.isHidden = true localVideoView.isHidden = true diff --git a/Signal/src/call/CallAudioService.swift b/Signal/src/call/CallAudioService.swift index d791606e8..9a446dbd0 100644 --- a/Signal/src/call/CallAudioService.swift +++ b/Signal/src/call/CallAudioService.swift @@ -418,8 +418,11 @@ struct AudioSource: Hashable { guard let availableInputs = session.availableInputs else { // I'm not sure when this would happen. - // TODO this happens when the call ends (at leas ton iOS8) -// owsFail("No available inputs or inputs not ready") + if #available(iOS 9.0, *) { + // Fails on iOS8. We *could* remove this assert, but it might + // still be helfpul in catching a bug in a more used platform. + owsFail("No available inputs or inputs not ready") + } return [AudioSource.builtInSpeaker] } diff --git a/Signal/src/views/RemoteVideoView.h b/Signal/src/views/RemoteVideoView.h index 6e84de450..46db91fc9 100644 --- a/Signal/src/views/RemoteVideoView.h +++ b/Signal/src/views/RemoteVideoView.h @@ -5,8 +5,8 @@ #import /** - * Drives the full screen remote video, this class is backed by either the modern MetalKit backed view on supported - * systems or the leagacy EAGL view. MetalKit is supported on 64bit systems running iOS8 or newer. + * Drives the full screen remote video. This is *not* a swift class + * so we can take advantage of some compile time constants from WebRTC */ @interface RemoteVideoView : UIView diff --git a/Signal/src/views/RemoteVideoView.m b/Signal/src/views/RemoteVideoView.m index 7f5cb8f07..8f1e6f8f2 100644 --- a/Signal/src/views/RemoteVideoView.m +++ b/Signal/src/views/RemoteVideoView.m @@ -79,12 +79,13 @@ NS_ASSUME_NONNULL_BEGIN return self; } + // On iOS8: prints a message saying the feature is unavailable. if (!SYSTEM_VERSION_GREATER_THAN_OR_EQUAL_TO(9, 0)) { _videoRenderer = [NullVideoRenderer new]; } + // On 64-bit, iOS9+: uses the MetalKit backed view for improved battery/rendering performance. if (_videoRenderer == nil) { -// This class is defined in objc in order to access this compile time macro // Currently RTC only supports metal on 64bit machines #if defined(RTC_SUPPORTS_METAL) // RTCMTLVideoView requires the MTKView class, available in the iOS9+ MetalKit framework @@ -94,7 +95,9 @@ NS_ASSUME_NONNULL_BEGIN #endif } + // On 32-bit: uses the legacy EAGL backed view. if (_videoRenderer == nil) { + OWSAssert(!__arm64__); RTCEAGLVideoView *eaglVideoView = [[RTCEAGLVideoView alloc] initWithFrame:CGRectZero]; eaglVideoView.delegate = self; _videoRenderer = eaglVideoView; From 15f6135639c3fd75fb13b4fa3b4a5a70612ec794 Mon Sep 17 00:00:00 2001 From: Michael Kirk Date: Wed, 4 Oct 2017 16:36:31 -0400 Subject: [PATCH 09/15] Fix AspectRatio on legacy video view // FREEBIE --- Signal/src/UIView+OWS.h | 1 + Signal/src/UIView+OWS.m | 7 +- .../ViewControllers/CallViewController.swift | 1 - Signal/src/views/RemoteVideoView.h | 4 + Signal/src/views/RemoteVideoView.m | 107 ++++++++++++++---- 5 files changed, 94 insertions(+), 26 deletions(-) diff --git a/Signal/src/UIView+OWS.h b/Signal/src/UIView+OWS.h index 62ec64920..498d04421 100644 --- a/Signal/src/UIView+OWS.h +++ b/Signal/src/UIView+OWS.h @@ -36,6 +36,7 @@ CGFloat ScaleFromIPhone5(CGFloat iPhone5Value); - (void)autoPinHeightToHeightOfView:(UIView *)view; - (NSLayoutConstraint *)autoPinToSquareAspectRatio; +- (NSLayoutConstraint *)autoPinToAspectRatio:(CGFloat)ratio; #pragma mark - Content Hugging and Compression Resistance diff --git a/Signal/src/UIView+OWS.m b/Signal/src/UIView+OWS.m index e3bdbaaef..7f14abc90 100644 --- a/Signal/src/UIView+OWS.m +++ b/Signal/src/UIView+OWS.m @@ -99,6 +99,11 @@ CGFloat ScaleFromIPhone5(CGFloat iPhone5Value) } - (NSLayoutConstraint *)autoPinToSquareAspectRatio +{ + return [self autoPinToAspectRatio:1.0]; +} + +- (NSLayoutConstraint *)autoPinToAspectRatio:(CGFloat)ratio { self.translatesAutoresizingMaskIntoConstraints = NO; @@ -107,7 +112,7 @@ CGFloat ScaleFromIPhone5(CGFloat iPhone5Value) relatedBy:NSLayoutRelationEqual toItem:self attribute:NSLayoutAttributeHeight - multiplier:1.f + multiplier:ratio constant:0.f]; [constraint autoInstall]; return constraint; diff --git a/Signal/src/ViewControllers/CallViewController.swift b/Signal/src/ViewControllers/CallViewController.swift index 750186916..7873aa072 100644 --- a/Signal/src/ViewControllers/CallViewController.swift +++ b/Signal/src/ViewControllers/CallViewController.swift @@ -589,7 +589,6 @@ class CallViewController: OWSViewController, CallObserver, CallServiceObserver { internal func updateRemoteVideoLayout() { remoteVideoView.isHidden = !self.hasRemoteVideoTrack - self.remoteVideoView.setSize(self.view.frame.size) updateCallUI(callState: call.state) } diff --git a/Signal/src/views/RemoteVideoView.h b/Signal/src/views/RemoteVideoView.h index 46db91fc9..353789890 100644 --- a/Signal/src/views/RemoteVideoView.h +++ b/Signal/src/views/RemoteVideoView.h @@ -4,6 +4,8 @@ #import +NS_ASSUME_NONNULL_BEGIN + /** * Drives the full screen remote video. This is *not* a swift class * so we can take advantage of some compile time constants from WebRTC @@ -11,3 +13,5 @@ @interface RemoteVideoView : UIView @end + +NS_ASSUME_NONNULL_END diff --git a/Signal/src/views/RemoteVideoView.m b/Signal/src/views/RemoteVideoView.m index 8f1e6f8f2..f266ad1ec 100644 --- a/Signal/src/views/RemoteVideoView.m +++ b/Signal/src/views/RemoteVideoView.m @@ -7,14 +7,16 @@ #import "UIView+OWS.h" #import #import +#import #import #import #import NS_ASSUME_NONNULL_BEGIN -// As of RTC M61, iOS8 crashes when ending call while de-alloc'ing the EAGLVideoView -// WebRTC doesn't seem to support iOS8 - e.g. their Podfile requires iOS9+ +// As of RTC M61, iOS8 crashes when ending calls while de-alloc'ing the EAGLVideoView. +// WebRTC doesn't seem to support iOS8 - e.g. their Podfile requires iOS9+, and they +// unconditionally require MetalKit on a 64bit iOS8 device (which crashes). // Until WebRTC supports iOS8, we show a "upgrade iOS to see remote video" view // to our few remaining iOS8 users @interface NullVideoRenderer : UIView @@ -63,11 +65,13 @@ NS_ASSUME_NONNULL_BEGIN @end - @interface RemoteVideoView () @property (nonatomic, readonly) __kindof UIView *videoRenderer; +// Used for legacy EAGLVideoView +@property (nullable, nonatomic) NSMutableArray *remoteVideoConstraints; + @end @implementation RemoteVideoView @@ -82,44 +86,46 @@ NS_ASSUME_NONNULL_BEGIN // On iOS8: prints a message saying the feature is unavailable. if (!SYSTEM_VERSION_GREATER_THAN_OR_EQUAL_TO(9, 0)) { _videoRenderer = [NullVideoRenderer new]; + [self addSubview:_videoRenderer]; + [_videoRenderer autoPinEdgesToSuperviewEdges]; } - // On 64-bit, iOS9+: uses the MetalKit backed view for improved battery/rendering performance. - if (_videoRenderer == nil) { // Currently RTC only supports metal on 64bit machines #if defined(RTC_SUPPORTS_METAL) - // RTCMTLVideoView requires the MTKView class, available in the iOS9+ MetalKit framework - if ([MTKView class]) { - _videoRenderer = [[RTCMTLVideoView alloc] initWithFrame:CGRectZero]; + // On 64-bit, iOS9+: uses the MetalKit backed view for improved battery/rendering performance. + if (_videoRenderer == nil) { + + // It is insufficient to check the RTC_SUPPORTS_METAL macro to determine Metal support. + // RTCMTLVideoView requires the MTKView class, available only in iOS9+ + // So check that it exists before proceeding. + if ([MTKView class]) { + _videoRenderer = [[RTCMTLVideoView alloc] initWithFrame:CGRectZero]; + [self addSubview:_videoRenderer]; + [_videoRenderer autoPinEdgesToSuperviewEdges]; + } } +#elif defined(__arm64__) + // Canary incase the upstream RTC_SUPPORTS_METAL macro changes semantics + OWSFail(@"should only use legacy video view on 32bit systems"); #endif - } - // On 32-bit: uses the legacy EAGL backed view. + // On 32-bit iOS9+ systems, use the legacy EAGL backed view. if (_videoRenderer == nil) { - OWSAssert(!__arm64__); - RTCEAGLVideoView *eaglVideoView = [[RTCEAGLVideoView alloc] initWithFrame:CGRectZero]; + RTCEAGLVideoView *eaglVideoView = [RTCEAGLVideoView new]; eaglVideoView.delegate = self; _videoRenderer = eaglVideoView; + [self addSubview:_videoRenderer]; + // Pinning legacy RTCEAGL view discards aspect ratio. + // So we have a more verbose layout in the RTCEAGLVideoViewDelegate methods + // [_videoRenderer autoPinEdgesToSuperviewEdges]; } - [self addSubview:_videoRenderer]; - + // We want the rendered video to go edge-to-edge. _videoRenderer.layoutMargins = UIEdgeInsetsZero; - [_videoRenderer autoPinEdgesToSuperviewEdges]; return self; } -#pragma mark - RTCEAGLVideoViewDelegate - -- (void)videoView:(RTCEAGLVideoView *)videoView didChangeVideoSize:(CGSize)size -{ - // Do nothing. In older versions of WebRTC we used this to fit the video to fullscreen, - // but now we use the RTCVideoRenderer.setSize. - // However setting a delegate is *required* when using EAGL view. -} - #pragma mark - RTCVideoRenderer /** The size of the frame. */ @@ -128,6 +134,59 @@ NS_ASSUME_NONNULL_BEGIN [self.videoRenderer setSize:size]; } +#pragma mark - RTCEAGLVideoViewDelegate + +- (void)videoView:(RTCEAGLVideoView *)videoView didChangeVideoSize:(CGSize)remoteVideoSize +{ + AssertIsOnMainThread(); + CGFloat aspectRatio = remoteVideoSize.width / remoteVideoSize.height; + + DDLogVerbose(@"%@ Remote video size: width: %f height: %f ratio: %f", + self.logTag, + remoteVideoSize.width, + remoteVideoSize.height, + aspectRatio); + + UIView *containingView = self.superview; + if (containingView == nil) { + DDLogDebug(@"%@ Cannot layout video view without superview", self.logTag); + return; + } + + if (![self.videoRenderer isKindOfClass:[RTCEAGLVideoView class]]) { + OWSFail(@"%@ Unexpected video renderer: %@", self.logTag, self.videoRenderer); + return; + } + + [NSLayoutConstraint deactivateConstraints:self.remoteVideoConstraints]; + + NSMutableArray *constraints = [NSMutableArray new]; + if (remoteVideoSize.width > 0 && remoteVideoSize.height > 0 && containingView.bounds.size.width > 0 + && containingView.bounds.size.height > 0) { + + [constraints addObject:[videoView autoPinToAspectRatio:aspectRatio]]; + [constraints addObject:[videoView autoSetDimension:ALDimensionWidth + toSize:containingView.width + relation:NSLayoutRelationGreaterThanOrEqual]]; + [constraints addObject:[videoView autoSetDimension:ALDimensionHeight + toSize:containingView.height + relation:NSLayoutRelationGreaterThanOrEqual]]; + [constraints addObjectsFromArray:[videoView autoCenterInSuperview]]; + } else { + [constraints addObjectsFromArray:[videoView autoPinEdgesToSuperviewEdges]]; + } + + self.remoteVideoConstraints = constraints; + // We need to force relayout to occur immediately (and not + // wait for a UIKit layout/render pass) or the remoteVideoView + // (which presumably is updating its CALayer directly) will + // ocassionally appear to have bad frames. + [videoView setNeedsLayout]; + [[videoView superview] setNeedsLayout]; + [videoView layoutIfNeeded]; + [[videoView superview] layoutIfNeeded]; +} + /** The frame to be displayed. */ - (void)renderFrame:(nullable RTCVideoFrame *)frame { From 14b6f31635c2f4a5d921ac408ff5c31db938e690 Mon Sep 17 00:00:00 2001 From: Michael Kirk Date: Wed, 4 Oct 2017 17:24:26 -0400 Subject: [PATCH 10/15] position video view below status bar // FREEBIE --- Signal/src/views/RemoteVideoView.m | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/Signal/src/views/RemoteVideoView.m b/Signal/src/views/RemoteVideoView.m index f266ad1ec..d6b175651 100644 --- a/Signal/src/views/RemoteVideoView.m +++ b/Signal/src/views/RemoteVideoView.m @@ -102,6 +102,15 @@ NS_ASSUME_NONNULL_BEGIN _videoRenderer = [[RTCMTLVideoView alloc] initWithFrame:CGRectZero]; [self addSubview:_videoRenderer]; [_videoRenderer autoPinEdgesToSuperviewEdges]; + // HACK: Although RTCMTLVideo view is positioned to the top edge of the screen + // It's inner (private) MTKView is below the status bar. + for (UIView *subview in [_videoRenderer subviews]) { + if ([subview isKindOfClass:[MTKView class]]) { + [subview autoPinEdgesToSuperviewEdges]; + } else { + OWSFail(@"New subviews added to MTLVideoView. Reconsider this hack."); + } + } } } #elif defined(__arm64__) From 580e82beae43ef0103c2206b19969161e2781da6 Mon Sep 17 00:00:00 2001 From: Michael Kirk Date: Thu, 5 Oct 2017 09:20:59 -0400 Subject: [PATCH 11/15] CR: clamp reasonable aspect ratio // FREEBIE --- Signal/src/UIView+OWS.m | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/Signal/src/UIView+OWS.m b/Signal/src/UIView+OWS.m index 7f14abc90..f8eaf9dad 100644 --- a/Signal/src/UIView+OWS.m +++ b/Signal/src/UIView+OWS.m @@ -105,14 +105,19 @@ CGFloat ScaleFromIPhone5(CGFloat iPhone5Value) - (NSLayoutConstraint *)autoPinToAspectRatio:(CGFloat)ratio { - self.translatesAutoresizingMaskIntoConstraints = NO; + // Clamp to ensure view has reasonable aspect ratio. + CGFloat clampedRatio = Clamp(ratio, 0.5, 95.0); + if (clampedRatio != ratio) { + OWSFail(@"Invalid aspect ratio: %f for view: %@", ratio, self); + } + self.translatesAutoresizingMaskIntoConstraints = NO; NSLayoutConstraint *constraint = [NSLayoutConstraint constraintWithItem:self attribute:NSLayoutAttributeWidth relatedBy:NSLayoutRelationEqual toItem:self attribute:NSLayoutAttributeHeight - multiplier:ratio + multiplier:clampedRatio constant:0.f]; [constraint autoInstall]; return constraint; From 3d3af2179282d3c9efd2198425c980e90582a787 Mon Sep 17 00:00:00 2001 From: Michael Kirk Date: Thu, 5 Oct 2017 09:21:22 -0400 Subject: [PATCH 12/15] CR: clarify comment, proper linewrap // FREEBIE --- Signal/src/call/CallAudioService.swift | 8 +++++--- Signal/src/views/RemoteVideoView.m | 1 + 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/Signal/src/call/CallAudioService.swift b/Signal/src/call/CallAudioService.swift index 9a446dbd0..8f38afc40 100644 --- a/Signal/src/call/CallAudioService.swift +++ b/Signal/src/call/CallAudioService.swift @@ -417,10 +417,12 @@ struct AudioSource: Hashable { let session = AVAudioSession.sharedInstance() guard let availableInputs = session.availableInputs else { - // I'm not sure when this would happen. + // I'm not sure why this would happen, but it may indicate an error. + // In practice, I haven't seen it on iOS9+. + // + // I *have* seen it on iOS8, but it doesn't seem to cause any problems, + // so we do *not* trigger the assert on that platform. if #available(iOS 9.0, *) { - // Fails on iOS8. We *could* remove this assert, but it might - // still be helfpul in catching a bug in a more used platform. owsFail("No available inputs or inputs not ready") } return [AudioSource.builtInSpeaker] diff --git a/Signal/src/views/RemoteVideoView.m b/Signal/src/views/RemoteVideoView.m index d6b175651..63a47067b 100644 --- a/Signal/src/views/RemoteVideoView.m +++ b/Signal/src/views/RemoteVideoView.m @@ -41,6 +41,7 @@ NS_ASSUME_NONNULL_BEGIN label.textAlignment = NSTextAlignmentCenter; label.font = [UIFont ows_boldFontWithSize:ScaleFromIPhone5(20)]; label.textColor = UIColor.whiteColor; + label.lineBreakMode = NSLineBreakByWordWrapping; [self addSubview:label]; [label autoVCenterInSuperview]; From 3864880a6fb49e92d9d52d2034d793ef7166a334 Mon Sep 17 00:00:00 2001 From: Michael Kirk Date: Thu, 5 Oct 2017 09:21:37 -0400 Subject: [PATCH 13/15] CR: ensure view doesn't grow indefinitely - explicitly mark constraint as required // FREEBIE --- Signal/src/views/RemoteVideoView.m | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/Signal/src/views/RemoteVideoView.m b/Signal/src/views/RemoteVideoView.m index 63a47067b..0b8568088 100644 --- a/Signal/src/views/RemoteVideoView.m +++ b/Signal/src/views/RemoteVideoView.m @@ -107,7 +107,10 @@ NS_ASSUME_NONNULL_BEGIN // It's inner (private) MTKView is below the status bar. for (UIView *subview in [_videoRenderer subviews]) { if ([subview isKindOfClass:[MTKView class]]) { - [subview autoPinEdgesToSuperviewEdges]; + [NSLayoutConstraint autoSetPriority:UILayoutPriorityRequired + forConstraints:^{ + [subview autoPinEdgesToSuperviewEdges]; + }]; } else { OWSFail(@"New subviews added to MTLVideoView. Reconsider this hack."); } @@ -174,6 +177,9 @@ NS_ASSUME_NONNULL_BEGIN if (remoteVideoSize.width > 0 && remoteVideoSize.height > 0 && containingView.bounds.size.width > 0 && containingView.bounds.size.height > 0) { + // to approximate "scale to fill" contentMode + // - Pin aspect ratio + // - Width and height is *at least* as wide as superview [constraints addObject:[videoView autoPinToAspectRatio:aspectRatio]]; [constraints addObject:[videoView autoSetDimension:ALDimensionWidth toSize:containingView.width @@ -182,6 +188,13 @@ NS_ASSUME_NONNULL_BEGIN toSize:containingView.height relation:NSLayoutRelationGreaterThanOrEqual]]; [constraints addObjectsFromArray:[videoView autoCenterInSuperview]]; + + // Low priority constraints force view to be no larger than necessary. + [NSLayoutConstraint autoSetPriority:UILayoutPriorityDefaultLow + forConstraints:^{ + [constraints addObjectsFromArray:[videoView autoPinEdgesToSuperviewEdges]]; + }]; + } else { [constraints addObjectsFromArray:[videoView autoPinEdgesToSuperviewEdges]]; } From 4c797151e623f8bd60f648eddddb7794bb8937cb Mon Sep 17 00:00:00 2001 From: Michael Kirk Date: Thu, 5 Oct 2017 12:14:25 -0400 Subject: [PATCH 14/15] Avoid divide by 0 error Didn't actually see this in the wild - just being defensive. // FREEBIE --- Signal/src/views/RemoteVideoView.m | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/Signal/src/views/RemoteVideoView.m b/Signal/src/views/RemoteVideoView.m index 0b8568088..39fc309d5 100644 --- a/Signal/src/views/RemoteVideoView.m +++ b/Signal/src/views/RemoteVideoView.m @@ -152,6 +152,11 @@ NS_ASSUME_NONNULL_BEGIN - (void)videoView:(RTCEAGLVideoView *)videoView didChangeVideoSize:(CGSize)remoteVideoSize { AssertIsOnMainThread(); + if (remoteVideoSize.height <= 0) { + OWSFail(@"Illegal video height: %f", remoteVideoSize.height); + return; + } + CGFloat aspectRatio = remoteVideoSize.width / remoteVideoSize.height; DDLogVerbose(@"%@ Remote video size: width: %f height: %f ratio: %f", From fe8c6346a95e2b5be4d92a839719168eb9bb74af Mon Sep 17 00:00:00 2001 From: Michael Kirk Date: Thu, 5 Oct 2017 12:52:48 -0400 Subject: [PATCH 15/15] Update carthage to use oak-built WebRTC.framework M61+Signal // FREEBIE --- Carthage | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Carthage b/Carthage index 31b7ea5a3..a3eede219 160000 --- a/Carthage +++ b/Carthage @@ -1 +1 @@ -Subproject commit 31b7ea5a35345833d06d61b7da2333827de3a084 +Subproject commit a3eede219e225d5ae420c3e46161a6363e45cefc