From 876814dd43090b98ce16252d2ef89646547c6c39 Mon Sep 17 00:00:00 2001 From: Niels Andriesse Date: Fri, 13 Aug 2021 13:47:22 +1000 Subject: [PATCH] Basic proof of concept --- Podfile | 10 ++++---- Podfile.lock | 10 ++++---- Session.xcodeproj/project.pbxproj | 4 ++-- Session/Calls/CallVC.swift | 24 +++++++------------ Session/Calls/CameraManager.swift | 3 ++- Session/Calls/VideoCallVC.swift | 6 +++-- .../Calls/CallManager+UI.swift | 1 + SessionMessagingKit/Calls/CallManager.swift | 12 ++++------ 8 files changed, 33 insertions(+), 37 deletions(-) diff --git a/Podfile b/Podfile index 1b004a8b1..496fb32c7 100644 --- a/Podfile +++ b/Podfile @@ -6,6 +6,7 @@ use_frameworks! target 'Session' do pod 'AFNetworking', inhibit_warnings: true pod 'CryptoSwift', :inhibit_warnings => true + pod 'GoogleWebRTC', :inhibit_warnings => true pod 'Mantle', git: 'https://github.com/signalapp/Mantle', branch: 'signal-master', :inhibit_warnings => true pod 'NVActivityIndicatorView', :inhibit_warnings => true pod 'PromiseKit', :inhibit_warnings => true @@ -13,7 +14,6 @@ target 'Session' do pod 'Reachability', :inhibit_warnings => true pod 'SocketRocket', '~> 0.5.1', :inhibit_warnings => true pod 'Sodium', '~> 0.8.0', :inhibit_warnings => true - pod 'WebRTC', '~> 63.11', :inhibit_warnings => true pod 'YapDatabase/SQLCipher', :git => 'https://github.com/loki-project/session-ios-yap-database.git', branch: 'signal-release', :inhibit_warnings => true pod 'YYImage', git: 'https://github.com/signalapp/YYImage', :inhibit_warnings => true pod 'ZXingObjC', :inhibit_warnings => true @@ -23,20 +23,20 @@ target 'SessionShareExtension' do pod 'AFNetworking', inhibit_warnings: true pod 'CryptoSwift', :inhibit_warnings => true pod 'Curve25519Kit', git: 'https://github.com/signalapp/Curve25519Kit.git', :inhibit_warnings => true + pod 'GoogleWebRTC', :inhibit_warnings => true pod 'Mantle', git: 'https://github.com/signalapp/Mantle', branch: 'signal-master', :inhibit_warnings => true pod 'PromiseKit', :inhibit_warnings => true pod 'PureLayout', '~> 3.1.8', :inhibit_warnings => true pod 'SignalCoreKit', git: 'https://github.com/signalapp/SignalCoreKit.git', :inhibit_warnings => true pod 'SocketRocket', '~> 0.5.1', :inhibit_warnings => true - pod 'WebRTC', '~> 63.11', :inhibit_warnings => true pod 'YapDatabase/SQLCipher', :git => 'https://github.com/loki-project/session-ios-yap-database.git', branch: 'signal-release', :inhibit_warnings => true end target 'SessionNotificationServiceExtension' do pod 'Curve25519Kit', git: 'https://github.com/signalapp/Curve25519Kit.git', :inhibit_warnings => true + pod 'GoogleWebRTC', :inhibit_warnings => true pod 'SignalCoreKit', git: 'https://github.com/signalapp/SignalCoreKit.git', :inhibit_warnings => true pod 'SocketRocket', '~> 0.5.1', :inhibit_warnings => true - pod 'WebRTC', '~> 63.11', :inhibit_warnings => true pod 'YapDatabase/SQLCipher', :git => 'https://github.com/loki-project/session-ios-yap-database.git', branch: 'signal-release', :inhibit_warnings => true end @@ -44,6 +44,7 @@ target 'SignalUtilitiesKit' do pod 'AFNetworking', inhibit_warnings: true pod 'CryptoSwift', :inhibit_warnings => true pod 'Curve25519Kit', git: 'https://github.com/signalapp/Curve25519Kit.git', :inhibit_warnings => true + pod 'GoogleWebRTC', :inhibit_warnings => true pod 'GRKOpenSSLFramework', :inhibit_warnings => true pod 'HKDFKit', :inhibit_warnings => true pod 'Mantle', git: 'https://github.com/signalapp/Mantle', branch: 'signal-master', :inhibit_warnings => true @@ -55,7 +56,6 @@ target 'SignalUtilitiesKit' do pod 'SignalCoreKit', git: 'https://github.com/signalapp/SignalCoreKit.git', :inhibit_warnings => true pod 'SocketRocket', '~> 0.5.1', :inhibit_warnings => true pod 'SwiftProtobuf', '~> 1.5.0', :inhibit_warnings => true - pod 'WebRTC', '~> 63.11', :inhibit_warnings => true pod 'YapDatabase/SQLCipher', :git => 'https://github.com/loki-project/session-ios-yap-database.git', branch: 'signal-release', :inhibit_warnings => true pod 'YYImage', git: 'https://github.com/signalapp/YYImage', :inhibit_warnings => true end @@ -68,6 +68,7 @@ target 'SessionMessagingKit' do pod 'AFNetworking', inhibit_warnings: true pod 'CryptoSwift', :inhibit_warnings => true pod 'Curve25519Kit', git: 'https://github.com/signalapp/Curve25519Kit.git', :inhibit_warnings => true + pod 'GoogleWebRTC', :inhibit_warnings => true pod 'HKDFKit', :inhibit_warnings => true pod 'Mantle', git: 'https://github.com/signalapp/Mantle', branch: 'signal-master', :inhibit_warnings => true pod 'PromiseKit', :inhibit_warnings => true @@ -78,7 +79,6 @@ target 'SessionMessagingKit' do pod 'SocketRocket', '~> 0.5.1', :inhibit_warnings => true pod 'Sodium', '~> 0.8.0', :inhibit_warnings => true pod 'SwiftProtobuf', '~> 1.5.0', :inhibit_warnings => true - pod 'WebRTC', '~> 63.11', :inhibit_warnings => true pod 'YapDatabase/SQLCipher', :git => 'https://github.com/loki-project/session-ios-yap-database.git', branch: 'signal-release', :inhibit_warnings => true end diff --git a/Podfile.lock b/Podfile.lock index 4507b28a4..0f47af5d0 100644 --- a/Podfile.lock +++ b/Podfile.lock @@ -21,6 +21,7 @@ PODS: - Curve25519Kit (2.1.0): - CocoaLumberjack - SignalCoreKit + - GoogleWebRTC (1.1.31999) - GRKOpenSSLFramework (1.0.2.20) - HKDFKit (0.0.3) - Mantle (2.1.0): @@ -52,7 +53,6 @@ PODS: - SQLCipher/standard (4.4.0): - SQLCipher/common - SwiftProtobuf (1.5.0) - - WebRTC (63.11.20455) - YapDatabase/SQLCipher (3.1.1): - YapDatabase/SQLCipher/Core (= 3.1.1) - YapDatabase/SQLCipher/Extensions (= 3.1.1) @@ -126,6 +126,7 @@ DEPENDENCIES: - AFNetworking - CryptoSwift - Curve25519Kit (from `https://github.com/signalapp/Curve25519Kit.git`) + - GoogleWebRTC - GRKOpenSSLFramework - HKDFKit - Mantle (from `https://github.com/signalapp/Mantle`, branch `signal-master`) @@ -138,7 +139,6 @@ DEPENDENCIES: - SocketRocket (~> 0.5.1) - Sodium (~> 0.8.0) - SwiftProtobuf (~> 1.5.0) - - WebRTC (~> 63.11) - YapDatabase/SQLCipher (from `https://github.com/loki-project/session-ios-yap-database.git`, branch `signal-release`) - YYImage (from `https://github.com/signalapp/YYImage`) - ZXingObjC @@ -148,6 +148,7 @@ SPEC REPOS: - AFNetworking - CocoaLumberjack - CryptoSwift + - GoogleWebRTC - GRKOpenSSLFramework - HKDFKit - NVActivityIndicatorView @@ -159,7 +160,6 @@ SPEC REPOS: - Sodium - SQLCipher - SwiftProtobuf - - WebRTC - ZXingObjC EXTERNAL SOURCES: @@ -198,6 +198,7 @@ SPEC CHECKSUMS: CocoaLumberjack: bd155f2dd06c0e0b03f876f7a3ee55693122ec94 CryptoSwift: 093499be1a94b0cae36e6c26b70870668cb56060 Curve25519Kit: e63f9859ede02438ae3defc5e1a87e09d1ec7ee6 + GoogleWebRTC: b39a78c4f5cc6b0323415b9233db03a2faa7b0f0 GRKOpenSSLFramework: dc635b0a9d4cd8af2a9ff80a61e779e21b69dfd8 HKDFKit: c058305d6f64b84f28c50bd7aa89574625bcb62a Mantle: 2fa750afa478cd625a94230fbf1c13462f29395b @@ -211,11 +212,10 @@ SPEC CHECKSUMS: Sodium: 63c0ca312a932e6da481689537d4b35568841bdc SQLCipher: e434ed542b24f38ea7b36468a13f9765e1b5c072 SwiftProtobuf: 241400280f912735c1e1b9fe675fdd2c6c4d42e2 - WebRTC: f2a6203584745fe53532633397557876b5d71640 YapDatabase: b418a4baa6906e8028748938f9159807fd039af4 YYImage: 6db68da66f20d9f169ceb94dfb9947c3867b9665 ZXingObjC: fdbb269f25dd2032da343e06f10224d62f537bdb -PODFILE CHECKSUM: 0f0ee15979921c085945dc1cacb0fd1fd380b77d +PODFILE CHECKSUM: 215e4f2af32f65d98d1442e6ec0df3576c994a02 COCOAPODS: 1.10.1 diff --git a/Session.xcodeproj/project.pbxproj b/Session.xcodeproj/project.pbxproj index beeb2b9f4..f7f3842fb 100644 --- a/Session.xcodeproj/project.pbxproj +++ b/Session.xcodeproj/project.pbxproj @@ -4251,6 +4251,7 @@ "${BUILT_PRODUCTS_DIR}/AFNetworking/AFNetworking.framework", "${BUILT_PRODUCTS_DIR}/CocoaLumberjack/CocoaLumberjack.framework", "${BUILT_PRODUCTS_DIR}/CryptoSwift/CryptoSwift.framework", + "${PODS_ROOT}/GoogleWebRTC/Frameworks/frameworks/WebRTC.framework", "${BUILT_PRODUCTS_DIR}/Mantle/Mantle.framework", "${BUILT_PRODUCTS_DIR}/NVActivityIndicatorView/NVActivityIndicatorView.framework", "${BUILT_PRODUCTS_DIR}/PromiseKit/PromiseKit.framework", @@ -4259,7 +4260,6 @@ "${BUILT_PRODUCTS_DIR}/SQLCipher/SQLCipher.framework", "${BUILT_PRODUCTS_DIR}/SocketRocket/SocketRocket.framework", "${BUILT_PRODUCTS_DIR}/Sodium/Sodium.framework", - "${PODS_ROOT}/WebRTC/WebRTC.framework", "${BUILT_PRODUCTS_DIR}/YYImage/YYImage.framework", "${BUILT_PRODUCTS_DIR}/YapDatabase/YapDatabase.framework", "${BUILT_PRODUCTS_DIR}/ZXingObjC/ZXingObjC.framework", @@ -4275,6 +4275,7 @@ "${TARGET_BUILD_DIR}/${FRAMEWORKS_FOLDER_PATH}/AFNetworking.framework", "${TARGET_BUILD_DIR}/${FRAMEWORKS_FOLDER_PATH}/CocoaLumberjack.framework", "${TARGET_BUILD_DIR}/${FRAMEWORKS_FOLDER_PATH}/CryptoSwift.framework", + "${TARGET_BUILD_DIR}/${FRAMEWORKS_FOLDER_PATH}/WebRTC.framework", "${TARGET_BUILD_DIR}/${FRAMEWORKS_FOLDER_PATH}/Mantle.framework", "${TARGET_BUILD_DIR}/${FRAMEWORKS_FOLDER_PATH}/NVActivityIndicatorView.framework", "${TARGET_BUILD_DIR}/${FRAMEWORKS_FOLDER_PATH}/PromiseKit.framework", @@ -4283,7 +4284,6 @@ "${TARGET_BUILD_DIR}/${FRAMEWORKS_FOLDER_PATH}/SQLCipher.framework", "${TARGET_BUILD_DIR}/${FRAMEWORKS_FOLDER_PATH}/SocketRocket.framework", "${TARGET_BUILD_DIR}/${FRAMEWORKS_FOLDER_PATH}/Sodium.framework", - "${TARGET_BUILD_DIR}/${FRAMEWORKS_FOLDER_PATH}/WebRTC.framework", "${TARGET_BUILD_DIR}/${FRAMEWORKS_FOLDER_PATH}/YYImage.framework", "${TARGET_BUILD_DIR}/${FRAMEWORKS_FOLDER_PATH}/YapDatabase.framework", "${TARGET_BUILD_DIR}/${FRAMEWORKS_FOLDER_PATH}/ZXingObjC.framework", diff --git a/Session/Calls/CallVC.swift b/Session/Calls/CallVC.swift index 77e44223d..111d1878b 100644 --- a/Session/Calls/CallVC.swift +++ b/Session/Calls/CallVC.swift @@ -4,6 +4,7 @@ import WebRTC final class CallVC : UIViewController, CameraCaptureDelegate, CallManagerDelegate, MockWebSocketDelegate { private let videoCallVC = VideoCallVC() + let videoCapturer: RTCVideoCapturer = RTCCameraVideoCapturer(delegate: CallManager.shared.localVideoSource) private var messageQueue: [String] = [] private var isConnected = false { didSet { @@ -43,31 +44,25 @@ final class CallVC : UIViewController, CameraCaptureDelegate, CallManagerDelegat // MARK: Lifecycle override func viewDidLoad() { super.viewDidLoad() - setUpCamera() - embedVideoCallVC() - view.addSubview(containerView) - containerView.pin(to: view) - } - - private func setUpCamera() { + touch(CallManager.shared) + CallManager.shared.delegate = self CameraManager.shared.delegate = self CameraManager.shared.prepare() - } - - private func embedVideoCallVC() { addChild(videoCallVC) containerView.addSubview(videoCallVC.view) videoCallVC.view.translatesAutoresizingMaskIntoConstraints = false videoCallVC.view.pin(to: containerView) + view.addSubview(containerView) + containerView.pin(to: view) } - override func viewWillAppear(_ animated: Bool) { - super.viewWillAppear(animated) + override func viewDidAppear(_ animated: Bool) { + super.viewDidAppear(animated) CameraManager.shared.start() } - override func viewDidDisappear(_ animated: Bool) { - super.viewDidDisappear(animated) + override func viewWillDisappear(_ animated: Bool) { + super.viewWillDisappear(animated) CameraManager.shared.stop() } @@ -157,7 +152,6 @@ final class CallVC : UIViewController, CameraCaptureDelegate, CallManagerDelegat ] guard let data = try? JSONSerialization.data(withJSONObject: message, options: [.prettyPrinted]) else { return } MockWebSocket.shared.send(data) - CallManager.shared.delegate = self if isInitiator { CallManager.shared.initiateCall().retainUntilComplete() } diff --git a/Session/Calls/CameraManager.swift b/Session/Calls/CameraManager.swift index 622fb13b0..019efab22 100644 --- a/Session/Calls/CameraManager.swift +++ b/Session/Calls/CameraManager.swift @@ -24,13 +24,14 @@ final class CameraManager : NSObject { private override init() { } func prepare() { + captureSession.sessionPreset = .low if let videoCaptureDevice = videoCaptureDevice, let videoInput = try? AVCaptureDeviceInput(device: videoCaptureDevice), captureSession.canAddInput(videoInput) { captureSession.addInput(videoInput) } if captureSession.canAddOutput(videoDataOutput) { captureSession.addOutput(videoDataOutput) - videoDataOutput.videoSettings = [kCVPixelBufferPixelFormatTypeKey as String: Int(kCVPixelFormatType_32BGRA)] + videoDataOutput.videoSettings = [ kCVPixelBufferPixelFormatTypeKey as String : Int(kCVPixelFormatType_32BGRA) ] videoDataOutput.setSampleBufferDelegate(self, queue: dataOutputQueue) videoDataOutput.connection(with: .video)?.videoOrientation = .portrait videoDataOutput.connection(with: .video)?.automaticallyAdjustsVideoMirroring = false diff --git a/Session/Calls/VideoCallVC.swift b/Session/Calls/VideoCallVC.swift index f757e5ecf..6e7caf8ca 100644 --- a/Session/Calls/VideoCallVC.swift +++ b/Session/Calls/VideoCallVC.swift @@ -51,8 +51,10 @@ extension VideoCallVC : CameraCaptureDelegate { func captureVideoOutput(sampleBuffer: CMSampleBuffer) { guard let pixelBuffer = CMSampleBufferGetImageBuffer(sampleBuffer) else { return } let rtcpixelBuffer = RTCCVPixelBuffer(pixelBuffer: pixelBuffer) - let timeStampNs = Int64(CMTimeGetSeconds(CMSampleBufferGetPresentationTimeStamp(sampleBuffer)) * 1000000000) - let videoFrame = RTCVideoFrame(buffer: rtcpixelBuffer, rotation: RTCVideoRotation._0, timeStampNs: timeStampNs) + let timestamp = CMTimeGetSeconds(CMSampleBufferGetPresentationTimeStamp(sampleBuffer)) + let timestampNs = Int64(timestamp * 1000000000) + let videoFrame = RTCVideoFrame(buffer: rtcpixelBuffer, rotation: RTCVideoRotation._0, timeStampNs: timestampNs) + videoFrame.timeStamp = Int32(timestamp) CallManager.shared.handleLocalFrameCaptured(videoFrame) } } diff --git a/SessionMessagingKit/Calls/CallManager+UI.swift b/SessionMessagingKit/Calls/CallManager+UI.swift index d9effa6c9..f9d6b73fd 100644 --- a/SessionMessagingKit/Calls/CallManager+UI.swift +++ b/SessionMessagingKit/Calls/CallManager+UI.swift @@ -11,6 +11,7 @@ extension CallManager { } public func handleLocalFrameCaptured(_ videoFrame: RTCVideoFrame) { + guard let videoCapturer = delegate?.videoCapturer else { return } localVideoSource.capturer(videoCapturer, didCapture: videoFrame) } } diff --git a/SessionMessagingKit/Calls/CallManager.swift b/SessionMessagingKit/Calls/CallManager.swift index b5808c01c..8bdcf2a4c 100644 --- a/SessionMessagingKit/Calls/CallManager.swift +++ b/SessionMessagingKit/Calls/CallManager.swift @@ -2,6 +2,7 @@ import PromiseKit import WebRTC public protocol CallManagerDelegate : AnyObject { + var videoCapturer: RTCVideoCapturer { get } func callManager(_ callManager: CallManager, sendData data: Data) } @@ -23,6 +24,7 @@ public final class CallManager : NSObject, RTCPeerConnectionDelegate { internal lazy var peerConnection: RTCPeerConnection = { let configuration = RTCConfiguration() configuration.iceServers = [ RTCIceServer(urlStrings: MockCallConfig.default.webRTCICEServers) ] + configuration.iceTransportPolicy = .all // TODO: Do these constraints make sense? let constraints = RTCMediaConstraints(mandatoryConstraints: [:], optionalConstraints: [ "DtlsSrtpKeyAgreement" : "true" ]) return factory.peerConnection(with: configuration, constraints: constraints, delegate: self) @@ -50,20 +52,16 @@ public final class CallManager : NSObject, RTCPeerConnectionDelegate { }() // Video - internal lazy var localVideoSource: RTCVideoSource = { - return factory.avFoundationVideoSource(with: nil) + public lazy var localVideoSource: RTCVideoSource = { + return factory.videoSource() }() internal lazy var localVideoTrack: RTCVideoTrack = { return factory.videoTrack(with: localVideoSource, trackId: "ARDAMSv0") }() - internal lazy var videoCapturer: RTCVideoCapturer = { - return RTCCameraVideoCapturer(delegate: localVideoSource) - }() - internal lazy var remoteVideoTrack: RTCVideoTrack? = { - return peerConnection.receivers.first { $0.track.kind == "video" }?.track as? RTCVideoTrack + return peerConnection.receivers.first { $0.track?.kind == "video" }?.track as? RTCVideoTrack }() // Stream