Darwin improvements

[Mac/iOS] feat: Add RTCYUVHelper for darwin. (#28)

Cross-platform `RTCMTLVideoView` for both iOS / macOS (#40)

rotationOverride should not be assign (#44)

[ObjC] Expose properties / methods required for AV1 codec support (#60)

Workaround: Render PixelBuffer in RTCMTLVideoView (#58)

Improve iOS/macOS H264 encoder (#70)

fix: fix video encoder not resuming correctly upon foregrounding (#75).

Fix camera rotation (#92)

add PrivacyInfo.xcprivacy to darwin frameworks. (#112)

Add NSPrivacyCollectedDataTypes key to xcprivacy file (#114)

Thread-safe `RTCInitFieldTrialDictionary` (#116)

Set RTCCameraVideoCapturer initial zoom factor (#121)

Unlock configuration before starting capture session #122

Co-authored-by: Hiroshi Horie <[email protected]>

Author: cloudwebrtc

sdk/BUILD.gn

@@ -144,12 +144,15 @@ if (is_ios || is_mac) {
       "objc/helpers/RTCDispatcher+Private.h",
       "objc/helpers/RTCDispatcher.h",
       "objc/helpers/RTCDispatcher.m",
+      "objc/helpers/RTCYUVHelper.h",
+      "objc/helpers/RTCYUVHelper.mm",
       "objc/helpers/scoped_cftyperef.h",
     ]
 
     deps = [
       ":base_objc",
       "../rtc_base:checks",
+      "//third_party/libyuv",
     ]
 
     absl_deps = [ "//third_party/abseil-cpp/absl/strings" ]
@@ -617,17 +620,13 @@ if (is_ios || is_mac) {
         "Metal.framework",
         "MetalKit.framework",
       ]
-      if (is_ios) {
+      if (is_ios || is_mac) {
         sources += [
           "objc/components/renderer/metal/RTCMTLVideoView.h",
           "objc/components/renderer/metal/RTCMTLVideoView.m",
         ]
       }
       if (is_mac) {
-        sources += [
-          "objc/components/renderer/metal/RTCMTLNSVideoView.h",
-          "objc/components/renderer/metal/RTCMTLNSVideoView.m",
-        ]
         frameworks += [ "AppKit.framework" ]
       }
       deps = [
@@ -1287,6 +1286,13 @@ if (is_ios || is_mac) {
       }
     }
 
+    bundle_data("darwin_privacy_info") {
+      sources = [
+        "objc/PrivacyInfo.xcprivacy",
+       ]
+       outputs = [ "{{bundle_resources_dir}}/{{source_file_part}}" ]
+    }
+
     if (is_ios) {
       apple_framework_bundle_with_umbrella_header("framework_objc") {
         info_plist = "objc/Info.plist"
@@ -1334,6 +1340,7 @@ if (is_ios || is_mac) {
           "objc/components/video_frame_buffer/RTCCVPixelBuffer.h",
           "objc/helpers/RTCCameraPreviewView.h",
           "objc/helpers/RTCDispatcher.h",
+          "objc/helpers/RTCYUVHelper.h",
           "objc/helpers/UIDevice+RTCDevice.h",
           "objc/api/peerconnection/RTCAudioDeviceModule.h",
           "objc/api/peerconnection/RTCIODevice.h",
@@ -1417,6 +1424,7 @@ if (is_ios || is_mac) {
           ":videocapture_objc",
           ":videocodec_objc",
           ":videotoolbox_objc",
+          ":darwin_privacy_info",
         ]
         if (!build_with_chromium) {
           deps += [
@@ -1526,6 +1534,7 @@ if (is_ios || is_mac) {
           "objc/base/RTCYUVPlanarBuffer.h",
           "objc/components/capturer/RTCCameraVideoCapturer.h",
           "objc/components/capturer/RTCFileVideoCapturer.h",
+          "objc/components/renderer/metal/RTCMTLVideoView.h",
           "objc/components/renderer/metal/RTCMTLNSVideoView.h",
           "objc/components/renderer/opengl/RTCVideoViewShading.h",
           "objc/components/video_codec/RTCCodecSpecificInfoH264.h",
@@ -1538,6 +1547,7 @@ if (is_ios || is_mac) {
           "objc/components/video_codec/RTCVideoEncoderH264.h",
           "objc/components/video_frame_buffer/RTCCVPixelBuffer.h",
           "objc/helpers/RTCDispatcher.h",
+          "objc/helpers/RTCYUVHelper.h",
           # Added for Simulcast support
           "objc/components/video_codec/RTCVideoEncoderFactorySimulcast.h",
           "objc/api/video_codec/RTCVideoEncoderSimulcast.h",
@@ -1558,6 +1568,7 @@ if (is_ios || is_mac) {
           ":videocapture_objc",
           ":videocodec_objc",
           ":videotoolbox_objc",
+          ":darwin_privacy_info",
         ]
         if (!build_with_chromium) {
           deps += [

sdk/objc/PrivacyInfo.xcprivacy

@@ -0,0 +1,28 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
+<plist version="1.0">
+<dict>
+	<key>NSPrivacyCollectedDataTypes</key>
+	<array/>
+	<key>NSPrivacyAccessedAPITypes</key>
+	<array>
+		<dict>
+			<key>NSPrivacyAccessedAPIType</key>
+			<string>NSPrivacyAccessedAPICategoryFileTimestamp</string>
+			<key>NSPrivacyAccessedAPITypeReasons</key>
+			<array>
+				<string>C617.1</string>
+			</array>
+		</dict>
+		<dict>
+			<key>NSPrivacyAccessedAPIType</key>
+			<string>NSPrivacyAccessedAPICategorySystemBootTime</string>
+			<key>NSPrivacyAccessedAPITypeReasons</key>
+			<array>
+				<string>35F9.1</string>
+				<string>8FFB.1</string>
+			</array>
+		</dict>
+	</array>
+</dict>
+</plist>

sdk/objc/api/peerconnection/RTCFieldTrials.mm

@@ -10,47 +10,58 @@
 
 #import "RTCFieldTrials.h"
 
+#import <os/lock.h>
 #include <memory>
-
 #import "base/RTCLogging.h"
 
 #include "system_wrappers/include/field_trial.h"
 
 NSString *const kRTCFieldTrialAudioForceABWENoTWCCKey = @"WebRTC-Audio-ABWENoTWCC";
-NSString * const kRTCFieldTrialFlexFec03AdvertisedKey = @"WebRTC-FlexFEC-03-Advertised";
-NSString * const kRTCFieldTrialFlexFec03Key = @"WebRTC-FlexFEC-03";
-NSString * const kRTCFieldTrialH264HighProfileKey = @"WebRTC-H264HighProfile";
-NSString * const kRTCFieldTrialMinimizeResamplingOnMobileKey =
+NSString *const kRTCFieldTrialFlexFec03AdvertisedKey = @"WebRTC-FlexFEC-03-Advertised";
+NSString *const kRTCFieldTrialFlexFec03Key = @"WebRTC-FlexFEC-03";
+NSString *const kRTCFieldTrialH264HighProfileKey = @"WebRTC-H264HighProfile";
+NSString *const kRTCFieldTrialMinimizeResamplingOnMobileKey =
     @"WebRTC-Audio-MinimizeResamplingOnMobile";
 NSString *const kRTCFieldTrialUseNWPathMonitor = @"WebRTC-Network-UseNWPathMonitor";
-NSString * const kRTCFieldTrialEnabledValue = @"Enabled";
+NSString *const kRTCFieldTrialEnabledValue = @"Enabled";
 
 // InitFieldTrialsFromString stores the char*, so the char array must outlive
 // the application.
 static char *gFieldTrialInitString = nullptr;
+static os_unfair_lock fieldTrialLock = OS_UNFAIR_LOCK_INIT;
 
 void RTCInitFieldTrialDictionary(NSDictionary<NSString *, NSString *> *fieldTrials) {
   if (!fieldTrials) {
     RTCLogWarning(@"No fieldTrials provided.");
     return;
   }
+
   // Assemble the keys and values into the field trial string.
-  // We don't perform any extra format checking. That should be done by the underlying WebRTC calls.
   NSMutableString *fieldTrialInitString = [NSMutableString string];
   for (NSString *key in fieldTrials) {
     NSString *fieldTrialEntry = [NSString stringWithFormat:@"%@/%@/", key, fieldTrials[key]];
     [fieldTrialInitString appendString:fieldTrialEntry];
   }
+
   size_t len = fieldTrialInitString.length + 1;
+
+  // Locking before modifying global variable
+  os_unfair_lock_lock(&fieldTrialLock);
   if (gFieldTrialInitString != nullptr) {
     delete[] gFieldTrialInitString;
+    gFieldTrialInitString = nullptr;
   }
+
   gFieldTrialInitString = new char[len];
-  if (![fieldTrialInitString getCString:gFieldTrialInitString
-                              maxLength:len
-                               encoding:NSUTF8StringEncoding]) {
+  bool success = [fieldTrialInitString getCString:gFieldTrialInitString
+                                        maxLength:len
+                                         encoding:NSUTF8StringEncoding];
+  if (!success) {
     RTCLogError(@"Failed to convert field trial string.");
+    os_unfair_lock_unlock(&fieldTrialLock);
     return;
   }
+
   webrtc::field_trial::InitFieldTrialsFromString(gFieldTrialInitString);
+  os_unfair_lock_unlock(&fieldTrialLock);
 }

sdk/objc/api/peerconnection/RTCPeerConnectionFactory.h

@@ -25,6 +25,10 @@ NS_ASSUME_NONNULL_BEGIN
 @class RTC_OBJC_TYPE(RTCVideoTrack);
 @class RTC_OBJC_TYPE(RTCPeerConnectionFactoryOptions);
 @class RTC_OBJC_TYPE(RTCAudioDeviceModule);
+@class RTC_OBJC_TYPE(RTCRtpCapabilities);
+
+typedef NS_ENUM(NSInteger, RTCRtpMediaType);
+
 @protocol RTC_OBJC_TYPE
 (RTCPeerConnectionDelegate);
 @protocol RTC_OBJC_TYPE

sdk/objc/api/peerconnection/RTCPeerConnectionFactory.mm

@@ -25,6 +25,9 @@
 #import "RTCPeerConnection+Private.h"
 #import "RTCVideoSource+Private.h"
 #import "RTCVideoTrack+Private.h"
+#import "RTCRtpReceiver+Private.h"
+#import "RTCRtpCapabilities+Private.h"
+#import "RTCRtpCodecCapability+Private.h"
 #import "base/RTCLogging.h"
 #import "base/RTCVideoDecoderFactory.h"
 #import "base/RTCVideoEncoderFactory.h"
@@ -130,6 +133,20 @@ - (instancetype)init {
 #endif
 }
 
+- (RTC_OBJC_TYPE(RTCRtpCapabilities) *)rtpSenderCapabilitiesFor:(RTCRtpMediaType)mediaType {
+
+  webrtc::RtpCapabilities capabilities = _nativeFactory->GetRtpSenderCapabilities([RTCRtpReceiver nativeMediaTypeForMediaType: mediaType]);
+
+  return [[RTCRtpCapabilities alloc] initWithNativeRtpCapabilities: capabilities];
+}
+
+- (RTC_OBJC_TYPE(RTCRtpCapabilities) *)rtpReceiverCapabilitiesFor:(RTCRtpMediaType)mediaType {
+
+  webrtc::RtpCapabilities capabilities = _nativeFactory->GetRtpReceiverCapabilities([RTCRtpReceiver nativeMediaTypeForMediaType: mediaType]);
+
+  return [[RTCRtpCapabilities alloc] initWithNativeRtpCapabilities: capabilities];
+}
+
 - (instancetype)
     initWithBypassVoiceProcessing:(BOOL)bypassVoiceProcessing
                    encoderFactory:(nullable id<RTC_OBJC_TYPE(RTCVideoEncoderFactory)>)encoderFactory

sdk/objc/api/peerconnection/RTCRtpCodecParameters.h

@@ -30,6 +30,7 @@ RTC_EXTERN const NSString *const kRTCComfortNoiseCodecName;
 RTC_EXTERN const NSString *const kRTCVp8CodecName;
 RTC_EXTERN const NSString *const kRTCVp9CodecName;
 RTC_EXTERN const NSString *const kRTCH264CodecName;
+RTC_EXTERN const NSString *const kRTCAv1CodecName;
 
 /** Defined in https://www.w3.org/TR/webrtc/#idl-def-rtcrtpcodecparameters */
 RTC_OBJC_EXPORT

sdk/objc/api/peerconnection/RTCRtpCodecParameters.mm

@@ -32,6 +32,7 @@
 const NSString * const kRTCVp8CodecName = @(cricket::kVp8CodecName);
 const NSString * const kRTCVp9CodecName = @(cricket::kVp9CodecName);
 const NSString * const kRTCH264CodecName = @(cricket::kH264CodecName);
+const NSString * const kRTCAv1CodecName = @(cricket::kAv1CodecName);
 
 @implementation RTC_OBJC_TYPE (RTCRtpCodecParameters)
 

sdk/objc/api/peerconnection/RTCRtpEncodingParameters.h

@@ -69,6 +69,10 @@ RTC_OBJC_EXPORT
  https://w3c.github.io/webrtc-extensions/#dom-rtcrtpencodingparameters-adaptiveptime */
 @property(nonatomic, assign) BOOL adaptiveAudioPacketTime;
 
+/** A case-sensitive identifier of the scalability mode to be used for this stream.
+  https://w3c.github.io/webrtc-svc/#rtcrtpencodingparameters */
+@property(nonatomic, copy, nullable) NSString *scalabilityMode;
+
 - (instancetype)init;
 
 @end

sdk/objc/api/peerconnection/RTCRtpEncodingParameters.mm

@@ -25,6 +25,7 @@ @implementation RTC_OBJC_TYPE (RTCRtpEncodingParameters)
 @synthesize bitratePriority = _bitratePriority;
 @synthesize networkPriority = _networkPriority;
 @synthesize adaptiveAudioPacketTime = _adaptiveAudioPacketTime;
+@synthesize scalabilityMode = _scalabilityMode;
 
 - (instancetype)init {
   webrtc::RtpEncodingParameters nativeParameters;
@@ -59,6 +60,9 @@ - (instancetype)initWithNativeParameters:
     if (nativeParameters.ssrc) {
       _ssrc = [NSNumber numberWithUnsignedLong:*nativeParameters.ssrc];
     }
+    if (nativeParameters.scalability_mode) {
+      _scalabilityMode = [NSString stringWithUTF8String:nativeParameters.scalability_mode->c_str()];
+    }
     _bitratePriority = nativeParameters.bitrate_priority;
     _networkPriority = [RTC_OBJC_TYPE(RTCRtpEncodingParameters)
         priorityFromNativePriority:nativeParameters.network_priority];
@@ -92,6 +96,9 @@ - (instancetype)initWithNativeParameters:
   if (_ssrc != nil) {
     parameters.ssrc = absl::optional<uint32_t>(_ssrc.unsignedLongValue);
   }
+  if (_scalabilityMode != nil) {
+    parameters.scalability_mode = absl::optional<std::string>(std::string([_scalabilityMode UTF8String]));
+  }
   parameters.bitrate_priority = _bitratePriority;
   parameters.network_priority =
       [RTC_OBJC_TYPE(RTCRtpEncodingParameters) nativePriorityFromPriority:_networkPriority];

sdk/objc/api/peerconnection/RTCRtpTransceiver.h

@@ -14,6 +14,8 @@
 #import "RTCRtpReceiver.h"
 #import "RTCRtpSender.h"
 
+@class RTC_OBJC_TYPE(RTCRtpCodecCapability);
+
 NS_ASSUME_NONNULL_BEGIN
 
 extern NSString *const kRTCRtpTransceiverErrorDomain;
@@ -105,6 +107,8 @@ RTC_OBJC_EXPORT
  */
 @property(nonatomic, readonly) RTCRtpTransceiverDirection direction;
 
+@property(nonatomic, copy) NSArray<RTC_OBJC_TYPE(RTCRtpCodecCapability) *> *codecPreferences;
+
 /** The currentDirection attribute indicates the current direction negotiated
  *  for this transceiver. If this transceiver has never been represented in an
  *  offer/answer exchange, or if the transceiver is stopped, the value is not