Simulcast support for iOS/Android.

Simulcast support for iOS SDK (#4)

Support for simulcast in Android SDK (#3)

include simulcast headers for mac also (#10)

Fix simulcast using hardware encoder on Android (#48)

Add scalabilityMode support for AV1/VP9. (#90)

Co-authored-by: Hiroshi Horie <[email protected]>
Co-authored-by: Angelika Serwa <[email protected]>

Author: 3 people

api/video_codecs/video_encoder_factory.h

@@ -85,13 +85,8 @@ class VideoEncoderFactory {
   virtual CodecSupport QueryCodecSupport(
       const SdpVideoFormat& format,
       absl::optional<std::string> scalability_mode) const {
-    // Default implementation, query for supported formats and check if the
-    // specified format is supported. Returns false if scalability_mode is
-    // specified.
     CodecSupport codec_support;
-    if (!scalability_mode) {
-      codec_support.is_supported = format.IsCodecInList(GetSupportedFormats());
-    }
+    codec_support.is_supported = format.IsCodecInList(GetSupportedFormats());
     return codec_support;
   }
 

sdk/BUILD.gn

@@ -723,6 +723,7 @@ if (is_ios || is_mac) {
       ]
 
       deps = [
+        ":simulcast",
         ":base_objc",
         ":native_video",
         ":videocodec_objc",
@@ -822,6 +823,22 @@ if (is_ios || is_mac) {
       ]
     }
 
+    rtc_library("simulcast") {
+      sources = [
+        "objc/components/video_codec/RTCVideoEncoderFactorySimulcast.h",
+        "objc/components/video_codec/RTCVideoEncoderFactorySimulcast.mm",
+        "objc/api/video_codec/RTCVideoEncoderSimulcast.h",
+        "objc/api/video_codec/RTCVideoEncoderSimulcast.mm",
+      ]
+
+      deps = [
+        ":base_objc",
+        ":wrapped_native_codec_objc",
+        "../media:rtc_media_base",
+        "../media:rtc_simulcast_encoder_adapter",
+      ]
+    }
+  
     rtc_library("mediaconstraints_objc") {
       configs += [ "..:no_global_constructors" ]
       sources = [
@@ -1368,6 +1385,9 @@ if (is_ios || is_mac) {
           "objc/api/video_codec/RTCVideoEncoderAV1.h",
           "objc/api/video_frame_buffer/RTCNativeI420Buffer.h",
           "objc/api/video_frame_buffer/RTCNativeMutableI420Buffer.h",
+          # Added for Simulcast support
+          "objc/components/video_codec/RTCVideoEncoderFactorySimulcast.h",
+          "objc/api/video_codec/RTCVideoEncoderSimulcast.h",
         ]
 
         if (!build_with_chromium) {
@@ -1518,6 +1538,9 @@ if (is_ios || is_mac) {
           "objc/components/video_codec/RTCVideoEncoderH264.h",
           "objc/components/video_frame_buffer/RTCCVPixelBuffer.h",
           "objc/helpers/RTCDispatcher.h",
+          # Added for Simulcast support
+          "objc/components/video_codec/RTCVideoEncoderFactorySimulcast.h",
+          "objc/api/video_codec/RTCVideoEncoderSimulcast.h",
         ]
         if (!build_with_chromium) {
           sources += [

sdk/android/BUILD.gn

@@ -530,6 +530,8 @@ if (is_android) {
     sources = [
       "api/org/webrtc/SoftwareVideoDecoderFactory.java",
       "api/org/webrtc/SoftwareVideoEncoderFactory.java",
+      "api/org/webrtc/SimulcastVideoEncoder.java",
+      "api/org/webrtc/SimulcastVideoEncoderFactory.java",
     ]
 
     deps = [
@@ -908,6 +910,21 @@ if (current_os == "linux" || is_android) {
     ]
   }
 
+  rtc_library("simulcast_jni") {
+    visibility = [ "*" ]
+    allow_poison = [ "software_video_codecs" ]
+    sources = [
+      "src/jni/simulcast_video_encoder.cc",
+      "src/jni/simulcast_video_encoder.h"
+    ]
+    deps = [
+      ":base_jni",
+      ":video_jni",
+      ":native_api_codecs",
+      "../../media:rtc_simulcast_encoder_adapter"
+    ]
+  }
+
   rtc_library("swcodecs_jni") {
     visibility = [ "*" ]
     allow_poison = [ "software_video_codecs" ]
@@ -921,6 +938,7 @@ if (current_os == "linux" || is_android) {
       ":libvpx_vp8_jni",
       ":libvpx_vp9_jni",
       ":native_api_jni",
+      ":simulcast_jni",
       ":video_jni",
       "../../api/environment",
       "../../api/video_codecs:builtin_video_decoder_factory",

sdk/android/api/org/webrtc/HardwareVideoEncoderFactory.java

@@ -143,11 +143,11 @@ public VideoCodecInfo[] getSupportedCodecs() {
         // supported by the decoder.
         if (type == VideoCodecMimeType.H264 && isH264HighProfileSupported(codec)) {
           supportedCodecInfos.add(new VideoCodecInfo(
-              name, MediaCodecUtils.getCodecProperties(type, /* highProfile= */ true)));
+              name, MediaCodecUtils.getCodecProperties(type, /* highProfile= */ true), new ArrayList<>()));
         }
 
         supportedCodecInfos.add(new VideoCodecInfo(
-            name, MediaCodecUtils.getCodecProperties(type, /* highProfile= */ false)));
+            name, MediaCodecUtils.getCodecProperties(type, /* highProfile= */ false), new ArrayList<>()));
       }
     }
 

sdk/android/api/org/webrtc/LibaomAv1Encoder.java

@@ -9,6 +9,7 @@
  */
 
 package org.webrtc;
+import java.util.List;
 
 public class LibaomAv1Encoder extends WrappedNativeVideoEncoder {
   @Override
@@ -22,4 +23,10 @@ public long createNative(long webrtcEnvRef) {
   public boolean isHardwareEncoder() {
     return false;
   }
+
+  static List<String> scalabilityModes() {
+    return nativeGetSupportedScalabilityModes();
+  }
+
+  static native List<String> nativeGetSupportedScalabilityModes();
 }

sdk/android/api/org/webrtc/LibvpxVp9Encoder.java

@@ -9,6 +9,7 @@
  */
 
 package org.webrtc;
+import java.util.List;
 
 public class LibvpxVp9Encoder extends WrappedNativeVideoEncoder {
   @Override
@@ -24,4 +25,10 @@ public boolean isHardwareEncoder() {
   }
 
   static native boolean nativeIsSupported();
+
+  static List<String> scalabilityModes() {
+    return nativeGetSupportedScalabilityModes();
+  }
+
+  static native List<String> nativeGetSupportedScalabilityModes();
 }

sdk/android/api/org/webrtc/RtpParameters.java

@@ -76,6 +76,8 @@ public static class Encoding {
     // If non-null, scale the width and height down by this factor for video. If null,
     // implementation default scaling factor will be used.
     @Nullable public Double scaleResolutionDownBy;
+    // Scalability modes are used to represent simulcast and SVC layers.
+    @Nullable public String scalabilityMode;
     // SSRC to be used by this encoding.
     // Can't be changed between getParameters/setParameters.
     public Long ssrc;
@@ -93,8 +95,8 @@ public Encoding(String rid, boolean active, Double scaleResolutionDownBy) {
     @CalledByNative("Encoding")
     Encoding(String rid, boolean active, double bitratePriority, @Priority int networkPriority,
         Integer maxBitrateBps, Integer minBitrateBps, Integer maxFramerate,
-        Integer numTemporalLayers, Double scaleResolutionDownBy, Long ssrc,
-        boolean adaptiveAudioPacketTime) {
+        Integer numTemporalLayers, Double scaleResolutionDownBy, String scalabilityMode,
+        Long ssrc, boolean adaptiveAudioPacketTime) {
       this.rid = rid;
       this.active = active;
       this.bitratePriority = bitratePriority;
@@ -104,6 +106,7 @@ public Encoding(String rid, boolean active, Double scaleResolutionDownBy) {
       this.maxFramerate = maxFramerate;
       this.numTemporalLayers = numTemporalLayers;
       this.scaleResolutionDownBy = scaleResolutionDownBy;
+      this.scalabilityMode = scalabilityMode;
       this.ssrc = ssrc;
       this.adaptiveAudioPacketTime = adaptiveAudioPacketTime;
     }
@@ -160,6 +163,12 @@ Double getScaleResolutionDownBy() {
       return scaleResolutionDownBy;
     }
 
+    @Nullable
+    @CalledByNative("Encoding")
+    String getScalabilityMode() {
+      return scalabilityMode;
+    }
+
     @CalledByNative("Encoding")
     Long getSsrc() {
       return ssrc;

sdk/android/api/org/webrtc/SimulcastVideoEncoder.java

@@ -0,0 +1,28 @@
+package org.webrtc;
+
+public class SimulcastVideoEncoder extends WrappedNativeVideoEncoder {
+
+    static native long nativeCreateEncoder(long webrtcEnvRef, VideoEncoderFactory primary, VideoEncoderFactory fallback, VideoCodecInfo info);
+
+    VideoEncoderFactory primary;
+    VideoEncoderFactory fallback;
+    VideoCodecInfo info;
+
+    public SimulcastVideoEncoder(VideoEncoderFactory primary, VideoEncoderFactory fallback, VideoCodecInfo info) {
+        this.primary = primary;
+        this.fallback = fallback;
+        this.info = info;
+    }
+
+    @Override
+    public long createNative(long webrtcEnvRef) {
+        return nativeCreateEncoder(primary, fallback, info);
+    }
+
+    @Override
+    public boolean isHardwareEncoder() {
+        return false;
+    }
+
+}
+

sdk/android/api/org/webrtc/SimulcastVideoEncoderFactory.java

@@ -0,0 +1,43 @@
+/*
+ *  Copyright 2017 The WebRTC project authors. All Rights Reserved.
+ *
+ *  Use of this source code is governed by a BSD-style license
+ *  that can be found in the LICENSE file in the root of the source
+ *  tree. An additional intellectual property rights grant can be found
+ *  in the file PATENTS.  All contributing project authors may
+ *  be found in the AUTHORS file in the root of the source tree.
+ */
+
+package org.webrtc;
+
+import androidx.annotation.Nullable;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Arrays;
+
+public class SimulcastVideoEncoderFactory implements VideoEncoderFactory {
+
+    VideoEncoderFactory primary;
+    VideoEncoderFactory fallback;
+
+    public SimulcastVideoEncoderFactory(VideoEncoderFactory primary, VideoEncoderFactory fallback) {
+        this.primary = primary;
+        this.fallback = fallback;
+    }
+
+    @Nullable
+    @Override
+    public VideoEncoder createEncoder(VideoCodecInfo info) {
+        return new SimulcastVideoEncoder(primary, fallback, info);
+    }
+
+    @Override
+    public VideoCodecInfo[] getSupportedCodecs() {
+        List<VideoCodecInfo> codecs = new ArrayList<VideoCodecInfo>();
+        codecs.addAll(Arrays.asList(primary.getSupportedCodecs()));
+        codecs.addAll(Arrays.asList(fallback.getSupportedCodecs()));
+        return codecs.toArray(new VideoCodecInfo[codecs.size()]);
+    }
+
+}

sdk/android/api/org/webrtc/VideoCodecInfo.java

@@ -14,6 +14,8 @@
 import java.util.Arrays;
 import java.util.Locale;
 import java.util.Map;
+import java.util.List;
+import java.util.ArrayList;
 
 /**
  * Represent a video codec as encoded in SDP.
@@ -34,20 +36,24 @@ public class VideoCodecInfo {
 
   public final String name;
   public final Map<String, String> params;
+  public final List<String> scalabilityModes;
+
   @Deprecated public final int payload;
 
   @CalledByNative
-  public VideoCodecInfo(String name, Map<String, String> params) {
+  public VideoCodecInfo(String name, Map<String, String> params, List<String> scalabilityModes) {
     this.payload = 0;
     this.name = name;
     this.params = params;
+    this.scalabilityModes = scalabilityModes;
   }
 
   @Deprecated
   public VideoCodecInfo(int payload, String name, Map<String, String> params) {
     this.payload = payload;
     this.name = name;
     this.params = params;
+    this.scalabilityModes = new ArrayList<>();
   }
 
   @Override
@@ -83,4 +89,9 @@ String getName() {
   Map getParams() {
     return params;
   }
+
+  @CalledByNative
+  List<String> getScalabilityModes() {
+    return scalabilityModes;
+  }
 }