Merge pull request #11 from taste1981/cloudgaming

fix issues in picture_id populating

Author: Chunbo

call/rtp_payload_params.cc

@@ -76,7 +76,7 @@ void PopulateRtpWithCodecSpecifics(const CodecSpecificInfo& info,
       h264_header.packetization_mode =
           info.codecSpecific.H264.packetization_mode;
       rtp->simulcastIdx = info.codecSpecific.H264.simulcast_idx;
-      rtp->picture_id = info.codecSpecific.H264.picture_id;
+      h264_header.picture_id = info.codecSpecific.H264.picture_id;
       return;
     }
     case kVideoCodecMultiplex:

media/engine/webrtcvideoengine.cc

@@ -597,6 +597,10 @@ RtpCapabilities WebRtcVideoEngine::GetCapabilities() const {
   // demuxing is completed.
   // capabilities.header_extensions.push_back(webrtc::RtpExtension(
   //     webrtc::RtpExtension::kMidUri, webrtc::RtpExtension::kMidDefaultId));
+  // Temporal scalability support.
+  capabilities.header_extensions.push_back(
+      webrtc::RtpExtension(webrtc::RtpExtension::kPictureIdUri,
+                           webrtc::RtpExtension::kPictureIdDefaultId));
   return capabilities;
 }
 

media/engine/webrtcvoiceengine.cc

@@ -615,10 +615,6 @@ RtpCapabilities WebRtcVoiceEngine::GetCapabilities() const {
   // demuxing is completed.
   // capabilities.header_extensions.push_back(webrtc::RtpExtension(
   //     webrtc::RtpExtension::kMidUri, webrtc::RtpExtension::kMidDefaultId));
-  // Temporal scalability support.
-  capabilities.header_extensions.push_back(
-      webrtc::RtpExtension(webrtc::RtpExtension::kPictureIdUri,
-                           webrtc::RtpExtension::kPictureIdDefaultId));
   return capabilities;
 }
 

modules/audio_processing/audio_processing_impl.cc

@@ -880,7 +880,7 @@ int AudioProcessingImpl::ProcessStream(const float* const* src,
   }
 
   capture_.capture_audio->CopyFrom(src, formats_.api_format.input_stream());
-  RETURN_ON_ERR(ProcessCaptureStreamLocked());
+  //RETURN_ON_ERR(ProcessCaptureStreamLocked());
   capture_.capture_audio->CopyTo(formats_.api_format.output_stream(), dest);
 
   if (aec_dump_) {

modules/pacing/paced_sender.cc

@@ -33,7 +33,8 @@
 namespace {
 // Time limit in milliseconds between packet bursts.
 const int64_t kMinPacketLimitMs = 5;
-const int64_t kMinPacketLimitLowLatency = 1; // For low latency mode we set process interval to 1ms instead.
+const int64_t kMinPacketLimitLowLatency =
+    1;  // For low latency mode we set process interval to 1ms instead.
 const int64_t kCongestedPacketIntervalMs = 500;
 const int64_t kPausedProcessIntervalMs = kCongestedPacketIntervalMs;
 const int64_t kMaxElapsedTimeMs = 2000;
@@ -48,6 +49,7 @@ namespace webrtc {
 
 const int64_t PacedSender::kMaxQueueLengthMs = 2000;
 const float PacedSender::kDefaultPaceMultiplier = 2.5f;
+const char kLogLatencyToFileFieldTrial[] = "OWT-Log-Latency-To-File";
 
 int64_t capture_timestamp = 0;
 int64_t end_timestamp = 0;
@@ -96,14 +98,21 @@ PacedSender::PacedSender(const Clock* clock,
   UpdateBudgetWithElapsedTime(kMinPacketLimitMs);
   if (low_latency_mode_)
     prober_->SetEnabled(false);
-  recorder = fopen("pacer.txt", "a+");
+  recorder = nullptr;
+  if (webrtc::field_trial::IsEnabled(kLogLatencyToFileFieldTrial))
+    recorder = fopen("pacer.txt", "a+");
 }
 
-PacedSender::~PacedSender() {}
+PacedSender::~PacedSender() {
+  if (recorder != nullptr) {
+    fclose(recorder);
+    recorder = nullptr;
+  }
+}
 
 void PacedSender::CreateProbeCluster(int bitrate_bps) {
   rtc::CritScope cs(&critsect_);
-  if (!low_latency_mode_) // Do not create probe cluster on low latency mode.
+  if (!low_latency_mode_)  // Do not create probe cluster on low latency mode.
     prober_->CreateProbeCluster(bitrate_bps, clock_->TimeInMilliseconds());
 }
 
@@ -300,7 +309,7 @@ void PacedSender::Process() {
     // congested state due to no feedback being received.
     int64_t elapsed_since_last_send_us = now_us - last_send_time_us_;
     if (elapsed_since_last_send_us >= kCongestedPacketIntervalMs * 1000) {
-      // We can not send padding unless a normal packet has first been sent. If
+      // We do not send padding unless a normal packet has first been sent. If
       // we do, timestamps get messed up.
       if (packet_counter_ > 0) {
         PacedPacketInfo pacing_info;
@@ -311,7 +320,7 @@ void PacedSender::Process() {
   }
   if (paused_) {
     RTC_LOG(LS_ERROR) << "Paused.";
-    //return;
+    // return;
   }
 
   if (elapsed_time_ms > 0) {
@@ -362,21 +371,24 @@ void PacedSender::Process() {
                         << ",sz: " << packet.bytes
                         << ", q: " << packets_->SizeInPackets();
 
-      if (capture_timestamp != packet.capture_time_ms) {  // New frame
-        int64_t last_frame_cost = end_timestamp - capture_timestamp;
-        int64_t time_since_last_frame =
-            clock_->TimeInMilliseconds() - end_timestamp;
-        fprintf(recorder, "%lld\t%lld\t%zd\t%lld\n", frame_count, last_frame_cost,
-                total_size, time_since_last_frame);
-        frame_count++;
-        total_size = 0;
-        end_timestamp = clock_->TimeInMilliseconds();
-        capture_timestamp = packet.capture_time_ms;
-	  } else {
-        capture_timestamp = packet.capture_time_ms;
-        total_size += packet.bytes;
-        end_timestamp = clock_->TimeInMilliseconds();
-	  }
+      if (webrtc::field_trial::IsEnabled(kLogLatencyToFileFieldTrial)) {
+        if (capture_timestamp != packet.capture_time_ms) {  // New frame
+          int64_t last_frame_cost = end_timestamp - capture_timestamp;
+          int64_t time_since_last_frame =
+              clock_->TimeInMilliseconds() - end_timestamp;
+          fprintf(recorder, "%lld\t%lld\t%zd\t%lld\n", frame_count,
+                  last_frame_cost, total_size, time_since_last_frame);
+          frame_count++;
+          total_size = 0;
+          end_timestamp = clock_->TimeInMilliseconds();
+          capture_timestamp = packet.capture_time_ms;
+        } else {
+          capture_timestamp = packet.capture_time_ms;
+          total_size += packet.bytes;
+          end_timestamp = clock_->TimeInMilliseconds();
+        }
+      }
+
       if (is_probing && bytes_sent > recommended_probe_size)
         break;
     } else {
@@ -389,12 +401,12 @@ void PacedSender::Process() {
 
   if (!IsLowLatencyMode()) {
     if (packets_->Empty() && !Congested()) {
-      // We can not send padding unless a normal packet has first been sent. If we
-      // do, timestamps get messed up.
+      // We can not send padding unless a normal packet has first been sent. If
+      // we do, timestamps get messed up.
       if (packet_counter_ > 0) {
         int padding_needed =
             static_cast<int>(is_probing ? (recommended_probe_size - bytes_sent)
-                                      : padding_budget_->bytes_remaining());
+                                        : padding_budget_->bytes_remaining());
         if (padding_needed > 0) {
           bytes_sent += SendPadding(padding_needed, pacing_info);
         }
@@ -422,9 +434,10 @@ bool PacedSender::SendPacket(const PacketQueueInterface::Packet& packet,
   bool apply_pacing =
       !audio_packet || account_for_audio_ || video_blocks_audio_;
   if (!IsLowLatencyMode()) {
-    if (apply_pacing && (Congested() || (media_budget_->bytes_remaining() == 0 &&
-                                       pacing_info.probe_cluster_id ==
-                                           PacedPacketInfo::kNotAProbe))) {
+    if (apply_pacing &&
+        (Congested() ||
+         (media_budget_->bytes_remaining() == 0 &&
+          pacing_info.probe_cluster_id == PacedPacketInfo::kNotAProbe))) {
       return false;
     }
   }

modules/video_coding/timing.cc

@@ -14,6 +14,8 @@
 
 #include "rtc_base/time/timestamp_extrapolator.h"
 #include "system_wrappers/include/clock.h"
+#include "system_wrappers/include/field_trial.h"
+#include "system_wrappers/include/runtime_enabled_features.h"
 
 namespace webrtc {
 
@@ -173,6 +175,10 @@ int64_t VCMTiming::RenderTimeMs(uint32_t frame_timestamp,
 
 int64_t VCMTiming::RenderTimeMsInternal(uint32_t frame_timestamp,
                                         int64_t now_ms) const {
+  // For low latency mode, render immediately
+  if (webrtc::field_trial::IsEnabled(runtime_enabled_features::kLowLatencyMode)) {
+    return 0;
+  }
   if (min_playout_delay_ms_ == 0 && max_playout_delay_ms_ == 0) {
     // Render as soon as possible.
     return 0;

system_wrappers/include/runtime_enabled_features.h

@@ -26,6 +26,7 @@ namespace webrtc {
 namespace runtime_enabled_features {
 
 const char kDualStreamModeFeatureName[] = "WebRtcDualStreamMode";
+const char kLowLatencyMode[] = "OWT-LowLatencyMode";
 
 bool IsFeatureEnabled(std::string feature_name);
 

video/video_receive_stream.cc

@@ -82,6 +82,12 @@ VideoCodec CreateDecoderVideoCodec(const VideoReceiveStream::Decoder& decoder) {
 
 namespace internal {
 
+const char kLogLatencyToFileFieldTrial[] = "OWT-Log-Latency-To-File";
+
+bool GetLogLatencyToFileIsFieldTrialEnabled() {
+  return webrtc::field_trial::IsEnabled(kLogLatencyToFileFieldTrial);
+}
+
 VideoReceiveStream::VideoReceiveStream(
     RtpStreamReceiverControllerInterface* receiver_controller,
     int num_cpu_cores,
@@ -150,11 +156,20 @@ VideoReceiveStream::VideoReceiveStream(
     rtx_receiver_ = receiver_controller->CreateReceiver(
         config_.rtp.rtx_ssrc, rtx_receive_stream_.get());
   }
+  recorder_pre_ = nullptr;
+  if (GetLogLatencyToFileIsFieldTrialEnabled()) {
+    char dump_file_name[128];
+    snprintf(dump_file_name, 128, "pre_decode_vrs-%p.txt", this);
+    recorder_pre_ = fopen(dump_file_name, "w+");
+  }
 }
 
 VideoReceiveStream::~VideoReceiveStream() {
   RTC_DCHECK_CALLED_SEQUENTIALLY(&worker_sequence_checker_);
   RTC_LOG(LS_INFO) << "~VideoReceiveStream: " << config_.ToString();
+  if (recorder_pre_ != nullptr) {
+    fclose(recorder_pre_);
+  }
   Stop();
 
   process_thread_->DeRegisterModule(&rtp_stream_sync_);
@@ -416,6 +431,10 @@ bool VideoReceiveStream::Decode() {
 
   if (frame) {
     int64_t now_ms = clock_->TimeInMilliseconds();
+    if (recorder_pre_ != nullptr) {
+      int32_t frame_ts = frame->Timestamp();
+      fprintf(recorder_pre_, "%d\t%lld\n", frame_ts, now_ms);
+    }
     RTC_DCHECK_EQ(res, video_coding::FrameBuffer::ReturnReason::kFrameFound);
     int decode_result = video_receiver_.Decode(frame.get());
     if (decode_result == WEBRTC_VIDEO_CODEC_OK ||

video/video_receive_stream.h

@@ -160,6 +160,7 @@ class VideoReceiveStream : public webrtc::VideoReceiveStream,
   bool frame_decoded_ = false;
 
   int64_t last_keyframe_request_ms_ = 0;
+  FILE* recorder_pre_;
 };
 }  // namespace internal
 }  // namespace webrtc