Skip to content

Commit 93487cf

Browse files
authored
fix: Use CVPixelBuffer to build DesktopCapture Frame, fix the crash caused by non-CVPixelBuffer frame in RTCVideoEncoderH264 that cannot be cropped. (#63)
1 parent f4ebcc0 commit 93487cf

File tree

3 files changed

+145
-140
lines changed

3 files changed

+145
-140
lines changed

sdk/objc/native/src/objc_desktop_capture.h

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -59,7 +59,6 @@ class ObjCDesktopCapturer : public DesktopCapturer::Callback, public rtc::Messag
5959
webrtc::DesktopCaptureOptions options_;
6060
std::unique_ptr<webrtc::DesktopAndCursorComposer> capturer_;
6161
std::unique_ptr<rtc::Thread> thread_;
62-
rtc::scoped_refptr<webrtc::I420Buffer> i420_buffer_;
6362
CaptureState capture_state_ = CS_STOPPED;
6463
DesktopType type_;
6564
webrtc::DesktopCapturer::SourceId source_id_;

sdk/objc/native/src/objc_desktop_capture.mm

Lines changed: 56 additions & 53 deletions
Original file line numberDiff line numberDiff line change
@@ -19,14 +19,15 @@
1919
#include "third_party/libyuv/include/libyuv.h"
2020

2121
#import "components/capturer/RTCDesktopCapturer+Private.h"
22+
#import "components/video_frame_buffer/RTCCVPixelBuffer.h"
2223

2324
namespace webrtc {
2425

2526
enum { kCaptureDelay = 33, kCaptureMessageId = 1000 };
2627

2728
ObjCDesktopCapturer::ObjCDesktopCapturer(DesktopType type,
28-
webrtc::DesktopCapturer::SourceId source_id,
29-
id<RTC_OBJC_TYPE(DesktopCapturerDelegate)> delegate)
29+
webrtc::DesktopCapturer::SourceId source_id,
30+
id<RTC_OBJC_TYPE(DesktopCapturerDelegate)> delegate)
3031
: thread_(rtc::Thread::Create()), source_id_(source_id), delegate_(delegate) {
3132
RTC_DCHECK(thread_);
3233
type_ = type;
@@ -36,53 +37,47 @@
3637
options_.set_allow_iosurface(true);
3738
thread_->Invoke<void>(RTC_FROM_HERE, [this, type] {
3839
if (type == kScreen) {
39-
capturer_ = std::make_unique<DesktopAndCursorComposer>(webrtc::DesktopCapturer::CreateScreenCapturer(options_), options_);
40-
} else {
41-
capturer_ = std::make_unique<DesktopAndCursorComposer>(webrtc::DesktopCapturer::CreateWindowCapturer(options_), options_);
40+
capturer_ = std::make_unique<DesktopAndCursorComposer>(
41+
webrtc::DesktopCapturer::CreateScreenCapturer(options_), options_);
42+
} else {
43+
capturer_ = std::make_unique<DesktopAndCursorComposer>(
44+
webrtc::DesktopCapturer::CreateWindowCapturer(options_), options_);
4245
}
4346
});
4447
}
4548

4649
ObjCDesktopCapturer::~ObjCDesktopCapturer() {
47-
thread_->Invoke<void>(RTC_FROM_HERE, [this] {
48-
capturer_.reset();
49-
});
50+
thread_->Invoke<void>(RTC_FROM_HERE, [this] { capturer_.reset(); });
5051
}
5152

5253
ObjCDesktopCapturer::CaptureState ObjCDesktopCapturer::Start(uint32_t fps) {
53-
54-
if(fps == 0) {
55-
capture_state_ = CS_FAILED;
56-
return capture_state_;
54+
if (fps == 0) {
55+
capture_state_ = CS_FAILED;
56+
return capture_state_;
5757
}
5858

59-
if(fps >= 60) {
59+
if (fps >= 60) {
6060
capture_delay_ = uint32_t(1000.0 / 60.0);
6161
} else {
6262
capture_delay_ = uint32_t(1000.0 / fps);
6363
}
6464

65-
if(source_id_ != -1) {
66-
if(!capturer_->SelectSource(source_id_)) {
67-
capture_state_ = CS_FAILED;
68-
return capture_state_;
65+
if (source_id_ != -1) {
66+
if (!capturer_->SelectSource(source_id_)) {
67+
capture_state_ = CS_FAILED;
68+
return capture_state_;
6969
}
70-
if(type_ == kWindow) {
71-
if(!capturer_->FocusOnSelectedSource()) {
70+
if (type_ == kWindow) {
71+
if (!capturer_->FocusOnSelectedSource()) {
7272
capture_state_ = CS_FAILED;
7373
return capture_state_;
7474
}
7575
}
7676
}
7777

78-
thread_->Invoke<void>(RTC_FROM_HERE, [this] {
79-
capturer_->Start(this);
80-
});
78+
thread_->Invoke<void>(RTC_FROM_HERE, [this] { capturer_->Start(this); });
8179
capture_state_ = CS_RUNNING;
82-
thread_->PostTask(ToQueuedTask(
83-
[this]{
84-
CaptureFrame();
85-
}));
80+
thread_->PostTask(ToQueuedTask([this] { CaptureFrame(); }));
8681
[delegate_ didSourceCaptureStart];
8782
return capture_state_;
8883
}
@@ -97,7 +92,7 @@
9792
}
9893

9994
void ObjCDesktopCapturer::OnCaptureResult(webrtc::DesktopCapturer::Result result,
100-
std::unique_ptr<webrtc::DesktopFrame> frame) {
95+
std::unique_ptr<webrtc::DesktopFrame> frame) {
10196
if (result != result_) {
10297
if (result == webrtc::DesktopCapturer::Result::ERROR_PERMANENT) {
10398
[delegate_ didSourceCaptureError];
@@ -118,14 +113,14 @@
118113
}
119114

120115
if (result == webrtc::DesktopCapturer::Result::ERROR_TEMPORARY) {
121-
return;
116+
return;
122117
}
123118

124119
int width = frame->size().width();
125120
int height = frame->size().height();
126121
int real_width = width;
127122

128-
if(type_ == kWindow) {
123+
if (type_ == kWindow) {
129124
int multiple = 0;
130125
#if defined(WEBRTC_ARCH_X86_FAMILY)
131126
multiple = 16;
@@ -134,24 +129,26 @@
134129
#endif
135130
// A multiple of $multiple must be used as the width of the src frame,
136131
// and the right black border needs to be cropped during conversion.
137-
if( multiple != 0 && (width % multiple) != 0 ) {
132+
if (multiple != 0 && (width % multiple) != 0) {
138133
width = (width / multiple + 1) * multiple;
139134
}
140135
}
141-
142-
if (!i420_buffer_ || !i420_buffer_.get() ||
143-
i420_buffer_->width() * i420_buffer_->height() != real_width * height) {
144-
i420_buffer_ = webrtc::I420Buffer::Create(real_width, height);
145-
}
146136

147-
libyuv::ConvertToI420(frame->data(),
148-
0,
149-
i420_buffer_->MutableDataY(),
150-
i420_buffer_->StrideY(),
151-
i420_buffer_->MutableDataU(),
152-
i420_buffer_->StrideU(),
153-
i420_buffer_->MutableDataV(),
154-
i420_buffer_->StrideV(),
137+
CVPixelBufferRef pixelBuffer = NULL;
138+
139+
NSDictionary *pixelAttributes = @{(NSString *)kCVPixelBufferIOSurfacePropertiesKey : @{}};
140+
CVReturn res = CVPixelBufferCreate(kCFAllocatorDefault,
141+
width,
142+
height,
143+
kCVPixelFormatType_32BGRA,
144+
(__bridge CFDictionaryRef)(pixelAttributes),
145+
&pixelBuffer);
146+
CVPixelBufferLockBaseAddress(pixelBuffer, 0);
147+
uint8_t *pxdata = (uint8_t *)CVPixelBufferGetBaseAddress(pixelBuffer);
148+
libyuv::ConvertToARGB(reinterpret_cast<uint8_t *>(frame->data()),
149+
real_width * height * 4,
150+
reinterpret_cast<uint8_t *>(pxdata),
151+
width * 4,
155152
0,
156153
0,
157154
width,
@@ -160,20 +157,26 @@
160157
height,
161158
libyuv::kRotate0,
162159
libyuv::FOURCC_ARGB);
160+
CVPixelBufferUnlockBaseAddress(pixelBuffer, 0);
161+
162+
if (res != kCVReturnSuccess) {
163+
NSLog(@"Unable to create cvpixelbuffer %d", res);
164+
return;
165+
}
166+
167+
RTC_OBJC_TYPE(RTCCVPixelBuffer) *rtcPixelBuffer =
168+
[[RTC_OBJC_TYPE(RTCCVPixelBuffer) alloc] initWithPixelBuffer:pixelBuffer];
163169
NSTimeInterval timeStampSeconds = CACurrentMediaTime();
164170
int64_t timeStampNs = lroundf(timeStampSeconds * NSEC_PER_SEC);
165-
RTCVideoFrame* rtc_video_frame =
166-
ToObjCVideoFrame(
167-
webrtc::VideoFrame::Builder()
168-
.set_video_frame_buffer(i420_buffer_)
169-
.set_rotation(webrtc::kVideoRotation_0)
170-
.set_timestamp_us(timeStampNs / 1000)
171-
.build()
172-
);
173-
[delegate_ didCaptureVideoFrame:rtc_video_frame];
171+
RTC_OBJC_TYPE(RTCVideoFrame) *videoFrame =
172+
[[RTC_OBJC_TYPE(RTCVideoFrame) alloc] initWithBuffer:rtcPixelBuffer
173+
rotation:RTCVideoRotation_0
174+
timeStampNs:timeStampNs];
175+
176+
[delegate_ didCaptureVideoFrame:videoFrame];
174177
}
175178

176-
void ObjCDesktopCapturer::OnMessage(rtc::Message* msg) {
179+
void ObjCDesktopCapturer::OnMessage(rtc::Message *msg) {
177180
if (msg->message_id == kCaptureMessageId) {
178181
CaptureFrame();
179182
}

0 commit comments

Comments
 (0)