Revert of Delete VideoFrameFactory, CapturedFrame, and related code. (patchset #9 id:160001 of https://codereview.webrtc.org/2262443003/ )
Reason for revert:
Breaks downstream testcode, still using CapturedFrame.
Original issue's description:
> Delete VideoFrameFactory, CapturedFrame, and related code.
>
> BUG=webrtc:5682
>
> Committed: https://crrev.com/66ac50e58c790624d51ede10ae438cbadbca9d2e
> Cr-Commit-Position: refs/heads/master@{#14315}
TBR=pthatcher@webrtc.org,perkj@webrtc.org
# Skipping CQ checks because original CL landed less than 1 days ago.
NOPRESUBMIT=true
NOTREECHECKS=true
NOTRY=true
BUG=webrtc:5682
Review-Url: https://codereview.webrtc.org/2357113002
Cr-Commit-Position: refs/heads/master@{#14320}
diff --git a/webrtc/media/BUILD.gn b/webrtc/media/BUILD.gn
index 14a3e2f..6179f67 100644
--- a/webrtc/media/BUILD.gn
+++ b/webrtc/media/BUILD.gn
@@ -84,6 +84,8 @@
"base/videocommon.h",
"base/videoframe.cc",
"base/videoframe.h",
+ "base/videoframefactory.cc",
+ "base/videoframefactory.h",
"base/videorenderer.h",
"base/videosourcebase.cc",
"base/videosourcebase.h",
@@ -106,6 +108,8 @@
"engine/webrtcvideoengine2.h",
"engine/webrtcvideoframe.cc",
"engine/webrtcvideoframe.h",
+ "engine/webrtcvideoframefactory.cc",
+ "engine/webrtcvideoframefactory.h",
"engine/webrtcvoe.h",
"engine/webrtcvoiceengine.cc",
"engine/webrtcvoiceengine.h",
@@ -319,6 +323,7 @@
"engine/webrtcvideocapturer_unittest.cc",
"engine/webrtcvideoengine2_unittest.cc",
"engine/webrtcvideoframe_unittest.cc",
+ "engine/webrtcvideoframefactory_unittest.cc",
"engine/webrtcvoiceengine_unittest.cc",
"sctp/sctpdataengine_unittest.cc",
]
diff --git a/webrtc/media/base/fakevideocapturer.h b/webrtc/media/base/fakevideocapturer.h
index 0f2d8ed..8ba56f1 100644
--- a/webrtc/media/base/fakevideocapturer.h
+++ b/webrtc/media/base/fakevideocapturer.h
@@ -20,6 +20,9 @@
#include "webrtc/media/base/videocapturer.h"
#include "webrtc/media/base/videocommon.h"
#include "webrtc/media/base/videoframe.h"
+#ifdef HAVE_WEBRTC_VIDEO
+#include "webrtc/media/engine/webrtcvideoframefactory.h"
+#endif
namespace cricket {
@@ -32,6 +35,9 @@
next_timestamp_(rtc::kNumNanosecsPerMillisec),
is_screencast_(is_screencast),
rotation_(webrtc::kVideoRotation_0) {
+#ifdef HAVE_WEBRTC_VIDEO
+ set_frame_factory(new cricket::WebRtcVideoFrameFactory());
+#endif
// Default supported formats. Use ResetSupportedFormats to over write.
std::vector<cricket::VideoFormat> formats;
formats.push_back(cricket::VideoFormat(1280, 720,
@@ -75,37 +81,46 @@
if (!running_) {
return false;
}
- RTC_CHECK(fourcc == FOURCC_I420);
- RTC_CHECK(width > 0);
- RTC_CHECK(height > 0);
-
- int adapted_width;
- int adapted_height;
- int crop_width;
- int crop_height;
- int crop_x;
- int crop_y;
-
- // TODO(nisse): It's a bit silly to have this logic in a fake
- // class. Child classes of VideoCapturer are expected to call
- // AdaptFrame, and the test case
- // VideoCapturerTest.SinkWantsMaxPixelAndMaxPixelCountStepUp
- // depends on this.
- if (AdaptFrame(width, height, 0, 0, &adapted_width, &adapted_height,
- &crop_width, &crop_height, &crop_x, &crop_y, nullptr)) {
- rtc::scoped_refptr<webrtc::I420Buffer> buffer(
- webrtc::I420Buffer::Create(adapted_width, adapted_height));
- buffer->InitializeData();
-
- OnFrame(WebRtcVideoFrame(buffer, rotation_,
- next_timestamp_ / rtc::kNumNanosecsPerMicrosec),
- width, height);
+ // Currently, |fourcc| is always I420 or ARGB.
+ uint32_t size = 0u;
+ if (fourcc == cricket::FOURCC_ARGB) {
+ size = width * 4 * height;
+ } else if (fourcc == cricket::FOURCC_I420) {
+ size = width * height + 2 * ((width + 1) / 2) * ((height + 1) / 2);
+ } else {
+ return false; // Unsupported FOURCC.
}
+ if (size == 0u) {
+ return false; // Width and/or Height were zero.
+ }
+
+ cricket::CapturedFrame frame;
+ frame.width = width;
+ frame.height = height;
+ frame.fourcc = fourcc;
+ frame.data_size = size;
+ frame.time_stamp = initial_timestamp_ + next_timestamp_;
next_timestamp_ += timestamp_interval;
+ std::unique_ptr<char[]> data(new char[size]);
+ frame.data = data.get();
+ // Copy something non-zero into the buffer so Validate wont complain that
+ // the frame is all duplicate.
+ memset(frame.data, 1, size / 2);
+ memset(reinterpret_cast<uint8_t*>(frame.data) + (size / 2), 2,
+ size - (size / 2));
+ memcpy(frame.data, reinterpret_cast<const uint8_t*>(&fourcc), 4);
+ frame.rotation = rotation_;
+ // TODO(zhurunz): SignalFrameCaptured carry returned value to be able to
+ // capture results from downstream.
+ SignalFrameCaptured(this, &frame);
return true;
}
+ void SignalCapturedFrame(cricket::CapturedFrame* frame) {
+ SignalFrameCaptured(this, frame);
+ }
+
sigslot::signal1<FakeVideoCapturer*> SignalDestroyed;
cricket::CaptureState Start(const cricket::VideoFormat& format) override {
diff --git a/webrtc/media/base/videoadapter_unittest.cc b/webrtc/media/base/videoadapter_unittest.cc
index 444ef54..e805679 100644
--- a/webrtc/media/base/videoadapter_unittest.cc
+++ b/webrtc/media/base/videoadapter_unittest.cc
@@ -31,18 +31,18 @@
capture_format_.interval = VideoFormat::FpsToInterval(30);
listener_.reset(new VideoCapturerListener(&adapter_));
- capturer_->AddOrUpdateSink(listener_.get(), rtc::VideoSinkWants());
+ capturer_->SignalFrameCaptured.connect(
+ listener_.get(), &VideoCapturerListener::OnFrameCaptured);
}
virtual void TearDown() {
// Explicitly disconnect the VideoCapturer before to avoid data races
// (frames delivered to VideoCapturerListener while it's being destructed).
- capturer_->RemoveSink(listener_.get());
+ capturer_->SignalFrameCaptured.disconnect_all();
}
protected:
- class VideoCapturerListener
- : public rtc::VideoSinkInterface<cricket::VideoFrame> {
+ class VideoCapturerListener: public sigslot::has_slots<> {
public:
struct Stats {
int captured_frames;
@@ -62,18 +62,19 @@
last_adapt_was_no_op_(false) {
}
- void OnFrame(const cricket::VideoFrame& frame) {
+ void OnFrameCaptured(VideoCapturer* capturer,
+ const CapturedFrame* captured_frame) {
rtc::CritScope lock(&crit_);
- const int in_width = frame.width();
- const int in_height = frame.height();
+ const int in_width = captured_frame->width;
+ const int in_height = abs(captured_frame->height);
int cropped_width;
int cropped_height;
int out_width;
int out_height;
- if (video_adapter_->AdaptFrameResolution(
- in_width, in_height,
- frame.timestamp_us() * rtc::kNumNanosecsPerMicrosec,
- &cropped_width, &cropped_height, &out_width, &out_height)) {
+ if (video_adapter_->AdaptFrameResolution(in_width, in_height,
+ captured_frame->time_stamp,
+ &cropped_width, &cropped_height,
+ &out_width, &out_height)) {
cropped_width_ = cropped_width;
cropped_height_ = cropped_height;
out_width_ = out_width;
@@ -182,7 +183,7 @@
capturer_->CaptureFrame();
EXPECT_GE(listener_->GetStats().captured_frames, 2);
- EXPECT_EQ(1, listener_->GetStats().dropped_frames);
+ EXPECT_EQ(0, listener_->GetStats().dropped_frames);
capturer_->CaptureFrame();
EXPECT_GE(listener_->GetStats().captured_frames, 3);
@@ -190,7 +191,7 @@
capturer_->CaptureFrame();
EXPECT_GE(listener_->GetStats().captured_frames, 4);
- EXPECT_EQ(2, listener_->GetStats().dropped_frames);
+ EXPECT_EQ(1, listener_->GetStats().dropped_frames);
capturer_->CaptureFrame();
EXPECT_GE(listener_->GetStats().captured_frames, 5);
@@ -198,7 +199,7 @@
capturer_->CaptureFrame();
EXPECT_GE(listener_->GetStats().captured_frames, 6);
- EXPECT_EQ(3, listener_->GetStats().dropped_frames);
+ EXPECT_EQ(2, listener_->GetStats().dropped_frames);
capturer_->CaptureFrame();
EXPECT_GE(listener_->GetStats().captured_frames, 7);
@@ -206,7 +207,7 @@
capturer_->CaptureFrame();
EXPECT_GE(listener_->GetStats().captured_frames, 8);
- EXPECT_EQ(4, listener_->GetStats().dropped_frames);
+ EXPECT_EQ(3, listener_->GetStats().dropped_frames);
capturer_->CaptureFrame();
EXPECT_GE(listener_->GetStats().captured_frames, 9);
@@ -214,7 +215,7 @@
capturer_->CaptureFrame();
EXPECT_GE(listener_->GetStats().captured_frames, 10);
- EXPECT_EQ(5, listener_->GetStats().dropped_frames);
+ EXPECT_EQ(4, listener_->GetStats().dropped_frames);
}
// Adapt the frame rate to be two thirds of the capture rate at the beginning.
diff --git a/webrtc/media/base/videocapturer.cc b/webrtc/media/base/videocapturer.cc
index 1d81d49..c340760 100644
--- a/webrtc/media/base/videocapturer.cc
+++ b/webrtc/media/base/videocapturer.cc
@@ -18,7 +18,9 @@
#include "webrtc/base/common.h"
#include "webrtc/base/logging.h"
#include "webrtc/base/systeminfo.h"
+#include "webrtc/media/base/videoframefactory.h"
#include "webrtc/media/engine/webrtcvideoframe.h"
+#include "webrtc/media/engine/webrtcvideoframefactory.h"
namespace cricket {
@@ -32,6 +34,29 @@
} // namespace
/////////////////////////////////////////////////////////////////////
+// Implementation of struct CapturedFrame
+/////////////////////////////////////////////////////////////////////
+CapturedFrame::CapturedFrame()
+ : width(0),
+ height(0),
+ fourcc(0),
+ pixel_width(0),
+ pixel_height(0),
+ time_stamp(0),
+ data_size(0),
+ rotation(webrtc::kVideoRotation_0),
+ data(NULL) {}
+
+// TODO(fbarchard): Remove this function once lmimediaengine stops using it.
+bool CapturedFrame::GetDataSize(uint32_t* size) const {
+ if (!size || data_size == CapturedFrame::kUnknownDataSize) {
+ return false;
+ }
+ *size = data_size;
+ return true;
+}
+
+/////////////////////////////////////////////////////////////////////
// Implementation of class VideoCapturer
/////////////////////////////////////////////////////////////////////
VideoCapturer::VideoCapturer() : apply_rotation_(false) {
@@ -42,9 +67,16 @@
void VideoCapturer::Construct() {
enable_camera_list_ = false;
capture_state_ = CS_STOPPED;
+ SignalFrameCaptured.connect(this, &VideoCapturer::OnFrameCaptured);
scaled_width_ = 0;
scaled_height_ = 0;
enable_video_adapter_ = true;
+ // There are lots of video capturers out there that don't call
+ // set_frame_factory. We can either go change all of them, or we
+ // can set this default.
+ // TODO(pthatcher): Remove this hack and require the frame factory
+ // to be passed in the constructor.
+ set_frame_factory(new WebRtcVideoFrameFactory());
}
const std::vector<VideoFormat>* VideoCapturer::GetSupportedFormats() const {
@@ -120,6 +152,29 @@
UpdateFilteredSupportedFormats();
}
+std::string VideoCapturer::ToString(const CapturedFrame* captured_frame) const {
+ std::string fourcc_name = GetFourccName(captured_frame->fourcc) + " ";
+ for (std::string::const_iterator i = fourcc_name.begin();
+ i < fourcc_name.end(); ++i) {
+ // Test character is printable; Avoid isprint() which asserts on negatives.
+ if (*i < 32 || *i >= 127) {
+ fourcc_name = "";
+ break;
+ }
+ }
+
+ std::ostringstream ss;
+ ss << fourcc_name << captured_frame->width << "x" << captured_frame->height;
+ return ss.str();
+}
+
+void VideoCapturer::set_frame_factory(VideoFrameFactory* frame_factory) {
+ frame_factory_.reset(frame_factory);
+ if (frame_factory) {
+ frame_factory->SetApplyRotation(apply_rotation_);
+ }
+}
+
bool VideoCapturer::GetInputSize(int* width, int* height) {
rtc::CritScope cs(&frame_stats_crit_);
if (!input_size_valid_) {
@@ -149,6 +204,9 @@
void VideoCapturer::OnSinkWantsChanged(const rtc::VideoSinkWants& wants) {
RTC_DCHECK(thread_checker_.CalledOnValidThread());
apply_rotation_ = wants.rotation_applied;
+ if (frame_factory_) {
+ frame_factory_->SetApplyRotation(apply_rotation_);
+ }
if (video_adapter()) {
video_adapter()->OnResolutionRequest(wants.max_pixel_count,
@@ -196,31 +254,54 @@
return true;
}
+void VideoCapturer::OnFrameCaptured(VideoCapturer*,
+ const CapturedFrame* captured_frame) {
+ int out_width;
+ int out_height;
+ int crop_width;
+ int crop_height;
+ int crop_x;
+ int crop_y;
+
+ // TODO(nisse): We don't do timestamp translation on this input
+ // path. It seems straight-forward to enable translation, but that
+ // breaks the WebRtcVideoEngine2Test.PropagatesInputFrameTimestamp
+ // test. Probably not worth the effort to fix, instead, try to
+ // delete or refactor all code using VideoFrameFactory and
+ // SignalCapturedFrame.
+ if (!AdaptFrame(captured_frame->width, captured_frame->height,
+ captured_frame->time_stamp / rtc::kNumNanosecsPerMicrosec,
+ 0,
+ &out_width, &out_height,
+ &crop_width, &crop_height, &crop_x, &crop_y, nullptr)) {
+ return;
+ }
+
+ if (!frame_factory_) {
+ LOG(LS_ERROR) << "No video frame factory.";
+ return;
+ }
+
+ // TODO(nisse): Reorganize frame factory methods. crop_x and crop_y
+ // are ignored for now.
+ std::unique_ptr<VideoFrame> adapted_frame(frame_factory_->CreateAliasedFrame(
+ captured_frame, crop_width, crop_height, out_width, out_height));
+
+ if (!adapted_frame) {
+ // TODO(fbarchard): LOG more information about captured frame attributes.
+ LOG(LS_ERROR) << "Couldn't convert to I420! "
+ << "From " << ToString(captured_frame) << " To "
+ << out_width << " x " << out_height;
+ return;
+ }
+
+ OnFrame(*adapted_frame, captured_frame->width, captured_frame->height);
+}
+
void VideoCapturer::OnFrame(const VideoFrame& frame,
int orig_width,
int orig_height) {
- // For a child class which implements rotation itself, we should
- // always have apply_rotation_ == false or frame.rotation() == 0.
- // Except possibly during races where apply_rotation_ is changed
- // mid-stream.
- if (apply_rotation_ && frame.rotation() != webrtc::kVideoRotation_0) {
- rtc::scoped_refptr<webrtc::VideoFrameBuffer> buffer(
- frame.video_frame_buffer());
- if (buffer->native_handle()) {
- // Sources producing native frames must handle apply_rotation
- // themselves. But even if they do, we may occasionally end up
- // in this case, for frames in flight at the time
- // applied_rotation is set to true. In that case, we just drop
- // the frame.
- LOG(LS_WARNING) << "Native frame requiring rotation. Discarding.";
- return;
- }
- broadcaster_.OnFrame(WebRtcVideoFrame(
- webrtc::I420Buffer::Rotate(buffer, frame.rotation()),
- webrtc::kVideoRotation_0, frame.timestamp_us()));
- } else {
- broadcaster_.OnFrame(frame);
- }
+ broadcaster_.OnFrame(frame);
UpdateInputSize(orig_width, orig_height);
}
diff --git a/webrtc/media/base/videocapturer.h b/webrtc/media/base/videocapturer.h
index bb25d2a..eb868d9 100644
--- a/webrtc/media/base/videocapturer.h
+++ b/webrtc/media/base/videocapturer.h
@@ -28,6 +28,7 @@
#include "webrtc/media/base/videoadapter.h"
#include "webrtc/media/base/videobroadcaster.h"
#include "webrtc/media/base/videocommon.h"
+#include "webrtc/media/base/videoframefactory.h"
namespace cricket {
@@ -44,6 +45,38 @@
class VideoFrame;
+struct CapturedFrame {
+ static const uint32_t kFrameHeaderSize = 40; // Size from width to data_size.
+ static const uint32_t kUnknownDataSize = 0xFFFFFFFF;
+
+ CapturedFrame();
+
+ // Get the number of bytes of the frame data. If data_size is known, return
+ // it directly. Otherwise, calculate the size based on width, height, and
+ // fourcc. Return true if succeeded.
+ bool GetDataSize(uint32_t* size) const;
+
+ // The width and height of the captured frame could be different from those
+ // of VideoFormat. Once the first frame is captured, the width, height,
+ // fourcc, pixel_width, and pixel_height should keep the same over frames.
+ int width; // in number of pixels
+ int height; // in number of pixels
+ uint32_t fourcc; // compression
+ uint32_t pixel_width; // width of a pixel, default is 1
+ uint32_t pixel_height; // height of a pixel, default is 1
+ int64_t time_stamp; // timestamp of when the frame was captured, in unix
+ // time with nanosecond units.
+ uint32_t data_size; // number of bytes of the frame data
+
+ webrtc::VideoRotation rotation; // rotation in degrees of the frame.
+
+ void* data; // pointer to the frame data. This object allocates the
+ // memory or points to an existing memory.
+
+ private:
+ RTC_DISALLOW_COPY_AND_ASSIGN(CapturedFrame);
+};
+
// VideoCapturer is an abstract class that defines the interfaces for video
// capturing. The subclasses implement the video capturer for various types of
// capturers and various platforms.
@@ -171,6 +204,13 @@
// Signal all capture state changes that are not a direct result of calling
// Start().
sigslot::signal2<VideoCapturer*, CaptureState> SignalStateChange;
+ // Frame callbacks are multithreaded to allow disconnect and connect to be
+ // called concurrently. It also ensures that it is safe to call disconnect
+ // at any time which is needed since the signal may be called from an
+ // unmarshalled thread owned by the VideoCapturer.
+ // Signal the captured frame to downstream.
+ sigslot::signal2<VideoCapturer*, const CapturedFrame*,
+ sigslot::multi_threaded_local> SignalFrameCaptured;
// If true, run video adaptation. By default, video adaptation is enabled
// and users must call video_adapter()->OnOutputFormatRequest()
@@ -180,6 +220,9 @@
enable_video_adapter_ = enable_video_adapter;
}
+ // Takes ownership.
+ void set_frame_factory(VideoFrameFactory* frame_factory);
+
bool GetInputSize(int* width, int* height);
// Implements VideoSourceInterface
@@ -220,6 +263,10 @@
int* crop_y,
int64_t* translated_camera_time_us);
+ // Callback attached to SignalFrameCaptured where SignalVideoFrames is called.
+ void OnFrameCaptured(VideoCapturer* video_capturer,
+ const CapturedFrame* captured_frame);
+
// Called when a frame has been captured and converted to a
// VideoFrame. OnFrame can be called directly by an implementation
// that does not use SignalFrameCaptured or OnFrameCaptured. The
@@ -244,6 +291,7 @@
}
void SetSupportedFormats(const std::vector<VideoFormat>& formats);
+ VideoFrameFactory* frame_factory() { return frame_factory_.get(); }
private:
void Construct();
@@ -253,6 +301,9 @@
int64_t GetFormatDistance(const VideoFormat& desired,
const VideoFormat& supported);
+ // Convert captured frame to readable string for LOG messages.
+ std::string ToString(const CapturedFrame* frame) const;
+
// Updates filtered_supported_formats_ so that it contains the formats in
// supported_formats_ that fulfill all applied restrictions.
void UpdateFilteredSupportedFormats();
@@ -264,6 +315,7 @@
rtc::ThreadChecker thread_checker_;
std::string id_;
CaptureState capture_state_;
+ std::unique_ptr<VideoFrameFactory> frame_factory_;
std::unique_ptr<VideoFormat> capture_format_;
std::vector<VideoFormat> supported_formats_;
std::unique_ptr<VideoFormat> max_format_;
@@ -283,8 +335,7 @@
int input_width_ GUARDED_BY(frame_stats_crit_);
int input_height_ GUARDED_BY(frame_stats_crit_);
- // Whether capturer should apply rotation to the frame before
- // passing it on to the registered sinks.
+ // Whether capturer should apply rotation to the frame before signaling it.
bool apply_rotation_;
// State for the timestamp translation.
diff --git a/webrtc/media/base/videocapturer_unittest.cc b/webrtc/media/base/videocapturer_unittest.cc
index 821de58..25230b5 100644
--- a/webrtc/media/base/videocapturer_unittest.cc
+++ b/webrtc/media/base/videocapturer_unittest.cc
@@ -91,14 +91,14 @@
std::vector<cricket::VideoFormat> formats;
formats.push_back(cricket::VideoFormat(kWidth, kHeight,
- cricket::VideoFormat::FpsToInterval(5),
- cricket::FOURCC_I420));
+ cricket::VideoFormat::FpsToInterval(5), cricket::FOURCC_ARGB));
capturer_->ResetSupportedFormats(formats);
- EXPECT_EQ(cricket::CS_RUNNING,
- capturer_->Start(cricket::VideoFormat(
- kWidth, kHeight, cricket::VideoFormat::FpsToInterval(30),
- cricket::FOURCC_ANY)));
+ EXPECT_EQ(cricket::CS_RUNNING, capturer_->Start(cricket::VideoFormat(
+ kWidth,
+ kHeight,
+ cricket::VideoFormat::FpsToInterval(30),
+ cricket::FOURCC_ARGB)));
EXPECT_TRUE(capturer_->IsRunning());
EXPECT_EQ(0, renderer_.num_rendered_frames());
EXPECT_TRUE(capturer_->CaptureFrame());
@@ -245,10 +245,6 @@
EXPECT_EQ(webrtc::kVideoRotation_0, renderer2.rotation());
}
-// TODO(nisse): This test doesn't quite fit here. It tests two things:
-// Aggregation of VideoSinkWants, which is the responsibility of
-// VideoBroadcaster, and translation of VideoSinkWants to actual
-// resolution, which is the responsibility of the VideoAdapter.
TEST_F(VideoCapturerTest, SinkWantsMaxPixelAndMaxPixelCountStepUp) {
EXPECT_EQ(cricket::CS_RUNNING,
capturer_->Start(cricket::VideoFormat(
diff --git a/webrtc/media/base/videoframefactory.cc b/webrtc/media/base/videoframefactory.cc
new file mode 100644
index 0000000..a4eeece
--- /dev/null
+++ b/webrtc/media/base/videoframefactory.cc
@@ -0,0 +1,51 @@
+/*
+ * Copyright (c) 2014 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#include "webrtc/media/base/videoframefactory.h"
+
+#include <algorithm>
+#include "webrtc/media/base/videocapturer.h"
+
+namespace cricket {
+
+VideoFrame* VideoFrameFactory::CreateAliasedFrame(
+ const CapturedFrame* input_frame,
+ int cropped_input_width,
+ int cropped_input_height,
+ int output_width,
+ int output_height) const {
+ std::unique_ptr<VideoFrame> cropped_input_frame(CreateAliasedFrame(
+ input_frame, cropped_input_width, cropped_input_height));
+ if (!cropped_input_frame)
+ return nullptr;
+
+ if (cropped_input_width == output_width &&
+ cropped_input_height == output_height) {
+ // No scaling needed.
+ return cropped_input_frame.release();
+ }
+
+ // If the frame is rotated, we need to switch the width and height.
+ if (apply_rotation_ &&
+ (input_frame->rotation == webrtc::kVideoRotation_90 ||
+ input_frame->rotation == webrtc::kVideoRotation_270)) {
+ std::swap(output_width, output_height);
+ }
+
+ rtc::scoped_refptr<webrtc::I420Buffer> scaled_buffer(
+ pool_.CreateBuffer(output_width, output_height));
+ scaled_buffer->CropAndScaleFrom(cropped_input_frame->video_frame_buffer());
+
+ return new WebRtcVideoFrame(scaled_buffer, cropped_input_frame->rotation(),
+ cropped_input_frame->timestamp_us(),
+ cropped_input_frame->transport_frame_id());
+}
+
+} // namespace cricket
diff --git a/webrtc/media/base/videoframefactory.h b/webrtc/media/base/videoframefactory.h
new file mode 100644
index 0000000..985199f
--- /dev/null
+++ b/webrtc/media/base/videoframefactory.h
@@ -0,0 +1,63 @@
+/*
+ * Copyright (c) 2014 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#ifndef WEBRTC_MEDIA_BASE_VIDEOFRAMEFACTORY_H_
+#define WEBRTC_MEDIA_BASE_VIDEOFRAMEFACTORY_H_
+
+#include <memory>
+
+#include "webrtc/common_video/include/i420_buffer_pool.h"
+#include "webrtc/media/base/videoframe.h"
+
+namespace cricket {
+
+struct CapturedFrame;
+class VideoFrame;
+
+// Creates cricket::VideoFrames, or a subclass of cricket::VideoFrame
+// depending on the subclass of VideoFrameFactory.
+class VideoFrameFactory {
+ public:
+ VideoFrameFactory() : apply_rotation_(false) {}
+ virtual ~VideoFrameFactory() {}
+
+ // The returned frame aliases the aliased_frame if the input color
+ // space allows for aliasing, otherwise a color conversion will
+ // occur. Returns NULL if conversion fails.
+
+ // The returned frame will be a center crop of |input_frame| with
+ // size |cropped_width| x |cropped_height|.
+ virtual VideoFrame* CreateAliasedFrame(const CapturedFrame* input_frame,
+ int cropped_width,
+ int cropped_height) const = 0;
+
+ // The returned frame will be a center crop of |input_frame| with size
+ // |cropped_width| x |cropped_height|, scaled to |output_width| x
+ // |output_height|.
+ virtual VideoFrame* CreateAliasedFrame(const CapturedFrame* input_frame,
+ int cropped_input_width,
+ int cropped_input_height,
+ int output_width,
+ int output_height) const;
+
+ void SetApplyRotation(bool enable) { apply_rotation_ = enable; }
+
+ protected:
+ bool apply_rotation_;
+
+ private:
+ // An internal pool to avoid reallocations. It is mutable because it
+ // does not affect behaviour, only performance.
+ mutable webrtc::I420BufferPool pool_;
+};
+
+} // namespace cricket
+
+#endif // WEBRTC_MEDIA_BASE_VIDEOFRAMEFACTORY_H_
diff --git a/webrtc/media/engine/webrtcvideocapturer.cc b/webrtc/media/engine/webrtcvideocapturer.cc
index 2b1b62b..ba44b13 100644
--- a/webrtc/media/engine/webrtcvideocapturer.cc
+++ b/webrtc/media/engine/webrtcvideocapturer.cc
@@ -19,6 +19,7 @@
#include "webrtc/base/thread.h"
#include "webrtc/base/timeutils.h"
#include "webrtc/media/engine/webrtcvideoframe.h"
+#include "webrtc/media/engine/webrtcvideoframefactory.h"
#include "webrtc/base/win32.h" // Need this to #include the impl files.
#include "webrtc/modules/video_capture/video_capture_factory.h"
@@ -112,14 +113,18 @@
module_(nullptr),
captured_frames_(0),
start_thread_(nullptr),
- async_invoker_(nullptr) {}
+ async_invoker_(nullptr) {
+ set_frame_factory(new WebRtcVideoFrameFactory());
+}
WebRtcVideoCapturer::WebRtcVideoCapturer(WebRtcVcmFactoryInterface* factory)
: factory_(factory),
module_(nullptr),
captured_frames_(0),
start_thread_(nullptr),
- async_invoker_(nullptr) {}
+ async_invoker_(nullptr) {
+ set_frame_factory(new WebRtcVideoFrameFactory());
+}
WebRtcVideoCapturer::~WebRtcVideoCapturer() {}
diff --git a/webrtc/media/engine/webrtcvideoengine2.h b/webrtc/media/engine/webrtcvideoengine2.h
index e568c1e..b8d17e8 100644
--- a/webrtc/media/engine/webrtcvideoengine2.h
+++ b/webrtc/media/engine/webrtcvideoengine2.h
@@ -60,6 +60,7 @@
class WebRtcVoiceEngine;
class WebRtcVoiceMediaChannel;
+struct CapturedFrame;
struct Device;
// Exposed here for unittests.
diff --git a/webrtc/media/engine/webrtcvideoframe.cc b/webrtc/media/engine/webrtcvideoframe.cc
index 2a9bbcc..7b5a680 100644
--- a/webrtc/media/engine/webrtcvideoframe.cc
+++ b/webrtc/media/engine/webrtcvideoframe.cc
@@ -66,6 +66,14 @@
true /*apply_rotation*/);
}
+bool WebRtcVideoFrame::Init(const CapturedFrame* frame, int dw, int dh,
+ bool apply_rotation) {
+ return Reset(frame->fourcc, frame->width, frame->height, dw, dh,
+ static_cast<uint8_t*>(frame->data), frame->data_size,
+ frame->time_stamp / rtc::kNumNanosecsPerMicrosec,
+ frame->rotation, apply_rotation);
+}
+
int WebRtcVideoFrame::width() const {
return video_frame_buffer_ ? video_frame_buffer_->width() : 0;
}
diff --git a/webrtc/media/engine/webrtcvideoframe.h b/webrtc/media/engine/webrtcvideoframe.h
index 79ad571..d0034e2 100644
--- a/webrtc/media/engine/webrtcvideoframe.h
+++ b/webrtc/media/engine/webrtcvideoframe.h
@@ -22,6 +22,8 @@
namespace cricket {
+struct CapturedFrame;
+
// TODO(nisse): This class will be deleted when the cricket::VideoFrame and
// webrtc::VideoFrame classes are merged. See
// https://bugs.chromium.org/p/webrtc/issues/detail?id=5682. Try to use only the
@@ -53,10 +55,6 @@
~WebRtcVideoFrame();
- // TODO(nisse): Init (and its helpers Reset and Validate) are used
- // only by the LoadFrame function used in the VideoFrame unittests.
- // Rewrite tests, and delete this function.
-
// Creates a frame from a raw sample with FourCC "format" and size "w" x "h".
// "h" can be negative indicating a vertically flipped image.
// "dh" is destination height if cropping is desired and is always positive.
@@ -71,6 +69,15 @@
int64_t timestamp_ns,
webrtc::VideoRotation rotation);
+ // TODO(nisse): We're moving to have all timestamps use the same
+ // time scale as rtc::TimeMicros. However, this method is used by
+ // WebRtcVideoFrameFactory::CreateAliasedFrame this code path
+ // currently does not conform to the new timestamp conventions and
+ // may use the camera's own clock instead. It's unclear if this
+ // should be fixed, or if instead all of the VideoFrameFactory
+ // abstraction should be eliminated.
+ bool Init(const CapturedFrame* frame, int dw, int dh, bool apply_rotation);
+
void InitToEmptyBuffer(int w, int h);
int width() const override;
diff --git a/webrtc/media/engine/webrtcvideoframe_unittest.cc b/webrtc/media/engine/webrtcvideoframe_unittest.cc
index 2385e44..3743e87 100644
--- a/webrtc/media/engine/webrtcvideoframe_unittest.cc
+++ b/webrtc/media/engine/webrtcvideoframe_unittest.cc
@@ -22,6 +22,52 @@
public:
WebRtcVideoFrameTest() {}
+ void TestInit(int cropped_width, int cropped_height,
+ webrtc::VideoRotation frame_rotation,
+ bool apply_rotation) {
+ const int frame_width = 1920;
+ const int frame_height = 1080;
+
+ // Build the CapturedFrame.
+ CapturedFrame captured_frame;
+ captured_frame.fourcc = FOURCC_I420;
+ captured_frame.time_stamp = rtc::TimeNanos();
+ captured_frame.rotation = frame_rotation;
+ captured_frame.width = frame_width;
+ captured_frame.height = frame_height;
+ captured_frame.data_size = (frame_width * frame_height) +
+ ((frame_width + 1) / 2) * ((frame_height + 1) / 2) * 2;
+ std::unique_ptr<uint8_t[]> captured_frame_buffer(
+ new uint8_t[captured_frame.data_size]);
+ // Initialize memory to satisfy DrMemory tests.
+ memset(captured_frame_buffer.get(), 0, captured_frame.data_size);
+ captured_frame.data = captured_frame_buffer.get();
+
+ // Create the new frame from the CapturedFrame.
+ WebRtcVideoFrame frame;
+ EXPECT_TRUE(
+ frame.Init(&captured_frame, cropped_width, cropped_height,
+ apply_rotation));
+
+ // Verify the new frame.
+ EXPECT_EQ(captured_frame.time_stamp / rtc::kNumNanosecsPerMicrosec,
+ frame.timestamp_us());
+ if (apply_rotation)
+ EXPECT_EQ(webrtc::kVideoRotation_0, frame.rotation());
+ else
+ EXPECT_EQ(frame_rotation, frame.rotation());
+ // If |apply_rotation| and the frame rotation is 90 or 270, width and
+ // height are flipped.
+ if (apply_rotation && (frame_rotation == webrtc::kVideoRotation_90
+ || frame_rotation == webrtc::kVideoRotation_270)) {
+ EXPECT_EQ(cropped_width, frame.height());
+ EXPECT_EQ(cropped_height, frame.width());
+ } else {
+ EXPECT_EQ(cropped_width, frame.width());
+ EXPECT_EQ(cropped_height, frame.height());
+ }
+ }
+
void SetFrameRotation(WebRtcVideoFrame* frame,
webrtc::VideoRotation rotation) {
frame->rotation_ = rotation;
@@ -108,6 +154,32 @@
// TEST_WEBRTCVIDEOFRAME(ConvertToI422Buffer)
// TEST_WEBRTCVIDEOFRAME(ConstructARGBBlackWhitePixel)
+// These functions test implementation-specific details.
+// Tests the Init function with different cropped size.
+TEST_F(WebRtcVideoFrameTest, InitEvenSize) {
+ TestInit(640, 360, webrtc::kVideoRotation_0, true);
+}
+
+TEST_F(WebRtcVideoFrameTest, InitOddWidth) {
+ TestInit(601, 480, webrtc::kVideoRotation_0, true);
+}
+
+TEST_F(WebRtcVideoFrameTest, InitOddHeight) {
+ TestInit(360, 765, webrtc::kVideoRotation_0, true);
+}
+
+TEST_F(WebRtcVideoFrameTest, InitOddWidthHeight) {
+ TestInit(355, 1021, webrtc::kVideoRotation_0, true);
+}
+
+TEST_F(WebRtcVideoFrameTest, InitRotated90ApplyRotation) {
+ TestInit(640, 360, webrtc::kVideoRotation_90, true);
+}
+
+TEST_F(WebRtcVideoFrameTest, InitRotated90DontApplyRotation) {
+ TestInit(640, 360, webrtc::kVideoRotation_90, false);
+}
+
TEST_F(WebRtcVideoFrameTest, TextureInitialValues) {
webrtc::test::FakeNativeHandle* dummy_handle =
new webrtc::test::FakeNativeHandle();
diff --git a/webrtc/media/engine/webrtcvideoframefactory.cc b/webrtc/media/engine/webrtcvideoframefactory.cc
new file mode 100644
index 0000000..2c08c63
--- /dev/null
+++ b/webrtc/media/engine/webrtcvideoframefactory.cc
@@ -0,0 +1,30 @@
+/*
+ * Copyright (c) 2014 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#include <memory>
+
+#include "webrtc/base/logging.h"
+#include "webrtc/media/engine/webrtcvideoframe.h"
+#include "webrtc/media/engine/webrtcvideoframefactory.h"
+
+namespace cricket {
+
+VideoFrame* WebRtcVideoFrameFactory::CreateAliasedFrame(
+ const CapturedFrame* aliased_frame, int width, int height) const {
+ std::unique_ptr<WebRtcVideoFrame> frame(new WebRtcVideoFrame());
+ if (!frame->Init(aliased_frame, width, height, apply_rotation_)) {
+ LOG(LS_ERROR) <<
+ "Failed to create WebRtcVideoFrame in CreateAliasedFrame.";
+ return NULL;
+ }
+ return frame.release();
+}
+
+} // namespace cricket
diff --git a/webrtc/media/engine/webrtcvideoframefactory.h b/webrtc/media/engine/webrtcvideoframefactory.h
new file mode 100644
index 0000000..39d813f
--- /dev/null
+++ b/webrtc/media/engine/webrtcvideoframefactory.h
@@ -0,0 +1,35 @@
+/*
+ * Copyright (c) 2014 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#ifndef WEBRTC_MEDIA_ENGINE_WEBRTCVIDEOFRAMEFACTORY_H_
+#define WEBRTC_MEDIA_ENGINE_WEBRTCVIDEOFRAMEFACTORY_H_
+
+#include "webrtc/media/base/videoframefactory.h"
+
+namespace cricket {
+
+struct CapturedFrame;
+
+// Creates instances of cricket::WebRtcVideoFrame.
+class WebRtcVideoFrameFactory : public VideoFrameFactory {
+ public:
+ // Note: Overriding a method name overrides all overloaded versions.
+ // Without this using-declaration, we would hide the 5-argument
+ // method we want to inherit.
+ using VideoFrameFactory::CreateAliasedFrame;
+
+ VideoFrame* CreateAliasedFrame(const CapturedFrame* aliased_frame,
+ int width,
+ int height) const override;
+};
+
+} // namespace cricket
+
+#endif // WEBRTC_MEDIA_ENGINE_WEBRTCVIDEOFRAMEFACTORY_H_
diff --git a/webrtc/media/engine/webrtcvideoframefactory_unittest.cc b/webrtc/media/engine/webrtcvideoframefactory_unittest.cc
new file mode 100644
index 0000000..197784b
--- /dev/null
+++ b/webrtc/media/engine/webrtcvideoframefactory_unittest.cc
@@ -0,0 +1,109 @@
+/*
+ * Copyright (c) 2015 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#include <string.h>
+
+#include <memory>
+
+#include "webrtc/base/gunit.h"
+#include "webrtc/media/base/videocapturer.h"
+#include "webrtc/media/engine/webrtcvideoframe.h"
+#include "webrtc/media/engine/webrtcvideoframefactory.h"
+
+class WebRtcVideoFrameFactoryTest : public testing::Test {
+ public:
+ WebRtcVideoFrameFactoryTest() {}
+
+ void InitFrame(webrtc::VideoRotation frame_rotation) {
+ const int frame_width = 1920;
+ const int frame_height = 1080;
+
+ // Build the CapturedFrame.
+ captured_frame_.fourcc = cricket::FOURCC_I420;
+ captured_frame_.pixel_width = 1;
+ captured_frame_.pixel_height = 1;
+ captured_frame_.time_stamp = rtc::TimeNanos();
+ captured_frame_.rotation = frame_rotation;
+ captured_frame_.width = frame_width;
+ captured_frame_.height = frame_height;
+ captured_frame_.data_size =
+ (frame_width * frame_height) +
+ ((frame_width + 1) / 2) * ((frame_height + 1) / 2) * 2;
+ captured_frame_buffer_.reset(new uint8_t[captured_frame_.data_size]);
+ // Initialize memory to satisfy DrMemory tests.
+ memset(captured_frame_buffer_.get(), 0, captured_frame_.data_size);
+ captured_frame_.data = captured_frame_buffer_.get();
+ }
+
+ void VerifyFrame(cricket::VideoFrame* dest_frame,
+ webrtc::VideoRotation src_rotation,
+ int src_width,
+ int src_height,
+ bool apply_rotation) {
+ if (!apply_rotation) {
+ EXPECT_EQ(dest_frame->rotation(), src_rotation);
+ EXPECT_EQ(dest_frame->width(), src_width);
+ EXPECT_EQ(dest_frame->height(), src_height);
+ } else {
+ EXPECT_EQ(dest_frame->rotation(), webrtc::kVideoRotation_0);
+ if (src_rotation == webrtc::kVideoRotation_90 ||
+ src_rotation == webrtc::kVideoRotation_270) {
+ EXPECT_EQ(dest_frame->width(), src_height);
+ EXPECT_EQ(dest_frame->height(), src_width);
+ } else {
+ EXPECT_EQ(dest_frame->width(), src_width);
+ EXPECT_EQ(dest_frame->height(), src_height);
+ }
+ }
+ }
+
+ void TestCreateAliasedFrame(bool apply_rotation) {
+ cricket::VideoFrameFactory& factory = factory_;
+ factory.SetApplyRotation(apply_rotation);
+ InitFrame(webrtc::kVideoRotation_270);
+ const cricket::CapturedFrame& captured_frame = get_captured_frame();
+ // Create the new frame from the CapturedFrame.
+ std::unique_ptr<cricket::VideoFrame> frame;
+ int new_width = captured_frame.width / 2;
+ int new_height = captured_frame.height / 2;
+ frame.reset(factory.CreateAliasedFrame(&captured_frame, new_width,
+ new_height, new_width, new_height));
+ VerifyFrame(frame.get(), webrtc::kVideoRotation_270, new_width, new_height,
+ apply_rotation);
+
+ frame.reset(factory.CreateAliasedFrame(
+ &captured_frame, new_width, new_height, new_width / 2, new_height / 2));
+ VerifyFrame(frame.get(), webrtc::kVideoRotation_270, new_width / 2,
+ new_height / 2, apply_rotation);
+
+ // Reset the frame first so it's exclusive hence we could go through the
+ // StretchToFrame code path in CreateAliasedFrame.
+ frame.reset();
+ frame.reset(factory.CreateAliasedFrame(
+ &captured_frame, new_width, new_height, new_width / 2, new_height / 2));
+ VerifyFrame(frame.get(), webrtc::kVideoRotation_270, new_width / 2,
+ new_height / 2, apply_rotation);
+ }
+
+ const cricket::CapturedFrame& get_captured_frame() { return captured_frame_; }
+
+ private:
+ cricket::CapturedFrame captured_frame_;
+ std::unique_ptr<uint8_t[]> captured_frame_buffer_;
+ cricket::WebRtcVideoFrameFactory factory_;
+};
+
+TEST_F(WebRtcVideoFrameFactoryTest, NoApplyRotation) {
+ TestCreateAliasedFrame(false);
+}
+
+TEST_F(WebRtcVideoFrameFactoryTest, ApplyRotation) {
+ TestCreateAliasedFrame(true);
+}
diff --git a/webrtc/media/media.gyp b/webrtc/media/media.gyp
index 9528441..9a9fbbb 100644
--- a/webrtc/media/media.gyp
+++ b/webrtc/media/media.gyp
@@ -63,6 +63,8 @@
'base/videocommon.h',
'base/videoframe.cc',
'base/videoframe.h',
+ 'base/videoframefactory.cc',
+ 'base/videoframefactory.h',
'base/videosourcebase.cc',
'base/videosourcebase.h',
'devices/videorendererfactory.h',
@@ -85,6 +87,8 @@
'engine/webrtcvideoengine2.h',
'engine/webrtcvideoframe.cc',
'engine/webrtcvideoframe.h',
+ 'engine/webrtcvideoframefactory.cc',
+ 'engine/webrtcvideoframefactory.h',
'engine/webrtcvoe.h',
'engine/webrtcvoiceengine.cc',
'engine/webrtcvoiceengine.h',
