Add color space information to webrtc::VideoFrame and extract from VP9
This CL is the first step for introducing color space information in webrtc.
- Add ColorSpace class listing color profiles.
- Add ColorSpace as a member of webrtc::VideoFrame.
- Make use of this class by extracting info from VP9 decoder.
Bug: webrtc:9522
Change-Id: I5e2514efee2a193bddb4459261387f2d40e936ad
Reviewed-on: https://webrtc-review.googlesource.com/88540
Reviewed-by: Niklas Enbom <niklas.enbom@webrtc.org>
Reviewed-by: Magnus Jedvert <magjed@webrtc.org>
Commit-Queue: Emircan Uysaler <emircan@webrtc.org>
Cr-Commit-Position: refs/heads/master@{#23988}diff --git a/api/video/BUILD.gn b/api/video/BUILD.gn
index 454e480..d767048 100644
--- a/api/video/BUILD.gn
+++ b/api/video/BUILD.gn
@@ -11,6 +11,8 @@
rtc_source_set("video_frame") {
visibility = [ "*" ]
sources = [
+ "color_space.cc",
+ "color_space.h",
"video_content_type.cc",
"video_content_type.h",
"video_frame.cc",
diff --git a/api/video/color_space.cc b/api/video/color_space.cc
new file mode 100644
index 0000000..a8be5cd
--- /dev/null
+++ b/api/video/color_space.cc
@@ -0,0 +1,42 @@
+/*
+ * Copyright (c) 2018 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#include "api/video/color_space.h"
+
+namespace webrtc {
+
+ColorSpace::ColorSpace() = default;
+
+ColorSpace::ColorSpace(PrimaryID primaries,
+ TransferID transfer,
+ MatrixID matrix,
+ RangeID range)
+ : primaries_(primaries),
+ transfer_(transfer),
+ matrix_(matrix),
+ range_(range) {}
+
+ColorSpace::PrimaryID ColorSpace::primaries() const {
+ return primaries_;
+}
+
+ColorSpace::TransferID ColorSpace::transfer() const {
+ return transfer_;
+}
+
+ColorSpace::MatrixID ColorSpace::matrix() const {
+ return matrix_;
+}
+
+ColorSpace::RangeID ColorSpace::range() const {
+ return range_;
+}
+
+} // namespace webrtc
diff --git a/api/video/color_space.h b/api/video/color_space.h
new file mode 100644
index 0000000..736a170
--- /dev/null
+++ b/api/video/color_space.h
@@ -0,0 +1,84 @@
+/*
+ * Copyright (c) 2018 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#ifndef API_VIDEO_COLOR_SPACE_H_
+#define API_VIDEO_COLOR_SPACE_H_
+
+namespace webrtc {
+
+// Used to represent a color space for the purpose of color conversion. This
+// class only represents color information that can be transferred through the
+// bitstream of WebRTC's internal supported codecs:
+// - VP9 supports color profiles, see VP9 Bitstream & Decoding Process
+// Specification Version 0.6 Section 7.2.2 "Color config semantics" available
+// from https://www.webmproject.org.
+// TODO(emircan): Extract these values from decode and add to the existing ones.
+// - VP8 only supports BT.601, see
+// https://tools.ietf.org/html/rfc6386#section-9.2
+// - H264 supports different color primaries, transfer characteristics, matrix
+// coefficients and range. See T-REC-H.264 E.2.1, "VUI parameters semantics",
+// available from https://www.itu.int/rec/T-REC-H.264.
+class ColorSpace {
+ public:
+ enum class PrimaryID {
+ kInvalid,
+ kBT709,
+ kSMPTE170M, // Identical to BT601
+ kSMPTE240M,
+ kBT2020,
+ };
+
+ enum class TransferID {
+ kInvalid,
+ kBT709,
+ kSMPTE170M,
+ kSMPTE240M,
+ kBT2020,
+ kBT2020_10,
+ kIEC61966_2_1,
+ };
+
+ enum class MatrixID {
+ kInvalid,
+ kBT709,
+ kSMPTE170M,
+ kSMPTE240M,
+ kBT2020_NCL,
+ };
+
+ enum class RangeID {
+ kInvalid,
+ // Limited Rec. 709 color range with RGB values ranging from 16 to 235.
+ kLimited,
+ // Full RGB color range with RGB valees from 0 to 255.
+ kFull,
+ };
+
+ ColorSpace();
+ ColorSpace(PrimaryID primaries,
+ TransferID transfer,
+ MatrixID matrix,
+ RangeID full_range);
+
+ PrimaryID primaries() const;
+ TransferID transfer() const;
+ MatrixID matrix() const;
+ RangeID range() const;
+
+ private:
+ PrimaryID primaries_ = PrimaryID::kInvalid;
+ TransferID transfer_ = TransferID::kInvalid;
+ MatrixID matrix_ = MatrixID::kInvalid;
+ RangeID range_ = RangeID::kInvalid;
+};
+
+} // namespace webrtc
+
+#endif // API_VIDEO_COLOR_SPACE_H_
diff --git a/api/video/video_frame.cc b/api/video/video_frame.cc
index e8fb29d..5521ad8 100644
--- a/api/video/video_frame.cc
+++ b/api/video/video_frame.cc
@@ -15,6 +15,55 @@
namespace webrtc {
+VideoFrame::Builder::Builder() = default;
+
+VideoFrame::Builder::~Builder() = default;
+
+VideoFrame VideoFrame::Builder::build() {
+ return VideoFrame(video_frame_buffer_, timestamp_us_, timestamp_rtp_,
+ ntp_time_ms_, rotation_, color_space_);
+}
+
+VideoFrame::Builder& VideoFrame::Builder::set_video_frame_buffer(
+ const rtc::scoped_refptr<VideoFrameBuffer>& buffer) {
+ video_frame_buffer_ = buffer;
+ return *this;
+}
+
+VideoFrame::Builder& VideoFrame::Builder::set_timestamp_ms(
+ int64_t timestamp_ms) {
+ timestamp_us_ = timestamp_ms * rtc::kNumMicrosecsPerMillisec;
+ return *this;
+}
+
+VideoFrame::Builder& VideoFrame::Builder::set_timestamp_us(
+ int64_t timestamp_us) {
+ timestamp_us_ = timestamp_us;
+ return *this;
+}
+
+VideoFrame::Builder& VideoFrame::Builder::set_timestamp_rtp(
+ uint32_t timestamp_rtp) {
+ timestamp_rtp_ = timestamp_rtp;
+ return *this;
+}
+
+VideoFrame::Builder& VideoFrame::Builder::set_ntp_time_ms(int64_t ntp_time_ms) {
+ ntp_time_ms_ = ntp_time_ms;
+ return *this;
+}
+
+VideoFrame::Builder& VideoFrame::Builder::set_rotation(VideoRotation rotation) {
+ rotation_ = rotation;
+ return *this;
+}
+
+VideoFrame::Builder& VideoFrame::Builder::set_color_space(
+ const ColorSpace& color_space) {
+ color_space_ = color_space;
+ return *this;
+}
+
VideoFrame::VideoFrame(const rtc::scoped_refptr<VideoFrameBuffer>& buffer,
webrtc::VideoRotation rotation,
int64_t timestamp_us)
@@ -36,6 +85,19 @@
RTC_DCHECK(buffer);
}
+VideoFrame::VideoFrame(const rtc::scoped_refptr<VideoFrameBuffer>& buffer,
+ int64_t timestamp_us,
+ uint32_t timestamp_rtp,
+ int64_t ntp_time_ms,
+ VideoRotation rotation,
+ const absl::optional<ColorSpace>& color_space)
+ : video_frame_buffer_(buffer),
+ timestamp_rtp_(timestamp_rtp),
+ ntp_time_ms_(ntp_time_ms),
+ timestamp_us_(timestamp_us),
+ rotation_(rotation),
+ color_space_(color_space) {}
+
VideoFrame::~VideoFrame() = default;
VideoFrame::VideoFrame(const VideoFrame&) = default;
diff --git a/api/video/video_frame.h b/api/video/video_frame.h
index bd08346..dcb533e 100644
--- a/api/video/video_frame.h
+++ b/api/video/video_frame.h
@@ -13,6 +13,8 @@
#include <stdint.h>
+#include "absl/types/optional.h"
+#include "api/video/color_space.h"
#include "api/video/video_frame_buffer.h"
#include "api/video/video_rotation.h"
@@ -20,12 +22,35 @@
class VideoFrame {
public:
- // Preferred constructor.
+ // Preferred way of building VideoFrame objects.
+ class Builder {
+ public:
+ Builder();
+ ~Builder();
+
+ VideoFrame build();
+ Builder& set_video_frame_buffer(
+ const rtc::scoped_refptr<VideoFrameBuffer>& buffer);
+ Builder& set_timestamp_ms(int64_t timestamp_ms);
+ Builder& set_timestamp_us(int64_t timestamp_us);
+ Builder& set_timestamp_rtp(uint32_t timestamp_rtp);
+ Builder& set_ntp_time_ms(int64_t ntp_time_ms);
+ Builder& set_rotation(VideoRotation rotation);
+ Builder& set_color_space(const ColorSpace& color_space);
+
+ private:
+ rtc::scoped_refptr<webrtc::VideoFrameBuffer> video_frame_buffer_;
+ int64_t timestamp_us_ = 0;
+ uint32_t timestamp_rtp_ = 0;
+ int64_t ntp_time_ms_ = 0;
+ VideoRotation rotation_ = kVideoRotation_0;
+ absl::optional<ColorSpace> color_space_;
+ };
+
+ // To be deprecated. Migrate all use to Builder.
VideoFrame(const rtc::scoped_refptr<VideoFrameBuffer>& buffer,
webrtc::VideoRotation rotation,
int64_t timestamp_us);
-
- // For use by the parts of the pipeline that needs the RTP 90kHz timestamp.
VideoFrame(const rtc::scoped_refptr<VideoFrameBuffer>& buffer,
uint32_t timestamp_rtp,
int64_t render_time_ms,
@@ -85,6 +110,9 @@
VideoRotation rotation() const { return rotation_; }
void set_rotation(VideoRotation rotation) { rotation_ = rotation; }
+ // Set Color space when available.
+ absl::optional<ColorSpace> color_space() const { return color_space_; }
+
// Get render time in milliseconds.
// TODO(nisse): Deprecated. Migrate all users to timestamp_us().
int64_t render_time_ms() const;
@@ -100,12 +128,20 @@
}
private:
+ VideoFrame(const rtc::scoped_refptr<VideoFrameBuffer>& buffer,
+ int64_t timestamp_us,
+ uint32_t timestamp_rtp,
+ int64_t ntp_time_ms,
+ VideoRotation rotation,
+ const absl::optional<ColorSpace>& color_space);
+
// An opaque reference counted handle that stores the pixel data.
rtc::scoped_refptr<webrtc::VideoFrameBuffer> video_frame_buffer_;
uint32_t timestamp_rtp_;
int64_t ntp_time_ms_;
int64_t timestamp_us_;
VideoRotation rotation_;
+ absl::optional<ColorSpace> color_space_;
};
} // namespace webrtc
diff --git a/modules/video_coding/BUILD.gn b/modules/video_coding/BUILD.gn
index 2a2c1de..5003799 100644
--- a/modules/video_coding/BUILD.gn
+++ b/modules/video_coding/BUILD.gn
@@ -479,6 +479,7 @@
":webrtc_vp9_helpers",
"..:module_api",
"../..:webrtc_common",
+ "../../api/video:video_frame",
"../../api/video:video_frame_i010",
"../../api/video_codecs:video_codecs_api",
"../../common_video",
@@ -718,6 +719,7 @@
"../../api:create_videocodec_test_fixture_api",
"../../api:mock_video_codec_factory",
"../../api:videocodec_test_fixture_api",
+ "../../api/video:video_frame",
"../../api/video:video_frame_i420",
"../../api/video_codecs:rtc_software_fallback_wrappers",
"../../api/video_codecs:video_codecs_api",
diff --git a/modules/video_coding/codecs/vp9/test/vp9_impl_unittest.cc b/modules/video_coding/codecs/vp9/test/vp9_impl_unittest.cc
index 1d899ae..ec4efe8 100644
--- a/modules/video_coding/codecs/vp9/test/vp9_impl_unittest.cc
+++ b/modules/video_coding/codecs/vp9/test/vp9_impl_unittest.cc
@@ -8,6 +8,7 @@
* be found in the AUTHORS file in the root of the source tree.
*/
+#include "api/video/color_space.h"
#include "api/video/i420_buffer.h"
#include "common_video/libyuv/include/webrtc_libyuv.h"
#include "media/base/vp9_profile.h"
@@ -85,6 +86,12 @@
ASSERT_TRUE(WaitForDecodedFrame(&decoded_frame, &decoded_qp));
ASSERT_TRUE(decoded_frame);
EXPECT_GT(I420PSNR(input_frame, decoded_frame.get()), 36);
+
+ const ColorSpace color_space = decoded_frame->color_space().value();
+ EXPECT_EQ(ColorSpace::PrimaryID::kInvalid, color_space.primaries());
+ EXPECT_EQ(ColorSpace::TransferID::kInvalid, color_space.transfer());
+ EXPECT_EQ(ColorSpace::MatrixID::kInvalid, color_space.matrix());
+ EXPECT_EQ(ColorSpace::RangeID::kLimited, color_space.range());
}
// We only test the encoder here, since the decoded frame rotation is set based
diff --git a/modules/video_coding/codecs/vp9/vp9_impl.cc b/modules/video_coding/codecs/vp9/vp9_impl.cc
index 56ff936..da0ec5b 100644
--- a/modules/video_coding/codecs/vp9/vp9_impl.cc
+++ b/modules/video_coding/codecs/vp9/vp9_impl.cc
@@ -21,6 +21,7 @@
#include "vpx/vpx_encoder.h"
#include "absl/memory/memory.h"
+#include "api/video/color_space.h"
#include "api/video/i010_buffer.h"
#include "common_video/include/video_frame_buffer.h"
#include "common_video/libyuv/include/webrtc_libyuv.h"
@@ -1178,10 +1179,72 @@
RTC_NOTREACHED();
return WEBRTC_VIDEO_CODEC_NO_OUTPUT;
}
- VideoFrame decoded_image(img_wrapped_buffer, timestamp,
- 0 /* render_time_ms */, webrtc::kVideoRotation_0);
- decoded_image.set_ntp_time_ms(ntp_time_ms);
+ ColorSpace::PrimaryID primaries = ColorSpace::PrimaryID::kInvalid;
+ ColorSpace::TransferID transfer = ColorSpace::TransferID::kInvalid;
+ ColorSpace::MatrixID matrix = ColorSpace::MatrixID::kInvalid;
+ switch (img->cs) {
+ case VPX_CS_BT_601:
+ case VPX_CS_SMPTE_170:
+ primaries = ColorSpace::PrimaryID::kSMPTE170M;
+ transfer = ColorSpace::TransferID::kSMPTE170M;
+ matrix = ColorSpace::MatrixID::kSMPTE170M;
+ break;
+ case VPX_CS_SMPTE_240:
+ primaries = ColorSpace::PrimaryID::kSMPTE240M;
+ transfer = ColorSpace::TransferID::kSMPTE240M;
+ matrix = ColorSpace::MatrixID::kSMPTE240M;
+ break;
+ case VPX_CS_BT_709:
+ primaries = ColorSpace::PrimaryID::kBT709;
+ transfer = ColorSpace::TransferID::kBT709;
+ matrix = ColorSpace::MatrixID::kBT709;
+ break;
+ case VPX_CS_BT_2020:
+ primaries = ColorSpace::PrimaryID::kBT2020;
+ switch (img->bit_depth) {
+ case 8:
+ transfer = ColorSpace::TransferID::kBT709;
+ break;
+ case 10:
+ transfer = ColorSpace::TransferID::kBT2020_10;
+ break;
+ default:
+ RTC_NOTREACHED();
+ break;
+ }
+ matrix = ColorSpace::MatrixID::kBT2020_NCL;
+ break;
+ case VPX_CS_SRGB:
+ primaries = ColorSpace::PrimaryID::kBT709;
+ transfer = ColorSpace::TransferID::kIEC61966_2_1;
+ matrix = ColorSpace::MatrixID::kBT709;
+ break;
+ default:
+ break;
+ }
+
+ ColorSpace::RangeID range = ColorSpace::RangeID::kInvalid;
+ switch (img->range) {
+ case VPX_CR_STUDIO_RANGE:
+ range = ColorSpace::RangeID::kLimited;
+ break;
+ case VPX_CR_FULL_RANGE:
+ range = ColorSpace::RangeID::kFull;
+ break;
+ default:
+ break;
+ }
+
+ VideoFrame decoded_image =
+ VideoFrame::Builder()
+ .set_video_frame_buffer(img_wrapped_buffer)
+ .set_timestamp_ms(0)
+ .set_timestamp_rtp(timestamp)
+ .set_ntp_time_ms(ntp_time_ms)
+ .set_rotation(webrtc::kVideoRotation_0)
+ .set_color_space(ColorSpace(primaries, transfer, matrix, range))
+ .build();
decode_complete_callback_->Decoded(decoded_image, absl::nullopt, qp);
return WEBRTC_VIDEO_CODEC_OK;
}
