blob: cf682c36a95de87b55aca148104dd335fcd52900 [file] [log] [blame]
/*
* Copyright 2018 The WebRTC project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#include "test/scenario/video_frame_matcher.h"
#include <utility>
#include "common_video/libyuv/include/webrtc_libyuv.h"
#include "rtc_base/checks.h"
#include "rtc_base/event.h"
namespace webrtc {
namespace test {
namespace {
constexpr int kThumbWidth = 96;
constexpr int kThumbHeight = 96;
} // namespace
VideoFrameMatcher::VideoFrameMatcher(
std::vector<std::function<void(const VideoFramePair&)> >
frame_pair_handlers)
: frame_pair_handlers_(std::move(frame_pair_handlers)),
task_queue_("VideoAnalyzer") {}
VideoFrameMatcher::~VideoFrameMatcher() {
task_queue_.SendTask([this] { Finalize(); }, RTC_FROM_HERE);
}
void VideoFrameMatcher::RegisterLayer(int layer_id) {
task_queue_.PostTask([this, layer_id] { layers_[layer_id] = VideoLayer(); });
}
void VideoFrameMatcher::OnCapturedFrame(const VideoFrame& frame,
Timestamp at_time) {
CapturedFrame captured;
captured.id = next_capture_id_++;
captured.capture_time = at_time;
captured.frame = frame.video_frame_buffer();
captured.thumb = ScaleVideoFrameBuffer(*frame.video_frame_buffer()->ToI420(),
kThumbWidth, kThumbHeight),
task_queue_.PostTask([this, captured]() {
for (auto& layer : layers_) {
CapturedFrame copy = captured;
if (layer.second.last_decode &&
layer.second.last_decode->frame->width() <= captured.frame->width()) {
copy.best_score = I420SSE(*captured.thumb->GetI420(),
*layer.second.last_decode->thumb->GetI420());
copy.best_decode = layer.second.last_decode;
}
layer.second.captured_frames.push_back(std::move(copy));
}
});
}
void VideoFrameMatcher::OnDecodedFrame(const VideoFrame& frame,
int layer_id,
Timestamp render_time,
Timestamp at_time) {
rtc::scoped_refptr<DecodedFrame> decoded(new DecodedFrame{});
decoded->decoded_time = at_time;
decoded->render_time = render_time;
decoded->frame = frame.video_frame_buffer();
decoded->thumb = ScaleVideoFrameBuffer(*frame.video_frame_buffer()->ToI420(),
kThumbWidth, kThumbHeight);
task_queue_.PostTask([this, decoded, layer_id] {
auto& layer = layers_[layer_id];
decoded->id = layer.next_decoded_id++;
layer.last_decode = decoded;
for (auto& captured : layer.captured_frames) {
// We can't match with a smaller capture.
if (captured.frame->width() < decoded->frame->width()) {
captured.matched = true;
continue;
}
double score =
I420SSE(*captured.thumb->GetI420(), *decoded->thumb->GetI420());
if (score < captured.best_score) {
captured.best_score = score;
captured.best_decode = decoded;
captured.matched = false;
} else {
captured.matched = true;
}
}
while (!layer.captured_frames.empty() &&
layer.captured_frames.front().matched) {
HandleMatch(std::move(layer.captured_frames.front()), layer_id);
layer.captured_frames.pop_front();
}
});
}
bool VideoFrameMatcher::Active() const {
return !frame_pair_handlers_.empty();
}
void VideoFrameMatcher::HandleMatch(VideoFrameMatcher::CapturedFrame captured,
int layer_id) {
VideoFramePair frame_pair;
frame_pair.layer_id = layer_id;
frame_pair.captured = captured.frame;
frame_pair.capture_id = captured.id;
frame_pair.capture_time = captured.capture_time;
if (captured.best_decode) {
frame_pair.decode_id = captured.best_decode->id;
frame_pair.decoded = captured.best_decode->frame;
frame_pair.decoded_time = captured.best_decode->decoded_time;
// We can't render frames before they have been decoded.
frame_pair.render_time = std::max(captured.best_decode->render_time,
captured.best_decode->decoded_time);
frame_pair.repeated = captured.best_decode->repeat_count++;
}
for (auto& handler : frame_pair_handlers_)
handler(frame_pair);
}
void VideoFrameMatcher::Finalize() {
for (auto& layer : layers_) {
while (!layer.second.captured_frames.empty()) {
HandleMatch(std::move(layer.second.captured_frames.front()), layer.first);
layer.second.captured_frames.pop_front();
}
}
}
CapturedFrameTap::CapturedFrameTap(Clock* clock, VideoFrameMatcher* matcher)
: clock_(clock), matcher_(matcher) {}
void CapturedFrameTap::OnFrame(const VideoFrame& frame) {
matcher_->OnCapturedFrame(frame, clock_->CurrentTime());
}
void CapturedFrameTap::OnDiscardedFrame() {
discarded_count_++;
}
ForwardingCapturedFrameTap::ForwardingCapturedFrameTap(
Clock* clock,
VideoFrameMatcher* matcher,
rtc::VideoSourceInterface<VideoFrame>* source)
: clock_(clock), matcher_(matcher), source_(source) {}
void ForwardingCapturedFrameTap::OnFrame(const VideoFrame& frame) {
RTC_CHECK(sink_);
matcher_->OnCapturedFrame(frame, clock_->CurrentTime());
sink_->OnFrame(frame);
}
void ForwardingCapturedFrameTap::OnDiscardedFrame() {
RTC_CHECK(sink_);
discarded_count_++;
sink_->OnDiscardedFrame();
}
void ForwardingCapturedFrameTap::AddOrUpdateSink(
VideoSinkInterface<VideoFrame>* sink,
const rtc::VideoSinkWants& wants) {
if (!sink_)
sink_ = sink;
RTC_DCHECK_EQ(sink_, sink);
source_->AddOrUpdateSink(this, wants);
}
void ForwardingCapturedFrameTap::RemoveSink(
VideoSinkInterface<VideoFrame>* sink) {
source_->RemoveSink(this);
sink_ = nullptr;
}
DecodedFrameTap::DecodedFrameTap(Clock* clock,
VideoFrameMatcher* matcher,
int layer_id)
: clock_(clock), matcher_(matcher), layer_id_(layer_id) {
matcher_->RegisterLayer(layer_id_);
}
void DecodedFrameTap::OnFrame(const VideoFrame& frame) {
matcher_->OnDecodedFrame(frame, layer_id_,
Timestamp::ms(frame.render_time_ms()),
clock_->CurrentTime());
}
} // namespace test
} // namespace webrtc