modules/video_coding/generic_encoder_unittest.cc

/*
 *  Copyright (c) 2017 The WebRTC project authors. All Rights Reserved.
 *
 *  Use of this source code is governed by a BSD-style license
 *  that can be found in the LICENSE file in the root of the source
 *  tree. An additional intellectual property rights grant can be found
 *  in the file PATENTS.  All contributing project authors may
 *  be found in the AUTHORS file in the root of the source tree.
 */

#include <cstddef>
#include <vector>

#include "api/video/video_timing.h"
#include "modules/video_coding/generic_encoder.h"
#include "modules/video_coding/include/video_coding_defines.h"
#include "rtc_base/fake_clock.h"
#include "rtc_base/time_utils.h"
#include "test/gtest.h"

namespace webrtc {
namespace test {
namespace {
inline size_t FrameSize(const size_t& min_frame_size,
                        const size_t& max_frame_size,
                        const int& s,
                        const int& i) {
  return min_frame_size + (s + 1) * i % (max_frame_size - min_frame_size);
}

class FakeEncodedImageCallback : public EncodedImageCallback {
 public:
  FakeEncodedImageCallback()
      : last_frame_was_timing_(false),
        num_frames_dropped_(0),
        last_capture_timestamp_(-1) {}
  Result OnEncodedImage(const EncodedImage& encoded_image,
                        const CodecSpecificInfo* codec_specific_info,
                        const RTPFragmentationHeader* fragmentation) override {
    last_frame_was_timing_ =
        encoded_image.timing_.flags != VideoSendTiming::kInvalid &&
        encoded_image.timing_.flags != VideoSendTiming::kNotTriggered;
    last_capture_timestamp_ = encoded_image.capture_time_ms_;
    return Result(Result::OK);
  };

  void OnDroppedFrame(DropReason reason) override { ++num_frames_dropped_; }

  bool WasTimingFrame() { return last_frame_was_timing_; }

  size_t GetNumFramesDropped() { return num_frames_dropped_; }

  int64_t GetLastCaptureTimestamp() { return last_capture_timestamp_; }

 private:
  bool last_frame_was_timing_;
  size_t num_frames_dropped_;
  int64_t last_capture_timestamp_;
};

enum class FrameType {
  kNormal,
  kTiming,
  kDropped,
};

// Emulates |num_frames| on |num_streams| frames with capture timestamps
// increased by 1 from 0. Size of each frame is between
// |min_frame_size| and |max_frame_size|, outliers are counted relatevely to
// |average_frame_sizes[]| for each stream.
std::vector<std::vector<FrameType>> GetTimingFrames(
    const int64_t delay_ms,
    const size_t min_frame_size,
    const size_t max_frame_size,
    std::vector<size_t> average_frame_sizes,
    const int num_streams,
    const int num_frames) {
  FakeEncodedImageCallback sink;
  VCMEncodedFrameCallback callback(&sink);
  const size_t kFramerate = 30;
  callback.SetTimingFramesThresholds(
      {delay_ms, kDefaultOutlierFrameSizePercent});
  callback.OnFrameRateChanged(kFramerate);
  int s, i;
  std::vector<uint8_t> frame_data(max_frame_size);
  std::vector<std::vector<FrameType>> result(num_streams);
  for (s = 0; s < num_streams; ++s)
    callback.OnTargetBitrateChanged(average_frame_sizes[s] * kFramerate, s);
  int64_t current_timestamp = 0;
  for (i = 0; i < num_frames; ++i) {
    current_timestamp += 1;
    for (s = 0; s < num_streams; ++s) {
      // every (5+s)-th frame is dropped on s-th stream by design.
      bool dropped = i % (5 + s) == 0;

      EncodedImage image;
      CodecSpecificInfo codec_specific;
      image.set_buffer(frame_data.data(), frame_data.size());
      image.set_size(FrameSize(min_frame_size, max_frame_size, s, i));
      image.capture_time_ms_ = current_timestamp;
      image.SetTimestamp(static_cast<uint32_t>(current_timestamp * 90));
      image.SetSpatialIndex(s);
      codec_specific.codecType = kVideoCodecGeneric;
      callback.OnEncodeStarted(static_cast<uint32_t>(current_timestamp * 90),
                               current_timestamp, s);
      if (dropped) {
        result[s].push_back(FrameType::kDropped);
        continue;
      }
      callback.OnEncodedImage(image, &codec_specific, nullptr);
      if (sink.WasTimingFrame()) {
        result[s].push_back(FrameType::kTiming);
      } else {
        result[s].push_back(FrameType::kNormal);
      }
    }
  }
  return result;
}
}  // namespace

TEST(TestVCMEncodedFrameCallback, MarksTimingFramesPeriodicallyTogether) {
  const int64_t kDelayMs = 29;
  const size_t kMinFrameSize = 10;
  const size_t kMaxFrameSize = 20;
  const int kNumFrames = 1000;
  const int kNumStreams = 3;
  // No outliers as 1000 is larger than anything from range [10,20].
  const std::vector<size_t> kAverageSize = {1000, 1000, 1000};
  auto frames = GetTimingFrames(kDelayMs, kMinFrameSize, kMaxFrameSize,
                                kAverageSize, kNumStreams, kNumFrames);
  // Timing frames should be tirggered every delayMs.
  // As no outliers are expected, frames on all streams have to be
  // marked together.
  int last_timing_frame = -1;
  for (int i = 0; i < kNumFrames; ++i) {
    int num_normal = 0;
    int num_timing = 0;
    int num_dropped = 0;
    for (int s = 0; s < kNumStreams; ++s) {
      if (frames[s][i] == FrameType::kTiming) {
        ++num_timing;
      } else if (frames[s][i] == FrameType::kNormal) {
        ++num_normal;
      } else {
        ++num_dropped;
      }
    }
    // Can't have both normal and timing frames at the same timstamp.
    EXPECT_TRUE(num_timing == 0 || num_normal == 0);
    if (num_dropped < kNumStreams) {
      if (last_timing_frame == -1 || i >= last_timing_frame + kDelayMs) {
        // If didn't have timing frames for a period, current sent frame has to
        // be one. No normal frames should be sent.
        EXPECT_EQ(num_normal, 0);
      } else {
        // No unneeded timing frames should be sent.
        EXPECT_EQ(num_timing, 0);
      }
    }
    if (num_timing > 0)
      last_timing_frame = i;
  }
}

TEST(TestVCMEncodedFrameCallback, MarksOutliers) {
  const int64_t kDelayMs = 29;
  const size_t kMinFrameSize = 2495;
  const size_t kMaxFrameSize = 2505;
  const int kNumFrames = 1000;
  const int kNumStreams = 3;
  // Possible outliers as 1000 lies in range [995, 1005].
  const std::vector<size_t> kAverageSize = {998, 1000, 1004};
  auto frames = GetTimingFrames(kDelayMs, kMinFrameSize, kMaxFrameSize,
                                kAverageSize, kNumStreams, kNumFrames);
  // All outliers should be marked.
  for (int i = 0; i < kNumFrames; ++i) {
    for (int s = 0; s < kNumStreams; ++s) {
      if (FrameSize(kMinFrameSize, kMaxFrameSize, s, i) >=
          kAverageSize[s] * kDefaultOutlierFrameSizePercent / 100) {
        // Too big frame. May be dropped or timing, but not normal.
        EXPECT_NE(frames[s][i], FrameType::kNormal);
      }
    }
  }
}

TEST(TestVCMEncodedFrameCallback, NoTimingFrameIfNoEncodeStartTime) {
  EncodedImage image;
  CodecSpecificInfo codec_specific;
  int64_t timestamp = 1;
  uint8_t frame_data[500];
  image.set_buffer(frame_data, sizeof(frame_data));
  image.set_size(sizeof(frame_data));
  image.capture_time_ms_ = timestamp;
  image.SetTimestamp(static_cast<uint32_t>(timestamp * 90));
  codec_specific.codecType = kVideoCodecGeneric;
  FakeEncodedImageCallback sink;
  VCMEncodedFrameCallback callback(&sink);
  VideoCodec::TimingFrameTriggerThresholds thresholds;
  thresholds.delay_ms = 1;  // Make all frames timing frames.
  callback.SetTimingFramesThresholds(thresholds);
  callback.OnTargetBitrateChanged(500, 0);

  // Verify a single frame works with encode start time set.
  callback.OnEncodeStarted(static_cast<uint32_t>(timestamp * 90), timestamp, 0);
  callback.OnEncodedImage(image, &codec_specific, nullptr);
  EXPECT_TRUE(sink.WasTimingFrame());

  // New frame, now skip OnEncodeStarted. Should not result in timing frame.
  image.capture_time_ms_ = ++timestamp;
  image.SetTimestamp(static_cast<uint32_t>(timestamp * 90));
  callback.OnEncodedImage(image, &codec_specific, nullptr);
  EXPECT_FALSE(sink.WasTimingFrame());
}

TEST(TestVCMEncodedFrameCallback, AdjustsCaptureTimeForInternalSourceEncoder) {
  rtc::ScopedFakeClock clock;
  clock.SetTimeMicros(1234567);
  EncodedImage image;
  CodecSpecificInfo codec_specific;
  const int64_t kEncodeStartDelayMs = 2;
  const int64_t kEncodeFinishDelayMs = 10;
  int64_t timestamp = 1;
  uint8_t frame_data[500];
  image.set_buffer(frame_data, sizeof(frame_data));
  image.set_size(sizeof(frame_data));
  image.capture_time_ms_ = timestamp;
  image.SetTimestamp(static_cast<uint32_t>(timestamp * 90));
  codec_specific.codecType = kVideoCodecGeneric;
  FakeEncodedImageCallback sink;
  VCMEncodedFrameCallback callback(&sink);
  callback.SetInternalSource(true);
  VideoCodec::TimingFrameTriggerThresholds thresholds;
  thresholds.delay_ms = 1;  // Make all frames timing frames.
  callback.SetTimingFramesThresholds(thresholds);
  callback.OnTargetBitrateChanged(500, 0);

  // Verify a single frame without encode timestamps isn't a timing frame.
  callback.OnEncodedImage(image, &codec_specific, nullptr);
  EXPECT_FALSE(sink.WasTimingFrame());

  // New frame, but this time with encode timestamps set in timing_.
  // This should be a timing frame.
  image.capture_time_ms_ = ++timestamp;
  image.SetTimestamp(static_cast<uint32_t>(timestamp * 90));
  image.timing_.encode_start_ms = timestamp + kEncodeStartDelayMs;
  image.timing_.encode_finish_ms = timestamp + kEncodeFinishDelayMs;
  callback.OnEncodedImage(image, &codec_specific, nullptr);
  EXPECT_TRUE(sink.WasTimingFrame());
  // Frame is captured kEncodeFinishDelayMs before it's encoded, so restored
  // capture timestamp should be kEncodeFinishDelayMs in the past.
  EXPECT_EQ(
      sink.GetLastCaptureTimestamp(),
      clock.TimeNanos() / rtc::kNumNanosecsPerMillisec - kEncodeFinishDelayMs);
}

TEST(TestVCMEncodedFrameCallback, NotifiesAboutDroppedFrames) {
  EncodedImage image;
  CodecSpecificInfo codec_specific;
  const int64_t kTimestampMs1 = 47721840;
  const int64_t kTimestampMs2 = 47721850;
  const int64_t kTimestampMs3 = 47721860;
  const int64_t kTimestampMs4 = 47721870;
  codec_specific.codecType = kVideoCodecGeneric;
  FakeEncodedImageCallback sink;
  VCMEncodedFrameCallback callback(&sink);
  // Any non-zero bitrate needed to be set before the first frame.
  callback.OnTargetBitrateChanged(500, 0);
  image.capture_time_ms_ = kTimestampMs1;
  image.SetTimestamp(static_cast<uint32_t>(image.capture_time_ms_ * 90));
  callback.OnEncodeStarted(image.Timestamp(), image.capture_time_ms_, 0);
  EXPECT_EQ(0u, sink.GetNumFramesDropped());
  callback.OnEncodedImage(image, &codec_specific, nullptr);

  image.capture_time_ms_ = kTimestampMs2;
  image.SetTimestamp(static_cast<uint32_t>(image.capture_time_ms_ * 90));
  callback.OnEncodeStarted(image.Timestamp(), image.capture_time_ms_, 0);
  // No OnEncodedImageCall for timestamp2. Yet, at this moment it's not known
  // that frame with timestamp2 was dropped.
  EXPECT_EQ(0u, sink.GetNumFramesDropped());

  image.capture_time_ms_ = kTimestampMs3;
  image.SetTimestamp(static_cast<uint32_t>(image.capture_time_ms_ * 90));
  callback.OnEncodeStarted(image.Timestamp(), image.capture_time_ms_, 0);
  callback.OnEncodedImage(image, &codec_specific, nullptr);
  EXPECT_EQ(1u, sink.GetNumFramesDropped());

  image.capture_time_ms_ = kTimestampMs4;
  image.SetTimestamp(static_cast<uint32_t>(image.capture_time_ms_ * 90));
  callback.OnEncodeStarted(image.Timestamp(), image.capture_time_ms_, 0);
  callback.OnEncodedImage(image, &codec_specific, nullptr);
  EXPECT_EQ(1u, sink.GetNumFramesDropped());
}

TEST(TestVCMEncodedFrameCallback, RestoresCaptureTimestamps) {
  EncodedImage image;
  CodecSpecificInfo codec_specific;
  const int64_t kTimestampMs = 123456;
  codec_specific.codecType = kVideoCodecGeneric;
  FakeEncodedImageCallback sink;
  VCMEncodedFrameCallback callback(&sink);
  // Any non-zero bitrate needed to be set before the first frame.
  callback.OnTargetBitrateChanged(500, 0);
  image.capture_time_ms_ = kTimestampMs;  // Incorrect timesetamp.
  image.SetTimestamp(static_cast<uint32_t>(image.capture_time_ms_ * 90));
  callback.OnEncodeStarted(image.Timestamp(), image.capture_time_ms_, 0);
  image.capture_time_ms_ = 0;  // Incorrect timesetamp.
  callback.OnEncodedImage(image, &codec_specific, nullptr);
  EXPECT_EQ(kTimestampMs, sink.GetLastCaptureTimestamp());
}

}  // namespace test
}  // namespace webrtc