Danil Chapovalov | e7b752b | 2021-04-21 12:35:54 | [diff] [blame] | 1 | /* |
| 2 | * Copyright (c) 2021 The WebRTC project authors. All Rights Reserved. |
| 3 | * |
| 4 | * Use of this source code is governed by a BSD-style license |
| 5 | * that can be found in the LICENSE file in the root of the source |
| 6 | * tree. An additional intellectual property rights grant can be found |
| 7 | * in the file PATENTS. All contributing project authors may |
| 8 | * be found in the AUTHORS file in the root of the source tree. |
| 9 | */ |
| 10 | |
| 11 | #include <stdint.h> |
| 12 | |
| 13 | #include "absl/algorithm/container.h" |
| 14 | #include "absl/base/macros.h" |
| 15 | #include "absl/container/inlined_vector.h" |
| 16 | #include "api/array_view.h" |
Jonas Oreland | e62c2f2 | 2022-03-29 09:04:48 | [diff] [blame] | 17 | #include "api/field_trials_view.h" |
Danil Chapovalov | e7b752b | 2021-04-21 12:35:54 | [diff] [blame] | 18 | #include "api/video/video_frame.h" |
| 19 | #include "api/video_codecs/video_codec.h" |
| 20 | #include "api/video_codecs/video_encoder.h" |
Florent Castelli | 816f5b1 | 2023-05-31 12:39:03 | [diff] [blame] | 21 | #include "media/base/media_constants.h" |
Danil Chapovalov | bf607e2 | 2022-07-27 14:50:21 | [diff] [blame] | 22 | #include "modules/video_coding/codecs/interface/libvpx_interface.h" |
Danil Chapovalov | e7b752b | 2021-04-21 12:35:54 | [diff] [blame] | 23 | #include "modules/video_coding/codecs/vp9/libvpx_vp9_encoder.h" |
| 24 | #include "modules/video_coding/frame_dependencies_calculator.h" |
| 25 | #include "rtc_base/numerics/safe_compare.h" |
| 26 | #include "test/fuzzers/fuzz_data_helper.h" |
Danil Chapovalov | e7b752b | 2021-04-21 12:35:54 | [diff] [blame] | 27 | |
| 28 | // Fuzzer simulates various svc configurations and libvpx encoder dropping |
| 29 | // layer frames. |
| 30 | // Validates vp9 encoder wrapper produces consistent frame references. |
| 31 | namespace webrtc { |
| 32 | namespace { |
| 33 | |
| 34 | using test::FuzzDataHelper; |
Danil Chapovalov | e7b752b | 2021-04-21 12:35:54 | [diff] [blame] | 35 | |
Emil Lundmark | 9103e8e | 2021-09-10 10:32:53 | [diff] [blame] | 36 | constexpr int kBitrateEnabledBps = 100'000; |
| 37 | |
Danil Chapovalov | e7b752b | 2021-04-21 12:35:54 | [diff] [blame] | 38 | class FrameValidator : public EncodedImageCallback { |
| 39 | public: |
| 40 | ~FrameValidator() override = default; |
| 41 | |
| 42 | Result OnEncodedImage(const EncodedImage& encoded_image, |
| 43 | const CodecSpecificInfo* codec_specific_info) override { |
| 44 | RTC_CHECK(codec_specific_info); |
| 45 | RTC_CHECK_EQ(codec_specific_info->codecType, kVideoCodecVP9); |
| 46 | if (codec_specific_info->codecSpecific.VP9.first_frame_in_picture) { |
| 47 | ++picture_id_; |
| 48 | } |
Danil Chapovalov | 1858995 | 2021-05-12 18:58:02 | [diff] [blame] | 49 | int64_t frame_id = frame_id_++; |
| 50 | LayerFrame& layer_frame = frames_[frame_id % kMaxFrameHistorySize]; |
Danil Chapovalov | e7b752b | 2021-04-21 12:35:54 | [diff] [blame] | 51 | layer_frame.picture_id = picture_id_; |
| 52 | layer_frame.spatial_id = encoded_image.SpatialIndex().value_or(0); |
Danil Chapovalov | 1858995 | 2021-05-12 18:58:02 | [diff] [blame] | 53 | layer_frame.frame_id = frame_id; |
Danil Chapovalov | 8d3396d | 2021-06-11 13:43:20 | [diff] [blame] | 54 | layer_frame.temporal_id = |
| 55 | codec_specific_info->codecSpecific.VP9.temporal_idx; |
| 56 | if (layer_frame.temporal_id == kNoTemporalIdx) { |
| 57 | layer_frame.temporal_id = 0; |
| 58 | } |
| 59 | layer_frame.vp9_non_ref_for_inter_layer_pred = |
| 60 | codec_specific_info->codecSpecific.VP9.non_ref_for_inter_layer_pred; |
| 61 | CheckVp9References(layer_frame, codec_specific_info->codecSpecific.VP9); |
Danil Chapovalov | e7b752b | 2021-04-21 12:35:54 | [diff] [blame] | 62 | |
Danil Chapovalov | 8d3396d | 2021-06-11 13:43:20 | [diff] [blame] | 63 | if (codec_specific_info->generic_frame_info.has_value()) { |
| 64 | absl::InlinedVector<int64_t, 5> frame_dependencies = |
Danil Chapovalov | e7b752b | 2021-04-21 12:35:54 | [diff] [blame] | 65 | dependencies_calculator_.FromBuffersUsage( |
Danil Chapovalov | 8d3396d | 2021-06-11 13:43:20 | [diff] [blame] | 66 | frame_id, |
| 67 | codec_specific_info->generic_frame_info->encoder_buffers); |
Danil Chapovalov | e7b752b | 2021-04-21 12:35:54 | [diff] [blame] | 68 | |
Danil Chapovalov | 8d3396d | 2021-06-11 13:43:20 | [diff] [blame] | 69 | CheckGenericReferences(frame_dependencies, |
| 70 | *codec_specific_info->generic_frame_info); |
| 71 | CheckGenericAndCodecSpecificReferencesAreConsistent( |
| 72 | frame_dependencies, *codec_specific_info, layer_frame); |
Danil Chapovalov | e7b752b | 2021-04-21 12:35:54 | [diff] [blame] | 73 | } |
| 74 | |
Danil Chapovalov | e7b752b | 2021-04-21 12:35:54 | [diff] [blame] | 75 | return Result(Result::OK); |
| 76 | } |
| 77 | |
| 78 | private: |
Emil Lundmark | 9103e8e | 2021-09-10 10:32:53 | [diff] [blame] | 79 | // With 4 spatial layers and patterns up to 8 pictures, it should be enough to |
| 80 | // keep the last 32 frames to validate dependencies. |
Danil Chapovalov | 1858995 | 2021-05-12 18:58:02 | [diff] [blame] | 81 | static constexpr size_t kMaxFrameHistorySize = 32; |
Danil Chapovalov | e7b752b | 2021-04-21 12:35:54 | [diff] [blame] | 82 | struct LayerFrame { |
Danil Chapovalov | 1858995 | 2021-05-12 18:58:02 | [diff] [blame] | 83 | int64_t frame_id; |
Danil Chapovalov | e7b752b | 2021-04-21 12:35:54 | [diff] [blame] | 84 | int64_t picture_id; |
| 85 | int spatial_id; |
Danil Chapovalov | 8d3396d | 2021-06-11 13:43:20 | [diff] [blame] | 86 | int temporal_id; |
| 87 | bool vp9_non_ref_for_inter_layer_pred; |
Danil Chapovalov | e7b752b | 2021-04-21 12:35:54 | [diff] [blame] | 88 | }; |
| 89 | |
Danil Chapovalov | 8d3396d | 2021-06-11 13:43:20 | [diff] [blame] | 90 | void CheckVp9References(const LayerFrame& layer_frame, |
| 91 | const CodecSpecificInfoVP9& vp9_info) { |
Danil Chapovalov | 1858995 | 2021-05-12 18:58:02 | [diff] [blame] | 92 | if (layer_frame.frame_id == 0) { |
Danil Chapovalov | 8d3396d | 2021-06-11 13:43:20 | [diff] [blame] | 93 | RTC_CHECK(!vp9_info.inter_layer_predicted); |
Danil Chapovalov | 1858995 | 2021-05-12 18:58:02 | [diff] [blame] | 94 | } else { |
| 95 | const LayerFrame& previous_frame = Frame(layer_frame.frame_id - 1); |
Danil Chapovalov | 8d3396d | 2021-06-11 13:43:20 | [diff] [blame] | 96 | if (vp9_info.inter_layer_predicted) { |
| 97 | RTC_CHECK(!previous_frame.vp9_non_ref_for_inter_layer_pred); |
Danil Chapovalov | 1858995 | 2021-05-12 18:58:02 | [diff] [blame] | 98 | RTC_CHECK_EQ(layer_frame.picture_id, previous_frame.picture_id); |
| 99 | } |
| 100 | if (previous_frame.picture_id == layer_frame.picture_id) { |
| 101 | RTC_CHECK_GT(layer_frame.spatial_id, previous_frame.spatial_id); |
| 102 | // The check below would fail for temporal shift structures. Remove it |
| 103 | // or move it to !flexible_mode section when vp9 encoder starts |
| 104 | // supporting such structures. |
Danil Chapovalov | 8d3396d | 2021-06-11 13:43:20 | [diff] [blame] | 105 | RTC_CHECK_EQ(layer_frame.temporal_id, previous_frame.temporal_id); |
Danil Chapovalov | 1858995 | 2021-05-12 18:58:02 | [diff] [blame] | 106 | } |
Danil Chapovalov | e7b752b | 2021-04-21 12:35:54 | [diff] [blame] | 107 | } |
Danil Chapovalov | 8d3396d | 2021-06-11 13:43:20 | [diff] [blame] | 108 | if (!vp9_info.flexible_mode) { |
| 109 | if (vp9_info.gof.num_frames_in_gof > 0) { |
| 110 | gof_.CopyGofInfoVP9(vp9_info.gof); |
Danil Chapovalov | e7b752b | 2021-04-21 12:35:54 | [diff] [blame] | 111 | } |
Danil Chapovalov | 8d3396d | 2021-06-11 13:43:20 | [diff] [blame] | 112 | RTC_CHECK_EQ(gof_.temporal_idx[vp9_info.gof_idx], |
| 113 | layer_frame.temporal_id); |
Danil Chapovalov | e7b752b | 2021-04-21 12:35:54 | [diff] [blame] | 114 | } |
| 115 | } |
| 116 | |
Danil Chapovalov | 8d3396d | 2021-06-11 13:43:20 | [diff] [blame] | 117 | void CheckGenericReferences(rtc::ArrayView<const int64_t> frame_dependencies, |
| 118 | const GenericFrameInfo& generic_info) const { |
| 119 | for (int64_t dependency_frame_id : frame_dependencies) { |
Danil Chapovalov | e7b752b | 2021-04-21 12:35:54 | [diff] [blame] | 120 | RTC_CHECK_GE(dependency_frame_id, 0); |
Danil Chapovalov | 1858995 | 2021-05-12 18:58:02 | [diff] [blame] | 121 | const LayerFrame& dependency = Frame(dependency_frame_id); |
Danil Chapovalov | 8d3396d | 2021-06-11 13:43:20 | [diff] [blame] | 122 | RTC_CHECK_GE(generic_info.spatial_id, dependency.spatial_id); |
| 123 | RTC_CHECK_GE(generic_info.temporal_id, dependency.temporal_id); |
Danil Chapovalov | e7b752b | 2021-04-21 12:35:54 | [diff] [blame] | 124 | } |
| 125 | } |
| 126 | |
| 127 | void CheckGenericAndCodecSpecificReferencesAreConsistent( |
Danil Chapovalov | 8d3396d | 2021-06-11 13:43:20 | [diff] [blame] | 128 | rtc::ArrayView<const int64_t> frame_dependencies, |
| 129 | const CodecSpecificInfo& info, |
Danil Chapovalov | e7b752b | 2021-04-21 12:35:54 | [diff] [blame] | 130 | const LayerFrame& layer_frame) const { |
Danil Chapovalov | 8d3396d | 2021-06-11 13:43:20 | [diff] [blame] | 131 | const CodecSpecificInfoVP9& vp9_info = info.codecSpecific.VP9; |
| 132 | const GenericFrameInfo& generic_info = *info.generic_frame_info; |
| 133 | |
Danil Chapovalov | e7b752b | 2021-04-21 12:35:54 | [diff] [blame] | 134 | RTC_CHECK_EQ(generic_info.spatial_id, layer_frame.spatial_id); |
Danil Chapovalov | 8d3396d | 2021-06-11 13:43:20 | [diff] [blame] | 135 | RTC_CHECK_EQ(generic_info.temporal_id, layer_frame.temporal_id); |
| 136 | auto picture_id_diffs = |
| 137 | rtc::MakeArrayView(vp9_info.p_diff, vp9_info.num_ref_pics); |
| 138 | RTC_CHECK_EQ( |
| 139 | frame_dependencies.size(), |
| 140 | picture_id_diffs.size() + (vp9_info.inter_layer_predicted ? 1 : 0)); |
| 141 | for (int64_t dependency_frame_id : frame_dependencies) { |
Danil Chapovalov | e7b752b | 2021-04-21 12:35:54 | [diff] [blame] | 142 | RTC_CHECK_GE(dependency_frame_id, 0); |
Danil Chapovalov | 1858995 | 2021-05-12 18:58:02 | [diff] [blame] | 143 | const LayerFrame& dependency = Frame(dependency_frame_id); |
Danil Chapovalov | e7b752b | 2021-04-21 12:35:54 | [diff] [blame] | 144 | if (dependency.spatial_id != layer_frame.spatial_id) { |
Danil Chapovalov | 8d3396d | 2021-06-11 13:43:20 | [diff] [blame] | 145 | RTC_CHECK(vp9_info.inter_layer_predicted); |
Danil Chapovalov | e7b752b | 2021-04-21 12:35:54 | [diff] [blame] | 146 | RTC_CHECK_EQ(layer_frame.picture_id, dependency.picture_id); |
| 147 | RTC_CHECK_GT(layer_frame.spatial_id, dependency.spatial_id); |
| 148 | } else { |
Danil Chapovalov | 8d3396d | 2021-06-11 13:43:20 | [diff] [blame] | 149 | RTC_CHECK(vp9_info.inter_pic_predicted); |
Danil Chapovalov | e7b752b | 2021-04-21 12:35:54 | [diff] [blame] | 150 | RTC_CHECK_EQ(layer_frame.spatial_id, dependency.spatial_id); |
| 151 | RTC_CHECK(absl::c_linear_search( |
| 152 | picture_id_diffs, layer_frame.picture_id - dependency.picture_id)); |
| 153 | } |
| 154 | } |
| 155 | } |
| 156 | |
Danil Chapovalov | 1858995 | 2021-05-12 18:58:02 | [diff] [blame] | 157 | const LayerFrame& Frame(int64_t frame_id) const { |
| 158 | auto& frame = frames_[frame_id % kMaxFrameHistorySize]; |
| 159 | RTC_CHECK_EQ(frame.frame_id, frame_id); |
| 160 | return frame; |
| 161 | } |
| 162 | |
Danil Chapovalov | e7b752b | 2021-04-21 12:35:54 | [diff] [blame] | 163 | GofInfoVP9 gof_; |
Danil Chapovalov | 1858995 | 2021-05-12 18:58:02 | [diff] [blame] | 164 | int64_t frame_id_ = 0; |
Danil Chapovalov | e8080f4 | 2021-04-22 09:43:27 | [diff] [blame] | 165 | int64_t picture_id_ = 1; |
Danil Chapovalov | e7b752b | 2021-04-21 12:35:54 | [diff] [blame] | 166 | FrameDependenciesCalculator dependencies_calculator_; |
Danil Chapovalov | 1858995 | 2021-05-12 18:58:02 | [diff] [blame] | 167 | LayerFrame frames_[kMaxFrameHistorySize]; |
Danil Chapovalov | e7b752b | 2021-04-21 12:35:54 | [diff] [blame] | 168 | }; |
| 169 | |
Jonas Oreland | e62c2f2 | 2022-03-29 09:04:48 | [diff] [blame] | 170 | class FieldTrials : public FieldTrialsView { |
Danil Chapovalov | e7b752b | 2021-04-21 12:35:54 | [diff] [blame] | 171 | public: |
| 172 | explicit FieldTrials(FuzzDataHelper& config) |
| 173 | : flags_(config.ReadOrDefaultValue<uint8_t>(0)) {} |
| 174 | |
| 175 | ~FieldTrials() override = default; |
| 176 | std::string Lookup(absl::string_view key) const override { |
| 177 | static constexpr absl::string_view kBinaryFieldTrials[] = { |
Danil Chapovalov | e7b752b | 2021-04-21 12:35:54 | [diff] [blame] | 178 | "WebRTC-Vp9ExternalRefCtrl", |
| 179 | "WebRTC-Vp9IssueKeyFrameOnLayerDeactivation", |
| 180 | }; |
| 181 | for (size_t i = 0; i < ABSL_ARRAYSIZE(kBinaryFieldTrials); ++i) { |
| 182 | if (key == kBinaryFieldTrials[i]) { |
| 183 | return (flags_ & (1u << i)) ? "Enabled" : "Disabled"; |
| 184 | } |
| 185 | } |
| 186 | |
| 187 | // Ignore following field trials. |
| 188 | if (key == "WebRTC-CongestionWindow" || |
| 189 | key == "WebRTC-UseBaseHeavyVP8TL3RateAllocation" || |
Danil Chapovalov | e7b752b | 2021-04-21 12:35:54 | [diff] [blame] | 190 | key == "WebRTC-VideoRateControl" || |
| 191 | key == "WebRTC-VP9-PerformanceFlags" || |
| 192 | key == "WebRTC-VP9VariableFramerateScreenshare" || |
| 193 | key == "WebRTC-VP9QualityScaler") { |
| 194 | return ""; |
| 195 | } |
Sergey Silkin | efea7bb | 2024-02-21 18:13:35 | [diff] [blame] | 196 | |
| 197 | // TODO: bugs.webrtc.org/15827 - Fuzz frame drop config. |
| 198 | if (key == "WebRTC-LibvpxVp9Encoder-SvcFrameDropConfig") { |
| 199 | return ""; |
| 200 | } |
| 201 | |
Danil Chapovalov | e7b752b | 2021-04-21 12:35:54 | [diff] [blame] | 202 | // Crash when using unexpected field trial to decide if it should be fuzzed |
| 203 | // or have a constant value. |
| 204 | RTC_CHECK(false) << "Unfuzzed field trial " << key << "\n"; |
| 205 | } |
| 206 | |
| 207 | private: |
| 208 | const uint8_t flags_; |
| 209 | }; |
| 210 | |
| 211 | VideoCodec CodecSettings(FuzzDataHelper& rng) { |
| 212 | uint16_t config = rng.ReadOrDefaultValue<uint16_t>(0); |
| 213 | // Test up to to 4 spatial and 4 temporal layers. |
| 214 | int num_spatial_layers = 1 + (config & 0b11); |
| 215 | int num_temporal_layers = 1 + ((config >> 2) & 0b11); |
| 216 | |
| 217 | VideoCodec codec_settings = {}; |
| 218 | codec_settings.codecType = kVideoCodecVP9; |
| 219 | codec_settings.maxFramerate = 30; |
| 220 | codec_settings.width = 320 << (num_spatial_layers - 1); |
| 221 | codec_settings.height = 180 << (num_spatial_layers - 1); |
| 222 | if (num_spatial_layers > 1) { |
| 223 | for (int sid = 0; sid < num_spatial_layers; ++sid) { |
| 224 | SpatialLayer& spatial_layer = codec_settings.spatialLayers[sid]; |
| 225 | codec_settings.width = 320 << sid; |
| 226 | codec_settings.height = 180 << sid; |
Emil Lundmark | 9103e8e | 2021-09-10 10:32:53 | [diff] [blame] | 227 | spatial_layer.width = codec_settings.width; |
| 228 | spatial_layer.height = codec_settings.height; |
| 229 | spatial_layer.targetBitrate = kBitrateEnabledBps * num_temporal_layers; |
Danil Chapovalov | e7b752b | 2021-04-21 12:35:54 | [diff] [blame] | 230 | spatial_layer.maxFramerate = codec_settings.maxFramerate; |
| 231 | spatial_layer.numberOfTemporalLayers = num_temporal_layers; |
| 232 | } |
| 233 | } |
| 234 | codec_settings.VP9()->numberOfSpatialLayers = num_spatial_layers; |
| 235 | codec_settings.VP9()->numberOfTemporalLayers = num_temporal_layers; |
| 236 | int inter_layer_pred = (config >> 4) & 0b11; |
| 237 | // There are only 3 valid values. |
| 238 | codec_settings.VP9()->interLayerPred = static_cast<InterLayerPredMode>( |
| 239 | inter_layer_pred < 3 ? inter_layer_pred : 0); |
| 240 | codec_settings.VP9()->flexibleMode = (config & (1u << 6)) != 0; |
Niels Möller | 807328f | 2022-05-12 14:16:39 | [diff] [blame] | 241 | codec_settings.SetFrameDropEnabled((config & (1u << 7)) != 0); |
Danil Chapovalov | e7b752b | 2021-04-21 12:35:54 | [diff] [blame] | 242 | codec_settings.mode = VideoCodecMode::kRealtimeVideo; |
| 243 | return codec_settings; |
| 244 | } |
| 245 | |
| 246 | VideoEncoder::Settings EncoderSettings() { |
| 247 | return VideoEncoder::Settings(VideoEncoder::Capabilities(false), |
| 248 | /*number_of_cores=*/1, |
| 249 | /*max_payload_size=*/0); |
| 250 | } |
| 251 | |
Danil Chapovalov | caef2b3 | 2021-09-22 15:19:46 | [diff] [blame] | 252 | bool IsSupported(int num_spatial_layers, |
| 253 | int num_temporal_layers, |
| 254 | const VideoBitrateAllocation& allocation) { |
| 255 | // VP9 encoder doesn't support certain configurations. |
| 256 | // BitrateAllocator shouldn't produce them. |
| 257 | if (allocation.get_sum_bps() == 0) { |
| 258 | // Ignore allocation that turns off all the layers. |
| 259 | // In such a case it is up to upper layer code not to call Encode. |
| 260 | return false; |
| 261 | } |
| 262 | |
| 263 | for (int tid = 0; tid < num_temporal_layers; ++tid) { |
| 264 | int min_enabled_spatial_id = -1; |
| 265 | int max_enabled_spatial_id = -1; |
| 266 | int num_enabled_spatial_layers = 0; |
| 267 | for (int sid = 0; sid < num_spatial_layers; ++sid) { |
| 268 | if (allocation.GetBitrate(sid, tid) > 0) { |
| 269 | if (min_enabled_spatial_id == -1) { |
| 270 | min_enabled_spatial_id = sid; |
| 271 | } |
| 272 | max_enabled_spatial_id = sid; |
| 273 | ++num_enabled_spatial_layers; |
| 274 | } |
| 275 | } |
| 276 | if (num_enabled_spatial_layers == 0) { |
| 277 | // Each temporal layer should be enabled because skipping a full frame is |
| 278 | // not supported in non-flexible mode. |
| 279 | return false; |
| 280 | } |
| 281 | if (max_enabled_spatial_id - min_enabled_spatial_id + 1 != |
| 282 | num_enabled_spatial_layers) { |
| 283 | // To avoid odd spatial dependencies, there should be no gaps in active |
| 284 | // spatial layers. |
| 285 | return false; |
| 286 | } |
| 287 | } |
| 288 | |
| 289 | return true; |
| 290 | } |
| 291 | |
Danil Chapovalov | e7b752b | 2021-04-21 12:35:54 | [diff] [blame] | 292 | struct LibvpxState { |
| 293 | LibvpxState() { |
| 294 | pkt.kind = VPX_CODEC_CX_FRAME_PKT; |
| 295 | pkt.data.frame.buf = pkt_buffer; |
| 296 | pkt.data.frame.sz = ABSL_ARRAYSIZE(pkt_buffer); |
| 297 | layer_id.spatial_layer_id = -1; |
| 298 | } |
| 299 | |
| 300 | uint8_t pkt_buffer[1000] = {}; |
| 301 | vpx_codec_enc_cfg_t config = {}; |
| 302 | vpx_codec_priv_output_cx_pkt_cb_pair_t callback = {}; |
| 303 | vpx_image_t img = {}; |
| 304 | vpx_svc_ref_frame_config_t ref_config = {}; |
| 305 | vpx_svc_layer_id_t layer_id = {}; |
| 306 | vpx_svc_frame_drop_t frame_drop = {}; |
| 307 | vpx_codec_cx_pkt pkt = {}; |
| 308 | }; |
| 309 | |
Danil Chapovalov | bf607e2 | 2022-07-27 14:50:21 | [diff] [blame] | 310 | class StubLibvpx : public LibvpxInterface { |
Danil Chapovalov | e7b752b | 2021-04-21 12:35:54 | [diff] [blame] | 311 | public: |
| 312 | explicit StubLibvpx(LibvpxState* state) : state_(state) { RTC_CHECK(state_); } |
| 313 | |
| 314 | vpx_codec_err_t codec_enc_config_default(vpx_codec_iface_t* iface, |
| 315 | vpx_codec_enc_cfg_t* cfg, |
| 316 | unsigned int usage) const override { |
| 317 | state_->config = *cfg; |
| 318 | return VPX_CODEC_OK; |
| 319 | } |
| 320 | |
| 321 | vpx_codec_err_t codec_enc_init(vpx_codec_ctx_t* ctx, |
| 322 | vpx_codec_iface_t* iface, |
| 323 | const vpx_codec_enc_cfg_t* cfg, |
| 324 | vpx_codec_flags_t flags) const override { |
| 325 | RTC_CHECK(ctx); |
| 326 | ctx->err = VPX_CODEC_OK; |
| 327 | return VPX_CODEC_OK; |
| 328 | } |
| 329 | |
| 330 | vpx_image_t* img_wrap(vpx_image_t* img, |
| 331 | vpx_img_fmt_t fmt, |
| 332 | unsigned int d_w, |
| 333 | unsigned int d_h, |
| 334 | unsigned int stride_align, |
| 335 | unsigned char* img_data) const override { |
| 336 | state_->img.fmt = fmt; |
| 337 | state_->img.d_w = d_w; |
| 338 | state_->img.d_h = d_h; |
| 339 | return &state_->img; |
| 340 | } |
| 341 | |
| 342 | vpx_codec_err_t codec_encode(vpx_codec_ctx_t* ctx, |
| 343 | const vpx_image_t* img, |
| 344 | vpx_codec_pts_t pts, |
| 345 | uint64_t duration, |
| 346 | vpx_enc_frame_flags_t flags, |
| 347 | uint64_t deadline) const override { |
| 348 | if (flags & VPX_EFLAG_FORCE_KF) { |
| 349 | state_->pkt.data.frame.flags = VPX_FRAME_IS_KEY; |
| 350 | } else { |
| 351 | state_->pkt.data.frame.flags = 0; |
| 352 | } |
| 353 | state_->pkt.data.frame.duration = duration; |
| 354 | return VPX_CODEC_OK; |
| 355 | } |
| 356 | |
| 357 | vpx_codec_err_t codec_control(vpx_codec_ctx_t* ctx, |
| 358 | vp8e_enc_control_id ctrl_id, |
| 359 | void* param) const override { |
| 360 | if (ctrl_id == VP9E_REGISTER_CX_CALLBACK) { |
| 361 | state_->callback = |
| 362 | *reinterpret_cast<vpx_codec_priv_output_cx_pkt_cb_pair_t*>(param); |
| 363 | } |
| 364 | return VPX_CODEC_OK; |
| 365 | } |
| 366 | |
| 367 | vpx_codec_err_t codec_control( |
| 368 | vpx_codec_ctx_t* ctx, |
| 369 | vp8e_enc_control_id ctrl_id, |
| 370 | vpx_svc_ref_frame_config_t* param) const override { |
| 371 | switch (ctrl_id) { |
| 372 | case VP9E_SET_SVC_REF_FRAME_CONFIG: |
| 373 | state_->ref_config = *param; |
| 374 | break; |
| 375 | case VP9E_GET_SVC_REF_FRAME_CONFIG: |
| 376 | *param = state_->ref_config; |
| 377 | break; |
| 378 | default: |
| 379 | break; |
| 380 | } |
| 381 | return VPX_CODEC_OK; |
| 382 | } |
| 383 | |
| 384 | vpx_codec_err_t codec_control(vpx_codec_ctx_t* ctx, |
| 385 | vp8e_enc_control_id ctrl_id, |
| 386 | vpx_svc_layer_id_t* param) const override { |
| 387 | switch (ctrl_id) { |
| 388 | case VP9E_SET_SVC_LAYER_ID: |
| 389 | state_->layer_id = *param; |
| 390 | break; |
| 391 | case VP9E_GET_SVC_LAYER_ID: |
| 392 | *param = state_->layer_id; |
| 393 | break; |
| 394 | default: |
| 395 | break; |
| 396 | } |
| 397 | return VPX_CODEC_OK; |
| 398 | } |
| 399 | |
| 400 | vpx_codec_err_t codec_control(vpx_codec_ctx_t* ctx, |
| 401 | vp8e_enc_control_id ctrl_id, |
| 402 | vpx_svc_frame_drop_t* param) const override { |
| 403 | if (ctrl_id == VP9E_SET_SVC_FRAME_DROP_LAYER) { |
| 404 | state_->frame_drop = *param; |
| 405 | } |
| 406 | return VPX_CODEC_OK; |
| 407 | } |
| 408 | |
| 409 | vpx_codec_err_t codec_enc_config_set( |
| 410 | vpx_codec_ctx_t* ctx, |
| 411 | const vpx_codec_enc_cfg_t* cfg) const override { |
| 412 | state_->config = *cfg; |
| 413 | return VPX_CODEC_OK; |
| 414 | } |
| 415 | |
Danil Chapovalov | bf607e2 | 2022-07-27 14:50:21 | [diff] [blame] | 416 | vpx_image_t* img_alloc(vpx_image_t* img, |
| 417 | vpx_img_fmt_t fmt, |
| 418 | unsigned int d_w, |
| 419 | unsigned int d_h, |
| 420 | unsigned int align) const override { |
| 421 | return nullptr; |
| 422 | } |
| 423 | void img_free(vpx_image_t* img) const override {} |
| 424 | vpx_codec_err_t codec_enc_init_multi(vpx_codec_ctx_t* ctx, |
| 425 | vpx_codec_iface_t* iface, |
| 426 | vpx_codec_enc_cfg_t* cfg, |
| 427 | int num_enc, |
| 428 | vpx_codec_flags_t flags, |
| 429 | vpx_rational_t* dsf) const override { |
| 430 | return VPX_CODEC_OK; |
| 431 | } |
| 432 | vpx_codec_err_t codec_destroy(vpx_codec_ctx_t* ctx) const override { |
| 433 | return VPX_CODEC_OK; |
| 434 | } |
| 435 | vpx_codec_err_t codec_control(vpx_codec_ctx_t* ctx, |
| 436 | vp8e_enc_control_id ctrl_id, |
| 437 | uint32_t param) const override { |
| 438 | return VPX_CODEC_OK; |
| 439 | } |
| 440 | vpx_codec_err_t codec_control(vpx_codec_ctx_t* ctx, |
| 441 | vp8e_enc_control_id ctrl_id, |
| 442 | int param) const override { |
| 443 | return VPX_CODEC_OK; |
| 444 | } |
| 445 | vpx_codec_err_t codec_control(vpx_codec_ctx_t* ctx, |
| 446 | vp8e_enc_control_id ctrl_id, |
| 447 | int* param) const override { |
| 448 | return VPX_CODEC_OK; |
| 449 | } |
| 450 | vpx_codec_err_t codec_control(vpx_codec_ctx_t* ctx, |
| 451 | vp8e_enc_control_id ctrl_id, |
| 452 | vpx_roi_map* param) const override { |
| 453 | return VPX_CODEC_OK; |
| 454 | } |
| 455 | vpx_codec_err_t codec_control(vpx_codec_ctx_t* ctx, |
| 456 | vp8e_enc_control_id ctrl_id, |
| 457 | vpx_active_map* param) const override { |
| 458 | return VPX_CODEC_OK; |
| 459 | } |
| 460 | vpx_codec_err_t codec_control(vpx_codec_ctx_t* ctx, |
| 461 | vp8e_enc_control_id ctrl_id, |
| 462 | vpx_scaling_mode* param) const override { |
| 463 | return VPX_CODEC_OK; |
| 464 | } |
| 465 | vpx_codec_err_t codec_control(vpx_codec_ctx_t* ctx, |
| 466 | vp8e_enc_control_id ctrl_id, |
| 467 | vpx_svc_extra_cfg_t* param) const override { |
| 468 | return VPX_CODEC_OK; |
| 469 | } |
| 470 | vpx_codec_err_t codec_control( |
| 471 | vpx_codec_ctx_t* ctx, |
| 472 | vp8e_enc_control_id ctrl_id, |
| 473 | vpx_svc_spatial_layer_sync_t* param) const override { |
| 474 | return VPX_CODEC_OK; |
| 475 | } |
| 476 | vpx_codec_err_t codec_control(vpx_codec_ctx_t* ctx, |
| 477 | vp8e_enc_control_id ctrl_id, |
| 478 | vpx_rc_funcs_t* param) const override { |
| 479 | return VPX_CODEC_OK; |
| 480 | } |
| 481 | const vpx_codec_cx_pkt_t* codec_get_cx_data( |
| 482 | vpx_codec_ctx_t* ctx, |
| 483 | vpx_codec_iter_t* iter) const override { |
| 484 | return nullptr; |
| 485 | } |
| 486 | const char* codec_error_detail(vpx_codec_ctx_t* ctx) const override { |
| 487 | return nullptr; |
| 488 | } |
| 489 | const char* codec_error(vpx_codec_ctx_t* ctx) const override { |
| 490 | return nullptr; |
| 491 | } |
| 492 | const char* codec_err_to_string(vpx_codec_err_t err) const override { |
| 493 | return nullptr; |
| 494 | } |
| 495 | |
Danil Chapovalov | e7b752b | 2021-04-21 12:35:54 | [diff] [blame] | 496 | private: |
| 497 | LibvpxState* const state_; |
| 498 | }; |
| 499 | |
| 500 | enum Actions { |
| 501 | kEncode, |
| 502 | kSetRates, |
| 503 | }; |
| 504 | |
| 505 | // When a layer frame is marked for drop, drops all layer frames from that |
| 506 | // pictures with larger spatial ids. |
| 507 | constexpr bool DropAbove(uint8_t layers_mask, int sid) { |
| 508 | uint8_t full_mask = (uint8_t{1} << (sid + 1)) - 1; |
| 509 | return (layers_mask & full_mask) != full_mask; |
| 510 | } |
| 511 | // inline unittests |
| 512 | static_assert(DropAbove(0b1011, /*sid=*/0) == false, ""); |
| 513 | static_assert(DropAbove(0b1011, /*sid=*/1) == false, ""); |
| 514 | static_assert(DropAbove(0b1011, /*sid=*/2) == true, ""); |
| 515 | static_assert(DropAbove(0b1011, /*sid=*/3) == true, ""); |
| 516 | |
| 517 | // When a layer frame is marked for drop, drops all layer frames from that |
| 518 | // pictures with smaller spatial ids. |
| 519 | constexpr bool DropBelow(uint8_t layers_mask, int sid, int num_layers) { |
| 520 | return (layers_mask >> sid) != (1 << (num_layers - sid)) - 1; |
| 521 | } |
| 522 | // inline unittests |
| 523 | static_assert(DropBelow(0b1101, /*sid=*/0, 4) == true, ""); |
| 524 | static_assert(DropBelow(0b1101, /*sid=*/1, 4) == true, ""); |
| 525 | static_assert(DropBelow(0b1101, /*sid=*/2, 4) == false, ""); |
| 526 | static_assert(DropBelow(0b1101, /*sid=*/3, 4) == false, ""); |
| 527 | |
| 528 | } // namespace |
| 529 | |
| 530 | void FuzzOneInput(const uint8_t* data, size_t size) { |
| 531 | FuzzDataHelper helper(rtc::MakeArrayView(data, size)); |
| 532 | |
| 533 | FrameValidator validator; |
| 534 | FieldTrials field_trials(helper); |
| 535 | // Setup call callbacks for the fake |
| 536 | LibvpxState state; |
| 537 | |
| 538 | // Initialize encoder |
Florent Castelli | 816f5b1 | 2023-05-31 12:39:03 | [diff] [blame] | 539 | LibvpxVp9Encoder encoder(cricket::CreateVideoCodec(cricket::kVp9CodecName), |
Danil Chapovalov | e7b752b | 2021-04-21 12:35:54 | [diff] [blame] | 540 | std::make_unique<StubLibvpx>(&state), field_trials); |
| 541 | VideoCodec codec = CodecSettings(helper); |
| 542 | if (encoder.InitEncode(&codec, EncoderSettings()) != WEBRTC_VIDEO_CODEC_OK) { |
| 543 | return; |
| 544 | } |
| 545 | RTC_CHECK_EQ(encoder.RegisterEncodeCompleteCallback(&validator), |
| 546 | WEBRTC_VIDEO_CODEC_OK); |
| 547 | { |
| 548 | // Enable all the layers initially. Encoder doesn't support producing |
| 549 | // frames when no layers are enabled. |
| 550 | LibvpxVp9Encoder::RateControlParameters parameters; |
| 551 | parameters.framerate_fps = 30.0; |
| 552 | for (int sid = 0; sid < codec.VP9()->numberOfSpatialLayers; ++sid) { |
| 553 | for (int tid = 0; tid < codec.VP9()->numberOfTemporalLayers; ++tid) { |
Emil Lundmark | 9103e8e | 2021-09-10 10:32:53 | [diff] [blame] | 554 | parameters.bitrate.SetBitrate(sid, tid, kBitrateEnabledBps); |
Danil Chapovalov | e7b752b | 2021-04-21 12:35:54 | [diff] [blame] | 555 | } |
| 556 | } |
| 557 | encoder.SetRates(parameters); |
| 558 | } |
| 559 | |
| 560 | std::vector<VideoFrameType> frame_types(1); |
| 561 | VideoFrame fake_image = VideoFrame::Builder() |
| 562 | .set_video_frame_buffer(I420Buffer::Create( |
| 563 | int{codec.width}, int{codec.height})) |
| 564 | .build(); |
| 565 | |
| 566 | // Start producing frames at random. |
| 567 | while (helper.CanReadBytes(1)) { |
| 568 | uint8_t action = helper.Read<uint8_t>(); |
| 569 | switch (action & 0b11) { |
| 570 | case kEncode: { |
| 571 | // bitmask of the action: SSSS-K00, where |
| 572 | // four S bit indicate which spatial layers should be produced, |
| 573 | // K bit indicates if frame should be a key frame. |
| 574 | frame_types[0] = (action & 0b100) ? VideoFrameType::kVideoFrameKey |
| 575 | : VideoFrameType::kVideoFrameDelta; |
| 576 | encoder.Encode(fake_image, &frame_types); |
| 577 | uint8_t encode_spatial_layers = (action >> 4); |
| 578 | for (size_t sid = 0; sid < state.config.ss_number_layers; ++sid) { |
Emil Lundmark | 9103e8e | 2021-09-10 10:32:53 | [diff] [blame] | 579 | if (state.config.ss_target_bitrate[sid] == 0) { |
| 580 | // Don't encode disabled spatial layers. |
| 581 | continue; |
| 582 | } |
Danil Chapovalov | e7b752b | 2021-04-21 12:35:54 | [diff] [blame] | 583 | bool drop = true; |
| 584 | switch (state.frame_drop.framedrop_mode) { |
| 585 | case FULL_SUPERFRAME_DROP: |
| 586 | drop = encode_spatial_layers == 0; |
| 587 | break; |
| 588 | case LAYER_DROP: |
| 589 | drop = (encode_spatial_layers & (1 << sid)) == 0; |
| 590 | break; |
| 591 | case CONSTRAINED_LAYER_DROP: |
| 592 | drop = DropBelow(encode_spatial_layers, sid, |
| 593 | state.config.ss_number_layers); |
| 594 | break; |
| 595 | case CONSTRAINED_FROM_ABOVE_DROP: |
| 596 | drop = DropAbove(encode_spatial_layers, sid); |
| 597 | break; |
| 598 | } |
| 599 | if (!drop) { |
| 600 | state.layer_id.spatial_layer_id = sid; |
| 601 | state.callback.output_cx_pkt(&state.pkt, state.callback.user_priv); |
| 602 | } |
| 603 | } |
| 604 | } break; |
| 605 | case kSetRates: { |
Emil Lundmark | 9103e8e | 2021-09-10 10:32:53 | [diff] [blame] | 606 | // bitmask of the action: (S2)(S1)(S0)01, |
Danil Chapovalov | e7b752b | 2021-04-21 12:35:54 | [diff] [blame] | 607 | // where Sx is number of temporal layers to enable for spatial layer x |
| 608 | // In pariculat Sx = 0 indicates spatial layer x should be disabled. |
| 609 | LibvpxVp9Encoder::RateControlParameters parameters; |
| 610 | parameters.framerate_fps = 30.0; |
| 611 | for (int sid = 0; sid < codec.VP9()->numberOfSpatialLayers; ++sid) { |
| 612 | int temporal_layers = (action >> ((1 + sid) * 2)) & 0b11; |
| 613 | for (int tid = 0; tid < temporal_layers; ++tid) { |
Emil Lundmark | 9103e8e | 2021-09-10 10:32:53 | [diff] [blame] | 614 | parameters.bitrate.SetBitrate(sid, tid, kBitrateEnabledBps); |
Danil Chapovalov | e7b752b | 2021-04-21 12:35:54 | [diff] [blame] | 615 | } |
| 616 | } |
Danil Chapovalov | caef2b3 | 2021-09-22 15:19:46 | [diff] [blame] | 617 | if (IsSupported(codec.VP9()->numberOfSpatialLayers, |
| 618 | codec.VP9()->numberOfTemporalLayers, |
| 619 | parameters.bitrate)) { |
Danil Chapovalov | e7b752b | 2021-04-21 12:35:54 | [diff] [blame] | 620 | encoder.SetRates(parameters); |
| 621 | } |
| 622 | } break; |
| 623 | default: |
| 624 | // Unspecificed values are noop. |
| 625 | break; |
| 626 | } |
| 627 | } |
| 628 | } |
| 629 | } // namespace webrtc |