diff --git a/call/bitrate_estimator_tests.cc b/call/bitrate_estimator_tests.cc
index af0e4c9..766e38c 100644
--- a/call/bitrate_estimator_tests.cc
+++ b/call/bitrate_estimator_tests.cc
@@ -170,7 +170,8 @@
           is_sending_receiving_(false),
           send_stream_(nullptr),
           frame_generator_capturer_(),
-          fake_decoder_() {
+          decoder_factory_(
+              []() { return absl::make_unique<test::FakeDecoder>(); }) {
       test_->GetVideoSendConfig()->rtp.ssrcs[0]++;
       send_stream_ = test_->sender_call_->CreateVideoSendStream(
           test_->GetVideoSendConfig()->Copy(),
@@ -185,7 +186,7 @@
       frame_generator_capturer_->Start();
 
       VideoReceiveStream::Decoder decoder;
-      decoder.decoder = &fake_decoder_;
+      decoder.decoder_factory = &decoder_factory_;
       decoder.payload_type = test_->GetVideoSendConfig()->rtp.payload_type;
       decoder.video_format =
           SdpVideoFormat(test_->GetVideoSendConfig()->rtp.payload_name);
@@ -229,7 +230,8 @@
     VideoSendStream* send_stream_;
     VideoReceiveStream* video_receive_stream_;
     std::unique_ptr<test::FrameGeneratorCapturer> frame_generator_capturer_;
-    test::FakeDecoder fake_decoder_;
+
+    test::FunctionVideoDecoderFactory decoder_factory_;
   };
 
   LogObserver receiver_log_;
diff --git a/call/call_perf_tests.cc b/call/call_perf_tests.cc
index b8b4601..05c39fe 100644
--- a/call/call_perf_tests.cc
+++ b/call/call_perf_tests.cc
@@ -33,7 +33,6 @@
 #include "test/call_test.h"
 #include "test/direct_transport.h"
 #include "test/drifting_clock.h"
-#include "test/encoder_proxy_factory.h"
 #include "test/encoder_settings.h"
 #include "test/fake_encoder.h"
 #include "test/field_trial.h"
@@ -44,6 +43,7 @@
 #include "test/single_threaded_task_queue.h"
 #include "test/testsupport/fileutils.h"
 #include "test/testsupport/perf_test.h"
+#include "test/video_encoder_proxy_factory.h"
 #include "video/transport_adapter.h"
 
 using webrtc::test::DriftingClock;
@@ -807,7 +807,7 @@
     uint32_t last_set_bitrate_kbps_;
     VideoSendStream* send_stream_;
     test::FrameGeneratorCapturer* frame_generator_;
-    test::EncoderProxyFactory encoder_factory_;
+    test::VideoEncoderProxyFactory encoder_factory_;
     VideoEncoderConfig encoder_config_;
   } test;
 
diff --git a/call/video_receive_stream.cc b/call/video_receive_stream.cc
index c4cdd49..672abd8 100644
--- a/call/video_receive_stream.cc
+++ b/call/video_receive_stream.cc
@@ -20,9 +20,8 @@
 std::string VideoReceiveStream::Decoder::ToString() const {
   char buf[1024];
   rtc::SimpleStringBuilder ss(buf);
-  ss << "{decoder: " << (decoder ? "(VideoDecoder)" : "nullptr");
-  ss << ", payload_type: " << payload_type;
-  ss << ", name: " << video_format.name;
+  ss << "{payload_type: " << payload_type;
+  ss << ", payload_name: " << video_format.name;
   ss << ", codec_params: {";
   for (const auto& it : video_format.parameters)
     ss << it.first << ": " << it.second;
diff --git a/call/video_receive_stream.h b/call/video_receive_stream.h
index b5c1649..d299cd8 100644
--- a/call/video_receive_stream.h
+++ b/call/video_receive_stream.h
@@ -33,7 +33,7 @@
 namespace webrtc {
 
 class RtpPacketSinkInterface;
-class VideoDecoder;
+class VideoDecoderFactory;
 
 class VideoReceiveStream {
  public:
@@ -45,8 +45,10 @@
     ~Decoder();
     std::string ToString() const;
 
-    // The actual decoder instance.
-    VideoDecoder* decoder = nullptr;
+    // Ownership stays with WebrtcVideoEngine (delegated from PeerConnection).
+    // TODO(nisse): Move one level out, to VideoReceiveStream::Config, and later
+    // to the configuration of VideoStreamDecoder.
+    VideoDecoderFactory* decoder_factory = nullptr;
     SdpVideoFormat video_format;
 
     // Received RTP packets with this payload type will be sent to this decoder
@@ -210,6 +212,10 @@
     // Target delay in milliseconds. A positive value indicates this stream is
     // used for streaming instead of a real-time call.
     int target_delay_ms = 0;
+
+    // TODO(nisse): Used with VideoDecoderFactory::LegacyCreateVideoDecoder.
+    // Delete when that method is retired.
+    std::string stream_id;
   };
 
   // Starts stream activity.
diff --git a/media/engine/webrtcvideoengine.cc b/media/engine/webrtcvideoengine.cc
index 501f5de..73585a8 100644
--- a/media/engine/webrtcvideoengine.cc
+++ b/media/engine/webrtcvideoengine.cc
@@ -16,14 +16,11 @@
 #include <string>
 #include <utility>
 
-#include "api/video/i420_buffer.h"
 #include "api/video_codecs/sdp_video_format.h"
-#include "api/video_codecs/video_decoder.h"
 #include "api/video_codecs/video_decoder_factory.h"
 #include "api/video_codecs/video_encoder.h"
 #include "api/video_codecs/video_encoder_factory.h"
 #include "call/call.h"
-#include "common_video/h264/profile_level_id.h"
 #include "media/engine/constants.h"
 #if defined(USE_BUILTIN_SW_CODECS)
 #include "media/engine/convert_legacy_video_factory.h"  // nogncheck
@@ -31,7 +28,6 @@
 #include "media/engine/simulcast.h"
 #include "media/engine/webrtcmediaengine.h"
 #include "media/engine/webrtcvoiceengine.h"
-#include "modules/video_coding/include/video_error_codes.h"
 #include "rtc_base/copyonwritebuffer.h"
 #include "rtc_base/logging.h"
 #include "rtc_base/strings/string_builder.h"
@@ -44,36 +40,6 @@
 
 namespace {
 
-// Video decoder class to be used for unknown codecs. Doesn't support decoding
-// but logs messages to LS_ERROR.
-class NullVideoDecoder : public webrtc::VideoDecoder {
- public:
-  int32_t InitDecode(const webrtc::VideoCodec* codec_settings,
-                     int32_t number_of_cores) override {
-    RTC_LOG(LS_ERROR) << "Can't initialize NullVideoDecoder.";
-    return WEBRTC_VIDEO_CODEC_OK;
-  }
-
-  int32_t Decode(const webrtc::EncodedImage& input_image,
-                 bool missing_frames,
-                 const webrtc::CodecSpecificInfo* codec_specific_info,
-                 int64_t render_time_ms) override {
-    RTC_LOG(LS_ERROR) << "The NullVideoDecoder doesn't support decoding.";
-    return WEBRTC_VIDEO_CODEC_OK;
-  }
-
-  int32_t RegisterDecodeCompleteCallback(
-      webrtc::DecodedImageCallback* callback) override {
-    RTC_LOG(LS_ERROR)
-        << "Can't register decode complete callback on NullVideoDecoder.";
-    return WEBRTC_VIDEO_CODEC_OK;
-  }
-
-  int32_t Release() override { return WEBRTC_VIDEO_CODEC_OK; }
-
-  const char* ImplementationName() const override { return "NullVideoDecoder"; }
-};
-
 // If this field trial is enabled, we will enable sending FlexFEC and disable
 // sending ULPFEC whenever the former has been negotiated in the SDPs.
 bool IsFlexfecFieldTrialEnabled() {
@@ -2194,12 +2160,10 @@
       first_frame_timestamp_(-1),
       estimated_remote_start_ntp_time_ms_(0) {
   config_.renderer = this;
-  DecoderMap old_decoders;
-  ConfigureCodecs(recv_codecs, &old_decoders);
+  ConfigureCodecs(recv_codecs);
   ConfigureFlexfecCodec(flexfec_config.payload_type);
   MaybeRecreateWebRtcFlexfecStream();
   RecreateWebRtcVideoStream();
-  RTC_DCHECK(old_decoders.empty());
 }
 
 WebRtcVideoChannel::WebRtcVideoReceiveStream::~WebRtcVideoReceiveStream() {
@@ -2208,7 +2172,6 @@
     call_->DestroyFlexfecReceiveStream(flexfec_stream_);
   }
   call_->DestroyVideoReceiveStream(stream_);
-  allocated_decoders_.clear();
 }
 
 const std::vector<uint32_t>&
@@ -2247,58 +2210,23 @@
 }
 
 void WebRtcVideoChannel::WebRtcVideoReceiveStream::ConfigureCodecs(
-    const std::vector<VideoCodecSettings>& recv_codecs,
-    DecoderMap* old_decoders) {
+    const std::vector<VideoCodecSettings>& recv_codecs) {
   RTC_DCHECK(!recv_codecs.empty());
-  *old_decoders = std::move(allocated_decoders_);
   config_.decoders.clear();
   config_.rtp.rtx_associated_payload_types.clear();
   for (const auto& recv_codec : recv_codecs) {
     webrtc::SdpVideoFormat video_format(recv_codec.codec.name,
                                         recv_codec.codec.params);
-    std::unique_ptr<webrtc::VideoDecoder> new_decoder;
-
-    if (allocated_decoders_.count(video_format) > 0) {
-      RTC_LOG(LS_WARNING)
-          << "VideoReceiveStream configured with duplicate codecs: "
-          << video_format.name;
-      continue;
-    }
-
-    auto it = old_decoders->find(video_format);
-    if (it != old_decoders->end()) {
-      new_decoder = std::move(it->second);
-      old_decoders->erase(it);
-    }
-
-    if (!new_decoder && decoder_factory_) {
-      new_decoder = decoder_factory_->LegacyCreateVideoDecoder(
-          webrtc::SdpVideoFormat(recv_codec.codec.name,
-                                 recv_codec.codec.params),
-          stream_params_.id);
-    }
-
-    // If we still have no valid decoder, we have to create a "Null" decoder
-    // that ignores all calls. The reason we can get into this state is that
-    // the old decoder factory interface doesn't have a way to query supported
-    // codecs.
-    if (!new_decoder)
-      new_decoder.reset(new NullVideoDecoder());
 
     webrtc::VideoReceiveStream::Decoder decoder;
-    decoder.decoder = new_decoder.get();
+    decoder.decoder_factory = decoder_factory_;
+    decoder.video_format = video_format;
     decoder.payload_type = recv_codec.codec.id;
     decoder.video_format =
         webrtc::SdpVideoFormat(recv_codec.codec.name, recv_codec.codec.params);
     config_.decoders.push_back(decoder);
     config_.rtp.rtx_associated_payload_types[recv_codec.rtx_payload_type] =
         recv_codec.codec.id;
-
-    const bool did_insert =
-        allocated_decoders_
-            .insert(std::make_pair(video_format, std::move(new_decoder)))
-            .second;
-    RTC_CHECK(did_insert);
   }
 
   const auto& codec = recv_codecs.front();
@@ -2378,9 +2306,8 @@
     const ChangedRecvParameters& params) {
   bool video_needs_recreation = false;
   bool flexfec_needs_recreation = false;
-  DecoderMap old_decoders;
   if (params.codec_settings) {
-    ConfigureCodecs(*params.codec_settings, &old_decoders);
+    ConfigureCodecs(*params.codec_settings);
     video_needs_recreation = true;
   }
   if (params.rtp_header_extensions) {
@@ -2413,6 +2340,7 @@
   }
   webrtc::VideoReceiveStream::Config config = config_.Copy();
   config.rtp.protected_by_flexfec = (flexfec_stream_ != nullptr);
+  config.stream_id = stream_params_.id;
   stream_ = call_->CreateVideoReceiveStream(std::move(config));
   MaybeAssociateFlexfecWithVideo();
   stream_->Start();
diff --git a/media/engine/webrtcvideoengine.h b/media/engine/webrtcvideoengine.h
index d949d78..40c39af 100644
--- a/media/engine/webrtcvideoengine.h
+++ b/media/engine/webrtcvideoengine.h
@@ -37,9 +37,7 @@
 #include "rtc_base/thread_checker.h"
 
 namespace webrtc {
-class VideoDecoder;
 class VideoDecoderFactory;
-class VideoEncoder;
 class VideoEncoderFactory;
 struct MediaConfig;
 }  // namespace webrtc
@@ -380,26 +378,13 @@
     VideoReceiverInfo GetVideoReceiverInfo(bool log_stats);
 
    private:
-    struct SdpVideoFormatCompare {
-      bool operator()(const webrtc::SdpVideoFormat& lhs,
-                      const webrtc::SdpVideoFormat& rhs) const {
-        return std::tie(lhs.name, lhs.parameters) <
-               std::tie(rhs.name, rhs.parameters);
-      }
-    };
-    typedef std::map<webrtc::SdpVideoFormat,
-                     std::unique_ptr<webrtc::VideoDecoder>,
-                     SdpVideoFormatCompare>
-        DecoderMap;
-
     void RecreateWebRtcVideoStream();
     void MaybeRecreateWebRtcFlexfecStream();
 
     void MaybeAssociateFlexfecWithVideo();
     void MaybeDissociateFlexfecFromVideo();
 
-    void ConfigureCodecs(const std::vector<VideoCodecSettings>& recv_codecs,
-                         DecoderMap* old_codecs);
+    void ConfigureCodecs(const std::vector<VideoCodecSettings>& recv_codecs);
     void ConfigureFlexfecCodec(int flexfec_payload_type);
 
     std::string GetCodecNameFromPayloadType(int payload_type);
@@ -407,7 +392,7 @@
     absl::optional<uint32_t> GetFirstPrimarySsrc() const;
 
     webrtc::Call* const call_;
-    StreamParams stream_params_;
+    const StreamParams stream_params_;
 
     // Both |stream_| and |flexfec_stream_| are managed by |this|. They are
     // destroyed by calling call_->DestroyVideoReceiveStream and
@@ -418,8 +403,7 @@
     webrtc::FlexfecReceiveStream::Config flexfec_config_;
     webrtc::FlexfecReceiveStream* flexfec_stream_;
 
-    webrtc::VideoDecoderFactory* decoder_factory_;
-    DecoderMap allocated_decoders_;
+    webrtc::VideoDecoderFactory* const decoder_factory_;
 
     rtc::CriticalSection sink_lock_;
     rtc::VideoSinkInterface<webrtc::VideoFrame>* sink_
diff --git a/test/BUILD.gn b/test/BUILD.gn
index 6e1898895..3fa25cf 100644
--- a/test/BUILD.gn
+++ b/test/BUILD.gn
@@ -540,7 +540,6 @@
     "constants.h",
     "drifting_clock.cc",
     "drifting_clock.h",
-    "encoder_proxy_factory.h",
     "encoder_settings.cc",
     "encoder_settings.h",
     "fake_videorenderer.h",
@@ -555,6 +554,8 @@
     "rtp_rtcp_observer.h",
     "statistics.cc",
     "statistics.h",
+    "video_decoder_proxy_factory.h",
+    "video_encoder_proxy_factory.h",
     "win/run_loop_win.cc",
   ]
   if (!is_win) {
diff --git a/test/call_test.cc b/test/call_test.cc
index 89ac329..1b1d356 100644
--- a/test/call_test.cc
+++ b/test/call_test.cc
@@ -52,6 +52,7 @@
         fake_encoder->SetMaxBitrate(fake_encoder_max_bitrate_);
         return fake_encoder;
       }),
+      fake_decoder_factory_([]() { return absl::make_unique<FakeDecoder>(); }),
       num_video_streams_(1),
       num_audio_streams_(0),
       num_flexfec_streams_(0),
@@ -366,11 +367,10 @@
     if (!decode_sub_stream || i == *decode_sub_stream) {
       decoder = test::CreateMatchingDecoder(video_send_config);
     } else {
-      decoder.decoder = new test::FakeDecoder();
       decoder.payload_type = video_send_config.rtp.payload_type;
       decoder.video_format = SdpVideoFormat(video_send_config.rtp.payload_name);
     }
-    allocated_decoders_.emplace_back(decoder.decoder);
+    decoder.decoder_factory = &fake_decoder_factory_;
     video_recv_config.decoders.push_back(decoder);
     receive_configs->emplace_back(std::move(video_recv_config));
   }
@@ -433,7 +433,6 @@
 
 void CallTest::CreateMatchingReceiveConfigs(Transport* rtcp_send_transport) {
   video_receive_configs_.clear();
-  allocated_decoders_.clear();
   for (VideoSendStream::Config& video_send_config : video_send_configs_) {
     CreateMatchingVideoReceiveConfigs(video_send_config, rtcp_send_transport);
   }
@@ -635,7 +634,6 @@
     receiver_call_->DestroyFlexfecReceiveStream(flexfec_recv_stream);
 
   video_receive_streams_.clear();
-  allocated_decoders_.clear();
 }
 
 void CallTest::DestroyVideoSendStreams() {
diff --git a/test/call_test.h b/test/call_test.h
index eb96cfd..0b2058b 100644
--- a/test/call_test.h
+++ b/test/call_test.h
@@ -22,6 +22,7 @@
 #include "test/fake_videorenderer.h"
 #include "test/fake_vp8_encoder.h"
 #include "test/frame_generator_capturer.h"
+#include "test/function_video_decoder_factory.h"
 #include "test/function_video_encoder_factory.h"
 #include "test/rtp_rtcp_observer.h"
 #include "test/single_threaded_task_queue.h"
@@ -201,7 +202,7 @@
 
   test::FunctionVideoEncoderFactory fake_encoder_factory_;
   int fake_encoder_max_bitrate_ = -1;
-  std::vector<std::unique_ptr<VideoDecoder>> allocated_decoders_;
+  test::FunctionVideoDecoderFactory fake_decoder_factory_;
   // Number of simulcast substreams.
   size_t num_video_streams_;
   size_t num_audio_streams_;
diff --git a/test/encoder_settings.cc b/test/encoder_settings.cc
index 1013433..945559b 100644
--- a/test/encoder_settings.cc
+++ b/test/encoder_settings.cc
@@ -12,13 +12,7 @@
 #include <algorithm>
 #include <string>
 
-#include "media/engine/internaldecoderfactory.h"
-#include "modules/video_coding/codecs/h264/include/h264.h"
-#include "modules/video_coding/codecs/multiplex/include/multiplex_decoder_adapter.h"
-#include "modules/video_coding/codecs/vp8/include/vp8.h"
-#include "modules/video_coding/codecs/vp9/include/vp9.h"
 #include "rtc_base/refcountedobject.h"
-#include "test/fake_decoder.h"
 
 namespace webrtc {
 namespace test {
@@ -128,18 +122,6 @@
   VideoReceiveStream::Decoder decoder;
   decoder.payload_type = payload_type;
   decoder.video_format = SdpVideoFormat(payload_name);
-  if (payload_name == "H264") {
-    decoder.decoder = H264Decoder::Create().release();
-  } else if (payload_name == "VP8") {
-    decoder.decoder = VP8Decoder::Create().release();
-  } else if (payload_name == "VP9") {
-    decoder.decoder = VP9Decoder::Create().release();
-  } else if (payload_name == "multiplex") {
-    decoder.decoder = new MultiplexDecoderAdapter(
-        new InternalDecoderFactory(), SdpVideoFormat(cricket::kVp9CodecName));
-  } else {
-    decoder.decoder = new FakeDecoder();
-  }
   return decoder;
 }
 
diff --git a/test/video_decoder_proxy_factory.h b/test/video_decoder_proxy_factory.h
new file mode 100644
index 0000000..250750c
--- /dev/null
+++ b/test/video_decoder_proxy_factory.h
@@ -0,0 +1,81 @@
+/*
+ *  Copyright (c) 2018 The WebRTC project authors. All Rights Reserved.
+ *
+ *  Use of this source code is governed by a BSD-style license
+ *  that can be found in the LICENSE file in the root of the source
+ *  tree. An additional intellectual property rights grant can be found
+ *  in the file PATENTS.  All contributing project authors may
+ *  be found in the AUTHORS file in the root of the source tree.
+ */
+
+#ifndef TEST_VIDEO_DECODER_PROXY_FACTORY_H_
+#define TEST_VIDEO_DECODER_PROXY_FACTORY_H_
+
+#include <memory>
+#include <vector>
+
+#include "absl/memory/memory.h"
+#include "api/video_codecs/video_decoder.h"
+#include "api/video_codecs/video_decoder_factory.h"
+
+namespace webrtc {
+namespace test {
+
+// An decoder factory with a single underlying VideoDecoder object, intended for
+// test purposes. Each call to CreateVideoDecoder returns a proxy for the same
+// decoder, typically an instance of FakeDecoder or MockEncoder.
+class VideoDecoderProxyFactory final : public VideoDecoderFactory {
+ public:
+  explicit VideoDecoderProxyFactory(VideoDecoder* decoder)
+      : decoder_(decoder) {}
+
+  // Unused by tests.
+  std::vector<SdpVideoFormat> GetSupportedFormats() const override {
+    RTC_NOTREACHED();
+    return {};
+  }
+
+  std::unique_ptr<VideoDecoder> CreateVideoDecoder(
+      const SdpVideoFormat& format) override {
+    return absl::make_unique<DecoderProxy>(decoder_);
+  }
+
+ private:
+  // Wrapper class, since CreateVideoDecoder needs to surrender
+  // ownership to the object it returns.
+  class DecoderProxy final : public VideoDecoder {
+   public:
+    explicit DecoderProxy(VideoDecoder* decoder) : decoder_(decoder) {}
+
+   private:
+    int32_t Decode(const EncodedImage& input_image,
+                   bool missing_frames,
+                   const CodecSpecificInfo* codec_specific_info,
+                   int64_t render_time_ms) override {
+      return decoder_->Decode(input_image, missing_frames, codec_specific_info,
+                              render_time_ms);
+    }
+    int32_t InitDecode(const VideoCodec* config,
+                       int32_t number_of_cores) override {
+      return decoder_->InitDecode(config, number_of_cores);
+    }
+    int32_t RegisterDecodeCompleteCallback(
+        DecodedImageCallback* callback) override {
+      return decoder_->RegisterDecodeCompleteCallback(callback);
+    }
+    int32_t Release() override { return decoder_->Release(); }
+    bool PrefersLateDecoding() const { return decoder_->PrefersLateDecoding(); }
+    const char* ImplementationName() const override {
+      return decoder_->ImplementationName();
+    }
+
+    VideoDecoder* const decoder_;
+  };
+
+  VideoDecoder* const decoder_;
+};
+
+}  // namespace test
+}  // namespace webrtc
+
+#endif  // TEST_VIDEO_DECODER_PROXY_FACTORY_H_
diff --git a/test/encoder_proxy_factory.h b/test/video_encoder_proxy_factory.h
similarity index 92%
rename from test/encoder_proxy_factory.h
rename to test/video_encoder_proxy_factory.h
index 8672a46..e7289c0 100644
--- a/test/encoder_proxy_factory.h
+++ b/test/video_encoder_proxy_factory.h
@@ -8,8 +8,8 @@
  *  be found in the AUTHORS file in the root of the source tree.
  */
 
-#ifndef TEST_ENCODER_PROXY_FACTORY_H_
-#define TEST_ENCODER_PROXY_FACTORY_H_
+#ifndef TEST_VIDEO_ENCODER_PROXY_FACTORY_H_
+#define TEST_VIDEO_ENCODER_PROXY_FACTORY_H_
 
 #include <memory>
 #include <vector>
@@ -24,9 +24,9 @@
 // An encoder factory with a single underlying VideoEncoder object,
 // intended for test purposes. Each call to CreateVideoEncoder returns
 // a proxy for the same encoder, typically an instance of FakeEncoder.
-class EncoderProxyFactory final : public VideoEncoderFactory {
+class VideoEncoderProxyFactory final : public VideoEncoderFactory {
  public:
-  explicit EncoderProxyFactory(VideoEncoder* encoder) : encoder_(encoder) {
+  explicit VideoEncoderProxyFactory(VideoEncoder* encoder) : encoder_(encoder) {
     codec_info_.is_hardware_accelerated = false;
     codec_info_.has_internal_source = false;
   }
@@ -100,4 +100,4 @@
 }  // namespace test
 }  // namespace webrtc
 
-#endif  // TEST_ENCODER_PROXY_FACTORY_H_
+#endif  // TEST_VIDEO_ENCODER_PROXY_FACTORY_H_
diff --git a/video/BUILD.gn b/video/BUILD.gn
index cecdd38..624d278 100644
--- a/video/BUILD.gn
+++ b/video/BUILD.gn
@@ -364,6 +364,7 @@
       "../call:call_interfaces",
       "../common_video",
       "../logging:rtc_event_log_api",
+      "../media:rtc_internal_video_codecs",
       "../modules/rtp_rtcp",
       "../rtc_base:checks",
       "../rtc_base:rtc_base_approved",
diff --git a/video/end_to_end_tests/bandwidth_tests.cc b/video/end_to_end_tests/bandwidth_tests.cc
index 5ab24fa..ab5e568 100644
--- a/video/end_to_end_tests/bandwidth_tests.cc
+++ b/video/end_to_end_tests/bandwidth_tests.cc
@@ -15,12 +15,12 @@
 #include "rtc_base/rate_limiter.h"
 #include "system_wrappers/include/sleep.h"
 #include "test/call_test.h"
-#include "test/encoder_proxy_factory.h"
 #include "test/fake_encoder.h"
 #include "test/field_trial.h"
 #include "test/gtest.h"
 #include "test/rtcp_packet_parser.h"
 #include "test/rtp_rtcp_observer.h"
+#include "test/video_encoder_proxy_factory.h"
 
 namespace webrtc {
 
@@ -344,7 +344,7 @@
     test::SingleThreadedTaskQueueForTesting* const task_queue_;
     rtc::CriticalSection crit_;
     VideoSendStream* send_stream_;
-    test::EncoderProxyFactory encoder_factory_;
+    test::VideoEncoderProxyFactory encoder_factory_;
     uint32_t bitrate_kbps_ RTC_GUARDED_BY(crit_);
   } test(&task_queue_);
 
diff --git a/video/end_to_end_tests/codec_tests.cc b/video/end_to_end_tests/codec_tests.cc
index 5c52555..fad3ac7 100644
--- a/video/end_to_end_tests/codec_tests.cc
+++ b/video/end_to_end_tests/codec_tests.cc
@@ -39,7 +39,7 @@
                 VideoRotation rotation_to_test,
                 const std::string& payload_name,
                 VideoEncoderFactory* encoder_factory,
-                std::unique_ptr<webrtc::VideoDecoder> decoder)
+                VideoDecoderFactory* decoder_factory)
       : EndToEndTest(4 * CodecEndToEndTest::kDefaultTimeoutMs),
         // TODO(hta): This timeout (120 seconds) is excessive.
         // https://bugs.webrtc.org/6830
@@ -47,7 +47,7 @@
         expected_rotation_(rotation_to_test),
         payload_name_(payload_name),
         encoder_factory_(encoder_factory),
-        decoder_(std::move(decoder)),
+        decoder_factory_(decoder_factory),
         frame_counter_(0) {}
 
   void PerformTest() override {
@@ -70,7 +70,7 @@
         send_config->rtp.payload_type;
     (*receive_configs)[0].decoders[0].video_format =
         SdpVideoFormat(send_config->rtp.payload_name);
-    (*receive_configs)[0].decoders[0].decoder = decoder_.get();
+    (*receive_configs)[0].decoders[0].decoder_factory = decoder_factory_;
   }
 
   void OnFrame(const VideoFrame& video_frame) override {
@@ -89,7 +89,7 @@
   VideoRotation expected_rotation_;
   std::string payload_name_;
   VideoEncoderFactory* encoder_factory_;
-  std::unique_ptr<webrtc::VideoDecoder> decoder_;
+  VideoDecoderFactory* decoder_factory_;
   int frame_counter_;
 };
 
@@ -101,16 +101,20 @@
 TEST_P(CodecEndToEndTest, SendsAndReceivesVP8) {
   test::FunctionVideoEncoderFactory encoder_factory(
       []() { return VP8Encoder::Create(); });
+  test::FunctionVideoDecoderFactory decoder_factory(
+      []() { return VP8Decoder::Create(); });
   CodecObserver test(5, kVideoRotation_0, "VP8", &encoder_factory,
-                     VP8Decoder::Create());
+                     &decoder_factory);
   RunBaseTest(&test);
 }
 
 TEST_P(CodecEndToEndTest, SendsAndReceivesVP8Rotation90) {
   test::FunctionVideoEncoderFactory encoder_factory(
       []() { return VP8Encoder::Create(); });
+  test::FunctionVideoDecoderFactory decoder_factory(
+      []() { return VP8Decoder::Create(); });
   CodecObserver test(5, kVideoRotation_90, "VP8", &encoder_factory,
-                     VP8Decoder::Create());
+                     &decoder_factory);
   RunBaseTest(&test);
 }
 
@@ -118,47 +122,58 @@
 TEST_P(CodecEndToEndTest, SendsAndReceivesVP9) {
   test::FunctionVideoEncoderFactory encoder_factory(
       []() { return VP9Encoder::Create(); });
+  test::FunctionVideoDecoderFactory decoder_factory(
+      []() { return VP9Decoder::Create(); });
   CodecObserver test(500, kVideoRotation_0, "VP9", &encoder_factory,
-                     VP9Decoder::Create());
+                     &decoder_factory);
   RunBaseTest(&test);
 }
 
 TEST_P(CodecEndToEndTest, SendsAndReceivesVP9VideoRotation90) {
   test::FunctionVideoEncoderFactory encoder_factory(
       []() { return VP9Encoder::Create(); });
+  test::FunctionVideoDecoderFactory decoder_factory(
+      []() { return VP9Decoder::Create(); });
   CodecObserver test(5, kVideoRotation_90, "VP9", &encoder_factory,
-                     VP9Decoder::Create());
+                     &decoder_factory);
   RunBaseTest(&test);
 }
 
 // Mutiplex tests are using VP9 as the underlying implementation.
 TEST_P(CodecEndToEndTest, SendsAndReceivesMultiplex) {
   InternalEncoderFactory internal_encoder_factory;
-  InternalDecoderFactory decoder_factory;
+  InternalDecoderFactory internal_decoder_factory;
   test::FunctionVideoEncoderFactory encoder_factory(
       [&internal_encoder_factory]() {
         return absl::make_unique<MultiplexEncoderAdapter>(
             &internal_encoder_factory, SdpVideoFormat(cricket::kVp9CodecName));
       });
-  CodecObserver test(
-      5, kVideoRotation_0, "multiplex", &encoder_factory,
-      absl::make_unique<MultiplexDecoderAdapter>(
-          &decoder_factory, SdpVideoFormat(cricket::kVp9CodecName)));
+  test::FunctionVideoDecoderFactory decoder_factory(
+      [&internal_decoder_factory]() {
+        return absl::make_unique<MultiplexDecoderAdapter>(
+            &internal_decoder_factory, SdpVideoFormat(cricket::kVp9CodecName));
+      });
+
+  CodecObserver test(5, kVideoRotation_0, "multiplex", &encoder_factory,
+                     &decoder_factory);
   RunBaseTest(&test);
 }
 
 TEST_P(CodecEndToEndTest, SendsAndReceivesMultiplexVideoRotation90) {
   InternalEncoderFactory internal_encoder_factory;
-  InternalDecoderFactory decoder_factory;
+  InternalDecoderFactory internal_decoder_factory;
   test::FunctionVideoEncoderFactory encoder_factory(
       [&internal_encoder_factory]() {
         return absl::make_unique<MultiplexEncoderAdapter>(
             &internal_encoder_factory, SdpVideoFormat(cricket::kVp9CodecName));
       });
-  CodecObserver test(
-      5, kVideoRotation_90, "multiplex", &encoder_factory,
-      absl::make_unique<MultiplexDecoderAdapter>(
-          &decoder_factory, SdpVideoFormat(cricket::kVp9CodecName)));
+  test::FunctionVideoDecoderFactory decoder_factory(
+      [&internal_decoder_factory]() {
+        return absl::make_unique<MultiplexDecoderAdapter>(
+            &internal_decoder_factory, SdpVideoFormat(cricket::kVp9CodecName));
+      });
+  CodecObserver test(5, kVideoRotation_90, "multiplex", &encoder_factory,
+                     &decoder_factory);
   RunBaseTest(&test);
 }
 
@@ -183,16 +198,20 @@
 TEST_P(EndToEndTestH264, SendsAndReceivesH264) {
   test::FunctionVideoEncoderFactory encoder_factory(
       []() { return H264Encoder::Create(cricket::VideoCodec("H264")); });
+  test::FunctionVideoDecoderFactory decoder_factory(
+      []() { return H264Decoder::Create(); });
   CodecObserver test(500, kVideoRotation_0, "H264", &encoder_factory,
-                     H264Decoder::Create());
+                     &decoder_factory);
   RunBaseTest(&test);
 }
 
 TEST_P(EndToEndTestH264, SendsAndReceivesH264VideoRotation90) {
   test::FunctionVideoEncoderFactory encoder_factory(
       []() { return H264Encoder::Create(cricket::VideoCodec("H264")); });
+  test::FunctionVideoDecoderFactory decoder_factory(
+      []() { return H264Decoder::Create(); });
   CodecObserver test(5, kVideoRotation_90, "H264", &encoder_factory,
-                     H264Decoder::Create());
+                     &decoder_factory);
   RunBaseTest(&test);
 }
 
@@ -201,8 +220,10 @@
   codec.SetParam(cricket::kH264FmtpPacketizationMode, "0");
   test::FunctionVideoEncoderFactory encoder_factory(
       [codec]() { return H264Encoder::Create(codec); });
+  test::FunctionVideoDecoderFactory decoder_factory(
+      []() { return H264Decoder::Create(); });
   CodecObserver test(500, kVideoRotation_0, "H264", &encoder_factory,
-                     H264Decoder::Create());
+                     &decoder_factory);
   RunBaseTest(&test);
 }
 
@@ -211,8 +232,10 @@
   codec.SetParam(cricket::kH264FmtpPacketizationMode, "1");
   test::FunctionVideoEncoderFactory encoder_factory(
       [codec]() { return H264Encoder::Create(codec); });
+  test::FunctionVideoDecoderFactory decoder_factory(
+      []() { return H264Decoder::Create(); });
   CodecObserver test(500, kVideoRotation_0, "H264", &encoder_factory,
-                     H264Decoder::Create());
+                     &decoder_factory);
   RunBaseTest(&test);
 }
 #endif  // defined(WEBRTC_USE_H264)
diff --git a/video/end_to_end_tests/fec_tests.cc b/video/end_to_end_tests/fec_tests.cc
index 000f6d7..0300d12 100644
--- a/video/end_to_end_tests/fec_tests.cc
+++ b/video/end_to_end_tests/fec_tests.cc
@@ -11,6 +11,7 @@
 #include "api/test/simulated_network.h"
 #include "call/fake_network_pipe.h"
 #include "call/simulated_network.h"
+#include "media/engine/internaldecoderfactory.h"
 #include "modules/rtp_rtcp/source/byte_io.h"
 #include "modules/video_coding/codecs/vp8/include/vp8.h"
 #include "test/call_test.h"
@@ -98,7 +99,7 @@
       encoder_config->codec_type = kVideoCodecVP8;
       VideoReceiveStream::Decoder decoder =
           test::CreateMatchingDecoder(*send_config);
-      decoder_.reset(decoder.decoder);
+      decoder.decoder_factory = &decoder_factory_;
       (*receive_configs)[0].decoders.clear();
       (*receive_configs)[0].decoders.push_back(decoder);
 
@@ -119,7 +120,7 @@
     rtc::CriticalSection crit_;
     std::unique_ptr<VideoEncoder> encoder_;
     test::FunctionVideoEncoderFactory encoder_factory_;
-    std::unique_ptr<VideoDecoder> decoder_;
+    InternalDecoderFactory decoder_factory_;
     std::set<uint32_t> dropped_sequence_numbers_ RTC_GUARDED_BY(crit_);
     // Several packets can have the same timestamp.
     std::multiset<uint32_t> dropped_timestamps_ RTC_GUARDED_BY(crit_);
@@ -329,8 +330,7 @@
           ulpfec_sequence_number_(0),
           has_last_sequence_number_(false),
           last_sequence_number_(0),
-          encoder_factory_([]() { return VP8Encoder::Create(); }),
-          decoder_(VP8Decoder::Create()) {}
+          encoder_factory_([]() { return VP8Encoder::Create(); }) {}
 
    private:
     Action OnSendRtp(const uint8_t* packet, size_t length) override {
@@ -461,7 +461,7 @@
           send_config->rtp.payload_type;
       (*receive_configs)[0].decoders[0].video_format =
           SdpVideoFormat(send_config->rtp.payload_name);
-      (*receive_configs)[0].decoders[0].decoder = decoder_.get();
+      (*receive_configs)[0].decoders[0].decoder_factory = &decoder_factory_;
     }
 
     void PerformTest() override {
@@ -482,9 +482,8 @@
     uint16_t ulpfec_sequence_number_ RTC_GUARDED_BY(&crit_);
     bool has_last_sequence_number_;
     uint16_t last_sequence_number_;
-    std::unique_ptr<webrtc::VideoEncoder> encoder_;
     test::FunctionVideoEncoderFactory encoder_factory_;
-    std::unique_ptr<webrtc::VideoDecoder> decoder_;
+    InternalDecoderFactory decoder_factory_;
   } test;
 
   RunBaseTest(&test);
diff --git a/video/end_to_end_tests/log_tests.cc b/video/end_to_end_tests/log_tests.cc
index aa62383..77466b6 100644
--- a/video/end_to_end_tests/log_tests.cc
+++ b/video/end_to_end_tests/log_tests.cc
@@ -8,6 +8,7 @@
  *  be found in the AUTHORS file in the root of the source tree.
  */
 
+#include "media/engine/internaldecoderfactory.h"
 #include "modules/video_coding/codecs/vp8/include/vp8.h"
 #include "rtc_base/file.h"
 #include "test/call_test.h"
@@ -66,8 +67,6 @@
         VideoSendStream::Config* send_config,
         std::vector<VideoReceiveStream::Config>* receive_configs,
         VideoEncoderConfig* encoder_config) override {
-      decoder_ = VP8Decoder::Create();
-
       send_config->post_encode_callback = this;
       send_config->rtp.payload_name = "VP8";
       send_config->encoder_settings.encoder_factory = &encoder_factory_;
@@ -78,7 +77,7 @@
           send_config->rtp.payload_type;
       (*receive_configs)[0].decoders[0].video_format =
           SdpVideoFormat(send_config->rtp.payload_name);
-      (*receive_configs)[0].decoders[0].decoder = decoder_.get();
+      (*receive_configs)[0].decoders[0].decoder_factory = &decoder_factory_;
     }
 
     void EncodedFrameCallback(const EncodedFrame& encoded_frame) override {
@@ -97,7 +96,7 @@
    private:
     LogEndToEndTest* const fixture_;
     test::FunctionVideoEncoderFactory encoder_factory_;
-    std::unique_ptr<VideoDecoder> decoder_;
+    InternalDecoderFactory decoder_factory_;
     rtc::CriticalSection crit_;
     int recorded_frames_ RTC_GUARDED_BY(crit_);
   } test(this);
diff --git a/video/end_to_end_tests/multi_codec_receive_tests.cc b/video/end_to_end_tests/multi_codec_receive_tests.cc
index 7399ef8..e13bc35 100644
--- a/video/end_to_end_tests/multi_codec_receive_tests.cc
+++ b/video/end_to_end_tests/multi_codec_receive_tests.cc
@@ -180,6 +180,7 @@
   struct CodecConfig {
     std::string payload_name;
     VideoEncoderFactory* encoder_factory;
+    VideoDecoderFactory* decoder_factory;
     size_t num_temporal_layers;
   };
 
@@ -197,18 +198,16 @@
 
 void MultiCodecReceiveTest::ConfigureDecoders(
     const std::vector<CodecConfig>& configs) {
+  video_receive_configs_[0].decoders.clear();
   // Placing the payload names in a std::set retains the unique names only.
   std::set<std::string> unique_payload_names;
   for (const auto& config : configs)
-    unique_payload_names.insert(config.payload_name);
+    if (unique_payload_names.insert(config.payload_name).second) {
+      VideoReceiveStream::Decoder decoder = test::CreateMatchingDecoder(
+          PayloadNameToPayloadType(config.payload_name), config.payload_name);
+      decoder.decoder_factory = config.decoder_factory;
 
-  video_receive_configs_[0].decoders.clear();
-  for (const auto& payload_name : unique_payload_names) {
-    VideoReceiveStream::Decoder decoder = test::CreateMatchingDecoder(
-        PayloadNameToPayloadType(payload_name), payload_name);
-    allocated_decoders_.push_back(
-        std::unique_ptr<VideoDecoder>(decoder.decoder));
-    video_receive_configs_[0].decoders.push_back(decoder);
+      video_receive_configs_[0].decoders.push_back(decoder);
   }
 }
 
@@ -270,9 +269,13 @@
       []() { return VP8Encoder::Create(); });
   test::FunctionVideoEncoderFactory vp9_encoder_factory(
       []() { return VP9Encoder::Create(); });
-  RunTestWithCodecs({{"VP8", &vp8_encoder_factory, 1},
-                     {"VP9", &vp9_encoder_factory, 1},
-                     {"VP8", &vp8_encoder_factory, 1}});
+  test::FunctionVideoDecoderFactory vp8_decoder_factory(
+      []() { return VP8Decoder::Create(); });
+  test::FunctionVideoDecoderFactory vp9_decoder_factory(
+      []() { return VP9Decoder::Create(); });
+  RunTestWithCodecs({{"VP8", &vp8_encoder_factory, &vp8_decoder_factory, 1},
+                     {"VP9", &vp9_encoder_factory, &vp9_decoder_factory, 1},
+                     {"VP8", &vp8_encoder_factory, &vp8_decoder_factory, 1}});
 }
 
 TEST_F(MultiCodecReceiveTest, SingleStreamReceivesVp8Vp9WithTl) {
@@ -280,9 +283,13 @@
       []() { return VP8Encoder::Create(); });
   test::FunctionVideoEncoderFactory vp9_encoder_factory(
       []() { return VP9Encoder::Create(); });
-  RunTestWithCodecs({{"VP8", &vp8_encoder_factory, 2},
-                     {"VP9", &vp9_encoder_factory, 2},
-                     {"VP8", &vp8_encoder_factory, 2}});
+  test::FunctionVideoDecoderFactory vp8_decoder_factory(
+      []() { return VP8Decoder::Create(); });
+  test::FunctionVideoDecoderFactory vp9_decoder_factory(
+      []() { return VP9Decoder::Create(); });
+  RunTestWithCodecs({{"VP8", &vp8_encoder_factory, &vp8_decoder_factory, 2},
+                     {"VP9", &vp9_encoder_factory, &vp9_decoder_factory, 2},
+                     {"VP8", &vp8_encoder_factory, &vp8_decoder_factory, 2}});
 }
 
 #if defined(WEBRTC_USE_H264)
@@ -291,9 +298,13 @@
       []() { return VP8Encoder::Create(); });
   test::FunctionVideoEncoderFactory h264_encoder_factory(
       []() { return H264Encoder::Create(cricket::VideoCodec("H264")); });
-  RunTestWithCodecs({{"VP8", &vp8_encoder_factory, 1},
-                     {"H264", &h264_encoder_factory, 1},
-                     {"VP8", &vp8_encoder_factory, 1}});
+  test::FunctionVideoDecoderFactory vp8_decoder_factory(
+      []() { return VP8Decoder::Create(); });
+  test::FunctionVideoDecoderFactory h264_decoder_factory(
+      []() { return H264Decoder::Create(); });
+  RunTestWithCodecs({{"VP8", &vp8_encoder_factory, &vp8_decoder_factory, 1},
+                     {"H264", &h264_encoder_factory, &h264_decoder_factory, 1},
+                     {"VP8", &vp8_encoder_factory, &vp8_decoder_factory, 1}});
 }
 
 TEST_F(MultiCodecReceiveTest, SingleStreamReceivesVp8H264WithTl) {
@@ -301,9 +312,13 @@
       []() { return VP8Encoder::Create(); });
   test::FunctionVideoEncoderFactory h264_encoder_factory(
       []() { return H264Encoder::Create(cricket::VideoCodec("H264")); });
-  RunTestWithCodecs({{"VP8", &vp8_encoder_factory, 3},
-                     {"H264", &h264_encoder_factory, 1},
-                     {"VP8", &vp8_encoder_factory, 3}});
+  test::FunctionVideoDecoderFactory vp8_decoder_factory(
+      []() { return VP8Decoder::Create(); });
+  test::FunctionVideoDecoderFactory h264_decoder_factory(
+      []() { return H264Decoder::Create(); });
+  RunTestWithCodecs({{"VP8", &vp8_encoder_factory, &vp8_decoder_factory, 3},
+                     {"H264", &h264_encoder_factory, &h264_decoder_factory, 1},
+                     {"VP8", &vp8_encoder_factory, &vp8_decoder_factory, 3}});
 }
 
 TEST_F(MultiCodecReceiveTest, SingleStreamReceivesVp8Vp9H264) {
@@ -313,10 +328,16 @@
       []() { return VP9Encoder::Create(); });
   test::FunctionVideoEncoderFactory h264_encoder_factory(
       []() { return H264Encoder::Create(cricket::VideoCodec("H264")); });
-  RunTestWithCodecs({{"VP8", &vp8_encoder_factory, 1},
-                     {"VP9", &vp9_encoder_factory, 1},
-                     {"H264", &h264_encoder_factory, 1},
-                     {"VP9", &vp9_encoder_factory, 1}});
+  test::FunctionVideoDecoderFactory vp8_decoder_factory(
+      []() { return VP8Decoder::Create(); });
+  test::FunctionVideoDecoderFactory vp9_decoder_factory(
+      []() { return VP9Decoder::Create(); });
+  test::FunctionVideoDecoderFactory h264_decoder_factory(
+      []() { return H264Decoder::Create(); });
+  RunTestWithCodecs({{"VP8", &vp8_encoder_factory, &vp8_decoder_factory, 1},
+                     {"VP9", &vp9_encoder_factory, &vp9_decoder_factory, 1},
+                     {"H264", &h264_encoder_factory, &h264_decoder_factory, 1},
+                     {"VP9", &vp9_encoder_factory, &vp9_decoder_factory, 1}});
 }
 
 TEST_F(MultiCodecReceiveTest, SingleStreamReceivesVp8Vp9H264WithTl) {
@@ -326,10 +347,16 @@
       []() { return VP9Encoder::Create(); });
   test::FunctionVideoEncoderFactory h264_encoder_factory(
       []() { return H264Encoder::Create(cricket::VideoCodec("H264")); });
-  RunTestWithCodecs({{"VP8", &vp8_encoder_factory, 3},
-                     {"VP9", &vp9_encoder_factory, 2},
-                     {"H264", &h264_encoder_factory, 1},
-                     {"VP9", &vp9_encoder_factory, 3}});
+  test::FunctionVideoDecoderFactory vp8_decoder_factory(
+      []() { return VP8Decoder::Create(); });
+  test::FunctionVideoDecoderFactory vp9_decoder_factory(
+      []() { return VP9Decoder::Create(); });
+  test::FunctionVideoDecoderFactory h264_decoder_factory(
+      []() { return H264Decoder::Create(); });
+  RunTestWithCodecs({{"VP8", &vp8_encoder_factory, &vp8_decoder_factory, 3},
+                     {"VP9", &vp9_encoder_factory, &vp9_decoder_factory, 2},
+                     {"H264", &h264_encoder_factory, &h264_decoder_factory, 1},
+                     {"VP9", &vp9_encoder_factory, &vp9_decoder_factory, 3}});
 }
 #endif  // defined(WEBRTC_USE_H264)
 
diff --git a/video/end_to_end_tests/multi_stream_tester.cc b/video/end_to_end_tests/multi_stream_tester.cc
index c4f99e6..7d66976 100644
--- a/video/end_to_end_tests/multi_stream_tester.cc
+++ b/video/end_to_end_tests/multi_stream_tester.cc
@@ -18,6 +18,7 @@
 #include "call/fake_network_pipe.h"
 #include "call/simulated_network.h"
 #include "logging/rtc_event_log/rtc_event_log.h"
+#include "media/engine/internaldecoderfactory.h"
 #include "modules/video_coding/codecs/vp8/include/vp8.h"
 #include "test/call_test.h"
 #include "test/encoder_settings.h"
@@ -51,9 +52,9 @@
   VideoSendStream* send_streams[kNumStreams];
   VideoReceiveStream* receive_streams[kNumStreams];
   test::FrameGeneratorCapturer* frame_generators[kNumStreams];
-  std::vector<std::unique_ptr<VideoDecoder>> allocated_decoders;
   test::FunctionVideoEncoderFactory encoder_factory(
       []() { return VP8Encoder::Create(); });
+  InternalDecoderFactory decoder_factory;
 
   task_queue_->SendTask([&]() {
     sender_call = absl::WrapUnique(Call::Create(config));
@@ -91,8 +92,7 @@
       receive_config.rtp.local_ssrc = test::CallTest::kReceiverLocalVideoSsrc;
       VideoReceiveStream::Decoder decoder =
           test::CreateMatchingDecoder(send_config);
-      allocated_decoders.push_back(
-          std::unique_ptr<VideoDecoder>(decoder.decoder));
+      decoder.decoder_factory = &decoder_factory;
       receive_config.decoders.push_back(decoder);
 
       UpdateReceiveConfig(i, &receive_config);
diff --git a/video/end_to_end_tests/network_state_tests.cc b/video/end_to_end_tests/network_state_tests.cc
index 59c2b51..937777f 100644
--- a/video/end_to_end_tests/network_state_tests.cc
+++ b/video/end_to_end_tests/network_state_tests.cc
@@ -13,9 +13,9 @@
 #include "call/simulated_network.h"
 #include "system_wrappers/include/sleep.h"
 #include "test/call_test.h"
-#include "test/encoder_proxy_factory.h"
 #include "test/fake_encoder.h"
 #include "test/gtest.h"
+#include "test/video_encoder_proxy_factory.h"
 
 namespace webrtc {
 namespace {
@@ -82,7 +82,7 @@
     MediaType network_to_bring_up,
     VideoEncoder* encoder,
     Transport* transport) {
-  test::EncoderProxyFactory encoder_factory(encoder);
+  test::VideoEncoderProxyFactory encoder_factory(encoder);
 
   task_queue_.SendTask([this, network_to_bring_up, &encoder_factory,
                         transport]() {
@@ -341,7 +341,7 @@
     rtc::Event packet_event_;
     Call* sender_call_;
     Call* receiver_call_;
-    test::EncoderProxyFactory encoder_factory_;
+    test::VideoEncoderProxyFactory encoder_factory_;
     NetworkState sender_state_ RTC_GUARDED_BY(test_crit_);
     int sender_rtp_ RTC_GUARDED_BY(test_crit_);
     int sender_padding_ RTC_GUARDED_BY(test_crit_);
diff --git a/video/replay.cc b/video/replay.cc
index a5eafae..10aec6f 100644
--- a/video/replay.cc
+++ b/video/replay.cc
@@ -19,6 +19,7 @@
 #include "call/call.h"
 #include "common_video/libyuv/include/webrtc_libyuv.h"
 #include "logging/rtc_event_log/rtc_event_log.h"
+#include "media/engine/internaldecoderfactory.h"
 #include "modules/rtp_rtcp/include/rtp_header_parser.h"
 #include "rtc_base/checks.h"
 #include "rtc_base/file.h"
@@ -31,6 +32,7 @@
 #include "test/call_test.h"
 #include "test/encoder_settings.h"
 #include "test/fake_decoder.h"
+#include "test/function_video_decoder_factory.h"
 #include "test/gtest.h"
 #include "test/null_transport.h"
 #include "test/rtp_file_reader.h"
@@ -330,6 +332,7 @@
     test::NullTransport transport;
     std::vector<std::unique_ptr<rtc::VideoSinkInterface<VideoFrame>>> sinks;
     std::vector<VideoReceiveStream*> receive_streams;
+    std::unique_ptr<VideoDecoderFactory> decoder_factory;
   };
 
   // Loads multiple configurations from the provided configuration file.
@@ -350,6 +353,7 @@
       return nullptr;
     }
 
+    stream_state->decoder_factory = absl::make_unique<InternalDecoderFactory>();
     size_t config_count = 0;
     for (const auto& json : json_configs) {
       // Create the configuration and parse the JSON into the config.
@@ -357,9 +361,9 @@
           &(stream_state->transport), json);
       // Instantiate the underlying decoder.
       for (auto& decoder : receive_config.decoders) {
-        decoder.decoder = test::CreateMatchingDecoder(decoder.payload_type,
-                                                      decoder.video_format.name)
-                              .decoder;
+        decoder = test::CreateMatchingDecoder(decoder.payload_type,
+                                              decoder.video_format.name);
+        decoder.decoder_factory = stream_state->decoder_factory.get();
       }
       // Create a window for this config.
       std::stringstream window_title;
@@ -417,13 +421,19 @@
     VideoReceiveStream::Decoder decoder;
     decoder =
         test::CreateMatchingDecoder(flags::MediaPayloadType(), flags::Codec());
-    if (!flags::DecoderBitstreamFilename().empty()) {
+    if (flags::DecoderBitstreamFilename().empty()) {
+      stream_state->decoder_factory =
+          absl::make_unique<InternalDecoderFactory>();
+    } else {
       // Replace decoder with file writer if we're writing the bitstream to a
       // file instead.
-      delete decoder.decoder;
-      decoder.decoder = new DecoderBitstreamFileWriter(
-          flags::DecoderBitstreamFilename().c_str());
+      stream_state->decoder_factory =
+          absl::make_unique<test::FunctionVideoDecoderFactory>([]() {
+            return absl::make_unique<DecoderBitstreamFileWriter>(
+                flags::DecoderBitstreamFilename().c_str());
+          });
     }
+    decoder.decoder_factory = stream_state->decoder_factory.get();
     receive_config.decoders.push_back(decoder);
 
     stream_state->receive_streams.emplace_back(
diff --git a/video/video_quality_test.cc b/video/video_quality_test.cc
index cd4c5f7..838825e 100644
--- a/video/video_quality_test.cc
+++ b/video/video_quality_test.cc
@@ -495,7 +495,6 @@
   video_receive_configs_.clear();
   video_send_configs_.clear();
   video_encoder_configs_.clear();
-  allocated_decoders_.clear();
   bool decode_all_receive_streams = true;
   size_t num_video_substreams = params_.ss[0].streams.size();
   RTC_CHECK(num_video_streams_ > 0);
diff --git a/video/video_receive_stream.cc b/video/video_receive_stream.cc
index 7facf85..186ce4b 100644
--- a/video/video_receive_stream.cc
+++ b/video/video_receive_stream.cc
@@ -18,6 +18,7 @@
 
 #include "absl/memory/memory.h"
 #include "absl/types/optional.h"
+#include "api/video_codecs/video_decoder_factory.h"
 #include "call/rtp_stream_receiver_controller_interface.h"
 #include "call/rtx_receive_stream.h"
 #include "common_types.h"  // NOLINT(build/include)
@@ -75,6 +76,37 @@
 
   return codec;
 }
+
+// Video decoder class to be used for unknown codecs. Doesn't support decoding
+// but logs messages to LS_ERROR.
+class NullVideoDecoder : public webrtc::VideoDecoder {
+ public:
+  int32_t InitDecode(const webrtc::VideoCodec* codec_settings,
+                     int32_t number_of_cores) override {
+    RTC_LOG(LS_ERROR) << "Can't initialize NullVideoDecoder.";
+    return WEBRTC_VIDEO_CODEC_OK;
+  }
+
+  int32_t Decode(const webrtc::EncodedImage& input_image,
+                 bool missing_frames,
+                 const webrtc::CodecSpecificInfo* codec_specific_info,
+                 int64_t render_time_ms) override {
+    RTC_LOG(LS_ERROR) << "The NullVideoDecoder doesn't support decoding.";
+    return WEBRTC_VIDEO_CODEC_OK;
+  }
+
+  int32_t RegisterDecodeCompleteCallback(
+      webrtc::DecodedImageCallback* callback) override {
+    RTC_LOG(LS_ERROR)
+        << "Can't register decode complete callback on NullVideoDecoder.";
+    return WEBRTC_VIDEO_CODEC_OK;
+  }
+
+  int32_t Release() override { return WEBRTC_VIDEO_CODEC_OK; }
+
+  const char* ImplementationName() const override { return "NullVideoDecoder"; }
+};
+
 }  // namespace
 
 namespace internal {
@@ -121,7 +153,7 @@
   RTC_DCHECK(!config_.decoders.empty());
   std::set<int> decoder_payload_types;
   for (const Decoder& decoder : config_.decoders) {
-    RTC_CHECK(decoder.decoder);
+    RTC_CHECK(decoder.decoder_factory);
     RTC_CHECK(decoder_payload_types.find(decoder.payload_type) ==
               decoder_payload_types.end())
         << "Duplicate payload type (" << decoder.payload_type
@@ -203,7 +235,19 @@
   RTC_DCHECK(renderer != nullptr);
 
   for (const Decoder& decoder : config_.decoders) {
-    video_receiver_.RegisterExternalDecoder(decoder.decoder,
+    std::unique_ptr<VideoDecoder> video_decoder =
+        decoder.decoder_factory->LegacyCreateVideoDecoder(decoder.video_format,
+                                                          config_.stream_id);
+    // If we still have no valid decoder, we have to create a "Null" decoder
+    // that ignores all calls. The reason we can get into this state is that the
+    // old decoder factory interface doesn't have a way to query supported
+    // codecs.
+    if (!video_decoder) {
+      video_decoder = absl::make_unique<NullVideoDecoder>();
+    }
+    video_decoders_.push_back(std::move(video_decoder));
+
+    video_receiver_.RegisterExternalDecoder(video_decoders_.back().get(),
                                             decoder.payload_type);
     VideoCodec codec = CreateDecoderVideoCodec(decoder);
     rtp_video_stream_receiver_.AddReceiveCodec(codec,
diff --git a/video/video_receive_stream.h b/video/video_receive_stream.h
index 76ee2ed..fb0ee3b 100644
--- a/video/video_receive_stream.h
+++ b/video/video_receive_stream.h
@@ -143,6 +143,10 @@
   std::unique_ptr<VideoStreamDecoder> video_stream_decoder_;
   RtpStreamsSynchronizer rtp_stream_sync_;
 
+  // TODO(nisse, philipel): Creation and ownership of video encoders should be
+  // moved to the new VideoStreamDecoder.
+  std::vector<std::unique_ptr<VideoDecoder>> video_decoders_;
+
   rtc::CriticalSection ivf_writer_lock_;
   std::unique_ptr<IvfFileWriter> ivf_writer_ RTC_GUARDED_BY(ivf_writer_lock_);
 
diff --git a/video/video_receive_stream_unittest.cc b/video/video_receive_stream_unittest.cc
index 4b433d2..e4ba69b 100644
--- a/video/video_receive_stream_unittest.cc
+++ b/video/video_receive_stream_unittest.cc
@@ -23,6 +23,7 @@
 #include "rtc_base/event.h"
 #include "system_wrappers/include/clock.h"
 #include "test/field_trial.h"
+#include "test/video_decoder_proxy_factory.h"
 #include "video/call_stats.h"
 #include "video/video_receive_stream.h"
 
@@ -65,7 +66,9 @@
   VideoReceiveStreamTest()
       : process_thread_(ProcessThread::Create("TestThread")),
         config_(&mock_transport_),
-        call_stats_(Clock::GetRealTimeClock(), process_thread_.get()) {}
+        call_stats_(Clock::GetRealTimeClock(), process_thread_.get()),
+        h264_decoder_factory_(&mock_h264_video_decoder_),
+        null_decoder_factory_(&mock_null_video_decoder_) {}
 
   void SetUp() {
     constexpr int kDefaultNumCpuCores = 2;
@@ -77,12 +80,12 @@
     h264_decoder.video_format = SdpVideoFormat("H264");
     h264_decoder.video_format.parameters.insert(
         {"sprop-parameter-sets", "Z0IACpZTBYmI,aMljiA=="});
-    h264_decoder.decoder = &mock_h264_video_decoder_;
+    h264_decoder.decoder_factory = &h264_decoder_factory_;
     config_.decoders.push_back(h264_decoder);
     VideoReceiveStream::Decoder null_decoder;
     null_decoder.payload_type = 98;
     null_decoder.video_format = SdpVideoFormat("null");
-    null_decoder.decoder = &mock_null_video_decoder_;
+    null_decoder.decoder_factory = &null_decoder_factory_;
     config_.decoders.push_back(null_decoder);
 
     video_receive_stream_.reset(new webrtc::internal::VideoReceiveStream(
@@ -96,6 +99,8 @@
   CallStats call_stats_;
   MockVideoDecoder mock_h264_video_decoder_;
   MockVideoDecoder mock_null_video_decoder_;
+  test::VideoDecoderProxyFactory h264_decoder_factory_;
+  test::VideoDecoderProxyFactory null_decoder_factory_;
   cricket::FakeVideoRenderer fake_renderer_;
   MockTransport mock_transport_;
   PacketRouter packet_router_;
diff --git a/video/video_send_stream_tests.cc b/video/video_send_stream_tests.cc
index c015f56..a4d5e19 100644
--- a/video/video_send_stream_tests.cc
+++ b/video/video_send_stream_tests.cc
@@ -36,7 +36,6 @@
 #include "system_wrappers/include/sleep.h"
 #include "test/call_test.h"
 #include "test/configurable_frame_size_encoder.h"
-#include "test/encoder_proxy_factory.h"
 #include "test/fake_encoder.h"
 #include "test/fake_texture_frame.h"
 #include "test/field_trial.h"
@@ -48,6 +47,7 @@
 #include "test/null_transport.h"
 #include "test/rtcp_packet_parser.h"
 #include "test/testsupport/perf_test.h"
+#include "test/video_encoder_proxy_factory.h"
 
 #include "call/video_send_stream.h"
 #include "video/send_statistics_proxy.h"
@@ -1173,7 +1173,7 @@
 
     std::unique_ptr<internal::TransportAdapter> transport_adapter_;
     test::ConfigurableFrameSizeEncoder encoder_;
-    test::EncoderProxyFactory encoder_factory_;
+    test::VideoEncoderProxyFactory encoder_factory_;
 
     const size_t max_packet_size_;
     const size_t stop_size_;
@@ -1961,7 +1961,7 @@
 
   test::NullTransport transport;
   EncoderObserver encoder;
-  test::EncoderProxyFactory encoder_factory(&encoder);
+  test::VideoEncoderProxyFactory encoder_factory(&encoder);
 
   task_queue_.SendTask([this, &transport, &encoder_factory]() {
     CreateSenderCall();
@@ -2039,7 +2039,7 @@
       bitrate_config);
 
   StartBitrateObserver encoder;
-  test::EncoderProxyFactory encoder_factory(&encoder);
+  test::VideoEncoderProxyFactory encoder_factory(&encoder);
   // Since this test does not use a capturer, set |internal_source| = true.
   // Encoder configuration is otherwise updated on the next video frame.
   encoder_factory.SetHasInternalSource(true);
@@ -2124,7 +2124,7 @@
 TEST_P(VideoSendStreamTest, VideoSendStreamStopSetEncoderRateToZero) {
   test::NullTransport transport;
   StartStopBitrateObserver encoder;
-  test::EncoderProxyFactory encoder_factory(&encoder);
+  test::VideoEncoderProxyFactory encoder_factory(&encoder);
   encoder_factory.SetHasInternalSource(true);
   test::FrameForwarder forwarder;
 
@@ -2167,7 +2167,7 @@
 TEST_P(VideoSendStreamTest, VideoSendStreamUpdateActiveSimulcastLayers) {
   test::NullTransport transport;
   StartStopBitrateObserver encoder;
-  test::EncoderProxyFactory encoder_factory(&encoder);
+  test::VideoEncoderProxyFactory encoder_factory(&encoder);
   encoder_factory.SetHasInternalSource(true);
   test::FrameForwarder forwarder;
 
@@ -2441,7 +2441,7 @@
     bool callback_registered_ RTC_GUARDED_BY(crit_);
     size_t num_releases_ RTC_GUARDED_BY(crit_);
     bool released_ RTC_GUARDED_BY(crit_);
-    test::EncoderProxyFactory encoder_factory_;
+    test::VideoEncoderProxyFactory encoder_factory_;
     VideoEncoderConfig encoder_config_;
   } test_encoder(&task_queue_);
 
@@ -2512,7 +2512,7 @@
     rtc::Event init_encode_event_;
     size_t num_initializations_;
     VideoSendStream* stream_;
-    test::EncoderProxyFactory encoder_factory_;
+    test::VideoEncoderProxyFactory encoder_factory_;
     VideoEncoderConfig encoder_config_;
   } test;
 
@@ -2622,7 +2622,7 @@
   rtc::Event init_encode_event_;
   size_t num_initializations_;
   VideoSendStream* stream_;
-  test::EncoderProxyFactory encoder_factory_;
+  test::VideoEncoderProxyFactory encoder_factory_;
   VideoEncoderConfig encoder_config_;
 };
 
@@ -2840,7 +2840,7 @@
       EXPECT_TRUE(Wait())
           << "Timed out while waiting for the encoder to be initialized.";
     }
-    test::EncoderProxyFactory encoder_factory_;
+    test::VideoEncoderProxyFactory encoder_factory_;
   } test;
 
   RunBaseTest(&test);
@@ -3020,7 +3020,7 @@
     int num_initializations_;
     webrtc::Call* call_;
     webrtc::VideoSendStream* send_stream_;
-    test::EncoderProxyFactory encoder_factory_;
+    test::VideoEncoderProxyFactory encoder_factory_;
     webrtc::VideoEncoderConfig encoder_config_;
   } test(&task_queue_);
 
@@ -3109,7 +3109,7 @@
     }
 
     VideoSendStream* send_stream_;
-    test::EncoderProxyFactory encoder_factory_;
+    test::VideoEncoderProxyFactory encoder_factory_;
   } test;
 
   RunBaseTest(&test);
@@ -3741,7 +3741,7 @@
 
    private:
     test::SingleThreadedTaskQueueForTesting* const task_queue_;
-    test::EncoderProxyFactory encoder_factory_;
+    test::VideoEncoderProxyFactory encoder_factory_;
     Call* call_;
     rtc::CriticalSection crit_;
     uint32_t max_bitrate_bps_ RTC_GUARDED_BY(&crit_);
diff --git a/video/video_stream_encoder_unittest.cc b/video/video_stream_encoder_unittest.cc
index b97d2cf..bcccbca 100644
--- a/video/video_stream_encoder_unittest.cc
+++ b/video/video_stream_encoder_unittest.cc
@@ -22,13 +22,13 @@
 #include "rtc_base/refcountedobject.h"
 #include "system_wrappers/include/metrics_default.h"
 #include "system_wrappers/include/sleep.h"
-#include "test/encoder_proxy_factory.h"
 #include "test/encoder_settings.h"
 #include "test/fake_encoder.h"
 #include "test/field_trial.h"
 #include "test/frame_generator.h"
 #include "test/gmock.h"
 #include "test/gtest.h"
+#include "test/video_encoder_proxy_factory.h"
 #include "video/send_statistics_proxy.h"
 #include "video/video_stream_encoder.h"
 
@@ -691,7 +691,7 @@
   int codec_height_;
   int max_framerate_;
   TestEncoder fake_encoder_;
-  test::EncoderProxyFactory encoder_factory_;
+  test::VideoEncoderProxyFactory encoder_factory_;
   std::unique_ptr<MockableSendStatisticsProxy> stats_proxy_;
   TestSink sink_;
   AdaptingFrameForwarder video_source_;