blob: eb868d9877c0fc664b3f7fc0a042c154f74f90b6 [file] [log] [blame]
/*
* Copyright (c) 2010 The WebRTC project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
// Declaration of abstract class VideoCapturer
#ifndef WEBRTC_MEDIA_BASE_VIDEOCAPTURER_H_
#define WEBRTC_MEDIA_BASE_VIDEOCAPTURER_H_
#include <algorithm>
#include <memory>
#include <string>
#include <vector>
#include "webrtc/base/basictypes.h"
#include "webrtc/base/constructormagic.h"
#include "webrtc/base/criticalsection.h"
#include "webrtc/media/base/videosourceinterface.h"
#include "webrtc/base/sigslot.h"
#include "webrtc/base/thread_checker.h"
#include "webrtc/base/timestampaligner.h"
#include "webrtc/media/base/videoadapter.h"
#include "webrtc/media/base/videobroadcaster.h"
#include "webrtc/media/base/videocommon.h"
#include "webrtc/media/base/videoframefactory.h"
namespace cricket {
// Current state of the capturer.
enum CaptureState {
CS_STOPPED, // The capturer has been stopped or hasn't started yet.
CS_STARTING, // The capturer is in the process of starting. Note, it may
// still fail to start.
CS_RUNNING, // The capturer has been started successfully and is now
// capturing.
CS_FAILED, // The capturer failed to start.
};
class VideoFrame;
struct CapturedFrame {
static const uint32_t kFrameHeaderSize = 40; // Size from width to data_size.
static const uint32_t kUnknownDataSize = 0xFFFFFFFF;
CapturedFrame();
// Get the number of bytes of the frame data. If data_size is known, return
// it directly. Otherwise, calculate the size based on width, height, and
// fourcc. Return true if succeeded.
bool GetDataSize(uint32_t* size) const;
// The width and height of the captured frame could be different from those
// of VideoFormat. Once the first frame is captured, the width, height,
// fourcc, pixel_width, and pixel_height should keep the same over frames.
int width; // in number of pixels
int height; // in number of pixels
uint32_t fourcc; // compression
uint32_t pixel_width; // width of a pixel, default is 1
uint32_t pixel_height; // height of a pixel, default is 1
int64_t time_stamp; // timestamp of when the frame was captured, in unix
// time with nanosecond units.
uint32_t data_size; // number of bytes of the frame data
webrtc::VideoRotation rotation; // rotation in degrees of the frame.
void* data; // pointer to the frame data. This object allocates the
// memory or points to an existing memory.
private:
RTC_DISALLOW_COPY_AND_ASSIGN(CapturedFrame);
};
// VideoCapturer is an abstract class that defines the interfaces for video
// capturing. The subclasses implement the video capturer for various types of
// capturers and various platforms.
//
// The captured frames may need to be adapted (for example, cropping).
// Video adaptation is built into and enabled by default. After a frame has
// been captured from the device, it is sent to the video adapter, then out to
// the sinks.
//
// Programming model:
// Create an object of a subclass of VideoCapturer
// Initialize
// SignalStateChange.connect()
// AddOrUpdateSink()
// Find the capture format for Start() by either calling GetSupportedFormats()
// and selecting one of the supported or calling GetBestCaptureFormat().
// video_adapter()->OnOutputFormatRequest(desired_encoding_format)
// Start()
// GetCaptureFormat() optionally
// Stop()
//
// Assumption:
// The Start() and Stop() methods are called by a single thread (E.g., the
// media engine thread). Hence, the VideoCapture subclasses dont need to be
// thread safe.
//
class VideoCapturer : public sigslot::has_slots<>,
public rtc::VideoSourceInterface<cricket::VideoFrame> {
public:
VideoCapturer();
virtual ~VideoCapturer() {}
// Gets the id of the underlying device, which is available after the capturer
// is initialized. Can be used to determine if two capturers reference the
// same device.
const std::string& GetId() const { return id_; }
// Get the capture formats supported by the video capturer. The supported
// formats are non empty after the device has been opened successfully.
const std::vector<VideoFormat>* GetSupportedFormats() const;
// Get the best capture format for the desired format. The best format is the
// same as one of the supported formats except that the frame interval may be
// different. If the application asks for 16x9 and the camera does not support
// 16x9 HD or the application asks for 16x10, we find the closest 4x3 and then
// crop; Otherwise, we find what the application asks for. Note that we assume
// that for HD, the desired format is always 16x9. The subclasses can override
// the default implementation.
// Parameters
// desired: the input desired format. If desired.fourcc is not kAnyFourcc,
// the best capture format has the exactly same fourcc. Otherwise,
// the best capture format uses a fourcc in GetPreferredFourccs().
// best_format: the output of the best capture format.
// Return false if there is no such a best format, that is, the desired format
// is not supported.
virtual bool GetBestCaptureFormat(const VideoFormat& desired,
VideoFormat* best_format);
// TODO(hellner): deprecate (make private) the Start API in favor of this one.
// Also remove CS_STARTING as it is implied by the return
// value of StartCapturing().
bool StartCapturing(const VideoFormat& capture_format);
// Start the video capturer with the specified capture format.
// Parameter
// capture_format: The caller got this parameter by either calling
// GetSupportedFormats() and selecting one of the supported
// or calling GetBestCaptureFormat().
// Return
// CS_STARTING: The capturer is trying to start. Success or failure will
// be notified via the |SignalStateChange| callback.
// CS_RUNNING: if the capturer is started and capturing.
// CS_FAILED: if the capturer failes to start..
// CS_NO_DEVICE: if the capturer has no device and fails to start.
virtual CaptureState Start(const VideoFormat& capture_format) = 0;
// Get the current capture format, which is set by the Start() call.
// Note that the width and height of the captured frames may differ from the
// capture format. For example, the capture format is HD but the captured
// frames may be smaller than HD.
const VideoFormat* GetCaptureFormat() const {
return capture_format_.get();
}
// Stop the video capturer.
virtual void Stop() = 0;
// Check if the video capturer is running.
virtual bool IsRunning() = 0;
CaptureState capture_state() const {
return capture_state_;
}
virtual bool apply_rotation() { return apply_rotation_; }
// Returns true if the capturer is screencasting. This can be used to
// implement screencast specific behavior.
virtual bool IsScreencast() const = 0;
// Indicates that the encoder should denoise video before encoding
// it, wired up to VideoCapturerTrackSource::needs_denoising. If it
// is not set, the default configuration is used which is different
// depending on video codec.
// TODO(nisse): This is a workaround needed to fix
// https://bugs.chromium.org/p/chromium/issues/detail?id=645907.
// Chrome should migrate to implement VideoTrackSourceInterface
// directly, and then this method is no longer needed.
virtual rtc::Optional<bool> NeedsDenoising() const {
return rtc::Optional<bool>();
}
// Caps the VideoCapturer's format according to max_format. It can e.g. be
// used to prevent cameras from capturing at a resolution or framerate that
// the capturer is capable of but not performing satisfactorily at.
// The capping is an upper bound for each component of the capturing format.
// The fourcc component is ignored.
void ConstrainSupportedFormats(const VideoFormat& max_format);
void set_enable_camera_list(bool enable_camera_list) {
enable_camera_list_ = enable_camera_list;
}
bool enable_camera_list() {
return enable_camera_list_;
}
// Signal all capture state changes that are not a direct result of calling
// Start().
sigslot::signal2<VideoCapturer*, CaptureState> SignalStateChange;
// Frame callbacks are multithreaded to allow disconnect and connect to be
// called concurrently. It also ensures that it is safe to call disconnect
// at any time which is needed since the signal may be called from an
// unmarshalled thread owned by the VideoCapturer.
// Signal the captured frame to downstream.
sigslot::signal2<VideoCapturer*, const CapturedFrame*,
sigslot::multi_threaded_local> SignalFrameCaptured;
// If true, run video adaptation. By default, video adaptation is enabled
// and users must call video_adapter()->OnOutputFormatRequest()
// to receive frames.
bool enable_video_adapter() const { return enable_video_adapter_; }
void set_enable_video_adapter(bool enable_video_adapter) {
enable_video_adapter_ = enable_video_adapter;
}
// Takes ownership.
void set_frame_factory(VideoFrameFactory* frame_factory);
bool GetInputSize(int* width, int* height);
// Implements VideoSourceInterface
void AddOrUpdateSink(rtc::VideoSinkInterface<cricket::VideoFrame>* sink,
const rtc::VideoSinkWants& wants) override;
void RemoveSink(rtc::VideoSinkInterface<cricket::VideoFrame>* sink) override;
protected:
// OnSinkWantsChanged can be overridden to change the default behavior
// when a sink changes its VideoSinkWants by calling AddOrUpdateSink.
virtual void OnSinkWantsChanged(const rtc::VideoSinkWants& wants);
// Reports the appropriate frame size after adaptation. Returns true
// if a frame is wanted. Returns false if there are no interested
// sinks, or if the VideoAdapter decides to drop the frame.
// This function also implements timestamp translation/filtering.
// |camera_time_ns| is the camera's timestamp for the captured
// frame; it is expected to have good accuracy, but it may use an
// arbitrary epoch and a small possibly free-running with a frequency
// slightly different from the system clock. |system_time_us| is the
// monotonic system time (in the same scale as rtc::TimeMicros) when
// the frame was captured; the application is expected to read the
// system time as soon as possible after frame capture, but it may
// suffer scheduling jitter or poor system clock resolution. The
// output |translated_camera_time_us| is a combined timestamp,
// taking advantage of the supposedly higher accuracy in the camera
// timestamp, but using the same epoch and frequency as system time.
bool AdaptFrame(int width,
int height,
int64_t camera_time_us,
int64_t system_time_us,
int* out_width,
int* out_height,
int* crop_width,
int* crop_height,
int* crop_x,
int* crop_y,
int64_t* translated_camera_time_us);
// Callback attached to SignalFrameCaptured where SignalVideoFrames is called.
void OnFrameCaptured(VideoCapturer* video_capturer,
const CapturedFrame* captured_frame);
// Called when a frame has been captured and converted to a
// VideoFrame. OnFrame can be called directly by an implementation
// that does not use SignalFrameCaptured or OnFrameCaptured. The
// orig_width and orig_height are used only to produce stats.
void OnFrame(const VideoFrame& frame, int orig_width, int orig_height);
VideoAdapter* video_adapter() { return &video_adapter_; }
void SetCaptureState(CaptureState state);
// subclasses override this virtual method to provide a vector of fourccs, in
// order of preference, that are expected by the media engine.
virtual bool GetPreferredFourccs(std::vector<uint32_t>* fourccs) = 0;
// mutators to set private attributes
void SetId(const std::string& id) {
id_ = id;
}
void SetCaptureFormat(const VideoFormat* format) {
capture_format_.reset(format ? new VideoFormat(*format) : NULL);
}
void SetSupportedFormats(const std::vector<VideoFormat>& formats);
VideoFrameFactory* frame_factory() { return frame_factory_.get(); }
private:
void Construct();
// Get the distance between the desired format and the supported format.
// Return the max distance if they mismatch. See the implementation for
// details.
int64_t GetFormatDistance(const VideoFormat& desired,
const VideoFormat& supported);
// Convert captured frame to readable string for LOG messages.
std::string ToString(const CapturedFrame* frame) const;
// Updates filtered_supported_formats_ so that it contains the formats in
// supported_formats_ that fulfill all applied restrictions.
void UpdateFilteredSupportedFormats();
// Returns true if format doesn't fulfill all applied restrictions.
bool ShouldFilterFormat(const VideoFormat& format) const;
void UpdateInputSize(int width, int height);
rtc::ThreadChecker thread_checker_;
std::string id_;
CaptureState capture_state_;
std::unique_ptr<VideoFrameFactory> frame_factory_;
std::unique_ptr<VideoFormat> capture_format_;
std::vector<VideoFormat> supported_formats_;
std::unique_ptr<VideoFormat> max_format_;
std::vector<VideoFormat> filtered_supported_formats_;
bool enable_camera_list_;
int scaled_width_; // Current output size from ComputeScale.
int scaled_height_;
rtc::VideoBroadcaster broadcaster_;
bool enable_video_adapter_;
VideoAdapter video_adapter_;
rtc::CriticalSection frame_stats_crit_;
// The captured frame size before potential adapation.
bool input_size_valid_ GUARDED_BY(frame_stats_crit_) = false;
int input_width_ GUARDED_BY(frame_stats_crit_);
int input_height_ GUARDED_BY(frame_stats_crit_);
// Whether capturer should apply rotation to the frame before signaling it.
bool apply_rotation_;
// State for the timestamp translation.
rtc::TimestampAligner timestamp_aligner_;
RTC_DISALLOW_COPY_AND_ASSIGN(VideoCapturer);
};
} // namespace cricket
#endif // WEBRTC_MEDIA_BASE_VIDEOCAPTURER_H_