modules/desktop_capture/desktop_and_cursor_composer_unittest.cc

/*
 *  Copyright (c) 2013 The WebRTC project authors. All Rights Reserved.
 *
 *  Use of this source code is governed by a BSD-style license
 *  that can be found in the LICENSE file in the root of the source
 *  tree. An additional intellectual property rights grant can be found
 *  in the file PATENTS.  All contributing project authors may
 *  be found in the AUTHORS file in the root of the source tree.
 */

#include "modules/desktop_capture/desktop_and_cursor_composer.h"

#include <stdint.h>
#include <string.h>

#include <memory>
#include <utility>

#include "modules/desktop_capture/desktop_capturer.h"
#include "modules/desktop_capture/desktop_frame.h"
#include "modules/desktop_capture/mouse_cursor.h"
#include "modules/desktop_capture/shared_desktop_frame.h"
#include "rtc_base/arraysize.h"
#include "test/gtest.h"

namespace webrtc {

namespace {

const int kFrameXCoord = 100;
const int kFrameYCoord = 200;
const int kScreenWidth = 100;
const int kScreenHeight = 100;
const int kCursorWidth = 10;
const int kCursorHeight = 10;

const int kTestCursorSize = 3;
const uint32_t kTestCursorData[kTestCursorSize][kTestCursorSize] = {
    {
        0xffffffff,
        0x99990000,
        0xaa222222,
    },
    {
        0x88008800,
        0xaa0000aa,
        0xaa333333,
    },
    {
        0x00000000,
        0xaa0000aa,
        0xaa333333,
    },
};

uint32_t GetFakeFramePixelValue(const DesktopVector& p) {
  uint32_t r = 100 + p.x();
  uint32_t g = 100 + p.y();
  uint32_t b = 100 + p.x() + p.y();
  return b + (g << 8) + (r << 16) + 0xff000000;
}

uint32_t GetFramePixel(const DesktopFrame& frame, const DesktopVector& pos) {
  return *reinterpret_cast<uint32_t*>(frame.GetFrameDataAtPos(pos));
}

// Blends two pixel values taking into account alpha.
uint32_t BlendPixels(uint32_t dest, uint32_t src) {
  uint8_t alpha = 255 - ((src & 0xff000000) >> 24);
  uint32_t r =
      ((dest & 0x00ff0000) >> 16) * alpha / 255 + ((src & 0x00ff0000) >> 16);
  uint32_t g =
      ((dest & 0x0000ff00) >> 8) * alpha / 255 + ((src & 0x0000ff00) >> 8);
  uint32_t b = (dest & 0x000000ff) * alpha / 255 + (src & 0x000000ff);
  return b + (g << 8) + (r << 16) + 0xff000000;
}

DesktopFrame* CreateTestFrame() {
  DesktopFrame* frame =
      new BasicDesktopFrame(DesktopSize(kScreenWidth, kScreenHeight));
  uint32_t* data = reinterpret_cast<uint32_t*>(frame->data());
  for (int y = 0; y < kScreenHeight; ++y) {
    for (int x = 0; x < kScreenWidth; ++x) {
      *(data++) = GetFakeFramePixelValue(DesktopVector(x, y));
    }
  }
  return frame;
}

MouseCursor* CreateTestCursor(DesktopVector hotspot) {
  std::unique_ptr<DesktopFrame> image(
      new BasicDesktopFrame(DesktopSize(kCursorWidth, kCursorHeight)));
  uint32_t* data = reinterpret_cast<uint32_t*>(image->data());
  // Set four pixels near the hotspot and leave all other blank.
  for (int y = 0; y < kTestCursorSize; ++y) {
    for (int x = 0; x < kTestCursorSize; ++x) {
      data[(hotspot.y() + y) * kCursorWidth + (hotspot.x() + x)] =
          kTestCursorData[y][x];
    }
  }
  return new MouseCursor(image.release(), hotspot);
}

class FakeScreenCapturer : public DesktopCapturer {
 public:
  FakeScreenCapturer() {}

  void Start(Callback* callback) override { callback_ = callback; }

  void CaptureFrame() override {
    callback_->OnCaptureResult(
        next_frame_ ? Result::SUCCESS : Result::ERROR_TEMPORARY,
        std::move(next_frame_));
  }

  void SetNextFrame(std::unique_ptr<DesktopFrame> next_frame) {
    next_frame_ = std::move(next_frame);
  }

  bool IsOccluded(const DesktopVector& pos) override { return is_occluded_; }

  void set_is_occluded(bool value) { is_occluded_ = value; }

 private:
  Callback* callback_ = nullptr;

  std::unique_ptr<DesktopFrame> next_frame_;
  bool is_occluded_ = false;
};

class FakeMouseMonitor : public MouseCursorMonitor {
 public:
  FakeMouseMonitor() : changed_(true) {}

  void SetState(CursorState state, const DesktopVector& pos) {
    state_ = state;
    position_ = pos;
  }

  void SetHotspot(const DesktopVector& hotspot) {
    if (!hotspot_.equals(hotspot))
      changed_ = true;
    hotspot_ = hotspot;
  }

  void Init(Callback* callback, Mode mode) override { callback_ = callback; }

  void Capture() override {
    if (changed_) {
      callback_->OnMouseCursor(CreateTestCursor(hotspot_));
    }
    callback_->OnMouseCursorPosition(position_);
  }

 private:
  Callback* callback_;
  CursorState state_;
  DesktopVector position_;
  DesktopVector hotspot_;
  bool changed_;
};

void VerifyFrame(const DesktopFrame& frame,
                 MouseCursorMonitor::CursorState state,
                 const DesktopVector& pos) {
  // Verify that all other pixels are set to their original values.
  DesktopRect image_rect =
      DesktopRect::MakeWH(kTestCursorSize, kTestCursorSize);
  image_rect.Translate(pos);

  for (int y = 0; y < kScreenHeight; ++y) {
    for (int x = 0; x < kScreenWidth; ++x) {
      DesktopVector p(x, y);
      if (state == MouseCursorMonitor::INSIDE && image_rect.Contains(p)) {
        EXPECT_EQ(BlendPixels(GetFakeFramePixelValue(p),
                              kTestCursorData[y - pos.y()][x - pos.x()]),
                  GetFramePixel(frame, p));
      } else {
        EXPECT_EQ(GetFakeFramePixelValue(p), GetFramePixel(frame, p));
      }
    }
  }
}

}  // namespace

class DesktopAndCursorComposerTest : public ::testing::Test,
                                     public DesktopCapturer::Callback {
 public:
  DesktopAndCursorComposerTest(bool include_cursor = true)
      : fake_screen_(new FakeScreenCapturer()),
        fake_cursor_(include_cursor ? new FakeMouseMonitor() : nullptr),
        blender_(fake_screen_, fake_cursor_) {
    blender_.Start(this);
  }

  // DesktopCapturer::Callback interface
  void OnCaptureResult(DesktopCapturer::Result result,
                       std::unique_ptr<DesktopFrame> frame) override {
    frame_ = std::move(frame);
  }

 protected:
  // Owned by `blender_`.
  FakeScreenCapturer* fake_screen_;
  FakeMouseMonitor* fake_cursor_;

  DesktopAndCursorComposer blender_;
  std::unique_ptr<DesktopFrame> frame_;
};

class DesktopAndCursorComposerNoCursorMonitorTest
    : public DesktopAndCursorComposerTest {
 public:
  DesktopAndCursorComposerNoCursorMonitorTest()
      : DesktopAndCursorComposerTest(false) {}
};

TEST_F(DesktopAndCursorComposerTest, CursorShouldBeIgnoredIfNoFrameCaptured) {
  struct {
    int x, y;
    int hotspot_x, hotspot_y;
    bool inside;
  } tests[] = {
      {0, 0, 0, 0, true},    {50, 50, 0, 0, true},   {100, 50, 0, 0, true},
      {50, 100, 0, 0, true}, {100, 100, 0, 0, true}, {0, 0, 2, 5, true},
      {1, 1, 2, 5, true},    {50, 50, 2, 5, true},   {100, 100, 2, 5, true},
      {0, 0, 5, 2, true},    {50, 50, 5, 2, true},   {100, 100, 5, 2, true},
      {0, 0, 0, 0, false},
  };

  for (size_t i = 0; i < arraysize(tests); i++) {
    SCOPED_TRACE(i);

    DesktopVector hotspot(tests[i].hotspot_x, tests[i].hotspot_y);
    fake_cursor_->SetHotspot(hotspot);

    MouseCursorMonitor::CursorState state = tests[i].inside
                                                ? MouseCursorMonitor::INSIDE
                                                : MouseCursorMonitor::OUTSIDE;
    DesktopVector pos(tests[i].x, tests[i].y);
    fake_cursor_->SetState(state, pos);

    std::unique_ptr<SharedDesktopFrame> frame(
        SharedDesktopFrame::Wrap(CreateTestFrame()));

    blender_.CaptureFrame();
    // If capturer captured nothing, then cursor should be ignored, not matter
    // its state or position.
    EXPECT_EQ(frame_, nullptr);
  }
}

TEST_F(DesktopAndCursorComposerTest, CursorShouldBeIgnoredIfFrameMayContainIt) {
  // We can't use a shared frame because we need to detect modifications
  // compared to a control.
  std::unique_ptr<DesktopFrame> control_frame(CreateTestFrame());
  control_frame->set_top_left(DesktopVector(kFrameXCoord, kFrameYCoord));

  struct {
    int x;
    int y;
    bool may_contain_cursor;
  } tests[] = {
      {100, 200, true},
      {100, 200, false},
      {150, 250, true},
      {150, 250, false},
  };

  for (size_t i = 0; i < arraysize(tests); i++) {
    SCOPED_TRACE(i);

    std::unique_ptr<DesktopFrame> frame(CreateTestFrame());
    frame->set_top_left(DesktopVector(kFrameXCoord, kFrameYCoord));
    frame->set_may_contain_cursor(tests[i].may_contain_cursor);
    fake_screen_->SetNextFrame(std::move(frame));

    const DesktopVector abs_pos(tests[i].x, tests[i].y);
    fake_cursor_->SetState(MouseCursorMonitor::INSIDE, abs_pos);
    blender_.CaptureFrame();

    // If the frame may already have contained the cursor, then `CaptureFrame()`
    // should not have modified it, so it should be the same as the control.
    EXPECT_TRUE(frame_);
    const DesktopVector rel_pos(abs_pos.subtract(control_frame->top_left()));
    if (tests[i].may_contain_cursor) {
      EXPECT_EQ(
          *reinterpret_cast<uint32_t*>(frame_->GetFrameDataAtPos(rel_pos)),
          *reinterpret_cast<uint32_t*>(
              control_frame->GetFrameDataAtPos(rel_pos)));

    } else {
      // `CaptureFrame()` should have modified the frame to have the cursor.
      EXPECT_NE(
          *reinterpret_cast<uint32_t*>(frame_->GetFrameDataAtPos(rel_pos)),
          *reinterpret_cast<uint32_t*>(
              control_frame->GetFrameDataAtPos(rel_pos)));
      EXPECT_TRUE(frame_->may_contain_cursor());
    }
  }
}

TEST_F(DesktopAndCursorComposerTest,
       CursorShouldBeIgnoredIfItIsOutOfDesktopFrame) {
  std::unique_ptr<SharedDesktopFrame> frame(
      SharedDesktopFrame::Wrap(CreateTestFrame()));
  frame->set_top_left(DesktopVector(kFrameXCoord, kFrameYCoord));
  // The frame covers (100, 200) - (200, 300).

  struct {
    int x;
    int y;
  } tests[] = {
      {0, 0},    {50, 50},         {50, 150},      {100, 150}, {50, 200},
      {99, 200}, {100, 199},       {200, 300},     {200, 299}, {199, 300},
      {-1, -1},  {-10000, -10000}, {10000, 10000},
  };
  for (size_t i = 0; i < arraysize(tests); i++) {
    SCOPED_TRACE(i);

    fake_screen_->SetNextFrame(frame->Share());
    // The CursorState is ignored when using absolute cursor position.
    fake_cursor_->SetState(MouseCursorMonitor::OUTSIDE,
                           DesktopVector(tests[i].x, tests[i].y));
    blender_.CaptureFrame();
    VerifyFrame(*frame_, MouseCursorMonitor::OUTSIDE, DesktopVector(0, 0));
  }
}

TEST_F(DesktopAndCursorComposerTest, IsOccludedShouldBeConsidered) {
  std::unique_ptr<SharedDesktopFrame> frame(
      SharedDesktopFrame::Wrap(CreateTestFrame()));
  frame->set_top_left(DesktopVector(kFrameXCoord, kFrameYCoord));
  // The frame covers (100, 200) - (200, 300).

  struct {
    int x;
    int y;
  } tests[] = {
      {100, 200}, {101, 200}, {100, 201}, {101, 201}, {150, 250}, {199, 299},
  };
  fake_screen_->set_is_occluded(true);
  for (size_t i = 0; i < arraysize(tests); i++) {
    SCOPED_TRACE(i);

    fake_screen_->SetNextFrame(frame->Share());
    // The CursorState is ignored when using absolute cursor position.
    fake_cursor_->SetState(MouseCursorMonitor::OUTSIDE,
                           DesktopVector(tests[i].x, tests[i].y));
    blender_.CaptureFrame();
    VerifyFrame(*frame_, MouseCursorMonitor::OUTSIDE, DesktopVector());
  }
}

TEST_F(DesktopAndCursorComposerTest, CursorIncluded) {
  std::unique_ptr<SharedDesktopFrame> frame(
      SharedDesktopFrame::Wrap(CreateTestFrame()));
  frame->set_top_left(DesktopVector(kFrameXCoord, kFrameYCoord));
  // The frame covers (100, 200) - (200, 300).

  struct {
    int x;
    int y;
  } tests[] = {
      {100, 200}, {101, 200}, {100, 201}, {101, 201}, {150, 250}, {199, 299},
  };
  for (size_t i = 0; i < arraysize(tests); i++) {
    SCOPED_TRACE(i);

    const DesktopVector abs_pos(tests[i].x, tests[i].y);
    const DesktopVector rel_pos(abs_pos.subtract(frame->top_left()));

    fake_screen_->SetNextFrame(frame->Share());
    // The CursorState is ignored when using absolute cursor position.
    fake_cursor_->SetState(MouseCursorMonitor::OUTSIDE, abs_pos);
    blender_.CaptureFrame();
    VerifyFrame(*frame_, MouseCursorMonitor::INSIDE, rel_pos);

    // Verify that the cursor is erased before the frame buffer is returned to
    // the screen capturer.
    frame_.reset();
    VerifyFrame(*frame, MouseCursorMonitor::OUTSIDE, DesktopVector());
  }
}

TEST_F(DesktopAndCursorComposerNoCursorMonitorTest,
       UpdatedRegionIncludesOldAndNewCursorRectsIfMoved) {
  std::unique_ptr<SharedDesktopFrame> frame(
      SharedDesktopFrame::Wrap(CreateTestFrame()));
  DesktopRect first_cursor_rect;
  {
    // Block to scope test_cursor, which is invalidated by OnMouseCursor.
    MouseCursor* test_cursor = CreateTestCursor(DesktopVector(0, 0));
    first_cursor_rect = DesktopRect::MakeSize(test_cursor->image()->size());
    blender_.OnMouseCursor(test_cursor);
  }
  blender_.OnMouseCursorPosition(DesktopVector(0, 0));
  fake_screen_->SetNextFrame(frame->Share());
  blender_.CaptureFrame();

  DesktopVector cursor_move_offset(1, 1);
  DesktopRect second_cursor_rect = first_cursor_rect;
  second_cursor_rect.Translate(cursor_move_offset);
  blender_.OnMouseCursorPosition(cursor_move_offset);
  fake_screen_->SetNextFrame(frame->Share());
  blender_.CaptureFrame();

  EXPECT_TRUE(frame->updated_region().is_empty());
  DesktopRegion expected_region;
  expected_region.AddRect(first_cursor_rect);
  expected_region.AddRect(second_cursor_rect);
  EXPECT_TRUE(frame_->updated_region().Equals(expected_region));
}

TEST_F(DesktopAndCursorComposerNoCursorMonitorTest,
       UpdatedRegionIncludesOldAndNewCursorRectsIfShapeChanged) {
  std::unique_ptr<SharedDesktopFrame> frame(
      SharedDesktopFrame::Wrap(CreateTestFrame()));
  DesktopRect first_cursor_rect;
  {
    // Block to scope test_cursor, which is invalidated by OnMouseCursor.
    MouseCursor* test_cursor = CreateTestCursor(DesktopVector(0, 0));
    first_cursor_rect = DesktopRect::MakeSize(test_cursor->image()->size());
    blender_.OnMouseCursor(test_cursor);
  }
  blender_.OnMouseCursorPosition(DesktopVector(0, 0));
  fake_screen_->SetNextFrame(frame->Share());
  blender_.CaptureFrame();

  // Create a second cursor, the same shape as the first. Since the code doesn't
  // compare the cursor pixels, this is sufficient, and avoids needing two test
  // cursor bitmaps.
  DesktopRect second_cursor_rect;
  {
    MouseCursor* test_cursor = CreateTestCursor(DesktopVector(0, 0));
    second_cursor_rect = DesktopRect::MakeSize(test_cursor->image()->size());
    blender_.OnMouseCursor(test_cursor);
  }
  fake_screen_->SetNextFrame(frame->Share());
  blender_.CaptureFrame();

  EXPECT_TRUE(frame->updated_region().is_empty());
  DesktopRegion expected_region;
  expected_region.AddRect(first_cursor_rect);
  expected_region.AddRect(second_cursor_rect);
  EXPECT_TRUE(frame_->updated_region().Equals(expected_region));
}

TEST_F(DesktopAndCursorComposerNoCursorMonitorTest,
       UpdatedRegionUnchangedIfCursorUnchanged) {
  std::unique_ptr<SharedDesktopFrame> frame(
      SharedDesktopFrame::Wrap(CreateTestFrame()));
  blender_.OnMouseCursor(CreateTestCursor(DesktopVector(0, 0)));
  blender_.OnMouseCursorPosition(DesktopVector(0, 0));
  fake_screen_->SetNextFrame(frame->Share());
  blender_.CaptureFrame();
  fake_screen_->SetNextFrame(frame->Share());
  blender_.CaptureFrame();

  EXPECT_TRUE(frame->updated_region().is_empty());
  EXPECT_TRUE(frame_->updated_region().is_empty());
}

}  // namespace webrtc