AudioDeviceWindowsCore: fix mutex recursion.

This change fixes a few cases of mutex recursions resulting in CHECK failures.

Bug: webrtc:11864
Change-Id: I392152e0aed88e100089a09e85504dd0abef62a1
Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/182083
Reviewed-by: Henrik Andreassson <henrika@webrtc.org>
Commit-Queue: Markus Handell <handellm@webrtc.org>
Cr-Commit-Position: refs/heads/master@{#31970}
diff --git a/modules/audio_device/win/audio_device_core_win.cc b/modules/audio_device/win/audio_device_core_win.cc
index 548158c..771dcfa 100644
--- a/modules/audio_device/win/audio_device_core_win.cc
+++ b/modules/audio_device/win/audio_device_core_win.cc
@@ -641,7 +641,10 @@
 
 int32_t AudioDeviceWindowsCore::InitSpeaker() {
   MutexLock lock(&mutex_);
+  return InitSpeakerLocked();
+}
 
+int32_t AudioDeviceWindowsCore::InitSpeakerLocked() {
   if (_playing) {
     return -1;
   }
@@ -651,7 +654,7 @@
   }
 
   if (_usingOutputDeviceIndex) {
-    int16_t nDevices = PlayoutDevices();
+    int16_t nDevices = PlayoutDevicesLocked();
     if (_outputDeviceIndex > (nDevices - 1)) {
       RTC_LOG(LS_ERROR) << "current device selection is invalid => unable to"
                            " initialize";
@@ -710,7 +713,10 @@
 
 int32_t AudioDeviceWindowsCore::InitMicrophone() {
   MutexLock lock(&mutex_);
+  return InitMicrophoneLocked();
+}
 
+int32_t AudioDeviceWindowsCore::InitMicrophoneLocked() {
   if (_recording) {
     return -1;
   }
@@ -720,7 +726,7 @@
   }
 
   if (_usingInputDeviceIndex) {
-    int16_t nDevices = RecordingDevices();
+    int16_t nDevices = RecordingDevicesLocked();
     if (_inputDeviceIndex > (nDevices - 1)) {
       RTC_LOG(LS_ERROR) << "current device selection is invalid => unable to"
                            " initialize";
@@ -1368,10 +1374,12 @@
 // ----------------------------------------------------------------------------
 //  PlayoutDevices
 // ----------------------------------------------------------------------------
-
 int16_t AudioDeviceWindowsCore::PlayoutDevices() {
   MutexLock lock(&mutex_);
+  return PlayoutDevicesLocked();
+}
 
+int16_t AudioDeviceWindowsCore::PlayoutDevicesLocked() {
   if (_RefreshDeviceList(eRender) != -1) {
     return (_DeviceListCount(eRender));
   }
@@ -1634,7 +1642,10 @@
 
 int16_t AudioDeviceWindowsCore::RecordingDevices() {
   MutexLock lock(&mutex_);
+  return RecordingDevicesLocked();
+}
 
+int16_t AudioDeviceWindowsCore::RecordingDevicesLocked() {
   if (_RefreshDeviceList(eCapture) != -1) {
     return (_DeviceListCount(eCapture));
   }
@@ -1800,7 +1811,7 @@
   }
 
   // Initialize the speaker (devices might have been added or removed)
-  if (InitSpeaker() == -1) {
+  if (InitSpeakerLocked() == -1) {
     RTC_LOG(LS_WARNING) << "InitSpeaker() failed";
   }
 
@@ -2119,7 +2130,7 @@
   }
 
   // Initialize the microphone (devices might have been added or removed)
-  if (InitMicrophone() == -1) {
+  if (InitMicrophoneLocked() == -1) {
     RTC_LOG(LS_WARNING) << "InitMicrophone() failed";
   }
 
diff --git a/modules/audio_device/win/audio_device_core_win.h b/modules/audio_device/win/audio_device_core_win.h
index 4803362..e3826f9 100644
--- a/modules/audio_device/win/audio_device_core_win.h
+++ b/modules/audio_device/win/audio_device_core_win.h
@@ -88,83 +88,94 @@
       AudioDeviceModule::AudioLayer& audioLayer) const;
 
   // Main initializaton and termination
-  virtual InitStatus Init();
-  virtual int32_t Terminate();
+  virtual InitStatus Init() RTC_LOCKS_EXCLUDED(mutex_);
+  virtual int32_t Terminate() RTC_LOCKS_EXCLUDED(mutex_);
   virtual bool Initialized() const;
 
   // Device enumeration
-  virtual int16_t PlayoutDevices();
-  virtual int16_t RecordingDevices();
+  virtual int16_t PlayoutDevices() RTC_LOCKS_EXCLUDED(mutex_);
+  virtual int16_t RecordingDevices() RTC_LOCKS_EXCLUDED(mutex_);
   virtual int32_t PlayoutDeviceName(uint16_t index,
                                     char name[kAdmMaxDeviceNameSize],
-                                    char guid[kAdmMaxGuidSize]);
+                                    char guid[kAdmMaxGuidSize])
+      RTC_LOCKS_EXCLUDED(mutex_);
   virtual int32_t RecordingDeviceName(uint16_t index,
                                       char name[kAdmMaxDeviceNameSize],
-                                      char guid[kAdmMaxGuidSize]);
+                                      char guid[kAdmMaxGuidSize])
+      RTC_LOCKS_EXCLUDED(mutex_);
 
   // Device selection
-  virtual int32_t SetPlayoutDevice(uint16_t index);
+  virtual int32_t SetPlayoutDevice(uint16_t index) RTC_LOCKS_EXCLUDED(mutex_);
   virtual int32_t SetPlayoutDevice(AudioDeviceModule::WindowsDeviceType device);
-  virtual int32_t SetRecordingDevice(uint16_t index);
+  virtual int32_t SetRecordingDevice(uint16_t index) RTC_LOCKS_EXCLUDED(mutex_);
   virtual int32_t SetRecordingDevice(
-      AudioDeviceModule::WindowsDeviceType device);
+      AudioDeviceModule::WindowsDeviceType device) RTC_LOCKS_EXCLUDED(mutex_);
 
   // Audio transport initialization
   virtual int32_t PlayoutIsAvailable(bool& available);
-  virtual int32_t InitPlayout();
+  virtual int32_t InitPlayout() RTC_LOCKS_EXCLUDED(mutex_);
   virtual bool PlayoutIsInitialized() const;
   virtual int32_t RecordingIsAvailable(bool& available);
-  virtual int32_t InitRecording();
+  virtual int32_t InitRecording() RTC_LOCKS_EXCLUDED(mutex_);
   virtual bool RecordingIsInitialized() const;
 
   // Audio transport control
-  virtual int32_t StartPlayout();
-  virtual int32_t StopPlayout();
+  virtual int32_t StartPlayout() RTC_LOCKS_EXCLUDED(mutex_);
+  virtual int32_t StopPlayout() RTC_LOCKS_EXCLUDED(mutex_);
   virtual bool Playing() const;
-  virtual int32_t StartRecording();
+  virtual int32_t StartRecording() RTC_LOCKS_EXCLUDED(mutex_);
   virtual int32_t StopRecording();
   virtual bool Recording() const;
 
   // Audio mixer initialization
-  virtual int32_t InitSpeaker();
+  virtual int32_t InitSpeaker() RTC_LOCKS_EXCLUDED(mutex_);
   virtual bool SpeakerIsInitialized() const;
-  virtual int32_t InitMicrophone();
+  virtual int32_t InitMicrophone() RTC_LOCKS_EXCLUDED(mutex_);
   virtual bool MicrophoneIsInitialized() const;
 
   // Speaker volume controls
-  virtual int32_t SpeakerVolumeIsAvailable(bool& available);
-  virtual int32_t SetSpeakerVolume(uint32_t volume);
-  virtual int32_t SpeakerVolume(uint32_t& volume) const;
+  virtual int32_t SpeakerVolumeIsAvailable(bool& available)
+      RTC_LOCKS_EXCLUDED(mutex_);
+  virtual int32_t SetSpeakerVolume(uint32_t volume) RTC_LOCKS_EXCLUDED(mutex_);
+  virtual int32_t SpeakerVolume(uint32_t& volume) const
+      RTC_LOCKS_EXCLUDED(mutex_);
   virtual int32_t MaxSpeakerVolume(uint32_t& maxVolume) const;
   virtual int32_t MinSpeakerVolume(uint32_t& minVolume) const;
 
   // Microphone volume controls
-  virtual int32_t MicrophoneVolumeIsAvailable(bool& available);
-  virtual int32_t SetMicrophoneVolume(uint32_t volume);
-  virtual int32_t MicrophoneVolume(uint32_t& volume) const;
+  virtual int32_t MicrophoneVolumeIsAvailable(bool& available)
+      RTC_LOCKS_EXCLUDED(mutex_);
+  virtual int32_t SetMicrophoneVolume(uint32_t volume)
+      RTC_LOCKS_EXCLUDED(mutex_, volume_mutex_);
+  virtual int32_t MicrophoneVolume(uint32_t& volume) const
+      RTC_LOCKS_EXCLUDED(mutex_, volume_mutex_);
   virtual int32_t MaxMicrophoneVolume(uint32_t& maxVolume) const;
   virtual int32_t MinMicrophoneVolume(uint32_t& minVolume) const;
 
   // Speaker mute control
-  virtual int32_t SpeakerMuteIsAvailable(bool& available);
-  virtual int32_t SetSpeakerMute(bool enable);
+  virtual int32_t SpeakerMuteIsAvailable(bool& available)
+      RTC_LOCKS_EXCLUDED(mutex_);
+  virtual int32_t SetSpeakerMute(bool enable) RTC_LOCKS_EXCLUDED(mutex_);
   virtual int32_t SpeakerMute(bool& enabled) const;
 
   // Microphone mute control
-  virtual int32_t MicrophoneMuteIsAvailable(bool& available);
+  virtual int32_t MicrophoneMuteIsAvailable(bool& available)
+      RTC_LOCKS_EXCLUDED(mutex_);
   virtual int32_t SetMicrophoneMute(bool enable);
   virtual int32_t MicrophoneMute(bool& enabled) const;
 
   // Stereo support
   virtual int32_t StereoPlayoutIsAvailable(bool& available);
-  virtual int32_t SetStereoPlayout(bool enable);
+  virtual int32_t SetStereoPlayout(bool enable) RTC_LOCKS_EXCLUDED(mutex_);
   virtual int32_t StereoPlayout(bool& enabled) const;
   virtual int32_t StereoRecordingIsAvailable(bool& available);
-  virtual int32_t SetStereoRecording(bool enable);
-  virtual int32_t StereoRecording(bool& enabled) const;
+  virtual int32_t SetStereoRecording(bool enable) RTC_LOCKS_EXCLUDED(mutex_);
+  virtual int32_t StereoRecording(bool& enabled) const
+      RTC_LOCKS_EXCLUDED(mutex_);
 
   // Delay information and control
-  virtual int32_t PlayoutDelay(uint16_t& delayMS) const;
+  virtual int32_t PlayoutDelay(uint16_t& delayMS) const
+      RTC_LOCKS_EXCLUDED(mutex_);
 
   virtual bool BuiltInAECIsAvailable() const;
 
@@ -184,13 +195,17 @@
   bool _winSupportAvrt;
 
  private:  // thread functions
+  int32_t InitSpeakerLocked() RTC_EXCLUSIVE_LOCKS_REQUIRED(mutex_);
+  int32_t InitMicrophoneLocked() RTC_EXCLUSIVE_LOCKS_REQUIRED(mutex_);
+  int16_t PlayoutDevicesLocked() RTC_EXCLUSIVE_LOCKS_REQUIRED(mutex_);
+  int16_t RecordingDevicesLocked() RTC_EXCLUSIVE_LOCKS_REQUIRED(mutex_);
   DWORD InitCaptureThreadPriority();
   void RevertCaptureThreadPriority();
   static DWORD WINAPI WSAPICaptureThread(LPVOID context);
   DWORD DoCaptureThread();
 
   static DWORD WINAPI WSAPICaptureThreadPollDMO(LPVOID context);
-  DWORD DoCaptureThreadPollDMO();
+  DWORD DoCaptureThreadPollDMO() RTC_LOCKS_EXCLUDED(mutex_);
 
   static DWORD WINAPI WSAPIRenderThread(LPVOID context);
   DWORD DoRenderThread();
@@ -238,7 +253,7 @@
   ScopedCOMInitializer _comInit;
   AudioDeviceBuffer* _ptrAudioBuffer;
   mutable Mutex mutex_;
-  mutable Mutex volume_mutex_;
+  mutable Mutex volume_mutex_ RTC_ACQUIRED_AFTER(mutex_);
 
   IMMDeviceEnumerator* _ptrEnumerator;
   IMMDeviceCollection* _ptrRenderCollection;