diff --git a/video_engine/stream_synchronization.cc b/video_engine/stream_synchronization.cc
index 6ad579c..9490d10 100644
--- a/video_engine/stream_synchronization.cc
+++ b/video_engine/stream_synchronization.cc
@@ -29,12 +29,14 @@
     extra_video_delay_ms = 0;
     last_video_delay_ms = 0;
     extra_audio_delay_ms = 0;
+    last_audio_delay_ms = 0;
     network_delay = 120;
   }
 
   int extra_video_delay_ms;
   int last_video_delay_ms;
   int extra_audio_delay_ms;
+  int last_audio_delay_ms;
   int network_delay;
 };
 
@@ -87,9 +89,9 @@
 
 bool StreamSynchronization::ComputeDelays(int relative_delay_ms,
                                           int current_audio_delay_ms,
-                                          int* extra_audio_delay_ms,
+                                          int* total_audio_delay_target_ms,
                                           int* total_video_delay_target_ms) {
-  assert(extra_audio_delay_ms && total_video_delay_target_ms);
+  assert(total_audio_delay_target_ms && total_video_delay_target_ms);
 
   int current_video_delay_ms = *total_video_delay_target_ms;
   WEBRTC_TRACE(webrtc::kTraceInfo, webrtc::kTraceVideo, video_channel_id_,
@@ -173,17 +175,26 @@
   new_video_delay_ms =
       std::min(new_video_delay_ms, base_target_delay_ms_ + kMaxDeltaDelayMs);
 
-  // Make sure that audio is never below our target.
-  channel_delay_->extra_audio_delay_ms =
-      std::max(base_target_delay_ms_, channel_delay_->extra_audio_delay_ms);
+  int new_audio_delay_ms;
+  if (channel_delay_->extra_audio_delay_ms > base_target_delay_ms_) {
+    new_audio_delay_ms = channel_delay_->extra_audio_delay_ms;
+  } else {
+    // No change to the audio delay. We are changing video and we only
+    // allow to change one at the time.
+    new_audio_delay_ms = channel_delay_->last_audio_delay_ms;
+  }
+
+  // Make sure that we don't go below the extra audio delay.
+  new_audio_delay_ms = std::max(
+      new_audio_delay_ms, channel_delay_->extra_audio_delay_ms);
 
   // Verify we don't go above the maximum allowed audio delay.
-  channel_delay_->extra_audio_delay_ms = std::min(
-      channel_delay_->extra_audio_delay_ms,
-      base_target_delay_ms_ + kMaxDeltaDelayMs);
+  new_audio_delay_ms =
+      std::min(new_audio_delay_ms, base_target_delay_ms_ + kMaxDeltaDelayMs);
 
-  // Remember our last video delay.
+  // Remember our last audio and video delays.
   channel_delay_->last_video_delay_ms = new_video_delay_ms;
+  channel_delay_->last_audio_delay_ms = new_audio_delay_ms;
 
   WEBRTC_TRACE(webrtc::kTraceInfo, webrtc::kTraceVideo, video_channel_id_,
       "Sync video delay %d ms for video channel and audio delay %d for audio "
@@ -192,8 +203,8 @@
       audio_channel_id_);
 
   // Return values.
-  *extra_audio_delay_ms = channel_delay_->extra_audio_delay_ms;
   *total_video_delay_target_ms = new_video_delay_ms;
+  *total_audio_delay_target_ms = new_audio_delay_ms;
   return true;
 }
 
@@ -201,6 +212,8 @@
   // Initial extra delay for audio (accounting for existing extra delay).
   channel_delay_->extra_audio_delay_ms +=
       target_delay_ms - base_target_delay_ms_;
+  channel_delay_->last_audio_delay_ms +=
+      target_delay_ms - base_target_delay_ms_;
 
   // The video delay is compared to the last value (and how much we can update
   // is limited by that as well).