Drain audio processors on reconfiguration

Previously we would get a new AudioTrack and flush all audio processors if any
AudioProcessor needed to be flushed on reconfiguration. This was problematic for
the case of TrimmingAudioProcessor because it could become active or inactive
due to transitioning to a period with gapless metadata or without it (we don't
keep it active all the time because it is wasteful to populate its end buffer
for content that is not gapless).

This change handles the case where we don't need an AudioTrack but
AudioProcessors do need to be flushed. In this case we drain all the audio
processors when next handling data then switch to the new configuration.

This avoids truncation when period transitions change whether
TrimmingAudioProcessor is active but don't require a new AudioTrack, and is also
a step towards draining the AudioTrack when transitioning between periods if we
do need a new AudioTrack.

To do this, it needs to be possible to drain any pending output data from an
AudioProcessor after it's configured to a new format, so this change makes sure
AudioProcessors allow calling playToEndOfStream and getOutput after
reconfiguration and before flush.

PiperOrigin-RevId: 234033552

RELEASENOTES.md

@@ -37,6 +37,12 @@
   * Replace DownloadState.action with DownloadAction fields.
 * DRM: Fix black flicker when keys rotate in DRM protected content
   ([#3561](https://github.com/google/ExoPlayer/issues/3561)).
+* Audio:
+  * Allow `AudioProcessor`s to be drained of pending output after they are
+    reconfigured.
+  * Fix an issue that caused audio to be truncated at the end of a period
+    when switching to a new period where gapless playback information was newly
+    present or newly absent.
 * Add support for SHOUTcast ICY metadata
   ([#3735](https://github.com/google/ExoPlayer/issues/3735)).
 * CEA-608: Improved conformance to the specification

extensions/gvr/src/main/java/com/google/android/exoplayer2/ext/gvr/GvrAudioProcessor.java

@@ -38,9 +38,11 @@ public final class GvrAudioProcessor implements AudioProcessor {
   private static final int FRAMES_PER_OUTPUT_BUFFER = 1024;
   private static final int OUTPUT_CHANNEL_COUNT = 2;
   private static final int OUTPUT_FRAME_SIZE = OUTPUT_CHANNEL_COUNT * 2; // 16-bit stereo output.
+  private static final int NO_SURROUND_FORMAT = GvrAudioSurround.SurroundFormat.INVALID;
 
   private int sampleRateHz;
   private int channelCount;
+  private int pendingGvrAudioSurroundFormat;
   @Nullable private GvrAudioSurround gvrAudioSurround;
   private ByteBuffer buffer;
   private boolean inputEnded;
@@ -57,6 +59,7 @@ public GvrAudioProcessor() {
     sampleRateHz = Format.NO_VALUE;
     channelCount = Format.NO_VALUE;
     buffer = EMPTY_BUFFER;
+    pendingGvrAudioSurroundFormat = NO_SURROUND_FORMAT;
   }
 
   /**
@@ -92,33 +95,28 @@ public synchronized boolean configure(
     }
     this.sampleRateHz = sampleRateHz;
     this.channelCount = channelCount;
-    maybeReleaseGvrAudioSurround();
-    int surroundFormat;
     switch (channelCount) {
       case 1:
-        surroundFormat = GvrAudioSurround.SurroundFormat.SURROUND_MONO;
+        pendingGvrAudioSurroundFormat = GvrAudioSurround.SurroundFormat.SURROUND_MONO;
         break;
       case 2:
-        surroundFormat = GvrAudioSurround.SurroundFormat.SURROUND_STEREO;
+        pendingGvrAudioSurroundFormat = GvrAudioSurround.SurroundFormat.SURROUND_STEREO;
         break;
       case 4:
-        surroundFormat = GvrAudioSurround.SurroundFormat.FIRST_ORDER_AMBISONICS;
+        pendingGvrAudioSurroundFormat = GvrAudioSurround.SurroundFormat.FIRST_ORDER_AMBISONICS;
         break;
       case 6:
-        surroundFormat = GvrAudioSurround.SurroundFormat.SURROUND_FIVE_DOT_ONE;
+        pendingGvrAudioSurroundFormat = GvrAudioSurround.SurroundFormat.SURROUND_FIVE_DOT_ONE;
         break;
       case 9:
-        surroundFormat = GvrAudioSurround.SurroundFormat.SECOND_ORDER_AMBISONICS;
+        pendingGvrAudioSurroundFormat = GvrAudioSurround.SurroundFormat.SECOND_ORDER_AMBISONICS;
         break;
       case 16:
-        surroundFormat = GvrAudioSurround.SurroundFormat.THIRD_ORDER_AMBISONICS;
+        pendingGvrAudioSurroundFormat = GvrAudioSurround.SurroundFormat.THIRD_ORDER_AMBISONICS;
         break;
       default:
         throw new UnhandledFormatException(sampleRateHz, channelCount, encoding);
     }
-    gvrAudioSurround = new GvrAudioSurround(surroundFormat, sampleRateHz, channelCount,
-        FRAMES_PER_OUTPUT_BUFFER);
-    gvrAudioSurround.updateNativeOrientation(w, x, y, z);
     if (buffer == EMPTY_BUFFER) {
       buffer = ByteBuffer.allocateDirect(FRAMES_PER_OUTPUT_BUFFER * OUTPUT_FRAME_SIZE)
           .order(ByteOrder.nativeOrder());
@@ -128,7 +126,7 @@ public synchronized boolean configure(
 
   @Override
   public boolean isActive() {
-    return gvrAudioSurround != null;
+    return pendingGvrAudioSurroundFormat != NO_SURROUND_FORMAT || gvrAudioSurround != null;
   }
 
   @Override
@@ -156,28 +154,38 @@ public void queueInput(ByteBuffer input) {
 
   @Override
   public void queueEndOfStream() {
-    Assertions.checkNotNull(gvrAudioSurround);
+    if (gvrAudioSurround != null) {
+      gvrAudioSurround.triggerProcessing();
+    }
     inputEnded = true;
-    gvrAudioSurround.triggerProcessing();
   }
 
   @Override
   public ByteBuffer getOutput() {
-    Assertions.checkNotNull(gvrAudioSurround);
+    if (gvrAudioSurround == null) {
+      return EMPTY_BUFFER;
+    }
     int writtenBytes = gvrAudioSurround.getOutput(buffer, 0, buffer.capacity());
     buffer.position(0).limit(writtenBytes);
     return buffer;
   }
 
   @Override
   public boolean isEnded() {
-    Assertions.checkNotNull(gvrAudioSurround);
-    return inputEnded && gvrAudioSurround.getAvailableOutputSize() == 0;
+    return inputEnded
+        && (gvrAudioSurround == null || gvrAudioSurround.getAvailableOutputSize() == 0);
   }
 
   @Override
   public void flush() {
-    if (gvrAudioSurround != null) {
+    if (pendingGvrAudioSurroundFormat != NO_SURROUND_FORMAT) {
+      maybeReleaseGvrAudioSurround();
+      gvrAudioSurround =
+          new GvrAudioSurround(
+              pendingGvrAudioSurroundFormat, sampleRateHz, channelCount, FRAMES_PER_OUTPUT_BUFFER);
+      gvrAudioSurround.updateNativeOrientation(w, x, y, z);
+      pendingGvrAudioSurroundFormat = NO_SURROUND_FORMAT;
+    } else if (gvrAudioSurround != null) {
       gvrAudioSurround.flush();
     }
     inputEnded = false;
@@ -191,6 +199,7 @@ public synchronized void reset() {
     sampleRateHz = Format.NO_VALUE;
     channelCount = Format.NO_VALUE;
     buffer = EMPTY_BUFFER;
+    pendingGvrAudioSurroundFormat = NO_SURROUND_FORMAT;
   }
 
   private void maybeReleaseGvrAudioSurround() {

library/core/src/main/java/com/google/android/exoplayer2/audio/AudioProcessor.java

@@ -24,11 +24,10 @@
  * modifying its channel count, encoding and/or sample rate.
  *
  * <p>Call {@link #configure(int, int, int)} to configure the processor to receive input audio, then
- * call {@link #isActive()} to determine whether the processor is active. {@link
- * #queueInput(ByteBuffer)}, {@link #queueEndOfStream()}, {@link #getOutput()}, {@link #isEnded()},
- * {@link #getOutputChannelCount()}, {@link #getOutputEncoding()} and {@link
- * #getOutputSampleRateHz()} may only be called if the processor is active. Call {@link #reset()} to
- * reset the processor to its unconfigured state and release any resources.
+ * call {@link #isActive()} to determine whether the processor is active in the new configuration.
+ * {@link #queueInput(ByteBuffer)}, {@link #getOutputChannelCount()}, {@link #getOutputEncoding()}
+ * and {@link #getOutputSampleRateHz()} may only be called if the processor is active. Call {@link
+ * #reset()} to reset the processor to its unconfigured state and release any resources.
  *
  * <p>In addition to being able to modify the format of audio, implementations may allow parameters
  * to be set that affect the output audio and whether the processor is active/inactive.
@@ -50,15 +49,21 @@ public UnhandledFormatException(
   ByteBuffer EMPTY_BUFFER = ByteBuffer.allocateDirect(0).order(ByteOrder.nativeOrder());
 
   /**
-   * Configures the processor to process input audio with the specified format and returns whether
-   * to {@link #flush()} it. After calling this method, if the processor is active, {@link
-   * #getOutputSampleRateHz()}, {@link #getOutputChannelCount()} and {@link #getOutputEncoding()}
-   * return its output format.
+   * Configures the processor to process input audio with the specified format. After calling this
+   * method, call {@link #isActive()} to determine whether the audio processor is active.
+   *
+   * <p>If the audio processor is active after configuration, call {@link #getOutputSampleRateHz()},
+   * {@link #getOutputChannelCount()} and {@link #getOutputEncoding()} to get its new output format.
+   *
+   * <p>If this method returns {@code true}, it is necessary to {@link #flush()} the processor
+   * before queueing more data, but you can (optionally) first drain output in the previous
+   * configuration by calling {@link #queueEndOfStream()} and {@link #getOutput()}. If this method
+   * returns {@code false}, it is safe to queue new input immediately.
    *
    * @param sampleRateHz The sample rate of input audio in Hz.
    * @param channelCount The number of interleaved channels in input audio.
    * @param encoding The encoding of input audio.
-   * @return Whether to {@link #flush()} the processor.
+   * @return Whether the processor must be {@link #flush() flushed} before queueing more input.
    * @throws UnhandledFormatException Thrown if the specified format can't be handled as input.
    */
   boolean configure(int sampleRateHz, int channelCount, @C.PcmEncoding int encoding)
@@ -69,23 +74,20 @@ boolean configure(int sampleRateHz, int channelCount, @C.PcmEncoding int encodin
 
   /**
    * Returns the number of audio channels in the data output by the processor. The value may change
-   * as a result of calling {@link #configure(int, int, int)} and is undefined if the instance is
-   * not active.
+   * as a result of calling {@link #configure(int, int, int)}.
    */
   int getOutputChannelCount();
 
   /**
    * Returns the audio encoding used in the data output by the processor. The value may change as a
-   * result of calling {@link #configure(int, int, int)} and is undefined if the instance is not
-   * active.
+   * result of calling {@link #configure(int, int, int)}.
    */
   @C.PcmEncoding
   int getOutputEncoding();
 
   /**
    * Returns the sample rate of audio output by the processor, in hertz. The value may change as a
-   * result of calling {@link #configure(int, int, int)} and is undefined if the instance is not
-   * active.
+   * result of calling {@link #configure(int, int, int)}.
    */
   int getOutputSampleRateHz();
 
@@ -124,7 +126,10 @@ boolean configure(int sampleRateHz, int channelCount, @C.PcmEncoding int encodin
    */
   boolean isEnded();
 
-  /** Clears any state in preparation for receiving a new stream of input buffers. */
+  /**
+   * Clears any buffered data and pending output. If the audio processor is active, also prepares
+   * the audio processor to receive a new stream of input in the last configured (pending) format.
+   */
   void flush();
 
   /** Resets the processor to its unconfigured state. */

library/core/src/main/java/com/google/android/exoplayer2/audio/DefaultAudioSink.java

@@ -242,6 +242,7 @@ public long getSkippedOutputFrameCount() {
   /** Used to keep the audio session active on pre-V21 builds (see {@link #initialize()}). */
   @Nullable private AudioTrack keepSessionIdAudioTrack;
 
+  @Nullable private Configuration pendingConfiguration;
   private Configuration configuration;
   private AudioTrack audioTrack;
 
@@ -423,13 +424,13 @@ && supportsOutput(inputChannelCount, C.ENCODING_PCM_FLOAT)
         shouldConvertHighResIntPcmToFloat
             ? toFloatPcmAvailableAudioProcessors
             : toIntPcmAvailableAudioProcessors;
-    boolean flush = false;
+    boolean flushAudioProcessors = false;
     if (processingEnabled) {
       trimmingAudioProcessor.setTrimFrameCount(trimStartFrames, trimEndFrames);
       channelMappingAudioProcessor.setChannelMap(outputChannels);
       for (AudioProcessor audioProcessor : availableAudioProcessors) {
         try {
-          flush |= audioProcessor.configure(sampleRate, channelCount, encoding);
+          flushAudioProcessors |= audioProcessor.configure(sampleRate, channelCount, encoding);
         } catch (AudioProcessor.UnhandledFormatException e) {
           throw new ConfigurationException(e);
         }
@@ -464,8 +465,14 @@ && supportsOutput(inputChannelCount, C.ENCODING_PCM_FLOAT)
             processingEnabled,
             canApplyPlaybackParameters,
             availableAudioProcessors);
-    if (flush || configuration == null || !pendingConfiguration.canReuseAudioTrack(configuration)) {
+    if (configuration == null || !pendingConfiguration.canReuseAudioTrack(configuration)) {
+      // We need a new AudioTrack before we can handle more input. We should first stop() the track
+      // (if we have one) and wait for audio to play out. Tracked by [Internal: b/33161961].
       flush();
+    } else if (flushAudioProcessors) {
+      // We don't need a new AudioTrack but audio processors need to be flushed.
+      this.pendingConfiguration = pendingConfiguration;
+      return;
     }
     configuration = pendingConfiguration;
   }
@@ -567,6 +574,21 @@ public void handleDiscontinuity() {
   public boolean handleBuffer(ByteBuffer buffer, long presentationTimeUs)
       throws InitializationException, WriteException {
     Assertions.checkArgument(inputBuffer == null || buffer == inputBuffer);
+
+    if (pendingConfiguration != null) {
+      // We are waiting for audio processors to drain before applying a the new configuration.
+      if (!drainAudioProcessorsToEndOfStream()) {
+        return false;
+      }
+      configuration = pendingConfiguration;
+      pendingConfiguration = null;
+      playbackParameters =
+          configuration.canApplyPlaybackParameters
+              ? audioProcessorChain.applyPlaybackParameters(playbackParameters)
+              : PlaybackParameters.DEFAULT;
+      setupAudioProcessors();
+    }
+
     if (!isInitialized()) {
       initialize();
       if (playing) {
@@ -948,9 +970,9 @@ public void flush() {
       playbackParametersOffsetUs = 0;
       playbackParametersPositionUs = 0;
       trimmingAudioProcessor.resetTrimmedFrameCount();
+      flushAudioProcessors();
       inputBuffer = null;
       outputBuffer = null;
-      flushAudioProcessors();
       handledEndOfStream = false;
       drainingAudioProcessorIndex = C.INDEX_UNSET;
       avSyncHeader = null;
@@ -962,6 +984,10 @@ public void flush() {
       // AudioTrack.release can take some time, so we call it on a background thread.
       final AudioTrack toRelease = audioTrack;
       audioTrack = null;
+      if (pendingConfiguration != null) {
+        configuration = pendingConfiguration;
+        pendingConfiguration = null;
+      }
       audioTrackPositionTracker.reset();
       releasingConditionVariable.close();
       new Thread() {

library/core/src/main/java/com/google/android/exoplayer2/audio/SonicAudioProcessor.java

@@ -69,6 +69,7 @@ public final class SonicAudioProcessor implements AudioProcessor {
   private int outputSampleRateHz;
   private int pendingOutputSampleRateHz;
 
+  private boolean pendingSonicRecreation;
   @Nullable private Sonic sonic;
   private ByteBuffer buffer;
   private ShortBuffer shortBuffer;
@@ -103,7 +104,7 @@ public float setSpeed(float speed) {
     speed = Util.constrainValue(speed, MINIMUM_SPEED, MAXIMUM_SPEED);
     if (this.speed != speed) {
       this.speed = speed;
-      sonic = null;
+      pendingSonicRecreation = true;
     }
     flush();
     return speed;
@@ -120,7 +121,7 @@ public float setPitch(float pitch) {
     pitch = Util.constrainValue(pitch, MINIMUM_PITCH, MAXIMUM_PITCH);
     if (this.pitch != pitch) {
       this.pitch = pitch;
-      sonic = null;
+      pendingSonicRecreation = true;
     }
     flush();
     return pitch;
@@ -172,7 +173,7 @@ public boolean configure(int sampleRateHz, int channelCount, @Encoding int encod
     this.sampleRateHz = sampleRateHz;
     this.channelCount = channelCount;
     this.outputSampleRateHz = outputSampleRateHz;
-    sonic = null;
+    pendingSonicRecreation = true;
     return true;
   }
 
@@ -227,7 +228,9 @@ public void queueInput(ByteBuffer inputBuffer) {
 
   @Override
   public void queueEndOfStream() {
-    Assertions.checkNotNull(sonic).queueEndOfStream();
+    if (sonic != null) {
+      sonic.queueEndOfStream();
+    }
     inputEnded = true;
   }
 
@@ -246,9 +249,9 @@ public boolean isEnded() {
   @Override
   public void flush() {
     if (isActive()) {
-      if (sonic == null) {
+      if (pendingSonicRecreation) {
         sonic = new Sonic(sampleRateHz, channelCount, speed, pitch, outputSampleRateHz);
-      } else {
+      } else if (sonic != null) {
         sonic.flush();
       }
     }
@@ -269,6 +272,7 @@ public void reset() {
     shortBuffer = buffer.asShortBuffer();
     outputBuffer = EMPTY_BUFFER;
     pendingOutputSampleRateHz = SAMPLE_RATE_NO_CHANGE;
+    pendingSonicRecreation = false;
     sonic = null;
     inputBytes = 0;
     outputBytes = 0;

library/core/src/main/java/com/google/android/exoplayer2/audio/TrimmingAudioProcessor.java

@@ -147,10 +147,10 @@ public void queueInput(ByteBuffer inputBuffer) {
   public ByteBuffer getOutput() {
     if (super.isEnded() && endBufferSize > 0) {
       // Because audio processors may be drained in the middle of the stream we assume that the
-      // contents of the end buffer need to be output. Gapless transitions don't involve a call to
-      // queueEndOfStream so won't be affected. When audio is actually ending we play the padding
-      // data which is incorrect. This behavior can be fixed once we have the timestamps associated
-      // with input buffers.
+      // contents of the end buffer need to be output. For gapless transitions, configure will be
+      // always be called, which clears the end buffer as needed. When audio is actually ending we
+      // play the padding data which is incorrect. This behavior can be fixed once we have the
+      // timestamps associated with input buffers.
       replaceOutputBuffer(endBufferSize).put(endBuffer, 0, endBufferSize).flip();
       endBufferSize = 0;
     }