media/java/android/media/MediaSync.java

/*
 * Copyright (C) 2015 The Android Open Source Project
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package android.media;

import android.annotation.IntDef;
import android.annotation.NonNull;
import android.media.AudioTrack;
import android.os.Handler;
import android.os.Looper;
import android.os.Message;
import android.view.Surface;

import java.lang.annotation.Retention;
import java.lang.annotation.RetentionPolicy;
import java.nio.ByteBuffer;
import java.util.LinkedList;
import java.util.List;

/**
 * MediaSync class can be used to synchronously playback audio and video streams.
 * It can be used to play audio-only or video-only stream, too.
 *
 * <p>MediaSync is generally used like this:
 * <pre>
 * MediaSync sync = new MediaSync();
 * sync.configureSurface(surface);
 * Surface inputSurface = sync.createInputSurface();
 * ...
 * // MediaCodec videoDecoder = ...;
 * videoDecoder.configure(format, inputSurface, ...);
 * ...
 * sync.configureAudioTrack(audioTrack);
 * sync.setCallback(new MediaSync.Callback() {
 *     {@literal @Override}
 *     public void onReturnAudioBuffer(MediaSync sync, ByteBuffer audioBuffer, int bufferIndex) {
 *         ...
 *     }
 * }, null);
 * // This needs to be done since sync is paused on creation.
 * sync.setPlaybackRate(1.0f, MediaSync.PLAYBACK_RATE_AUDIO_MODE_RESAMPLE);
 *
 * for (;;) {
 *   ...
 *   // send video frames to surface for rendering, e.g., call
 *   // videoDecoder.releaseOutputBuffer(videoOutputBufferIx, videoPresentationTimeNs);
 *   // More details are available as below.
 *   ...
 *   sync.queueAudio(audioByteBuffer, bufferIndex, size, audioPresentationTimeUs); // non-blocking.
 *   // The audioByteBuffer and bufferIndex will be returned via callback.
 *   // More details are available as below.
 *   ...
 *     ...
 * }
 * sync.setPlaybackRate(0.0f, MediaSync.PLAYBACK_RATE_AUDIO_MODE_RESAMPLE);
 * sync.release();
 * sync = null;
 *
 * // The following code snippet illustrates how video/audio raw frames are created by
 * // MediaCodec's, how they are fed to MediaSync and how they are returned by MediaSync.
 * // This is the callback from MediaCodec.
 * onOutputBufferAvailable(MediaCodec codec, int bufferIndex, BufferInfo info) {
 *     // ...
 *     if (codec == videoDecoder) {
 *         // surface timestamp must contain media presentation time in nanoseconds.
 *         codec.releaseOutputBuffer(bufferIndex, 1000 * info.presentationTime);
 *     } else {
 *         ByteBuffer audioByteBuffer = codec.getOutputBuffer(bufferIndex);
 *         sync.queueByteBuffer(audioByteBuffer, bufferIndex, info.size, info.presentationTime);
 *     }
 *     // ...
 * }
 *
 * // This is the callback from MediaSync.
 * onReturnAudioBuffer(MediaSync sync, ByteBuffer buffer, int bufferIndex) {
 *     // ...
 *     audioDecoder.releaseBuffer(bufferIndex, false);
 *     // ...
 * }
 *
 * </pre>
 *
 * The client needs to configure corresponding sink (i.e., Surface and AudioTrack) based on
 * the stream type it will play.
 * <p>
 * For video, the client needs to call {@link #createInputSurface} to obtain a surface on
 * which it will render video frames.
 * <p>
 * For audio, the client needs to set up audio track correctly, e.g., using {@link
 * AudioTrack#MODE_STREAM}. The audio buffers are sent to MediaSync directly via {@link
 * #queueAudio}, and are returned to the client via {@link Callback#onReturnAudioBuffer}
 * asynchronously. The client should not modify an audio buffer till it's returned.
 * <p>
 * The client can optionally pre-fill audio/video buffers by setting playback rate to 0.0,
 * and then feed audio/video buffers to corresponding components. This can reduce possible
 * initial underrun.
 * <p>
 */
final public class MediaSync {
    /**
     * MediaSync callback interface. Used to notify the user asynchronously
     * of various MediaSync events.
     */
    public static abstract class Callback {
        /**
         * Called when returning an audio buffer which has been consumed.
         *
         * @param sync The MediaSync object.
         * @param audioBuffer The returned audio buffer.
         */
        public abstract void onReturnAudioBuffer(
                MediaSync sync, ByteBuffer audioBuffer, int bufferIndex);
    }

    private static final String TAG = "MediaSync";

    private static final int EVENT_CALLBACK = 1;
    private static final int EVENT_SET_CALLBACK = 2;

    private static final int CB_RETURN_AUDIO_BUFFER = 1;

    private static class AudioBuffer {
        public ByteBuffer mByteBuffer;
        public int mBufferIndex;
        public int mSizeInBytes;
        long mPresentationTimeUs;

        public AudioBuffer(ByteBuffer byteBuffer, int bufferIndex,
                           int sizeInBytes, long presentationTimeUs) {
            mByteBuffer = byteBuffer;
            mBufferIndex = bufferIndex;
            mSizeInBytes = sizeInBytes;
            mPresentationTimeUs = presentationTimeUs;
        }
    }

    private final Object mCallbackLock = new Object();
    private Handler mCallbackHandler = null;
    private MediaSync.Callback mCallback = null;

    private Thread mAudioThread = null;
    // Created on mAudioThread when mAudioThread is started. When used on user thread, they should
    // be guarded by checking mAudioThread.
    private Handler mAudioHandler = null;
    private Looper mAudioLooper = null;

    private final Object mAudioLock = new Object();
    private AudioTrack mAudioTrack = null;
    private List<AudioBuffer> mAudioBuffers = new LinkedList<AudioBuffer>();
    private float mPlaybackRate = 0.0f;

    private long mNativeContext;

    /**
     * Class constructor. On creation, MediaSync is paused, i.e., playback rate is 0.0f.
     */
    public MediaSync() {
        native_setup();
    }

    private native final void native_setup();

    @Override
    protected void finalize() {
        native_finalize();
    }

    private native final void native_finalize();

    /**
     * Make sure you call this when you're done to free up any opened
     * component instance instead of relying on the garbage collector
     * to do this for you at some point in the future.
     */
    public final void release() {
        returnAudioBuffers();
        if (mAudioThread != null) {
            if (mAudioLooper != null) {
                mAudioLooper.quit();
            }
        }
        setCallback(null, null);
        native_release();
    }

    private native final void native_release();

    /**
     * Sets an asynchronous callback for actionable MediaSync events.
     * It shouldn't be called inside callback.
     *
     * @param cb The callback that will run.
     * @param handler The Handler that will run the callback. Using null means to use MediaSync's
     *     internal handler if it exists.
     */
    public void setCallback(/* MediaSync. */ Callback cb, Handler handler) {
        synchronized(mCallbackLock) {
            if (handler != null) {
                mCallbackHandler = handler;
            } else {
                Looper looper;
                if ((looper = Looper.myLooper()) == null) {
                    looper = Looper.getMainLooper();
                }
                if (looper == null) {
                    mCallbackHandler = null;
                } else {
                    mCallbackHandler = new Handler(looper);
                }
            }

            mCallback = cb;
        }
    }

    /**
     * Configures the output surface for MediaSync.
     *
     * @param surface Specify a surface on which to render the video data.
     * @throws IllegalArgumentException if the surface has been released, or is invalid.
     *     or can not be connected.
     * @throws IllegalStateException if not in the Initialized state, or another surface
     *     has already been configured.
     */
    public void configureSurface(Surface surface) {
        native_configureSurface(surface);
    }

    private native final void native_configureSurface(Surface surface);

    /**
     * Configures the audio track for MediaSync.
     *
     * @param audioTrack Specify an AudioTrack through which to render the audio data.
     * @throws IllegalArgumentException if the audioTrack has been released, or is invalid.
     * @throws IllegalStateException if not in the Initialized state, or another audio track
     *     has already been configured.
     */
    public void configureAudioTrack(AudioTrack audioTrack) {
        // AudioTrack has sanity check for configured sample rate.
        int nativeSampleRateInHz = (audioTrack == null ? 0 : audioTrack.getSampleRate());

        native_configureAudioTrack(audioTrack, nativeSampleRateInHz);
        mAudioTrack = audioTrack;
        if (audioTrack != null && mAudioThread == null) {
            createAudioThread();
        }
    }

    private native final void native_configureAudioTrack(
            AudioTrack audioTrack, int nativeSampleRateInHz);

    /**
     * Requests a Surface to use as the input. This may only be called after
     * {@link #configureSurface}.
     * <p>
     * The application is responsible for calling release() on the Surface when
     * done.
     * @throws IllegalStateException if not configured, or another input surface has
     *     already been created.
     */
    public native final Surface createInputSurface();

    /**
     * Specifies resampling as audio mode for variable rate playback, i.e.,
     * resample the waveform based on the requested playback rate to get
     * a new waveform, and play back the new waveform at the original sampling
     * frequency.
     * When rate is larger than 1.0, pitch becomes higher.
     * When rate is smaller than 1.0, pitch becomes lower.
     */
    public static final int PLAYBACK_RATE_AUDIO_MODE_RESAMPLE = 0;

    /**
     * Specifies time stretching as audio mode for variable rate playback.
     * Time stretching changes the duration of the audio samples without
     * affecting its pitch.
     * FIXME: implement time strectching.
     * @hide
     */
    public static final int PLAYBACK_RATE_AUDIO_MODE_STRETCH = 1;

    /** @hide */
    @IntDef(
        value = {
            PLAYBACK_RATE_AUDIO_MODE_RESAMPLE,
            PLAYBACK_RATE_AUDIO_MODE_STRETCH })
    @Retention(RetentionPolicy.SOURCE)
    public @interface PlaybackRateAudioMode {}

    /**
     * Sets playback rate. It does same as {@link #setPlaybackRate(float, int)},
     * except that it always uses {@link #PLAYBACK_RATE_AUDIO_MODE_STRETCH} for audioMode.
     *
     * @param rate the ratio between desired playback rate and normal one. 1.0 means normal
     *     playback speed. 0.0 means stop or pause. Value larger than 1.0 means faster playback,
     *     while value between 0.0 and 1.0 for slower playback.
     *
     * @throws IllegalStateException if the internal sync engine or the audio track has not
     *     been initialized.
     * TODO: unhide when PLAYBACK_RATE_AUDIO_MODE_STRETCH is supported.
     * @hide
     */
    public void setPlaybackRate(float rate) {
        setPlaybackRate(rate, PLAYBACK_RATE_AUDIO_MODE_STRETCH);
    }

    /**
     * Sets playback rate and audio mode.
     *
     * <p> The supported audio modes are:
     * <ul>
     * <li> {@link #PLAYBACK_RATE_AUDIO_MODE_RESAMPLE}
     * </ul>
     *
     * @param rate the ratio between desired playback rate and normal one. 1.0 means normal
     *     playback speed. 0.0 means stop or pause. Value larger than 1.0 means faster playback,
     *     while value between 0.0 and 1.0 for slower playback.
     * @param audioMode audio playback mode. Must be one of the supported
     *     audio modes.
     *
     * @throws IllegalStateException if the internal sync engine or the audio track has not
     *     been initialized.
     * @throws IllegalArgumentException if audioMode is not supported.
     */
    public void setPlaybackRate(float rate, @PlaybackRateAudioMode int audioMode) {
        if (!isAudioPlaybackModeSupported(audioMode)) {
            final String msg = "Audio playback mode " + audioMode + " is not supported";
            throw new IllegalArgumentException(msg);
        }

        int status = AudioTrack.SUCCESS;
        if (mAudioTrack != null) {
            int nativeSampleRateInHz = mAudioTrack.getSampleRate();
            int playbackSampleRate = (int)(rate * nativeSampleRateInHz + 0.5);
            rate = playbackSampleRate / (float)nativeSampleRateInHz;

            try {
                if (rate == 0.0) {
                    mAudioTrack.pause();
                } else {
                    status = mAudioTrack.setPlaybackRate(playbackSampleRate);
                    mAudioTrack.play();
                }
            } catch (IllegalStateException e) {
                throw e;
            }
        }

        if (status != AudioTrack.SUCCESS) {
            throw new IllegalArgumentException("Fail to set playback rate in audio track");
        }

        synchronized(mAudioLock) {
            mPlaybackRate = rate;
        }
        if (mPlaybackRate != 0.0 && mAudioThread != null) {
            postRenderAudio(0);
        }
        native_setPlaybackRate(mPlaybackRate);
    }

    private native final void native_setPlaybackRate(float rate);

    /*
     * Test whether a given audio playback mode is supported.
     * TODO query supported AudioPlaybackMode from audio track.
     */
    private boolean isAudioPlaybackModeSupported(int mode) {
        return (mode == PLAYBACK_RATE_AUDIO_MODE_RESAMPLE);
    }

   /**
    * Get current playback position.
    * <p>
    * The MediaTimestamp represents a clock ticking during media playback. It's represented
    * by an anchor frame ({@link MediaTimestamp#mediaTimeUs} and {@link MediaTimestamp#nanoTime})
    * and clock speed ({@link MediaTimestamp#clockRate}). For continous playback with
    * constant speed, its anchor frame doesn't change that often. Thereafter, it's recommended
    * to not call this method often.
    * <p>
    * To help users to get current playback position, this method always returns the timestamp of
    * just-rendered frame, i.e., {@link System#nanoTime} and its corresponding media time. They
    * can be used as current playback position.
    *
    * @param timestamp a reference to a non-null MediaTimestamp instance allocated
    *         and owned by caller.
    * @return true if a timestamp is available, or false if no timestamp is available.
    *         If a timestamp if available, the MediaTimestamp instance is filled in with
    *         playback rate, together with the current media timestamp and the system nanoTime
    *         corresponding to the measured media timestamp.
    *         In the case that no timestamp is available, any supplied instance is left unaltered.
    */
    public boolean getTimestamp(@NonNull MediaTimestamp timestamp)
    {
        if (timestamp == null) {
            throw new IllegalArgumentException();
        }
        return native_getTimestamp(timestamp);
    }

    private native final boolean native_getTimestamp(MediaTimestamp timestamp);

    /**
     * Queues the audio data asynchronously for playback (AudioTrack must be in streaming mode).
     * @param audioData the buffer that holds the data to play. This buffer will be returned
     *     to the client via registered callback.
     * @param bufferIndex the buffer index used to identify audioData. It will be returned to
     *     the client along with audioData. This helps applications to keep track of audioData.
     * @param sizeInBytes number of bytes to queue.
     * @param presentationTimeUs the presentation timestamp in microseconds for the first frame
     *     in the buffer.
     * @throws IllegalStateException if audio track is not configured or internal configureation
     *     has not been done correctly.
     */
    public void queueAudio(
            ByteBuffer audioData, int bufferIndex, int sizeInBytes, long presentationTimeUs) {
        if (mAudioTrack == null || mAudioThread == null) {
            throw new IllegalStateException(
                    "AudioTrack is NOT configured or audio thread is not created");
        }

        synchronized(mAudioLock) {
            mAudioBuffers.add(new AudioBuffer(
                    audioData, bufferIndex, sizeInBytes, presentationTimeUs));
        }

        if (mPlaybackRate != 0.0) {
            postRenderAudio(0);
        }
    }

    // When called on user thread, make sure to check mAudioThread != null.
    private void postRenderAudio(long delayMillis) {
        mAudioHandler.postDelayed(new Runnable() {
            public void run() {
                synchronized(mAudioLock) {
                    if (mPlaybackRate == 0.0) {
                        return;
                    }

                    if (mAudioBuffers.isEmpty()) {
                        return;
                    }

                    AudioBuffer audioBuffer = mAudioBuffers.get(0);
                    int sizeWritten = mAudioTrack.write(
                            audioBuffer.mByteBuffer,
                            audioBuffer.mSizeInBytes,
                            AudioTrack.WRITE_NON_BLOCKING);
                    if (sizeWritten > 0) {
                        if (audioBuffer.mPresentationTimeUs != -1) {
                            native_updateQueuedAudioData(
                                    audioBuffer.mSizeInBytes, audioBuffer.mPresentationTimeUs);
                            audioBuffer.mPresentationTimeUs = -1;
                        }

                        if (sizeWritten == audioBuffer.mSizeInBytes) {
                            postReturnByteBuffer(audioBuffer);
                            mAudioBuffers.remove(0);
                            if (!mAudioBuffers.isEmpty()) {
                                postRenderAudio(0);
                            }
                            return;
                        }

                        audioBuffer.mSizeInBytes -= sizeWritten;
                    }
                    // TODO: wait time depends on fullness of audio track.
                    postRenderAudio(10);
                }
            }
        }, delayMillis);
    }

    private native final void native_updateQueuedAudioData(
            int sizeInBytes, long presentationTimeUs);

    private final void postReturnByteBuffer(final AudioBuffer audioBuffer) {
        synchronized(mCallbackLock) {
            if (mCallbackHandler != null) {
                final MediaSync sync = this;
                mCallbackHandler.post(new Runnable() {
                    public void run() {
                        synchronized(mCallbackLock) {
                            if (mCallbackHandler == null
                                    || mCallbackHandler.getLooper().getThread()
                                            != Thread.currentThread()) {
                                // callback handler has been changed.
                                return;
                            }
                            if (mCallback != null) {
                                mCallback.onReturnAudioBuffer(sync, audioBuffer.mByteBuffer,
                                        audioBuffer.mBufferIndex);
                            }
                        }
                    }
                });
            }
        }
    }

    private final void returnAudioBuffers() {
        synchronized(mAudioLock) {
            for (AudioBuffer audioBuffer: mAudioBuffers) {
                postReturnByteBuffer(audioBuffer);
            }
            mAudioBuffers.clear();
        }
    }

    private void createAudioThread() {
        mAudioThread = new Thread() {
            @Override
            public void run() {
                Looper.prepare();
                synchronized(mAudioLock) {
                    mAudioLooper = Looper.myLooper();
                    mAudioHandler = new Handler();
                    mAudioLock.notify();
                }
                Looper.loop();
            }
        };
        mAudioThread.start();

        synchronized(mAudioLock) {
            try {
                mAudioLock.wait();
            } catch(InterruptedException e) {
            }
        }
    }

    static {
        System.loadLibrary("media_jni");
        native_init();
    }

    private static native final void native_init();
}