| /* |
| * Copyright (C) 2012 The Android Open Source Project |
| * |
| * Licensed under the Apache License, Version 2.0 (the "License"); |
| * you may not use this file except in compliance with the License. |
| * You may obtain a copy of the License at |
| * |
| * http://www.apache.org/licenses/LICENSE-2.0 |
| * |
| * Unless required by applicable law or agreed to in writing, software |
| * distributed under the License is distributed on an "AS IS" BASIS, |
| * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| * See the License for the specific language governing permissions and |
| * limitations under the License. |
| */ |
| |
| package androidx.media.filterfw.decoder; |
| |
| import android.annotation.TargetApi; |
| import android.graphics.SurfaceTexture; |
| import android.graphics.SurfaceTexture.OnFrameAvailableListener; |
| import android.media.MediaCodec; |
| import android.media.MediaCodec.BufferInfo; |
| import android.media.MediaFormat; |
| import android.view.Surface; |
| |
| import androidx.media.filterfw.FrameImage2D; |
| import androidx.media.filterfw.ImageShader; |
| import androidx.media.filterfw.TextureSource; |
| |
| import java.nio.ByteBuffer; |
| |
| /** |
| * {@link TrackDecoder} that decodes a video track and renders the frames onto a |
| * {@link SurfaceTexture}. |
| * |
| * This implementation uses the GPU for image operations such as copying |
| * and color-space conversion. |
| */ |
| @TargetApi(16) |
| public class GpuVideoTrackDecoder extends VideoTrackDecoder { |
| |
| /** |
| * Identity fragment shader for external textures. |
| */ |
| private static final String COPY_FRAGMENT_SHADER = |
| "#extension GL_OES_EGL_image_external : require\n" + |
| "precision mediump float;\n" + |
| "uniform samplerExternalOES tex_sampler_0;\n" + |
| "varying vec2 v_texcoord;\n" + |
| "void main() {\n" + |
| " gl_FragColor = texture2D(tex_sampler_0, v_texcoord);\n" + |
| "}\n"; |
| |
| private final TextureSource mTextureSource; |
| private final SurfaceTexture mSurfaceTexture; // Access guarded by mFrameMonitor. |
| private final float[] mTransformMatrix; |
| |
| private final int mOutputWidth; |
| private final int mOutputHeight; |
| |
| private ImageShader mImageShader; |
| |
| private long mCurrentPresentationTimeUs; |
| |
| public GpuVideoTrackDecoder( |
| int trackIndex, MediaFormat format, Listener listener) { |
| super(trackIndex, format, listener); |
| |
| // Create a surface texture to be used by the video track decoder. |
| mTextureSource = TextureSource.newExternalTexture(); |
| mSurfaceTexture = new SurfaceTexture(mTextureSource.getTextureId()); |
| mSurfaceTexture.detachFromGLContext(); |
| mSurfaceTexture.setOnFrameAvailableListener(new OnFrameAvailableListener() { |
| @Override |
| public void onFrameAvailable(SurfaceTexture surfaceTexture) { |
| markFrameAvailable(); |
| } |
| }); |
| |
| mOutputWidth = format.getInteger(MediaFormat.KEY_WIDTH); |
| mOutputHeight = format.getInteger(MediaFormat.KEY_HEIGHT); |
| |
| mTransformMatrix = new float[16]; |
| } |
| |
| @Override |
| protected MediaCodec initMediaCodec(MediaFormat format) { |
| Surface surface = new Surface(mSurfaceTexture); |
| MediaCodec mediaCodec = MediaCodec.createDecoderByType( |
| format.getString(MediaFormat.KEY_MIME)); |
| mediaCodec.configure(format, surface, null, 0); |
| surface.release(); |
| return mediaCodec; |
| } |
| |
| @Override |
| protected boolean onDataAvailable( |
| MediaCodec codec, ByteBuffer[] buffers, int bufferIndex, BufferInfo info) { |
| boolean textureAvailable = waitForFrameGrab(); |
| |
| mCurrentPresentationTimeUs = info.presentationTimeUs; |
| |
| // Only render the next frame if we weren't interrupted. |
| codec.releaseOutputBuffer(bufferIndex, textureAvailable); |
| |
| if (textureAvailable) { |
| if (updateTexture()) { |
| notifyListener(); |
| } |
| } |
| |
| return false; |
| } |
| |
| /** |
| * Waits for the texture's {@link OnFrameAvailableListener} to be notified and then updates |
| * the internal {@link SurfaceTexture}. |
| */ |
| private boolean updateTexture() { |
| // Wait for the frame we just released to appear in the texture. |
| synchronized (mFrameMonitor) { |
| try { |
| while (!mFrameAvailable) { |
| mFrameMonitor.wait(); |
| } |
| mSurfaceTexture.attachToGLContext(mTextureSource.getTextureId()); |
| mSurfaceTexture.updateTexImage(); |
| mSurfaceTexture.detachFromGLContext(); |
| return true; |
| } catch (InterruptedException e) { |
| return false; |
| } |
| } |
| } |
| |
| @Override |
| protected void copyFrameDataTo(FrameImage2D outputVideoFrame, int rotation) { |
| TextureSource targetTexture = TextureSource.newExternalTexture(); |
| mSurfaceTexture.attachToGLContext(targetTexture.getTextureId()); |
| mSurfaceTexture.getTransformMatrix(mTransformMatrix); |
| |
| ImageShader imageShader = getImageShader(); |
| imageShader.setSourceTransform(mTransformMatrix); |
| |
| int outputWidth = mOutputWidth; |
| int outputHeight = mOutputHeight; |
| if (rotation != 0) { |
| float[] targetCoords = getRotationCoords(rotation); |
| imageShader.setTargetCoords(targetCoords); |
| if (needSwapDimension(rotation)) { |
| outputWidth = mOutputHeight; |
| outputHeight = mOutputWidth; |
| } |
| } |
| outputVideoFrame.resize(new int[] { outputWidth, outputHeight }); |
| imageShader.process( |
| targetTexture, |
| outputVideoFrame.lockRenderTarget(), |
| outputWidth, |
| outputHeight); |
| outputVideoFrame.setTimestamp(mCurrentPresentationTimeUs * 1000); |
| outputVideoFrame.unlock(); |
| targetTexture.release(); |
| |
| mSurfaceTexture.detachFromGLContext(); |
| } |
| |
| @Override |
| public void release() { |
| super.release(); |
| synchronized (mFrameMonitor) { |
| mTextureSource.release(); |
| mSurfaceTexture.release(); |
| } |
| } |
| |
| /* |
| * This method has to be called on the MFF processing thread. |
| */ |
| private ImageShader getImageShader() { |
| if (mImageShader == null) { |
| mImageShader = new ImageShader(COPY_FRAGMENT_SHADER); |
| mImageShader.setTargetRect(0f, 1f, 1f, -1f); |
| } |
| return mImageShader; |
| } |
| |
| /** |
| * Get the quad coords for rotation. |
| * @param rotation applied to the frame, value is one of |
| * {ROTATE_NONE, ROTATE_90_RIGHT, ROTATE_180, ROTATE_90_LEFT} |
| * @return coords the calculated quad coords for the given rotation |
| */ |
| private static float[] getRotationCoords(int rotation) { |
| switch(rotation) { |
| case MediaDecoder.ROTATE_90_RIGHT: |
| return new float[] { 0f, 0f, 0f, 1f, 1f, 0f, 1f, 1f }; |
| case MediaDecoder.ROTATE_180: |
| return new float[] { 1f, 0f, 0f, 0f, 1f, 1f, 0f, 1f }; |
| case MediaDecoder.ROTATE_90_LEFT: |
| return new float[] { 1f, 1f, 1f, 0f, 0f, 1f, 0f, 0f }; |
| case MediaDecoder.ROTATE_NONE: |
| return new float[] { 0f, 1f, 1f, 1f, 0f, 0f, 1f, 0f }; |
| default: |
| throw new IllegalArgumentException("Unsupported rotation angle."); |
| } |
| } |
| |
| } |