dolphin - opengl es 3.1 android download

Androide. Cómo reproducir video en Surface(OpenGL)

Question

Necesito ayuda

1. ¿Cómo reproducir video en Surface(OpenGL) en Android? Intenté reproducir video en mySurfaceView extends SurfaceView con el método de ayuda setSurface() en MediaPlayer .

   SurfaceTexture mTexture = new SurfaceTexture(texture_id);
   Surface mSurface = new Surface(mTexture);
   MediaPlayer mp = new MediaPlayer();
   mp.setSurface(mSurface);

Sólo tengo reproducción de audio - video no se reproduce.

2. ¿Cómo obtener el búfer de video para enviar en OpenGL ?
3. ¿Cómo se reproduce vídeo en GLTexture ?

Answer 1

Acabo de convertir la versión de Java a Kotlin

internal inline fun <T> glRun(message: String = "", block: (() -> T)): T {
    return block().also {
        var error: Int = GLES20.glGetError()
        while (error != GLES20.GL_NO_ERROR) {
            error = GLES20.glGetError()
            Log.d("MOVIE_GL_ERROR", "$message: $error")
            throw RuntimeException("GL Error: $message")
        }
    }
}

class MovieRenderer: GLSurfaceView.Renderer, SurfaceTexture.OnFrameAvailableListener {

    private var program = 0
    private var textureId = 0

    // Handles
    private var mvpMatrixHandle = 0
    private var stMatrixHandle = 0
    private var positionHandle = 0
    private var textureHandle = 0

    // Surface Texture
    private var updateSurface = false
    private lateinit var surfaceTexture: SurfaceTexture

    // Matrices
    private var mvpMatrix = FloatArray(16)
    private var stMatrix = FloatArray(16)

    // float buffer
    private val vertices: FloatBuffer = ByteBuffer.allocateDirect(VERTICES_DATA.size * FLOAT_SIZE_BYTES)
        .order(ByteOrder.nativeOrder())
        .asFloatBuffer().also {
            it.put(VERTICES_DATA).position(0)
        }

    var mediaPlayer: MediaPlayer? = null

    @Synchronized
    override fun onFrameAvailable(surfaceTexture: SurfaceTexture?) {
        updateSurface = true
    }

    override fun onDrawFrame(gl: GL10?) {
        synchronized(this) {
            if (updateSurface) {
                surfaceTexture.updateTexImage()
                surfaceTexture.getTransformMatrix(stMatrix)
                updateSurface = false
            }
        }

        GLES20.glClearColor(0.0f, 0.0f, 0.0f, 1.0f)
        GLES20.glClear(GLES20.GL_DEPTH_BUFFER_BIT or GLES20.GL_COLOR_BUFFER_BIT)

        glRun("glUseProgram: $program") {
            GLES20.glUseProgram(program)
        }

        vertices.position(VERTICES_POS_OFFSET);

        glRun("glVertexAttribPointer: Stride bytes") {
            GLES20.glVertexAttribPointer(positionHandle, 3, GLES20.GL_FLOAT, false,
                VERTICES_STRIDE_BYTES, vertices)
        }

        glRun("glEnableVertexAttribArray") {
            GLES20.glEnableVertexAttribArray(positionHandle)
        }

        vertices.position(VERTICES_UV_OFFSET)

        glRun("glVertexAttribPointer: texture handle") {
            GLES20.glVertexAttribPointer(textureHandle, 3, GLES20.GL_FLOAT, false,
                VERTICES_STRIDE_BYTES, vertices)
        }

        glRun("glEnableVertexAttribArray") {
            GLES20.glEnableVertexAttribArray(textureHandle)
        }

        Matrix.setIdentityM(mvpMatrix, 0)

        glRun("glUniformMatrix4fv: mvpMatrix") {
            GLES20.glUniformMatrix4fv(mvpMatrixHandle, 1, false, mvpMatrix, 0)
        }

        glRun("glUniformMatrix4fv: stMatrix") {
            GLES20.glUniformMatrix4fv(stMatrixHandle, 1, false, stMatrix, 0)
        }

        glRun("glDrawArrays: GL_TRIANGLE_STRIP") {
            GLES20.glDrawArrays(GLES20.GL_TRIANGLE_STRIP, 0, 4)
        }

        GLES20.glFinish()
    }

    override fun onSurfaceChanged(gl: GL10?, width: Int, height: Int) {
        GLES20.glViewport(0, 0, width, height)
    }

    override fun onSurfaceCreated(gl: GL10?, config: EGLConfig?) {
        program = createProgram()
        positionHandle = "aPosition".attr()
        textureHandle = "aTextureCoord".attr()
        mvpMatrixHandle = "uMVPMatrix".uniform()
        stMatrixHandle = "uSTMatrix".uniform()
        createTexture()
    }

    private fun createTexture() {
        val textures = IntArray(1)
        GLES20.glGenTextures(1, textures, 0)
        textureId = textures.first()
        glRun("glBindTexture textureId") { GLES20.glBindTexture(GL_TEXTURE_EXTERNAL_OES, textureId) }

        GLES20.glTexParameteri(GL_TEXTURE_EXTERNAL_OES, GLES20.GL_TEXTURE_MIN_FILTER, GLES20.GL_NEAREST)
        GLES20.glTexParameteri(GL_TEXTURE_EXTERNAL_OES, GLES20.GL_TEXTURE_MAG_FILTER, GLES20.GL_LINEAR)

        surfaceTexture = SurfaceTexture(textureId)
        surfaceTexture.setOnFrameAvailableListener(this)

        val surface = Surface(surfaceTexture)
        mediaPlayer?.setSurface(surface)
        surface.release()

        try {
            mediaPlayer?.prepare()
        } catch (error: IOException) {
            Log.e("MovieRenderer", "media player prepare failed");
            throw error
        }

        synchronized(this) {
            updateSurface = false
        }

        mediaPlayer?.start()
    }



    private fun String.attr(): Int {
        return glRun("Get attribute location: $this") {
            GLES20.glGetAttribLocation(program, this).also {
                if (it == -1) fail("Error Attribute: $this not found!")
            }
        }
    }

    private fun String.uniform(): Int {
        return glRun("Get uniform location: $this") {
            GLES20.glGetUniformLocation(program, this).also {
                if (it == -1) fail("Error Uniform: $this not found!")
            }
        }
    }

    companion object {
        private const val GL_TEXTURE_EXTERNAL_OES = 0x8D65

        private const val FLOAT_SIZE_BYTES = 4
        private const val VERTICES_STRIDE_BYTES = 5 * FLOAT_SIZE_BYTES
        private const val VERTICES_POS_OFFSET = 0
        private const val VERTICES_UV_OFFSET = 3

        private val VERTICES_DATA = floatArrayOf(
            -1.0f, -1.0f, 0f, 0.0f, 0.0f,
            1.0f, -1.0f, 0f, 1.0f, 0.0f,
            -1.0f,  1.0f, 0f, 0.0f, 1.0f,
            1.0f,  1.0f, 0f, 1.0f, 1.0f
        )

        private const val VERTEX_SHADER = """
            uniform mat4 uMVPMatrix;
            uniform mat4 uSTMatrix;
            attribute vec4 aPosition;
            attribute vec4 aTextureCoord;
            varying vec2 vTextureCoord;
            void main() {
                gl_Position = uMVPMatrix * aPosition;
                vTextureCoord = (uSTMatrix * aTextureCoord).xy;
            }
        """

        private const val FRAGMENT_SHADER = """
            #extension GL_OES_EGL_image_external : require
            precision mediump float;
            varying vec2 vTextureCoord;
            uniform samplerExternalOES sTexture;
            void main() {
              gl_FragColor = texture2D(sTexture, vTextureCoord);
            }
        """

        private fun createShader(type: Int, source: String): Int {
            val shader = GLES20.glCreateShader(type)
            if (shader == 0) throw RuntimeException("Cannot create shader $type/n$source")
            GLES20.glShaderSource(shader, source)
            GLES20.glCompileShader(shader)

            val args = IntArray(1)
            GLES20.glGetShaderiv(shader, GLES20.GL_COMPILE_STATUS, args, 0)
            if (args.first() == 0) {
                Log.e("MOVIE_SHADER", "Failed to compile shader source")
                Log.e("MOVIE_SHADER", GLES20.glGetShaderInfoLog(shader))
                GLES20.glDeleteShader(shader)
                throw RuntimeException("Could not compile shader $source/n$type")
            }

            return shader
        }

        private fun createProgram(vertexShaderSource: String = VERTEX_SHADER,
                                  fragmentShaderSource: String = FRAGMENT_SHADER): Int {

            val vertexShader = createShader(GLES20.GL_VERTEX_SHADER, vertexShaderSource)
            val fragmentShader = createShader(GLES20.GL_FRAGMENT_SHADER, fragmentShaderSource)

            val program = GLES20.glCreateProgram()
            if (program == 0) throw RuntimeException("Cannot create program")

            glRun("Attach vertex shader to program") {
                GLES20.glAttachShader(program, vertexShader)
            }

            glRun("Attach fragment shader to program") {
                GLES20.glAttachShader(program, fragmentShader)
            }

            GLES20.glLinkProgram(program)
            val args = IntArray(1)
            GLES20.glGetProgramiv(program, GLES20.GL_LINK_STATUS, args, 0)

            if (args.first() != GLES20.GL_TRUE) {
                val info = GLES20.glGetProgramInfoLog(program)
                GLES20.glDeleteProgram(program)
                throw RuntimeException("Cannot link program $program, Info: $info")
            }

            return program
        }


        private fun fail(message: String): Nothing {
            throw RuntimeException(message)
        }

    }
}

Answer 2

Desde el código fuente de Android ...

import java.io.IOException;
import java.nio.ByteBuffer;
import java.nio.ByteOrder;
import java.nio.FloatBuffer;

import javax.microedition.khronos.egl.EGLConfig;
import javax.microedition.khronos.opengles.GL10;

import android.content.Context;
import android.graphics.SurfaceTexture;
import android.media.MediaPlayer;
import android.opengl.GLES20;
import android.opengl.GLSurfaceView;
import android.opengl.Matrix;
import android.util.Log;
import android.view.Surface;

class VideoSurfaceView extends GLSurfaceView {

    VideoRender mRenderer;
    private MediaPlayer mMediaPlayer = null;

    public VideoSurfaceView(Context context, MediaPlayer mp) {
        super(context);

        setEGLContextClientVersion(2);
        mMediaPlayer = mp;
        mRenderer = new VideoRender(context);
        setRenderer(mRenderer);
    }

    @Override
    public void onResume() {
        queueEvent(new Runnable(){
                public void run() {
                    mRenderer.setMediaPlayer(mMediaPlayer);
                }});

        super.onResume();
    }

    private static class VideoRender
        implements GLSurfaceView.Renderer, SurfaceTexture.OnFrameAvailableListener {
        private static String TAG = "VideoRender";

        private static final int FLOAT_SIZE_BYTES = 4;
        private static final int TRIANGLE_VERTICES_DATA_STRIDE_BYTES = 5 * FLOAT_SIZE_BYTES;
        private static final int TRIANGLE_VERTICES_DATA_POS_OFFSET = 0;
        private static final int TRIANGLE_VERTICES_DATA_UV_OFFSET = 3;
        private final float[] mTriangleVerticesData = {
            // X, Y, Z, U, V
            -1.0f, -1.0f, 0, 0.f, 0.f,
            1.0f, -1.0f, 0, 1.f, 0.f,
            -1.0f,  1.0f, 0, 0.f, 1.f,
            1.0f,  1.0f, 0, 1.f, 1.f,
        };

        private FloatBuffer mTriangleVertices;

        private final String mVertexShader =
                "uniform mat4 uMVPMatrix;/n" +
                "uniform mat4 uSTMatrix;/n" +
                "attribute vec4 aPosition;/n" +
                "attribute vec4 aTextureCoord;/n" +
                "varying vec2 vTextureCoord;/n" +
                "void main() {/n" +
                "  gl_Position = uMVPMatrix * aPosition;/n" +
                "  vTextureCoord = (uSTMatrix * aTextureCoord).xy;/n" +
                "}/n";

        private final String mFragmentShader =
                "#extension GL_OES_EGL_image_external : require/n" +
                "precision mediump float;/n" +
                "varying vec2 vTextureCoord;/n" +
                "uniform samplerExternalOES sTexture;/n" +
                "void main() {/n" +
                "  gl_FragColor = texture2D(sTexture, vTextureCoord);/n" +
                "}/n";

        private float[] mMVPMatrix = new float[16];
        private float[] mSTMatrix = new float[16];

        private int mProgram;
        private int mTextureID;
        private int muMVPMatrixHandle;
        private int muSTMatrixHandle;
        private int maPositionHandle;
        private int maTextureHandle;

        private SurfaceTexture mSurface;
        private boolean updateSurface = false;

        private static int GL_TEXTURE_EXTERNAL_OES = 0x8D65;

        private MediaPlayer mMediaPlayer;

        public VideoRender(Context context) {
            mTriangleVertices = ByteBuffer.allocateDirect(
                mTriangleVerticesData.length * FLOAT_SIZE_BYTES)
                    .order(ByteOrder.nativeOrder()).asFloatBuffer();
            mTriangleVertices.put(mTriangleVerticesData).position(0);

            Matrix.setIdentityM(mSTMatrix, 0);
        }

        public void setMediaPlayer(MediaPlayer player) {
            mMediaPlayer = player;
        }

        public void onDrawFrame(GL10 glUnused) {
            synchronized(this) {
                if (updateSurface) {
                    mSurface.updateTexImage();
                    mSurface.getTransformMatrix(mSTMatrix);
                    updateSurface = false;
                }
            }

            GLES20.glClearColor(0.0f, 1.0f, 0.0f, 1.0f);
            GLES20.glClear( GLES20.GL_DEPTH_BUFFER_BIT | GLES20.GL_COLOR_BUFFER_BIT);

            GLES20.glUseProgram(mProgram);
            checkGlError("glUseProgram");

            GLES20.glActiveTexture(GLES20.GL_TEXTURE0);
            GLES20.glBindTexture(GL_TEXTURE_EXTERNAL_OES, mTextureID);

            mTriangleVertices.position(TRIANGLE_VERTICES_DATA_POS_OFFSET);
            GLES20.glVertexAttribPointer(maPositionHandle, 3, GLES20.GL_FLOAT, false,
                TRIANGLE_VERTICES_DATA_STRIDE_BYTES, mTriangleVertices);
            checkGlError("glVertexAttribPointer maPosition");
            GLES20.glEnableVertexAttribArray(maPositionHandle);
            checkGlError("glEnableVertexAttribArray maPositionHandle");

            mTriangleVertices.position(TRIANGLE_VERTICES_DATA_UV_OFFSET);
            GLES20.glVertexAttribPointer(maTextureHandle, 3, GLES20.GL_FLOAT, false,
                TRIANGLE_VERTICES_DATA_STRIDE_BYTES, mTriangleVertices);
            checkGlError("glVertexAttribPointer maTextureHandle");
            GLES20.glEnableVertexAttribArray(maTextureHandle);
            checkGlError("glEnableVertexAttribArray maTextureHandle");

            Matrix.setIdentityM(mMVPMatrix, 0);
            GLES20.glUniformMatrix4fv(muMVPMatrixHandle, 1, false, mMVPMatrix, 0);
            GLES20.glUniformMatrix4fv(muSTMatrixHandle, 1, false, mSTMatrix, 0);

            GLES20.glDrawArrays(GLES20.GL_TRIANGLE_STRIP, 0, 4);
            checkGlError("glDrawArrays");
            GLES20.glFinish();

        }

        public void onSurfaceChanged(GL10 glUnused, int width, int height) {

        }

        public void onSurfaceCreated(GL10 glUnused, EGLConfig config) {
            mProgram = createProgram(mVertexShader, mFragmentShader);
            if (mProgram == 0) {
                return;
            }
            maPositionHandle = GLES20.glGetAttribLocation(mProgram, "aPosition");
            checkGlError("glGetAttribLocation aPosition");
            if (maPositionHandle == -1) {
                throw new RuntimeException("Could not get attrib location for aPosition");
            }
            maTextureHandle = GLES20.glGetAttribLocation(mProgram, "aTextureCoord");
            checkGlError("glGetAttribLocation aTextureCoord");
            if (maTextureHandle == -1) {
                throw new RuntimeException("Could not get attrib location for aTextureCoord");
            }

            muMVPMatrixHandle = GLES20.glGetUniformLocation(mProgram, "uMVPMatrix");
            checkGlError("glGetUniformLocation uMVPMatrix");
            if (muMVPMatrixHandle == -1) {
                throw new RuntimeException("Could not get attrib location for uMVPMatrix");
            }

            muSTMatrixHandle = GLES20.glGetUniformLocation(mProgram, "uSTMatrix");
            checkGlError("glGetUniformLocation uSTMatrix");
            if (muSTMatrixHandle == -1) {
                throw new RuntimeException("Could not get attrib location for uSTMatrix");
            }


            int[] textures = new int[1];
            GLES20.glGenTextures(1, textures, 0);

            mTextureID = textures[0];
            GLES20.glBindTexture(GL_TEXTURE_EXTERNAL_OES, mTextureID);
            checkGlError("glBindTexture mTextureID");

            GLES20.glTexParameterf(GL_TEXTURE_EXTERNAL_OES, GLES20.GL_TEXTURE_MIN_FILTER,
                                   GLES20.GL_NEAREST);
            GLES20.glTexParameterf(GL_TEXTURE_EXTERNAL_OES, GLES20.GL_TEXTURE_MAG_FILTER,
                                   GLES20.GL_LINEAR);

            /*
             * Create the SurfaceTexture that will feed this textureID,
             * and pass it to the MediaPlayer
             */
            mSurface = new SurfaceTexture(mTextureID);
            mSurface.setOnFrameAvailableListener(this);

            Surface surface = new Surface(mSurface);
            mMediaPlayer.setSurface(surface);
            surface.release();

            try {
                mMediaPlayer.prepare();
            } catch (IOException t) {
                Log.e(TAG, "media player prepare failed");
            }

            synchronized(this) {
                updateSurface = false;
            }

            mMediaPlayer.start();
        }

        synchronized public void onFrameAvailable(SurfaceTexture surface) {
            updateSurface = true;
        }

        private int loadShader(int shaderType, String source) {
            int shader = GLES20.glCreateShader(shaderType);
            if (shader != 0) {
                GLES20.glShaderSource(shader, source);
                GLES20.glCompileShader(shader);
                int[] compiled = new int[1];
                GLES20.glGetShaderiv(shader, GLES20.GL_COMPILE_STATUS, compiled, 0);
                if (compiled[0] == 0) {
                    Log.e(TAG, "Could not compile shader " + shaderType + ":");
                    Log.e(TAG, GLES20.glGetShaderInfoLog(shader));
                    GLES20.glDeleteShader(shader);
                    shader = 0;
                }
            }
            return shader;
        }

        private int createProgram(String vertexSource, String fragmentSource) {
            int vertexShader = loadShader(GLES20.GL_VERTEX_SHADER, vertexSource);
            if (vertexShader == 0) {
                return 0;
            }
            int pixelShader = loadShader(GLES20.GL_FRAGMENT_SHADER, fragmentSource);
            if (pixelShader == 0) {
                return 0;
            }

            int program = GLES20.glCreateProgram();
            if (program != 0) {
                GLES20.glAttachShader(program, vertexShader);
                checkGlError("glAttachShader");
                GLES20.glAttachShader(program, pixelShader);
                checkGlError("glAttachShader");
                GLES20.glLinkProgram(program);
                int[] linkStatus = new int[1];
                GLES20.glGetProgramiv(program, GLES20.GL_LINK_STATUS, linkStatus, 0);
                if (linkStatus[0] != GLES20.GL_TRUE) {
                    Log.e(TAG, "Could not link program: ");
                    Log.e(TAG, GLES20.glGetProgramInfoLog(program));
                    GLES20.glDeleteProgram(program);
                    program = 0;
                }
            }
            return program;
        }

        private void checkGlError(String op) {
            int error;
            while ((error = GLES20.glGetError()) != GLES20.GL_NO_ERROR) {
                Log.e(TAG, op + ": glError " + error);
                throw new RuntimeException(op + ": glError " + error);
            }
        }

    }  // End of class VideoRender.

}  // End of class VideoSurfaceView.

Answer 3

Supongo que no puedes. Al menos eso es lo que descubrí. Mi plan era tener algún tipo de escena OpenGL (ticker de texto) mientras se reproduce un video. Dado que Android usa decodificación de HW para mostrar un video, no se hará con OpenGL. También intenté reproducir el video en OpenGL usando ffmpeg pero descubrí que ningún dispositivo con el que probé tenía el rendimiento suficiente para descodificar SW a través de ffmpeg.

Así que tuve que usar un VideoView para mostrar mi video y poner un GLSurfaceView encima de él, para ver el texto de mi ticker. Pero tienes que hacer que el GLSurfaceView sea translúcido como en ApiDemos hecho con ''TranslucentGLSurfaceViewActivity''.

Otra cosa que me di cuenta: si pones un GLSurfaceView encima de un VideoView, tus fps se descomponen dramáticamente de 60fps (opengl) a unos 30-40fps. Esto se aplica a todas las versiones 2.x de Android i probado. La semana pasada tuve la oportunidad de probarlo en Android 4 y esta vez no tuve una falla en fps. Tal vez realmente mejoraron la tubería de gráficos para ICS.

Saludos, -chris-

Answer 4

mMediaPlayer.setSurface(new Surface(mSurfaceTexture));

Puede usar la línea de código anterior para usarla en su mediaPlayerObject sobre su superficie deseada, que es una textura aplicada sobre su vista de superficie.

Espero que ayude.