android使用MediaCodec實現異步視頻編解碼
android使用MediaCodec實現異步視頻編解碼
最近在作屏幕投影的項目中須要對H.264的視頻流作解碼播放顯示，項目基於Android 7.0的系統，雖然android系統已經提供了一套以MediaCodec爲核心的硬解碼實現方案。可是在實際運用過程當中，遇到許多問題，在這裏進行一個總結，方便本身之後查閱，主要涉及如下內容：java
- TextureView與SurfaceView
- MediaCodec介紹
- 異步編解碼實現方式
- 同步編解碼實現方式
TextureView與SurfaceView
咱們在作解碼的過程當中，須要建立一個Surface來顯示咱們的解碼內容，Surface的來源View用的較多的有兩種，一種是TextureView，還有一種是SurfaceView，那這兩種view有什麼區別呢？android
SurfaceView： 與普通View不一樣的是，它有本身的Surface，而且Surface的渲染能夠放到單獨線程去作，這對於一些遊戲、視頻等性能相關的應用很是有益，由於它不會影響主線程對事件的響應。但它也有缺點，由於這個Surface不在View hierachy中，它的顯示也不受View的屬性控制，因此不能進行平移，縮放等變換，也不能放在其它ViewGroup中，一些View中的特性也沒法使用。git
TextureView： 它能夠將內容流直接投影到View中，能夠用於實現Live preview等功能。和SurfaceView不一樣，它不會在WMS中單首創建窗口，而是做爲View hierachy中的一個普通View，所以能夠和其它普通View同樣進行移動，旋轉，縮放，動畫等變化。值得注意的是TextureView必須在硬件加速的窗口中。它顯示的內容流數據能夠來自App進程或是遠端進程.github
MediaCodec介紹
MediaCodec是Android系統提供開發者用於訪問底層多媒體編解碼的組件，做爲Android底層多媒體支持基礎一般與MediaExtractor, MediaSync, MediaMuxer，MediaCrypto, MediaDrm, Image, Surface, AudioTrack一塊兒使用，後面我將一一介紹這些功能組件的使用方式。MediaCodec的工做流程：
web
當MediaCodec開始工做時, 首先會提供一個空的輸入buff給到Client端，Client端把要編碼的數據填充到這個空buff中回給MediaCodec，MediaCodec內部將數據編碼完成後，再填充一個buff提供給client端進行處理。針對編解碼兩種狀況，MediaCodec流程以下：app
編碼時：
1.將數據填充到MediaCodec提供的空buffer中。填充方式，同步與異步的實現有所差異。
2.客戶端通知MediaCodec數據已填好
3.MediaCodec內部對數據進行硬編碼
4.填充完成後將數據輸出給客戶端，客戶端根據需求傳輸數據ssh
解碼時：
1. 將編碼後的數據流填充buffer提供給MediaCodec。
2. 客戶端通知MediaCodec數據已填好
3. MediaCodec內部對數據硬解碼
4. 將解碼後的數據輸出給Surface顯示異步
MediaCodec狀態維護：
ide
異步編解碼實現方式
異步編碼器實現代碼以下：svg
package com.zdragon.videoio;
import android.media.MediaCodec;
import android.media.MediaCodecInfo;
import android.media.MediaFormat;
import android.os.Handler;
import android.os.HandlerThread;
import android.support.annotation.NonNull;
import android.util.Log;
import android.view.Surface;
import java.io.IOException;
import java.nio.ByteBuffer;
import java.util.concurrent.ArrayBlockingQueue;
/** * This class use for Encode Video Frame Data. * Created by zj on 2018/7/29 0029. */
public class VideoEncoder {
private final static String TAG = "VideoEncoder";
private final static int CONFIGURE_FLAG_ENCODE = MediaCodec.CONFIGURE_FLAG_ENCODE;
private final static int CACHE_BUFFER_SIZE = 8;
private MediaCodec mMediaCodec;
private MediaFormat mMediaFormat;
private int mViewWidth;
private int mViewHeight;
private Handler mVideoEncoderHandler;
private HandlerThread mVideoEncoderHandlerThread = new HandlerThread("VideoEncoder");
//This video stream format must be I420
private final static ArrayBlockingQueue<byte []> mInputDatasQueue = new ArrayBlockingQueue<byte []>(CACHE_BUFFER_SIZE);
//Cachhe video stream which has been encoded.
private final static ArrayBlockingQueue<byte []> mOutputDatasQueue = new ArrayBlockingQueue<byte[]>(CACHE_BUFFER_SIZE);
private MediaCodec.Callback mCallback = new MediaCodec.Callback() {
@Override
public void onInputBufferAvailable(@NonNull MediaCodec mediaCodec, int id) {
ByteBuffer inputBuffer = mediaCodec.getInputBuffer(id);
inputBuffer.clear();
byte [] dataSources = mInputDatasQueue.poll();
int length = 0;
if(dataSources != null) {
inputBuffer.put(dataSources);
length = dataSources.length;
}
mediaCodec.queueInputBuffer(id,0, length,0,0);
}
@Override
public void onOutputBufferAvailable(@NonNull MediaCodec mediaCodec, int id, @NonNull MediaCodec.BufferInfo bufferInfo) {
ByteBuffer outputBuffer = mMediaCodec.getOutputBuffer(id);
MediaFormat outputFormat = mMediaCodec.getOutputFormat(id);
if(outputBuffer != null && bufferInfo.size > 0){
byte [] buffer = new byte[outputBuffer.remaining()];
outputBuffer.get(buffer);
boolean result = mOutputDatasQueue.offer(buffer);
if(!result){
Log.d(TAG, "Offer to queue failed, queue in full state");
}
}
mMediaCodec.releaseOutputBuffer(id, true);
}
@Override
public void onError(@NonNull MediaCodec mediaCodec, @NonNull MediaCodec.CodecException e) {
Log.d(TAG, "------> onError");
}
@Override
public void onOutputFormatChanged(@NonNull MediaCodec mediaCodec, @NonNull MediaFormat mediaFormat) {
Log.d(TAG, "------> onOutputFormatChanged");
}
};
public VideoEncoder(String mimeType, int viewwidth, int viewheight){
try {
mMediaCodec = MediaCodec.createEncoderByType(mimeType);
} catch (IOException e) {
Log.e(TAG, Log.getStackTraceString(e));
mMediaCodec = null;
return;
}
this.mViewWidth = viewwidth;
this.mViewHeight = viewheight;
mVideoEncoderHandlerThread.start();
mVideoEncoderHandler = new Handler(mVideoEncoderHandlerThread.getLooper());
mMediaFormat = MediaFormat.createVideoFormat(mimeType, mViewWidth, mViewHeight);
mMediaFormat.setInteger(MediaFormat.KEY_COLOR_FORMAT, MediaCodecInfo.CodecCapabilities.COLOR_FormatYUV420Flexible);
mMediaFormat.setInteger(MediaFormat.KEY_BIT_RATE, 1920 * 1280);
mMediaFormat.setInteger(MediaFormat.KEY_FRAME_RATE, 30);
mMediaFormat.setInteger(MediaFormat.KEY_I_FRAME_INTERVAL, 1);
}
/** * Input Video stream which need encode to Queue * @param needEncodeData I420 format stream */
public void inputFrameToEncoder(byte [] needEncodeData){
boolean inputResult = mInputDatasQueue.offer(needEncodeData);
Log.d(TAG, "-----> inputEncoder queue result = " + inputResult + " queue current size = " + mInputDatasQueue.size());
}
/** * Get Encoded frame from queue * @return a encoded frame; it would be null when the queue is empty. */
public byte [] pollFrameFromEncoder(){
return mOutputDatasQueue.poll();
}
/** * start the MediaCodec to encode video data */
public void startEncoder(){
if(mMediaCodec != null){
mMediaCodec.setCallback(mCallback, mVideoEncoderHandler);
mMediaCodec.configure(mMediaFormat, null, null, CONFIGURE_FLAG_ENCODE);
mMediaCodec.start();
}else{
throw new IllegalArgumentException("startEncoder failed,is the MediaCodec has been init correct?");
}
}
/** * stop encode the video data */
public void stopEncoder(){
if(mMediaCodec != null){
mMediaCodec.stop();
mMediaCodec.setCallback(null);
}
}
/** * release all resource that used in Encoder */
public void release(){
if(mMediaCodec != null){
mInputDatasQueue.clear();
mOutputDatasQueue.clear();
mMediaCodec.release();
}
}
}
異步解碼器實現代碼以下：
package com.zdragon.videoio;
import android.media.MediaCodec;
import android.media.MediaCodecInfo;
import android.media.MediaFormat;
import android.os.Handler;
import android.os.HandlerThread;
import android.support.annotation.NonNull;
import android.util.Log;
import android.view.Surface;
import java.io.IOException;
import java.nio.ByteBuffer;
import java.util.concurrent.ArrayBlockingQueue;
/** * This class use for Decode Video Frame Data and show to SurfaceTexture * Created by zj on 2018/7/29 0029. */
public class VideoDecoder {
private final static String TAG = "VideoEncoder";
private final static int CONFIGURE_FLAG_DECODE = 0;
private MediaCodec mMediaCodec;
private MediaFormat mMediaFormat;
private Surface mSurface;
private int mViewWidth;
private int mViewHeight;
private VideoEncoder mVideoEncoder;
private Handler mVideoDecoderHandler;
private HandlerThread mVideoDecoderHandlerThread = new HandlerThread("VideoDecoder");
private MediaCodec.Callback mCallback = new MediaCodec.Callback() {
@Override
public void onInputBufferAvailable(@NonNull MediaCodec mediaCodec, int id) {
ByteBuffer inputBuffer = mediaCodec.getInputBuffer(id);
inputBuffer.clear();
byte [] dataSources = null;
if(mVideoEncoder != null) {
dataSources = mVideoEncoder.pollFrameFromEncoder();
}
int length = 0;
if(dataSources != null) {
inputBuffer.put(dataSources);
length = dataSources.length;
}
mediaCodec.queueInputBuffer(id,0, length,0,0);
}
@Override
public void onOutputBufferAvailable(@NonNull MediaCodec mediaCodec, int id, @NonNull MediaCodec.BufferInfo bufferInfo) {
ByteBuffer outputBuffer = mMediaCodec.getOutputBuffer(id);
MediaFormat outputFormat = mMediaCodec.getOutputFormat(id);
if(mMediaFormat == outputFormat && outputBuffer != null && bufferInfo.size > 0){
byte [] buffer = new byte[outputBuffer.remaining()];
outputBuffer.get(buffer);
}
mMediaCodec.releaseOutputBuffer(id, true);
}
@Override
public void onError(@NonNull MediaCodec mediaCodec, @NonNull MediaCodec.CodecException e) {
Log.d(TAG, "------> onError");
}
@Override
public void onOutputFormatChanged(@NonNull MediaCodec mediaCodec, @NonNull MediaFormat mediaFormat) {
Log.d(TAG, "------> onOutputFormatChanged");
}
};
public VideoDecoder(String mimeType, Surface surface, int viewwidth, int viewheight){
try {
mMediaCodec = MediaCodec.createDecoderByType(mimeType);
} catch (IOException e) {
Log.e(TAG, Log.getStackTraceString(e));
mMediaCodec = null;
return;
}
if(surface == null){
return;
}
this.mViewWidth = viewwidth;
this.mViewHeight = viewheight;
this.mSurface = surface;
mVideoDecoderHandlerThread.start();
mVideoDecoderHandler = new Handler(mVideoDecoderHandlerThread.getLooper());
mMediaFormat = MediaFormat.createVideoFormat(mimeType, mViewWidth, mViewHeight);
mMediaFormat.setInteger(MediaFormat.KEY_COLOR_FORMAT, MediaCodecInfo.CodecCapabilities.COLOR_FormatYUV420Flexible);
mMediaFormat.setInteger(MediaFormat.KEY_BIT_RATE, 1920 * 1280);
mMediaFormat.setInteger(MediaFormat.KEY_FRAME_RATE, 30);
mMediaFormat.setInteger(MediaFormat.KEY_I_FRAME_INTERVAL, 1);
}
public void setEncoder(VideoEncoder videoEncoder){
this.mVideoEncoder = videoEncoder;
}
public void startDecoder(){
if(mMediaCodec != null && mSurface != null){
mMediaCodec.setCallback(mCallback, mVideoDecoderHandler);
mMediaCodec.configure(mMediaFormat, mSurface,null,CONFIGURE_FLAG_DECODE);
mMediaCodec.start();
}else{
throw new IllegalArgumentException("startDecoder failed, please check the MediaCodec is init correct");
}
}
public void stopDecoder(){
if(mMediaCodec != null){
mMediaCodec.stop();
}
}
/** * release all resource that used in Encoder */
public void release(){
if(mMediaCodec != null){
mMediaCodec.release();
mMediaCodec = null;
}
}
}
驗證編解碼的正確性，我這裏作了一個demo, 在華爲榮耀V8上打開相機預覽，並將預覽數據編解碼後，在同界面的TextureView上顯示出來。具體實現方式以下：
1.界面UI，第一個TextureView用於預覽攝像頭，第二個TextureView用於顯示解碼後畫面
<?xml version="1.0" encoding="utf-8"?>
<LinearLayout xmlns:android="http://schemas.android.com/apk/res/android" xmlns:tools="http://schemas.android.com/tools" android:layout_width="match_parent" android:layout_height="match_parent" android:orientation="vertical" tools:context="com.zdragon.videoio.MainActivity">
<TextureView android:id="@+id/camera" android:layout_width="wrap_content" android:layout_height="0dp" android:layout_weight="1"/>
<TextureView android:id="@+id/decode" android:layout_width="wrap_content" android:layout_height="0dp" android:layout_weight="1" />
</LinearLayout>
- 在Activity的實現
package com.zdragon.videoio;
import android.Manifest;
import android.content.pm.PackageManager;
import android.graphics.ImageFormat;
import android.graphics.SurfaceTexture;
import android.hardware.Camera;
import android.support.annotation.NonNull;
import android.support.v4.app.ActivityCompat;
import android.support.v4.content.ContextCompat;
import android.support.v7.app.AppCompatActivity;
import android.os.Bundle;
import android.util.Log;
import android.util.Size;
import android.view.Surface;
import android.view.TextureView;
import java.io.IOException;
import java.util.List;
public class MainActivity extends AppCompatActivity {
private final static String TAG = "VideoIO";
private final static String MIME_FORMAT = "video/avc"; //support h.264
private TextureView mCameraTexture;
private TextureView mDecodeTexture;
private VideoDecoder mVideoDecoder;
private VideoEncoder mVideoEncoder;
private Camera mCamera;
private int mPreviewWidth;
private int mPreviewHeight;
private Camera.PreviewCallback mPreviewCallBack = new Camera.PreviewCallback() {
@Override
public void onPreviewFrame(byte[] bytes, Camera camera) {
byte[] i420bytes = new byte[bytes.length];
//from YV20 to i420
System.arraycopy(bytes, 0, i420bytes, 0, mPreviewWidth * mPreviewHeight);
System.arraycopy(bytes, mPreviewWidth * mPreviewHeight + mPreviewWidth * mPreviewHeight / 4, i420bytes, mPreviewWidth * mPreviewHeight, mPreviewWidth * mPreviewHeight / 4);
System.arraycopy(bytes, mPreviewWidth * mPreviewHeight, i420bytes, mPreviewWidth * mPreviewHeight + mPreviewWidth * mPreviewHeight / 4, mPreviewWidth * mPreviewHeight / 4);
if(mVideoEncoder != null) {
mVideoEncoder.inputFrameToEncoder(i420bytes);
}
}
};
private TextureView.SurfaceTextureListener mCameraTextureListener = new TextureView.SurfaceTextureListener() {
@Override
public void onSurfaceTextureAvailable(SurfaceTexture surfaceTexture, int i, int i1) {
openCamera(surfaceTexture,i, i1);
mVideoEncoder = new VideoEncoder(MIME_FORMAT, mPreviewWidth, mPreviewHeight);
mVideoEncoder.startEncoder();
}
@Override
public void onSurfaceTextureSizeChanged(SurfaceTexture surfaceTexture, int i, int i1) {
}
@Override
public boolean onSurfaceTextureDestroyed(SurfaceTexture surfaceTexture) {
if(mVideoEncoder != null){
mVideoEncoder.release();
}
closeCamera();
return true;
}
@Override
public void onSurfaceTextureUpdated(SurfaceTexture surfaceTexture) {
}
};
private TextureView.SurfaceTextureListener mDecodeTextureListener = new TextureView.SurfaceTextureListener() {
@Override
public void onSurfaceTextureAvailable(SurfaceTexture surfaceTexture, int i, int i1) {
System.out.println("----------" + i + " ," + i1);
mVideoDecoder = new VideoDecoder(MIME_FORMAT, new Surface(surfaceTexture), mPreviewWidth, mPreviewHeight);
mVideoDecoder.setEncoder(mVideoEncoder);
mVideoDecoder.startDecoder();
}
@Override
public void onSurfaceTextureSizeChanged(SurfaceTexture surfaceTexture, int i, int i1) {
}
@Override
public boolean onSurfaceTextureDestroyed(SurfaceTexture surfaceTexture) {
mVideoDecoder.stopDecoder();
mVideoDecoder.release();
return true;
}
@Override
public void onSurfaceTextureUpdated(SurfaceTexture surfaceTexture) {
}
};
@Override
public void onRequestPermissionsResult(int requestCode, @NonNull String[] permissions, @NonNull int[] grantResults) {
super.onRequestPermissionsResult(requestCode, permissions, grantResults);
initView();
}
@Override
protected void onCreate(Bundle savedInstanceState) {
super.onCreate(savedInstanceState);
setContentView(R.layout.activity_main);
if (ContextCompat.checkSelfPermission(this, Manifest.permission.CAMERA) == PackageManager.PERMISSION_GRANTED) {
Log.i("TEST","Granted");
initView();
} else {
ActivityCompat.requestPermissions(this, new String[]{Manifest.permission.CAMERA}, 1);//1 can be another integer
}
}
private void initView(){
mCameraTexture = (TextureView)findViewById(R.id.camera);
mDecodeTexture = (TextureView)findViewById(R.id.decode);
mCameraTexture.setSurfaceTextureListener(mCameraTextureListener);
mDecodeTexture.setSurfaceTextureListener(mDecodeTextureListener);
}
private void openCamera(SurfaceTexture texture,int width, int height){
if(texture == null){
Log.e(TAG, "openCamera need SurfaceTexture");
return;
}
mCamera = Camera.open(0);
try{
mCamera.setPreviewTexture(texture);
Camera.Parameters parameters = mCamera.getParameters();
parameters.setPreviewFormat(ImageFormat.YV12);
List<Camera.Size> list = parameters.getSupportedPreviewSizes();
for(Camera.Size size: list){
System.out.println("----size width = " + size.width + " size height = " + size.height);
}
mPreviewWidth = 640;
mPreviewHeight = 480;
parameters.setPreviewSize(mPreviewWidth,mPreviewHeight);
mCamera.setParameters(parameters);
mCamera.setPreviewCallback(mPreviewCallBack);
mCamera.startPreview();
}catch(IOException e){
Log.e(TAG, Log.getStackTraceString(e));
mCamera = null;
}
}
private void closeCamera(){
if(mCamera == null){
Log.e(TAG, "Camera not open");
return;
}
mCamera.stopPreview();
mCamera.release();
}
}
最終實現效果如圖，這樣處理的速度幾乎與攝像頭同步：
同步編解碼的方式能夠參考Android API文檔，MediaCodec的使用說明上有詳細使用用例。同步的時候有時候在不一樣的機型上，有的機器能夠正確播放，有的機器不能出畫面，這時候須要針對機型，在初始化Codec的時候，延時200ms調用start函數便可。
工程代碼下載路徑：
http下載：
ssh下載：
git@github.com:RunningWay/android.git