前文中，咱們已經利用 FFmpeg + OpenGLES + OpenSLES 實現了一個多媒體播放器，本文將在視頻渲染方面對播放器進行優化。android
視頻渲染優化
前文中，咱們都是將解碼的視頻幀經過 swscale 庫轉換爲 RGBA 格式，而後在送給 OpenGL 渲染，而視頻幀一般的格式是 YUV420P/YUV420SP ，因此大部分狀況下都須要 swscale 進行格式轉換。web
當視頻尺寸比較大時，再用 swscale 進行格式轉化的話，就會存在性能瓶頸，因此本文將 YUV 到 RGBA 的格式轉換放到 shader 裏，用 GPU 來實現格式轉換，提高渲染效率。微信
本文視頻渲染優化，實質上是對 OpenGLRender 視頻渲染器進行改進，使其支持 YUV420P 、 NV21 以及 NV12 這些經常使用格式圖像的渲染。app
咱們在前文一文掌握 YUV 的圖像的基本處理中知道，YUV420P 格式的圖像在內存中有 3 個平面，YUV420SP （NV2一、NV12）格式的圖像在內存中有 2 個平面，而 RGBA 格式的圖像只有一個平面。編輯器
因此，OpenGLRender 視頻渲染器要兼容 YUV420P、 YUV420SP 以及 RGBA 格式，須要建立 3 個紋理存儲待渲染的數據，渲染 YUV420P 格式的圖像須要用到 3 個紋理，渲染 YUV420SP 格式的圖像只需用到 2 個紋理便可，而渲染 RGBA 格式圖像只需一個紋理。ide
判斷解碼後視頻幀的格式，AVFrame 是解碼後的視頻幀。性能
void VideoDecoder::OnFrameAvailable(AVFrame *frame) {
if(m_VideoRender != nullptr && frame != nullptr) {
NativeImage image;
//YUV420P
if(GetCodecContext()->pix_fmt == AV_PIX_FMT_YUV420P || GetCodecContext()->pix_fmt == AV_PIX_FMT_YUVJ420P) {
image.format = IMAGE_FORMAT_I420;
image.width = frame->width;
image.height = frame->height;
image.pLineSize[0] = frame->linesize[0];
image.pLineSize[1] = frame->linesize[1];
image.pLineSize[2] = frame->linesize[2];
image.ppPlane[0] = frame->data[0];
image.ppPlane[1] = frame->data[1];
image.ppPlane[2] = frame->data[2];
if(frame->data[0] && frame->data[1] && !frame->data[2] && frame->linesize[0] == frame->linesize[1] && frame->linesize[2] == 0) {
// on some android device, output of h264 mediacodec decoder is NV12 兼容某些設備可能出現的格式不匹配問題
image.format = IMAGE_FORMAT_NV12;
}
} else if (GetCodecContext()->pix_fmt == AV_PIX_FMT_NV12) { //NV12
image.format = IMAGE_FORMAT_NV12;
image.width = frame->width;
image.height = frame->height;
image.pLineSize[0] = frame->linesize[0];
image.pLineSize[1] = frame->linesize[1];
image.ppPlane[0] = frame->data[0];
image.ppPlane[1] = frame->data[1];
} else if (GetCodecContext()->pix_fmt == AV_PIX_FMT_NV21) { //NV21
image.format = IMAGE_FORMAT_NV21;
image.width = frame->width;
image.height = frame->height;
image.pLineSize[0] = frame->linesize[0];
image.pLineSize[1] = frame->linesize[1];
image.ppPlane[0] = frame->data[0];
image.ppPlane[1] = frame->data[1];
} else if (GetCodecContext()->pix_fmt == AV_PIX_FMT_RGBA) { //RGBA
image.format = IMAGE_FORMAT_RGBA;
image.width = frame->width;
image.height = frame->height;
image.pLineSize[0] = frame->linesize[0];
image.ppPlane[0] = frame->data[0];
} else { //其餘格式由 swscale 轉換爲 RGBA
sws_scale(m_SwsContext, frame->data, frame->linesize, 0,
m_VideoHeight, m_RGBAFrame->data, m_RGBAFrame->linesize);
image.format = IMAGE_FORMAT_RGBA;
image.width = m_RenderWidth;
image.height = m_RenderHeight;
image.ppPlane[0] = m_RGBAFrame->data[0];
}
//將圖像傳遞給渲染器進行渲染
m_VideoRender->RenderVideoFrame(&image);
}
}
建立 3 個紋理，可是不指定要加載圖像的格式。學習
// TEXTURE_NUM = 3
glGenTextures(TEXTURE_NUM, m_TextureIds);
for (int i = 0; i < TEXTURE_NUM ; ++i) {
glActiveTexture(GL_TEXTURE0 + i);
glBindTexture(GL_TEXTURE_2D, m_TextureIds[i]);
glTexParameterf(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE);
glTexParameterf(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE);
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR);
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_LINEAR);
glBindTexture(GL_TEXTURE_2D, GL_NONE);
}
加載不一樣格式的數據到紋理。flex
switch (m_RenderImage.format)
{
//加載 RGBA 類型的數據
case IMAGE_FORMAT_RGBA:
glActiveTexture(GL_TEXTURE0);
glBindTexture(GL_TEXTURE_2D, m_TextureIds[0]);
glTexImage2D(GL_TEXTURE_2D, 0, GL_RGBA, m_RenderImage.width, m_RenderImage.height, 0, GL_RGBA, GL_UNSIGNED_BYTE, m_RenderImage.ppPlane[0]);
glBindTexture(GL_TEXTURE_2D, GL_NONE);
break;
//加載 YUV420SP 類型的數據
case IMAGE_FORMAT_NV21:
case IMAGE_FORMAT_NV12:
//upload Y plane data
glActiveTexture(GL_TEXTURE0);
glBindTexture(GL_TEXTURE_2D, m_TextureIds[0]);
glTexImage2D(GL_TEXTURE_2D, 0, GL_LUMINANCE, m_RenderImage.width,
m_RenderImage.height, 0, GL_LUMINANCE, GL_UNSIGNED_BYTE,
m_RenderImage.ppPlane[0]);
glBindTexture(GL_TEXTURE_2D, GL_NONE);
//update UV plane data
glActiveTexture(GL_TEXTURE1);
glBindTexture(GL_TEXTURE_2D, m_TextureIds[1]);
glTexImage2D(GL_TEXTURE_2D, 0, GL_LUMINANCE_ALPHA, m_RenderImage.width >> 1,
m_RenderImage.height >> 1, 0, GL_LUMINANCE_ALPHA, GL_UNSIGNED_BYTE,
m_RenderImage.ppPlane[1]);
glBindTexture(GL_TEXTURE_2D, GL_NONE);
break;
//加載 YUV420P 類型的數據
case IMAGE_FORMAT_I420:
//upload Y plane data
glActiveTexture(GL_TEXTURE0);
glBindTexture(GL_TEXTURE_2D, m_TextureIds[0]);
glTexImage2D(GL_TEXTURE_2D, 0, GL_LUMINANCE, m_RenderImage.width,
m_RenderImage.height, 0, GL_LUMINANCE, GL_UNSIGNED_BYTE,
m_RenderImage.ppPlane[0]);
glBindTexture(GL_TEXTURE_2D, GL_NONE);
//update U plane data
glActiveTexture(GL_TEXTURE1);
glBindTexture(GL_TEXTURE_2D, m_TextureIds[1]);
glTexImage2D(GL_TEXTURE_2D, 0, GL_LUMINANCE, m_RenderImage.width >> 1,
m_RenderImage.height >> 1, 0, GL_LUMINANCE, GL_UNSIGNED_BYTE,
m_RenderImage.ppPlane[1]);
glBindTexture(GL_TEXTURE_2D, GL_NONE);
//update V plane data
glActiveTexture(GL_TEXTURE2);
glBindTexture(GL_TEXTURE_2D, m_TextureIds[2]);
glTexImage2D(GL_TEXTURE_2D, 0, GL_LUMINANCE, m_RenderImage.width >> 1,
m_RenderImage.height >> 1, 0, GL_LUMINANCE, GL_UNSIGNED_BYTE,
m_RenderImage.ppPlane[2]);
glBindTexture(GL_TEXTURE_2D, GL_NONE);
break;
default:
break;
}
對應的頂點着色器和片斷着色器，其中重要的是，片斷着色器須要針對不一樣的圖像格式採用不用的採樣策略。優化
//頂點着色器
#version 300 es
layout(location = 0) in vec4 a_Position;
layout(location = 1) in vec2 a_texCoord;
out vec2 v_texCoord;
void main()
{
gl_Position = a_Position;
v_texCoord = a_texCoord;
}
//片斷着色器
#version 300 es
precision highp float;
in vec2 v_texCoord;
layout(location = 0) out vec4 outColor;
uniform sampler2D s_texture0;
uniform sampler2D s_texture1;
uniform sampler2D s_texture2;
uniform int u_ImgType;// 1:RGBA, 2:NV21, 3:NV12, 4:I420
void main()
{
if(u_ImgType == 1) //RGBA
{
outColor = texture(s_texture0, v_texCoord);
}
else if(u_ImgType == 2) //NV21
{
vec3 yuv;
yuv.x = texture(s_texture0, v_texCoord).r;
yuv.y = texture(s_texture1, v_texCoord).a - 0.5;
yuv.z = texture(s_texture1, v_texCoord).r - 0.5;
highp vec3 rgb = mat3(1.0, 1.0, 1.0,
0.0, -0.344, 1.770,
1.403, -0.714, 0.0) * yuv;
outColor = vec4(rgb, 1.0);
}
else if(u_ImgType == 3) //NV12
{
vec3 yuv;
yuv.x = texture(s_texture0, v_texCoord).r;
yuv.y = texture(s_texture1, v_texCoord).r - 0.5;
yuv.z = texture(s_texture1, v_texCoord).a - 0.5;
highp vec3 rgb = mat3(1.0, 1.0, 1.0,
0.0, -0.344, 1.770,
1.403, -0.714, 0.0) * yuv;
outColor = vec4(rgb, 1.0);
}
else if(u_ImgType == 4) //I420
{
vec3 yuv;
yuv.x = texture(s_texture0, v_texCoord).r;
yuv.y = texture(s_texture1, v_texCoord).r - 0.5;
yuv.z = texture(s_texture2, v_texCoord).r - 0.5;
highp vec3 rgb = mat3(1.0, 1.0, 1.0,
0.0, -0.344, 1.770,
1.403, -0.714, 0.0) * yuv;
outColor = vec4(rgb, 1.0);
}
else
{
outColor = vec4(1.0);
}
}
其中片斷着色器 u_ImgType 變量用於設置待渲染圖像的格式類型，從而採用不一樣的採樣轉換策略。
須要注意的是，YUV 格式圖像 UV 份量的默認值分別是 127 ，Y 份量默認值是 0 ，8 個 bit 位的取值範圍是 0 ~ 255，因爲在 shader 中紋理採樣值須要進行歸一化，因此 UV 份量的採樣值須要分別減去 0.5 ，確保 YUV 到 RGB 正確轉換。
-- END --
技術交流掃碼添加個人微信：Byte-Flow
免費獲取視頻教程和源碼
推薦：
FFmpeg + OpenGL ES 實現 3D 全景播放器
FFmpeg + OpenGLES 實現視頻解碼播放和視頻濾鏡
以爲不錯，點個在看唄~
本文分享自微信公衆號 - 字節流動（google_developer）。
若有侵權，請聯繫 support@oschina.cn 刪除。
本文參與「OSC源創計劃」，歡迎正在閱讀的你也加入，一塊兒分享。