最近接觸了一個國內優秀的流媒體平臺Darwin深度裁剪版本，看了一段時間後就想跟本身的開發的一個android設備對接，以了卻我多年對多媒體更深刻的理解。（本人曾經本身開發一個H264的移動電視的軟解碼播放器）android
在搭建好服務器後，拿出了全志的T2開發板，開始使用其團隊的EasyPusher庫進行推送。首先我嘗試視頻直播的開發，在研究了兩天後，發現爲個人機器的系統的解碼器已經被封裝成庫，對外不開放。因此我不得不使用其android系統的mediaRecoder進行編碼，而系統已經把其直接打包成了MP4文件，這樣我沒法得到直接的H264數據。因而陷入了困頓之中。在等待方案公司的幫助的時候，我尋找了別的方法去得到推送的數據流，最後不得不採用localhost技術，該技術就是將mediaRecoder編碼的數據經過相似本地網絡管道的方式接受下來。在接受到數據後發現仍是沒法進行傳送。由於接受到數據沒有任何MP4相關的描述信息只有數據流。一個MP4文件的數據可以播放，是由於在MP4文件中有不少的BOX構成，其中尤爲有兩個box很是重要：moovbox和mdatbox其中前者包含了MP4內的宏觀的描述信息，後者包含的實際的數據。由於mediarecoder在處理數據後纔將其中的moovbox的數據一次性寫進MP4文件，因此我能得到只有mdatbox的數據。沒有辦法只好對從根據輸入參數來得到相關信息以及大體估計時間戳。具體作法以下：git
第一步：搜索MDAT數據頭github
第二步：將其中的數據解析成H264的數據包，其中數據幀的長度恰好四個字節，用H264的頭（0x00,0x00,0x00,0x01來代替。）服務器
第三步：調用EasyPusher將數據發送到服務器。這裏須要注意的是，遇到I幀的時候須要加上SPS和PPS。固然localhost的數據是不會有這些數據的，只好本身根據分辨率等信息寫好。以下網絡
const unsigned char SPSPPSCIF[] ={0X00,0X00,0X00,0X01, 0x67,0x42, 0X00, 0X1F, 0XE5, 0X40, 0XA0, 0xfc,0X80,0X00,0X00,0X00,0X01,0X68,0XCE,0x31,0X12};//320*240ide
const unsigned char SPSPPS576[] ={0X00,0X00,0X00,0X01, 0x67,0x42, 0X00, 0X1F, 0XE5, 0X40, 0X5A, 0x09,0X32,0X00,0X00,0X00,0X01,0X68,0XCE,0x31,0X12}; //720*576工具
這些數據最好出現I幀(幀頭第5個字節通常是0X67，參照H264數據規範)加上。我估計這對流媒體傳輸來講是很重要的。編碼
傳輸過去後，在客戶端能夠看到視頻，但丟幀很是嚴重。我估計主要是由於localhost以及編碼器效率過低，同時時間戳也是一個估計數。因此形成了丟幀現象。沒有什麼好的辦法了，只有等待硬件編碼器的相關資料直接編碼進行發送了。code
繞過直播，開始搞回放。首先選擇一個DEMUX工具，我選擇的是MP4V2，主要是這個工具WINDOWS和android NDK版本都有，因此相對來講比較省時間。折騰一段時間後發現數據能傳輸上去了。可是音頻和視頻徹底不一樣步。在跟EasyDarwin 團隊的一個朋友討論後我決定開始搞音視頻同步，策略是採用的是講音視頻幀都同步到系統時間。果真效果仍是很明顯的，除了偶爾會卡住基本上能流暢播放。我估計着須要開始瞭解播放器端的同步了。相關的代碼附錄上：視頻
MP4 DEMUX：獲取視頻和音頻幀：
typedef struct _Media_INFO_
{
MP4FileHandle mp4File;
MP4TrackId video_trId;
MP4TrackId audio_trId;
u_int32_t video_sample_max_size;
u_int32_t video_num_samples;
u_int32_t video_width;
u_int32_t video_height;
double video_frame_rate;
u_int32_t video_timescale;
u_int32_t audio_num_samples;
int audio_num_channels;
u_int32_t audio_timescale;
MP4Duration audio_duration;
u_int32_t avgBitRate;
u_int8_t AudioType;
u_int8_t *p_audio_config;
u_int32_t n_audio_config_size;
u_int32_t n_video_config_size;
u_int32_t audio_sample_max_size;
MP4Duration video_duration;
u_int8_t *p_Vediobuffer;
u_int8_t *p_Audio_buffer;
}MEDIA_INFO;
staticint GetVideoStreamHeader(MP4FileHandle mp4File, MP4TrackId video_trId, intvideo_codec,
unsigned char*strm_hdr_buf, int *strm_hdr_leng)
{
int b;
// for MPEG4
unsigned char *p_video_conf;
unsigned int n_video_conf;
// for H.264
unsigned char **pp_sps,**pp_pps;
unsigned int *pn_sps, *pn_pps;
unsigned int n_strm_size;
int i;
switch (video_codec) {
case RAW_STRM_TYPE_M4V: //MPEG4
p_video_conf = NULL;
n_video_conf = 0;
b = MP4GetTrackESConfiguration(mp4File, video_trId,
&p_video_conf, &n_video_conf);
if (!b)
return -1;
memcpy(strm_hdr_buf, p_video_conf, n_video_conf);
free(p_video_conf);
*strm_hdr_leng = n_video_conf;
break;
case RAW_STRM_TYPE_H263: //H.263
*strm_hdr_leng = 0;
break;
case RAW_STRM_TYPE_H264RAW: // H.264
pp_sps = pp_pps = NULL;
pn_sps = pn_pps = NULL;
n_strm_size = 0;
b = MP4GetTrackH264SeqPictHeaders(mp4File, video_trId, &pp_sps,&pn_sps, &pp_pps, &pn_pps);
if (!b)
return -1;
// SPS memcpy
if (pp_sps) {
for (i=0; *(pp_sps + i); i++) {
memcpy(strm_hdr_buf +n_strm_size, h264_delimiter, sizeof(h264_delimiter));
n_strm_size +=sizeof(h264_delimiter);
memcpy(strm_hdr_buf +n_strm_size, *(pp_sps + i), *(pn_sps + i));
/*
if(NAL_UNIT_TYPE_TYPE(strm_hdr_buf[n_strm_size]) == 7) {
strm_hdr_buf[n_strm_size +1] = 66;
}
*/
n_strm_size += *(pn_sps + i);
free(*(pp_sps + i));
}
free(pp_sps);
}
// PPS memcpy
if (pp_pps) {
for (i=0; *(pp_pps + i); i++) {
memcpy(strm_hdr_buf +n_strm_size, h264_delimiter, sizeof(h264_delimiter));
n_strm_size +=sizeof(h264_delimiter);
memcpy(strm_hdr_buf + n_strm_size,*(pp_pps + i), *(pn_pps + i));
n_strm_size += *(pn_pps + i);
free(*(pp_pps + i));
}
free(pp_pps);
}
*strm_hdr_leng = n_strm_size;
break;
default: // Unknown
*strm_hdr_leng = 0;
break;
}
return 0;
}
int GetAudioFrame(MEDIA_INFO* pMedia,unsigned int *pSize, intsample_id, unsigned long* timebase ,unsigned long* dutation)
{
int nRead;
u_int8_t *p_audio_sample;
u_int32_t n_audio_sample;
int b, ret;
MP4Timestamp TimeStamp;
MP4Duration Audio_duration;
p_audio_sample = (u_int8_t *) pMedia->p_Audio_buffer;;
*pSize= pMedia->audio_sample_max_size;
n_audio_sample = *pSize;
if ((sample_id <= 0) || (sample_id >pMedia->audio_num_samples)) {
*pSize = 0;
return -1;
}
/////////////////////
// MP4ReadSample //
/////////////////////
b= MP4ReadSample(pMedia->mp4File,pMedia->audio_trId, sample_id,
&p_audio_sample,&n_audio_sample,
&TimeStamp,&Audio_duration, NULL, NULL);
if (!b) {
*pSize = 0;
return -1;
}
*pSize = nRead = n_audio_sample;
*timebase= TimeStamp;
*dutation= Audio_duration;
return nRead;
}
int GetVideoFrame(MEDIA_INFO* pMedia, intm_video_codec, unsigned int *pSize,unsigned int *isIframe, int sample_id,unsigned long* timebase, unsigned long*duration )
{
int nRead;
int n_video_hdr;
u_int8_t *p_video_sample;
u_int32_t n_video_sample;
MP4Timestamp TimeStamp;
MP4Duration video_duration;
int b, ret;
p_video_sample = (u_int8_t *) pMedia->p_Vediobuffer;
*pSize= pMedia->video_sample_max_size;
n_video_sample = *pSize;
if((sample_id <= 0) || (sample_id > pMedia->video_num_samples)) {
*pSize = 0;
return -1;
}
*isIframe = MP4GetSampleSync(pMedia->mp4File,pMedia->video_trId, sample_id);
if (*isIframe ) {
ret = GetVideoStreamHeader(pMedia->mp4File, pMedia->video_trId,m_video_codec,
p_video_sample, &n_video_hdr);
p_video_sample += n_video_hdr;
}
else
n_video_hdr = 0;
/////////////////////
// MP4ReadSample //
/////////////////////
b= MP4ReadSample(pMedia->mp4File, pMedia->video_trId, sample_id,
&p_video_sample,&n_video_sample,
&TimeStamp,&video_duration, NULL, NULL);
if (!b) {
*pSize = 0;
return -1;
}
//if (codec == h.264), the first 4 bytes are the length octets.
// They need to be changed to H.264 delimiter (00 00 00 01).
if (m_video_codec == RAW_STRM_TYPE_H264RAW) {
int h264_nal_leng;
int nal_leng_acc = 0;
do {
h264_nal_leng = p_video_sample[0];
h264_nal_leng = (h264_nal_leng << 8) | p_video_sample[1];
h264_nal_leng = (h264_nal_leng << 8) | p_video_sample[2];
h264_nal_leng = (h264_nal_leng << 8) | p_video_sample[3];
memcpy(p_video_sample, h264_delimiter, 4);
nal_leng_acc += (h264_nal_leng +4);
p_video_sample += (h264_nal_leng + 4);
} while (nal_leng_acc < n_video_sample);
}
*timebase= TimeStamp;//TimeStamp+video_duration;
*duration= video_duration;
*pSize = nRead = (n_video_hdr + n_video_sample);
return nRead;
}
時鐘同步：
/返回須要延遲的時鐘
typedef struct _SYN_CLOCK_CTRL_
{
unsignedchar Vedioflag;
unsignedchar Audioflag;
unsignedlong ClockBase;
unsignedlong ClockCurr;
unsignedlong VedioBase;
unsignedlong AudioBase;
}Sync_clock_Ctl;
Sync_clock_Ctl g_clock;
unsigned long Sync_clock(MEDIA_INFO*pMedia, unsigned long timebase, unsigned long duration, int type, unsignedlong* OutTime)
{
unsignedlong TimebaseNew;
unsignedlong DiffClock;
doubleTimeCalbase;
doubleTimenext;
unsignedlong CurrentTime;
unsignedlong NextTime;
unsignedlong delay;
unsignedlong TimeScale;
#ifdef _WIN32
if(g_clock.ClockBase== 0)
{
g_clock.ClockBase= ::GetTickCount();
}
g_clock.ClockCurr= ::GetTickCount();
#else
{
structtimeval tv;
gettimeofday(&tv,NULL);
//return(int64_t)tv.tv_sec * 1000000 + tv.tv_usec;
g_clock.ClockCurr= (int64_t)tv.tv_sec * 1000 + tv.tv_usec/1000;
if(g_clock.ClockBase== 0)
{
g_clock.ClockBase= g_clock.ClockCurr;
}
}
#endif
DiffClock= g_clock.ClockCurr - g_clock.ClockBase;//時鐘的耗時間Tick數//微妙級別忽略不計
if(type== VEDIO_PUSH)
{
if(g_clock.Vedioflag== 0)
{
g_clock.VedioBase= timebase;
g_clock.Vedioflag= 1;
}
TimeScale= pMedia->video_timescale;
}else
{
if(g_clock.Audioflag== 0)
{
g_clock.AudioBase= timebase;
g_clock.Audioflag= 1;
}
TimeScale= pMedia->audio_timescale;
}
TimeCalbase= ((double)(timebase-g_clock.VedioBase))/TimeScale;
Timenext= ((double)(timebase-g_clock.VedioBase+duration))/TimeScale;
//開始計算當前和小一個Sample的時間估計決定延遲//
CurrentTime= (unsigned long)(TimeCalbase*1000);
NextTime= (unsigned long)(Timenext*1000);
*OutTime= CurrentTime;
if(DiffClock> NextTime) //已經落後，快進
{
delay= 0;
}else
{
delay= NextTime- DiffClock;//從新計算時間
}
returndelay;
}
源碼下載網址：https://github.com/Car-eye-admin/ 車載視頻開發能夠加QQ羣590411159。