Android 音视频——NuPlayer 的渲染模块
作者：程思扬
- 2022 年 6 月 04 日
本文字数：12709 字
阅读完需：约 42 分钟
渲染模块的主要功能如下。
将音视频原始数据缓存到队列。
音频数据消耗播放。
视频数据消耗显示。
音视频同步。
播放器控制。
下面将音视频原始数据 缓存到队列。在\frameworks\av\media\libmediaplayerservice\
nuplayer'NuPlayerRenderer.cpp中，存在一个QueueEntry结构体和两个队列，代码如下： struct QueueEntry { sp<ABuffer> mBuffer; sp<AMessage> mNotifyConsumed; size_t mOffset; status_t mFinalResult; int32_t mBufferOrdinal; };List<QueueEntry> mAudioQueue; //音频缓存队列List<QueueEntry> mVideoQueue; //视频缓存队列数据是在onQueueBuffer函数中进行添加：void NuPlayer::Renderer::onQueueBuffer(const sp<AMessage> &msg) { int32_t audio; CHECK(msg->findInt32("audio", &audio));
if (dropBufferIfStale(audio, msg)) { return; }
if (audio) { mHasAudio = true;//音频 } else { mHasVideo = true;//视频 }
if (mHasVideo) { if (mVideoScheduler == NULL) { mVideoScheduler = new VideoFrameScheduler(); mVideoScheduler->init(); } }
sp<ABuffer> buffer; CHECK(msg->findBuffer("buffer", &buffer));
sp<AMessage> notifyConsumed; CHECK(msg->findMessage("notifyConsumed", ¬ifyConsumed));
QueueEntry entry; entry.mBuffer = buffer; entry.mNotifyConsumed = notifyConsumed; entry.mOffset = 0; entry.mFinalResult = OK; entry.mBufferOrdinal = ++mTotalBuffersQueued;
if (audio) { Mutex::Autolock autoLock(mLock); mAudioQueue.push_back(entry); postDrainAudioQueue_l(); } else { mVideoQueue.push_back(entry); postDrainVideoQueue(); }
Mutex::Autolock autoLock(mLock); if (!mSyncQueues || mAudioQueue.empty() || mVideoQueue.empty()) { return; }
sp<ABuffer> firstAudioBuffer = (*mAudioQueue.begin()).mBuffer; sp<ABuffer> firstVideoBuffer = (*mVideoQueue.begin()).mBuffer;
if (firstAudioBuffer == NULL || firstVideoBuffer == NULL) { // EOS signalled on either queue. syncQueuesDone_l(); return; }
int64_t firstAudioTimeUs; int64_t firstVideoTimeUs; CHECK(firstAudioBuffer->meta() ->findInt64("timeUs", &firstAudioTimeUs)); CHECK(firstVideoBuffer->meta() ->findInt64("timeUs", &firstVideoTimeUs));
int64_t diff = firstVideoTimeUs - firstAudioTimeUs;
ALOGV("queueDiff = %.2f secs", diff / 1E6);
if (diff > 100000ll) { // Audio data starts More than 0.1 secs before video. // Drop some audio.
(*mAudioQueue.begin()).mNotifyConsumed->post(); mAudioQueue.erase(mAudioQueue.begin()); return; }
syncQueuesDone_l();}
那么音频是怎么播放的呢？在NuPlayerRenderer中，会先调用openAudioSink函数void NuPlayer::Renderer::onAudioTearDown(AudioTearDownReason reason) { if (mAudioTornDown) { return; } mAudioTornDown = true;
int64_t currentPositionUs; sp<AMessage> notify = mNotify->dup(); if (getCurrentPosition(¤tPositionUs) == OK) { notify->setInt64("positionUs", currentPositionUs); }
mAudioSink->stop(); mAudioSink->flush();
notify->setInt32("what", kWhatAudioTearDown); notify->setInt32("reason", reason); notify->post();}
void NuPlayer::Renderer::startAudioOffloadPauseTimeout() { if (offloadingAudio()) { mWakeLock->acquire(); sp<AMessage> msg = new AMessage(kWhatAudioOffloadPauseTimeout, this); msg->setInt32("drainGeneration", mAudioOffloadPauseTimeoutGeneration); msg->post(kOffloadPauseMaxUs); }}
void NuPlayer::Renderer::cancelAudioOffloadPauseTimeout() { if (offloadingAudio()) { mWakeLock->release(true); ++mAudioOffloadPauseTimeoutGeneration; }}
status_t NuPlayer::Renderer::onOpenAudioSink( const sp<AMessage> &format, bool offloadOnly, bool hasVideo, uint32_t flags) { ALOGV("openAudioSink: offloadOnly(%d) offloadingAudio(%d)", offloadOnly, offloadingAudio()); bool audioSinkChanged = false;
int32_t numChannels; CHECK(format->findInt32("channel-count", &numChannels));
int32_t channelMask; if (!format->findInt32("channel-mask", &channelMask)) { // signal to the AudioSink to derive the mask from count. channelMask = CHANNEL_MASK_USE_CHANNEL_ORDER; }
int32_t sampleRate; CHECK(format->findInt32("sample-rate", &sampleRate));
if (offloadingAudio()) { audio_format_t audioFormat = AUDIO_FORMAT_PCM_16_BIT; AString mime; CHECK(format->findString("mime", &mime)); status_t err = mapMimeToAudioFormat(audioFormat, mime.c_str());
if (err != OK) { ALOGE("Couldn't map mime \"%s\" to a valid " "audio_format", mime.c_str()); onDisableOffloadAudio(); } else { ALOGV("Mime \"%s\" mapped to audio_format 0x%x", mime.c_str(), audioFormat);
int avgBitRate = -1; format->findInt32("bit-rate", &avgBitRate);
int32_t aacProfile = -1; if (audioFormat == AUDIO_FORMAT_AAC && format->findInt32("aac-profile", &aacProfile)) { // Redefine AAC format as per aac profile mapAACProfileToAudioFormat( audioFormat, aacProfile); }
audio_offload_info_t offloadInfo = AUDIO_INFO_INITIALIZER; offloadInfo.duration_us = -1; format->findInt64( "durationUs", &offloadInfo.duration_us); offloadInfo.sample_rate = sampleRate; offloadInfo.channel_mask = channelMask; offloadInfo.format = audioFormat; offloadInfo.stream_type = AUDIO_STREAM_MUSIC; offloadInfo.bit_rate = avgBitRate; offloadInfo.has_video = hasVideo; offloadInfo.is_streaming = true;
if (memcmp(&mCurrentOffloadInfo, &offloadInfo, sizeof(offloadInfo)) == 0) { ALOGV("openAudioSink: no change in offload mode"); // no change from previous configuration, everything ok. return OK; } mCurrentPcmInfo = AUDIO_PCMINFO_INITIALIZER;
ALOGV("openAudioSink: try to open AudioSink in offload mode"); uint32_t offloadFlags = flags; offloadFlags |= AUDIO_OUTPUT_FLAG_COMPRESS_OFFLOAD; offloadFlags &= ~AUDIO_OUTPUT_FLAG_DEEP_BUFFER; audioSinkChanged = true; mAudioSink->close();
err = mAudioSink->open( sampleRate, numChannels, (audio_channel_mask_t)channelMask, audioFormat, 0 /* bufferCount - unused */, &NuPlayer::Renderer::AudioSinkCallback, this, (audio_output_flags_t)offloadFlags, &offloadInfo);
if (err == OK) { err = mAudioSink->setPlaybackRate(mPlaybackSettings); }
if (err == OK) { // If the playback is offloaded to h/w, we pass // the HAL some metadata information. // We don't want to do this for PCM because it // will be going through the AudioFlinger mixer // before reaching the hardware. // TODO mCurrentOffloadInfo = offloadInfo; if (!mPaused) { // for preview mode, don't start if paused err = mAudioSink->start(); } ALOGV_IF(err == OK, "openAudioSink: offload succeeded"); } if (err != OK) { // Clean up, fall back to non offload mode. mAudioSink->close(); onDisableOffloadAudio(); mCurrentOffloadInfo = AUDIO_INFO_INITIALIZER; ALOGV("openAudioSink: offload failed"); } else { mUseAudioCallback = true; // offload mode transfers data through callback ++mAudioDrainGeneration; // discard pending kWhatDrainAudioQueue message. } } } if (!offloadOnly && !offloadingAudio()) { ALOGV("openAudioSink: open AudioSink in NON-offload mode"); uint32_t pcmFlags = flags; pcmFlags &= ~AUDIO_OUTPUT_FLAG_COMPRESS_OFFLOAD;
const PcmInfo info = { (audio_channel_mask_t)channelMask, (audio_output_flags_t)pcmFlags, AUDIO_FORMAT_PCM_16_BIT, // TODO: change to audioFormat numChannels, sampleRate }; if (memcmp(&mCurrentPcmInfo, &info, sizeof(info)) == 0) { ALOGV("openAudioSink: no change in pcm mode"); // no change from previous configuration, everything ok. return OK; }
audioSinkChanged = true; mAudioSink->close(); mCurrentOffloadInfo = AUDIO_INFO_INITIALIZER; // Note: It is possible to set up the callback, but not use it to send audio data. // This requires a fix in AudioSink to explicitly specify the transfer mode. mUseAudioCallback = getUseAudioCallbackSetting(); if (mUseAudioCallback) { ++mAudioDrainGeneration; // discard pending kWhatDrainAudioQueue message. }
// Compute the desired buffer size. // For callback mode, the amount of time before wakeup is about half the buffer size. const uint32_t frameCount = (unsigned long long)sampleRate * getAudioSinkPcmMsSetting() / 1000;
// The doNotReconnect means AudioSink will signal back and let NuPlayer to re-construct // AudioSink. We don't want this when there's video because it will cause a video seek to // the previous I frame. But we do want this when there's only audio because it will give // NuPlayer a chance to switch from non-offload mode to offload mode. // So we only set doNotReconnect when there's no video. const bool doNotReconnect = !hasVideo; status_t err = mAudioSink->open( sampleRate, numChannels, (audio_channel_mask_t)channelMask, AUDIO_FORMAT_PCM_16_BIT, 0 /* bufferCount - unused */, mUseAudioCallback ? &NuPlayer::Renderer::AudioSinkCallback : NULL, mUseAudioCallback ? this : NULL, (audio_output_flags_t)pcmFlags, NULL, doNotReconnect, frameCount); if (err == OK) { err = mAudioSink->setPlaybackRate(mPlaybackSettings); } if (err != OK) { ALOGW("openAudioSink: non offloaded open failed status: %d", err); mAudioSink->close(); mCurrentPcmInfo = AUDIO_PCMINFO_INITIALIZER; return err; } mCurrentPcmInfo = info; if (!mPaused) { // for preview mode, don't start if paused mAudioSink->start(); } } if (audioSinkChanged) { onAudioSinkChanged(); } mAudioTornDown = false; return OK;}
void NuPlayer::Renderer::postDrainAudioQueue_l(int64_t delayUs) { if (mDrainAudioQueuePending || mSyncQueues || mUseAudioCallback) { return; }
if (mAudioQueue.empty()) { return; }
// FIXME: if paused, wait until AudioTrack stop() is complete before delivering data. if (mPaused) { const int64_t diffUs = mPauseDrainAudioAllowedUs - ALooper::GetNowUs(); if (diffUs > delayUs) { delayUs = diffUs; } }
mDrainAudioQueuePending = true; sp<AMessage> msg = new AMessage(kWhatDrainAudioQueue, this); msg->setInt32("drainGeneration", mAudioDrainGeneration); msg->post(delayUs);}
复制代码
上面主要是发送了一个消息 kWhatDrainAudioQueue,找到对应接收消息的地方，代码如 下：
case kWhatDrainAudioQueue: { mDrainAudioQueuePending = false;
int32_t generation; CHECK(msg->findInt32("drainGeneration", &generation)); if (generation != getDrainGeneration(true /* audio */)) { break; }
if (onDrainAudioQueue()) { uint32_t numFramesPlayed; CHECK_EQ(mAudioSink->getPosition(&numFramesPlayed), (status_t)OK);
uint32_t numFramesPendingPlayout = mNumFramesWritten - numFramesPlayed;
// This is how long the audio sink will have data to // play back. int64_t delayUs = mAudioSink->msecsPerFrame() * numFramesPendingPlayout * 1000ll; if (mPlaybackRate > 1.0f) { delayUs /= mPlaybackRate; }
// Let's give it more data after about half that time // has elapsed. Mutex::Autolock autoLock(mLock); postDrainAudioQueue_l(delayUs / 2); } break; }
复制代码
主要是有一个进行判断的 onDrainAudioQueue 函数，判断是否需要重新向 AudioSink 写入 数据，代码如下：
bool NuPlayer::Renderer::onDrainAudioQueue() { // do not drain audio during teardown as queued buffers may be invalid. if (mAudioTornDown) { return false; } // TODO: This call to getPosition checks if AudioTrack has been created // in AudioSink before draining audio. If AudioTrack doesn't exist, then // CHECKs on getPosition will fail. // We still need to figure out why AudioTrack is not created when // this function is called. One possible reason could be leftover // audio. Another possible place is to check whether decoder // has received INFO_FORMAT_CHANGED as the first buffer since // AudioSink is opened there, and possible interactions with flush // immediately after start. Investigate error message // "vorbis_dsp_synthesis returned -135", along with RTSP. uint32_t numFramesPlayed; if (mAudioSink->getPosition(&numFramesPlayed) != OK) { // When getPosition fails, renderer will not reschedule the draining // unless new samples are queued. // If we have pending EOS (or "eos" marker for discontinuities), we need // to post these now as NuPlayerDecoder might be waiting for it. drainAudioQueueUntilLastEOS();
ALOGW("onDrainAudioQueue(): audio sink is not ready"); return false; }
#if 0 ssize_t numFramesAvailableToWrite = mAudioSink->frameCount() - (mNumFramesWritten - numFramesPlayed);
if (numFramesAvailableToWrite == mAudioSink->frameCount()) { ALOGI("audio sink underrun"); } else { ALOGV("audio queue has %d frames left to play", mAudioSink->frameCount() - numFramesAvailableToWrite); }#endif
uint32_t prevFramesWritten = mNumFramesWritten; while (!mAudioQueue.empty()) { QueueEntry *entry = &*mAudioQueue.begin();
mLastAudioBufferDrained = entry->mBufferOrdinal;
if (entry->mBuffer == NULL) { // EOS int64_t postEOSDelayUs = 0; if (mAudioSink->needsTrailingPadding()) { postEOSDelayUs = getPendingAudioPlayoutDurationUs(ALooper::GetNowUs()); } notifyEOS(true /* audio */, entry->mFinalResult, postEOSDelayUs);
mAudioQueue.erase(mAudioQueue.begin()); entry = NULL; if (mAudioSink->needsTrailingPadding()) { // If we're not in gapless playback (i.e. through setNextPlayer), we // need to stop the track here, because that will play out the last // little bit at the end of the file. Otherwise short files won't play. mAudioSink->stop(); mNumFramesWritten = 0; } return false; }
// ignore 0-sized buffer which could be EOS marker with no data if (entry->mOffset == 0 && entry->mBuffer->size() > 0) { int64_t mediaTimeUs; CHECK(entry->mBuffer->meta()->findInt64("timeUs", &mediaTimeUs)); ALOGV("onDrainAudioQueue: rendering audio at media time %.2f secs", mediaTimeUs / 1E6); onNewAudioMediaTime(mediaTimeUs); }
size_t copy = entry->mBuffer->size() - entry->mOffset;
ssize_t written = mAudioSink->write(entry->mBuffer->data() + entry->mOffset, copy, false /* blocking */); if (written < 0) { // An error in AudioSink write. Perhaps the AudioSink was not properly opened. if (written == WOULD_BLOCK) { ALOGV("AudioSink write would block when writing %zu bytes", copy); } else { ALOGE("AudioSink write error(%zd) when writing %zu bytes", written, copy); // This can only happen when AudioSink was opened with doNotReconnect flag set to // true, in which case the NuPlayer will handle the reconnect. notifyAudioTearDown(); } break; }
entry->mOffset += written; if (entry->mOffset == entry->mBuffer->size()) { entry->mNotifyConsumed->post(); mAudioQueue.erase(mAudioQueue.begin());
entry = NULL; }
size_t copiedFrames = written / mAudioSink->frameSize(); mNumFramesWritten += copiedFrames;
{ Mutex::Autolock autoLock(mLock); int64_t maxTimeMedia; maxTimeMedia = mAnchorTimeMediaUs + (int64_t)(max((long long)mNumFramesWritten - mAnchorNumFramesWritten, 0LL) * 1000LL * mAudioSink->msecsPerFrame()); mMediaClock->updateMaxTimeMedia(maxTimeMedia);
notifyIfMediaRenderingStarted_l(); }
if (written != (ssize_t)copy) { // A short count was received from AudioSink::write() // // AudioSink write is called in non-blocking mode. // It may return with a short count when: // // 1) Size to be copied is not a multiple of the frame size. We consider this fatal. // 2) The data to be copied exceeds the available buffer in AudioSink. // 3) An error occurs and data has been partially copied to the buffer in AudioSink. // 4) AudioSink is an AudioCache for data retrieval, and the AudioCache is exceeded.
// (Case 1) // Must be a multiple of the frame size. If it is not a multiple of a frame size, it // needs to fail, as we should not carry over fractional frames between calls. CHECK_EQ(copy % mAudioSink->frameSize(), 0);
// (Case 2, 3, 4) // Return early to the caller. // Beware of calling immediately again as this may busy-loop if you are not careful. ALOGV("AudioSink write short frame count %zd < %zu", written, copy); break; } }
// calculate whether we need to reschedule another write. bool reschedule = !mAudioQueue.empty() && (!mPaused || prevFramesWritten != mNumFramesWritten); // permit pause to fill buffers //ALOGD("reschedule:%d empty:%d mPaused:%d prevFramesWritten:%u mNumFramesWritten:%u", // reschedule, mAudioQueue.empty(), mPaused, prevFramesWritten, mNumFramesWritten); return reschedule;}
复制代码
到这里，已经很清楚了，音频播放流程如下：先打开音频后端，然后当向音频队列中发送 数据时，音频队列同时向音频后端写入数据，以供播放音频。那么视频显示呢？同样是在视频原始数据进入视频队列后，开始执行 postDrain Video Queue 函数：
void NuPlayer::Renderer::postDrainVideoQueue() { if (mDrainVideoQueuePending || getSyncQueues() || (mPaused && mVideoSampleReceived)) { return; }
if (mVideoQueue.empty()) { return; }
QueueEntry &entry = *mVideoQueue.begin();
sp<AMessage> msg = new AMessage(kWhatDrainVideoQueue, this); msg->setInt32("drainGeneration", getDrainGeneration(false /* audio */));
if (entry.mBuffer == NULL) { // EOS doesn't carry a timestamp. msg->post(); mDrainVideoQueuePending = true; return; }
int64_t delayUs; int64_t nowUs = ALooper::GetNowUs(); int64_t realTimeUs; if (mFlags & FLAG_REAL_TIME) { int64_t mediaTimeUs; CHECK(entry.mBuffer->meta()->findInt64("timeUs", &mediaTimeUs)); realTimeUs = mediaTimeUs; } else { int64_t mediaTimeUs; CHECK(entry.mBuffer->meta()->findInt64("timeUs", &mediaTimeUs));
{ Mutex::Autolock autoLock(mLock); if (mAnchorTimeMediaUs < 0) { mMediaClock->updateAnchor(mediaTimeUs, nowUs, mediaTimeUs); mAnchorTimeMediaUs = mediaTimeUs; realTimeUs = nowUs; } else { realTimeUs = getRealTimeUs(mediaTimeUs, nowUs); } } if (!mHasAudio) { // smooth out videos >= 10fps mMediaClock->updateMaxTimeMedia(mediaTimeUs + 100000); }
// Heuristics to handle situation when media time changed without a // discontinuity. If we have not drained an audio buffer that was // received after this buffer, repost in 10 msec. Otherwise repost // in 500 msec. delayUs = realTimeUs - nowUs; if (delayUs > 500000) { int64_t postDelayUs = 500000; if (mHasAudio && (mLastAudioBufferDrained - entry.mBufferOrdinal) <= 0) { postDelayUs = 10000; } msg->setWhat(kWhatPostDrainVideoQueue); msg->post(postDelayUs); mVideoScheduler->restart(); ALOGI("possible video time jump of %dms, retrying in %dms", (int)(delayUs / 1000), (int)(postDelayUs / 1000)); mDrainVideoQueuePending = true; return; } }
realTimeUs = mVideoScheduler->schedule(realTimeUs * 1000) / 1000; int64_t twoVsyncsUs = 2 * (mVideoScheduler->getVsyncPeriod() / 1000);
delayUs = realTimeUs - nowUs;
ALOGW_IF(delayUs > 500000, "unusually high delayUs: %" PRId64, delayUs); // post 2 display refreshes before rendering is due msg->post(delayUs > twoVsyncsUs ? delayUs - twoVsyncsUs : 0);
mDrainVideoQueuePending = true;}
复制代码
这里先发送一个 kWhatDrainVideoQueue 的消息，然后进行音视频同步。发送 kWhatDrain- VideoQueue 消息后，会触发调用 onDrainVideoQueue 函数：
void NuPlayer::Renderer::onDrainVideoQueue() { if (mVideoQueue.empty()) { return; }
QueueEntry *entry = &*mVideoQueue.begin();
if (entry->mBuffer == NULL) { // EOS
notifyEOS(false /* audio */, entry->mFinalResult);
mVideoQueue.erase(mVideoQueue.begin()); entry = NULL;
setVideoLateByUs(0); return; }
int64_t nowUs = -1; int64_t realTimeUs; if (mFlags & FLAG_REAL_TIME) { CHECK(entry->mBuffer->meta()->findInt64("timeUs", &realTimeUs)); } else { int64_t mediaTimeUs; CHECK(entry->mBuffer->meta()->findInt64("timeUs", &mediaTimeUs));
nowUs = ALooper::GetNowUs(); realTimeUs = getRealTimeUs(mediaTimeUs, nowUs); }
bool tooLate = false;
if (!mPaused) { if (nowUs == -1) { nowUs = ALooper::GetNowUs(); } setVideoLateByUs(nowUs - realTimeUs); tooLate = (mVideoLateByUs > 40000);
if (tooLate) { ALOGV("video late by %lld us (%.2f secs)", (long long)mVideoLateByUs, mVideoLateByUs / 1E6); } else { int64_t mediaUs = 0; mMediaClock->getMediaTime(realTimeUs, &mediaUs); ALOGV("rendering video at media time %.2f secs", (mFlags & FLAG_REAL_TIME ? realTimeUs : mediaUs) / 1E6); } } else { setVideoLateByUs(0); if (!mVideoSampleReceived && !mHasAudio) { // This will ensure that the first frame after a flush won't be used as anchor // when renderer is in paused state, because resume can happen any time after seek. Mutex::Autolock autoLock(mLock); clearAnchorTime_l(); } }
entry->mNotifyConsumed->setInt64("timestampNs", realTimeUs * 1000ll); entry->mNotifyConsumed->setInt32("render", !tooLate); entry->mNotifyConsumed->post(); mVideoQueue.erase(mVideoQueue.begin()); entry = NULL;
mVideoSampleReceived = true;
if (!mPaused) { if (!mVideoRenderingStarted) { mVideoRenderingStarted = true; notifyVideoRenderingStart(); } Mutex::Autolock autoLock(mLock); notifyIfMediaRenderingStarted_l();//向上通知开始渲染视频 }}
复制代码
NuPlayer::Renderer 使用的是以视频为基准的同步机制，解码后的音频数据时间戳如果大 于视频数据时间戳，直接丢弃音频包，然后直接渲染视频。同步机制主要位于视频缓冲区处理 部分的 onDrainVideoQueue 和音频缓冲区处理部分的 onDrainVideoQueue 中。音视频的渲染都 釆用类似定时器的机制，只不过视频显示需要依赖于实际解码器，音频播放需要依赖于 AudioSink 的接口。
划线
评论
复制
发布于: 刚刚阅读数: 6
版权声明: 本文为 InfoQ 作者【程思扬】的原创文章。
原文链接:【http://xie.infoq.cn/article/903d3330d53c9f8aff54788a1】。文章转载请联系作者。
程思扬
关注
会的越多，不会的越多 2022.01.03 加入
还未添加个人简介
评论