用AI赚第一桶💰低成本搭建一套AI赚钱工具,源码可二开。 广告
## 前言 上一篇文章的丢帧是依据编码后的码率和目标码率来决定丢帧, > [http://www.jianshu.com/p/e2a9740b9877](https://www.jianshu.com/p/e2a9740b9877) 而本文介绍的丢帧依据是目标帧率。 ##### 由此可对丢帧策略分类如下: * 编码后的码率和目标码率来决定丢帧 * 目标帧率决定丢帧 ## 调用流程 前处理模块调用丢帧处理 ~~~cpp ViEEncoder::DeliverFrame ->int32_t VideoProcessingModuleImpl::PreprocessFrame ->int32_t VPMFramePreprocessor::PreprocessFrame ~~~ PreprocessFrame前处理先进行丢帧处理,然后进行重采样处理。丢帧处理中,每来一帧先进行输入帧率统计,然后进行丢帧判断。输入帧率统计和上一篇码率决定丢帧输入丢帧统计算法一样。 ~~~php int32_t VPMFramePreprocessor::PreprocessFrame(const I420VideoFrame& frame, I420VideoFrame** processed_frame) { if (frame.IsZeroSize()) { return VPM_PARAMETER_ERROR; } vd_->UpdateIncomingframe_rate(); //更新输入帧率 if (vd_->DropFrame()) {//判断是否丢帧 return 1; // drop 1 frame } // Resizing incoming frame if needed. Otherwise, remains NULL. // We are not allowed to resample the input frame (must make a copy of it). *processed_frame = NULL; if (spatial_resampler_->ApplyResample(frame.width(), frame.height())) { int32_t ret = spatial_resampler_->ResampleFrame(frame, &resampled_frame_); if (ret != VPM_OK) return ret; *processed_frame = &resampled_frame_; } // Perform content analysis on the frame to be encoded. if (enable_ca_) { // Compute new metrics every |kSkipFramesCA| frames, starting with // the first frame. if (frame_cnt_ % kSkipFrameCA == 0) { if (*processed_frame == NULL) { content_metrics_ = ca_->ComputeContentMetrics(frame); } else { content_metrics_ = ca_->ComputeContentMetrics(resampled_frame_); } } ++frame_cnt_; } return VPM_OK; }``` ##输入帧率统计 每来一帧都将此时的时间作为样本,记录在滑动窗口为kFrameCountHistory_size的incoming_frame_times_数组中。并进行ProcessIncomingframe_rate输入帧率的计算。 ~~~ void VPMVideoDecimator::UpdateIncomingframe\_rate() { int64\_t now = TickTime::MillisecondTimestamp(); if (incoming\_frame\_times\_\[0\] == 0) { // First no shift. } else { // Shift. for (int i = kFrameCountHistory\_size - 2; i >= 0; i--) { incoming\_frame\_times\_\[i+1\] = incoming\_frame\_times\_\[i\]; } } incoming\_frame\_times\_\[0\] = now; ProcessIncomingframe\_rate(now); }``` 统计最多不超过2秒钟的样本,计算输入帧率。 ~~~cpp void VPMVideoDecimator::ProcessIncomingframe_rate(int64_t now) { int32_t num = 0; int32_t nrOfFrames = 0; for (num = 1; num < (kFrameCountHistory_size - 1); num++) { // Don't use data older than 2sec. if (incoming_frame_times_[num] <= 0 || now - incoming_frame_times_[num] > kFrameHistoryWindowMs) { break; } else { nrOfFrames++; } } if (num > 1) { int64_t diff = now - incoming_frame_times_[num-1]; incoming_frame_rate_ = 1.0; if (diff > 0) { incoming_frame_rate_ = nrOfFrames * 1000.0f / static_cast<float>(diff); } } else { incoming_frame_rate_ = static_cast<float>(nrOfFrames); } }``` ##目标帧率丢帧核心 一、当2 * overshoot < (int32_t) incomingframe_rate,即输入帧率大于目标帧率,小于2倍目标帧率的情况下。具体细节看注释,这里假设incomingframe_rate=20,target_frame_rate_=15。具体就是实现均匀丢帧。 ~~~ bool VPMVideoDecimator::DropFrame() { if (!enable\_temporal\_decimation\_) return false; if (incoming\_frame\_rate\_ <= 0) return false; const uint32\_t incomingframe\_rate = static\_cast(incoming\_frame\_rate\_ + 0.5f); if (target\_frame\_rate\_ == 0) return true; bool drop = false; if (incomingframe\_rate > target\_frame\_rate\_) {//输入帧率大于目标帧率 int32\_t overshoot = overshoot\_modifier\_ + (incomingframe\_rate - target\_frame\_rate\_);//20-15=5,超出帧率 if (overshoot < 0) { overshoot = 0; overshoot\_modifier\_ = 0; } ~~~cpp if (overshoot && 2 * overshoot < (int32_t) incomingframe_rate) {//2*5<20,即20<2*15,输入帧率小于2倍目标帧率 if (drop_count_) { // Just got here so drop to be sure. drop_count_ = 0; return true; } const uint32_t dropVar = incomingframe_rate / overshoot;//20/5=4,丢帧比率 if (keep_count_ >= dropVar) {//均匀丢帧 drop = true; overshoot_modifier_ = -((int32_t) incomingframe_rate % overshoot) / 3;//-(20%5)/3=0,修正overshoot keep_count_ = 1;//重置保留帧数 } else { keep_count_++;//保留帧数 } } else { keep_count_ = 0; const uint32_t dropVar = overshoot / target_frame_rate_; if (drop_count_ < dropVar) { drop = true; drop_count_++; } else { overshoot_modifier_ = overshoot % target_frame_rate_; drop = false; drop_count_ = 0; } } ~~~ } return drop; }``` 二、当2 \* overshoot >=(int32\_t) incomingframe\_rate时,即输入帧率大于等于2倍目标帧率,此时,均匀丢帧每次丢1帧以上,具体丢帧方法和上一部分略有区别。 ~~~cpp bool VPMVideoDecimator::DropFrame() { if (!enable_temporal_decimation_) return false; if (incoming_frame_rate_ <= 0) return false; const uint32_t incomingframe_rate = static_cast<uint32_t>(incoming_frame_rate_ + 0.5f); if (target_frame_rate_ == 0) return true; bool drop = false; if (incomingframe_rate > target_frame_rate_) {//输入帧率大于目标帧率 int32_t overshoot = overshoot_modifier_ + (incomingframe_rate - target_frame_rate_); //30-10=20,超出帧率 if (overshoot < 0) { overshoot = 0; overshoot_modifier_ = 0; } if (overshoot && 2 * overshoot < (int32_t) incomingframe_rate) { //2*20>30,即30>2*10,输入帧率大于2倍目标帧率 if (drop_count_) { // Just got here so drop to be sure. drop_count_ = 0; return true; } const uint32_t dropVar = incomingframe_rate / overshoot; if (keep_count_ >= dropVar) { drop = true; overshoot_modifier_ = -((int32_t) incomingframe_rate % overshoot) / 3; keep_count_ = 1; } else { keep_count_++; } } else { keep_count_ = 0; const uint32_t dropVar = overshoot / target_frame_rate_; //20/10=2,丢帧比率 if (drop_count_ < dropVar) {//一次丢1帧以上 drop = true; drop_count_++; } else { overshoot_modifier_ = overshoot % target_frame_rate_; //20%10=0,overshoot修正 drop = false; drop_count_ = 0; } } } return drop; } ~~~