From 45fc6dfaaaea4f0a2429ad1fcff4d5e1412bf4c6 Mon Sep 17 00:00:00 2001 From: Minyue Li Date: Thu, 21 Jun 2018 11:47:14 +0200 Subject: [PATCH] Aligning time in audio jitter buffer plot to other plots in rtc event log visualizer. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bug: webrtc:9147 Change-Id: I4ddb3e93ea04a11a68e097ecad731d6d9d6842a9 Reviewed-on: https://webrtc-review.googlesource.com/75322 Reviewed-by: Björn Terelius Reviewed-by: Henrik Lundin Commit-Queue: Minyue Li Cr-Commit-Position: refs/heads/master@{#23712} --- .../neteq/tools/neteq_delay_analyzer.cc | 208 ++++++++---------- .../neteq/tools/neteq_delay_analyzer.h | 14 +- rtc_tools/event_log_visualizer/analyzer.cc | 75 +++---- rtc_tools/event_log_visualizer/main.cc | 8 +- 4 files changed, 133 insertions(+), 172 deletions(-) diff --git a/modules/audio_coding/neteq/tools/neteq_delay_analyzer.cc b/modules/audio_coding/neteq/tools/neteq_delay_analyzer.cc index 2b2152018b..e5bd7652ef 100644 --- a/modules/audio_coding/neteq/tools/neteq_delay_analyzer.cc +++ b/modules/audio_coding/neteq/tools/neteq_delay_analyzer.cc @@ -23,6 +23,13 @@ namespace webrtc { namespace test { namespace { +std::string kArrivalDelayX = "arrival_delay_x"; +std::string kArrivalDelayY = "arrival_delay_y"; +std::string kTargetDelayX = "target_delay_x"; +std::string kTargetDelayY = "target_delay_y"; +std::string kPlayoutDelayX = "playout_delay_x"; +std::string kPlayoutDelayY = "playout_delay_y"; + // Helper function for NetEqDelayAnalyzer::CreateGraphs. Returns the // interpolated value of a function at the point x. Vector x_vec contains the // sample points, and y_vec contains the function values at these points. The @@ -54,6 +61,26 @@ double LinearInterpolate(double x, } return y; } + +void PrintDelays(const NetEqDelayAnalyzer::Delays& delays, + int64_t ref_time_ms, + const std::string& var_name_x, + const std::string& var_name_y, + std::ofstream& output, + const std::string& terminator = "") { + output << var_name_x << " = [ "; + for (const std::pair& delay : delays) { + output << (delay.first - ref_time_ms) / 1000.f << ", "; + } + output << "]" << terminator << std::endl; + + output << var_name_y << " = [ "; + for (const std::pair& delay : delays) { + output << delay.second << ", "; + } + output << "]" << terminator << std::endl; +} + } // namespace void NetEqDelayAnalyzer::AfterInsertPacket( @@ -97,12 +124,10 @@ void NetEqDelayAnalyzer::AfterGetAudio(int64_t time_now_ms, ++get_audio_count_; } -void NetEqDelayAnalyzer::CreateGraphs( - std::vector* send_time_s, - std::vector* arrival_delay_ms, - std::vector* corrected_arrival_delay_ms, - std::vector>* playout_delay_ms, - std::vector>* target_delay_ms) const { +void NetEqDelayAnalyzer::CreateGraphs(Delays* arrival_delay_ms, + Delays* corrected_arrival_delay_ms, + Delays* playout_delay_ms, + Delays* target_delay_ms) const { if (get_audio_time_ms_.empty()) { return; } @@ -123,111 +148,76 @@ void NetEqDelayAnalyzer::CreateGraphs( // calculates the base offset. for (auto& d : data_) { rtp_timestamps_ms.push_back( - unwrapper.Unwrap(d.first) / + static_cast(unwrapper.Unwrap(d.first)) / rtc::CheckedDivExact(last_sample_rate_hz_, 1000)); offset = std::min(offset, d.second.arrival_time_ms - rtp_timestamps_ms.back()); } - // Calculate send times in seconds for each packet. This is the (unwrapped) - // RTP timestamp in ms divided by 1000. - send_time_s->resize(rtp_timestamps_ms.size()); - std::transform(rtp_timestamps_ms.begin(), rtp_timestamps_ms.end(), - send_time_s->begin(), [rtp_timestamps_ms](double x) { - return (x - rtp_timestamps_ms[0]) / 1000.f; - }); - RTC_DCHECK_EQ(send_time_s->size(), rtp_timestamps_ms.size()); - // This loop traverses the data again and populates the graph vectors. The // reason to have two loops and traverse twice is that the offset cannot be // known until the first traversal is done. Meanwhile, the final offset must // be known already at the start of this second loop. - auto data_it = data_.cbegin(); - for (size_t i = 0; i < send_time_s->size(); ++i, ++data_it) { - RTC_DCHECK(data_it != data_.end()); - const double offset_send_time_ms = rtp_timestamps_ms[i] + offset; - const auto& timing = data_it->second; - corrected_arrival_delay_ms->push_back( + size_t i = 0; + for (const auto& data : data_) { + const double offset_send_time_ms = rtp_timestamps_ms[i++] + offset; + const auto& timing = data.second; + corrected_arrival_delay_ms->push_back(std::make_pair( + timing.arrival_time_ms, LinearInterpolate(timing.arrival_time_ms, get_audio_time_ms_, nominal_get_audio_time_ms) - - offset_send_time_ms); - arrival_delay_ms->push_back(timing.arrival_time_ms - offset_send_time_ms); + offset_send_time_ms)); + arrival_delay_ms->push_back(std::make_pair( + timing.arrival_time_ms, timing.arrival_time_ms - offset_send_time_ms)); if (timing.decode_get_audio_count) { // This packet was decoded. RTC_DCHECK(timing.sync_delay_ms); - const float playout_ms = *timing.decode_get_audio_count * 10 + - get_audio_time_ms_[0] + *timing.sync_delay_ms - - offset_send_time_ms; - playout_delay_ms->push_back(playout_ms); + const int64_t get_audio_time = + *timing.decode_get_audio_count * 10 + get_audio_time_ms_[0]; + const float playout_ms = + get_audio_time + *timing.sync_delay_ms - offset_send_time_ms; + playout_delay_ms->push_back(std::make_pair(get_audio_time, playout_ms)); RTC_DCHECK(timing.target_delay_ms); RTC_DCHECK(timing.current_delay_ms); const float target = playout_ms - *timing.current_delay_ms + *timing.target_delay_ms; - target_delay_ms->push_back(target); - } else { - // This packet was never decoded. Mark target and playout delays as empty. - playout_delay_ms->push_back(absl::nullopt); - target_delay_ms->push_back(absl::nullopt); + target_delay_ms->push_back(std::make_pair(get_audio_time, target)); } } - RTC_DCHECK(data_it == data_.end()); - RTC_DCHECK_EQ(send_time_s->size(), corrected_arrival_delay_ms->size()); - RTC_DCHECK_EQ(send_time_s->size(), playout_delay_ms->size()); - RTC_DCHECK_EQ(send_time_s->size(), target_delay_ms->size()); } void NetEqDelayAnalyzer::CreateMatlabScript( const std::string& script_name) const { - std::vector send_time_s; - std::vector arrival_delay_ms; - std::vector corrected_arrival_delay_ms; - std::vector> playout_delay_ms; - std::vector> target_delay_ms; - CreateGraphs(&send_time_s, &arrival_delay_ms, &corrected_arrival_delay_ms, + Delays arrival_delay_ms; + Delays corrected_arrival_delay_ms; + Delays playout_delay_ms; + Delays target_delay_ms; + CreateGraphs(&arrival_delay_ms, &corrected_arrival_delay_ms, &playout_delay_ms, &target_delay_ms); + // Maybe better to find the actually smallest timestamp, to surely avoid + // x-axis starting from negative. + const int64_t ref_time_ms = arrival_delay_ms.front().first; + // Create an output file stream to Matlab script file. std::ofstream output(script_name); - // The iterator is used to batch-output comma-separated values from vectors. - std::ostream_iterator output_iterator(output, ","); - output << "send_time_s = [ "; - std::copy(send_time_s.begin(), send_time_s.end(), output_iterator); - output << "];" << std::endl; + PrintDelays(corrected_arrival_delay_ms, ref_time_ms, kArrivalDelayX, + kArrivalDelayY, output, ";"); - output << "arrival_delay_ms = [ "; - std::copy(arrival_delay_ms.begin(), arrival_delay_ms.end(), output_iterator); - output << "];" << std::endl; + // PrintDelays(corrected_arrival_delay_x, kCorrectedArrivalDelayX, + // kCorrectedArrivalDelayY, output); - output << "corrected_arrival_delay_ms = [ "; - std::copy(corrected_arrival_delay_ms.begin(), - corrected_arrival_delay_ms.end(), output_iterator); - output << "];" << std::endl; + PrintDelays(playout_delay_ms, ref_time_ms, kPlayoutDelayX, kPlayoutDelayY, + output, ";"); - output << "playout_delay_ms = [ "; - for (const auto& v : playout_delay_ms) { - if (!v) { - output << "nan, "; - } else { - output << *v << ", "; - } - } - output << "];" << std::endl; + PrintDelays(target_delay_ms, ref_time_ms, kTargetDelayX, kTargetDelayY, + output, ";"); - output << "target_delay_ms = [ "; - for (const auto& v : target_delay_ms) { - if (!v) { - output << "nan, "; - } else { - output << *v << ", "; - } - } - output << "];" << std::endl; - - output << "h=plot(send_time_s, arrival_delay_ms, " - << "send_time_s, target_delay_ms, 'g.', " - << "send_time_s, playout_delay_ms);" << std::endl; + output << "h=plot(" << kArrivalDelayX << ", " << kArrivalDelayY << ", " + << kTargetDelayX << ", " << kTargetDelayY << ", 'g.', " + << kPlayoutDelayX << ", " << kPlayoutDelayY << ");" << std::endl; output << "set(h(1),'color',0.75*[1 1 1]);" << std::endl; output << "set(h(2),'markersize',6);" << std::endl; output << "set(h(3),'linew',1.5);" << std::endl; @@ -235,7 +225,7 @@ void NetEqDelayAnalyzer::CreateMatlabScript( output << "axis tight" << std::endl; output << "ax2=axis;" << std::endl; output << "axis([ax2(1:3) ax1(4)])" << std::endl; - output << "xlabel('send time [s]');" << std::endl; + output << "xlabel('time [s]');" << std::endl; output << "ylabel('relative delay [ms]');" << std::endl; if (!ssrcs_.empty()) { auto ssrc_it = ssrcs_.cbegin(); @@ -255,65 +245,45 @@ void NetEqDelayAnalyzer::CreateMatlabScript( void NetEqDelayAnalyzer::CreatePythonScript( const std::string& script_name) const { - std::vector send_time_s; - std::vector arrival_delay_ms; - std::vector corrected_arrival_delay_ms; - std::vector> playout_delay_ms; - std::vector> target_delay_ms; - CreateGraphs(&send_time_s, &arrival_delay_ms, &corrected_arrival_delay_ms, + Delays arrival_delay_ms; + Delays corrected_arrival_delay_ms; + Delays playout_delay_ms; + Delays target_delay_ms; + CreateGraphs(&arrival_delay_ms, &corrected_arrival_delay_ms, &playout_delay_ms, &target_delay_ms); + // Maybe better to find the actually smallest timestamp, to surely avoid + // x-axis starting from negative. + const int64_t ref_time_ms = arrival_delay_ms.front().first; + // Create an output file stream to the python script file. std::ofstream output(script_name); - // The iterator is used to batch-output comma-separated values from vectors. - std::ostream_iterator output_iterator(output, ","); // Necessary includes output << "import numpy as np" << std::endl; output << "import matplotlib.pyplot as plt" << std::endl; - output << "send_time_s = ["; - std::copy(send_time_s.begin(), send_time_s.end(), output_iterator); - output << "]" << std::endl; + PrintDelays(corrected_arrival_delay_ms, ref_time_ms, kArrivalDelayX, + kArrivalDelayY, output); - output << "arrival_delay_ms = ["; - std::copy(arrival_delay_ms.begin(), arrival_delay_ms.end(), output_iterator); - output << "]" << std::endl; + // PrintDelays(corrected_arrival_delay_x, kCorrectedArrivalDelayX, + // kCorrectedArrivalDelayY, output); - output << "corrected_arrival_delay_ms = ["; - std::copy(corrected_arrival_delay_ms.begin(), - corrected_arrival_delay_ms.end(), output_iterator); - output << "]" << std::endl; + PrintDelays(playout_delay_ms, ref_time_ms, kPlayoutDelayX, kPlayoutDelayY, + output); - output << "playout_delay_ms = ["; - for (const auto& v : playout_delay_ms) { - if (!v) { - output << "float('nan'), "; - } else { - output << *v << ", "; - } - } - output << "]" << std::endl; - - output << "target_delay_ms = ["; - for (const auto& v : target_delay_ms) { - if (!v) { - output << "float('nan'), "; - } else { - output << *v << ", "; - } - } - output << "]" << std::endl; + PrintDelays(target_delay_ms, ref_time_ms, kTargetDelayX, kTargetDelayY, + output); output << "if __name__ == '__main__':" << std::endl; - output << " h=plt.plot(send_time_s, arrival_delay_ms, " - << "send_time_s, target_delay_ms, 'g.', " - << "send_time_s, playout_delay_ms)" << std::endl; + output << " h=plt.plot(" << kArrivalDelayX << ", " << kArrivalDelayY << ", " + << kTargetDelayX << ", " << kTargetDelayY << ", 'g.', " + << kPlayoutDelayX << ", " << kPlayoutDelayY << ")" << std::endl; output << " plt.setp(h[0],'color',[.75, .75, .75])" << std::endl; output << " plt.setp(h[1],'markersize',6)" << std::endl; output << " plt.setp(h[2],'linewidth',1.5)" << std::endl; output << " plt.axis('tight')" << std::endl; - output << " plt.xlabel('send time [s]')" << std::endl; + output << " plt.xlabel('time [s]')" << std::endl; output << " plt.ylabel('relative delay [ms]')" << std::endl; if (!ssrcs_.empty()) { auto ssrc_it = ssrcs_.cbegin(); diff --git a/modules/audio_coding/neteq/tools/neteq_delay_analyzer.h b/modules/audio_coding/neteq/tools/neteq_delay_analyzer.h index b74ff77f74..5099e03fcb 100644 --- a/modules/audio_coding/neteq/tools/neteq_delay_analyzer.h +++ b/modules/audio_coding/neteq/tools/neteq_delay_analyzer.h @@ -37,11 +37,11 @@ class NetEqDelayAnalyzer : public test::NetEqPostInsertPacket, bool muted, NetEq* neteq) override; - void CreateGraphs(std::vector* send_times_s, - std::vector* arrival_delay_ms, - std::vector* corrected_arrival_delay_ms, - std::vector>* playout_delay_ms, - std::vector>* target_delay_ms) const; + using Delays = std::vector>; + void CreateGraphs(Delays* arrival_delay_ms, + Delays* corrected_arrival_delay_ms, + Delays* playout_delay_ms, + Delays* target_delay_ms) const; // Creates a matlab script with file name script_name. When executed in // Matlab, the script will generate graphs with the same timing information @@ -55,8 +55,8 @@ class NetEqDelayAnalyzer : public test::NetEqPostInsertPacket, private: struct TimingData { - explicit TimingData(double at) : arrival_time_ms(at) {} - double arrival_time_ms; + explicit TimingData(int64_t at) : arrival_time_ms(at) {} + int64_t arrival_time_ms; absl::optional decode_get_audio_count; absl::optional sync_delay_ms; absl::optional target_delay_ms; diff --git a/rtc_tools/event_log_visualizer/analyzer.cc b/rtc_tools/event_log_visualizer/analyzer.cc index f20d301d16..9e86eb0ff5 100644 --- a/rtc_tools/event_log_visualizer/analyzer.cc +++ b/rtc_tools/event_log_visualizer/analyzer.cc @@ -1777,49 +1777,43 @@ EventLogAnalyzer::NetEqStatsGetterMap EventLogAnalyzer::SimulateNetEq( void EventLogAnalyzer::CreateAudioJitterBufferGraph( const NetEqStatsGetterMap& neteq_stats, Plot* plot) const { - if (neteq_stats.size() < 1) - return; - + RTC_CHECK(!neteq_stats.empty()); const uint32_t ssrc = neteq_stats.begin()->first; - std::vector send_times_s; - std::vector arrival_delay_ms; - std::vector corrected_arrival_delay_ms; - std::vector> playout_delay_ms; - std::vector> target_delay_ms; + test::NetEqDelayAnalyzer::Delays arrival_delay_ms; + test::NetEqDelayAnalyzer::Delays corrected_arrival_delay_ms; + test::NetEqDelayAnalyzer::Delays playout_delay_ms; + test::NetEqDelayAnalyzer::Delays target_delay_ms; + neteq_stats.at(ssrc)->delay_analyzer()->CreateGraphs( - &send_times_s, &arrival_delay_ms, &corrected_arrival_delay_ms, - &playout_delay_ms, &target_delay_ms); - RTC_DCHECK_EQ(send_times_s.size(), arrival_delay_ms.size()); - RTC_DCHECK_EQ(send_times_s.size(), corrected_arrival_delay_ms.size()); - RTC_DCHECK_EQ(send_times_s.size(), playout_delay_ms.size()); - RTC_DCHECK_EQ(send_times_s.size(), target_delay_ms.size()); + &arrival_delay_ms, &corrected_arrival_delay_ms, &playout_delay_ms, + &target_delay_ms); std::map time_series_packet_arrival; std::map time_series_relative_packet_arrival; std::map time_series_play_time; std::map time_series_target_time; - float min_y_axis = 0.f; - float max_y_axis = 0.f; - for (size_t i = 0; i < send_times_s.size(); ++i) { - time_series_packet_arrival[ssrc].points.emplace_back( - TimeSeriesPoint(send_times_s[i], arrival_delay_ms[i])); + + for (const auto& data : arrival_delay_ms) { + const float x = ToCallTimeSec(data.first * 1000); // ms to us. + const float y = data.second; + time_series_packet_arrival[ssrc].points.emplace_back(TimeSeriesPoint(x, y)); + } + for (const auto& data : corrected_arrival_delay_ms) { + const float x = ToCallTimeSec(data.first * 1000); // ms to us. + const float y = data.second; time_series_relative_packet_arrival[ssrc].points.emplace_back( - TimeSeriesPoint(send_times_s[i], corrected_arrival_delay_ms[i])); - min_y_axis = std::min(min_y_axis, corrected_arrival_delay_ms[i]); - max_y_axis = std::max(max_y_axis, corrected_arrival_delay_ms[i]); - if (playout_delay_ms[i]) { - time_series_play_time[ssrc].points.emplace_back( - TimeSeriesPoint(send_times_s[i], *playout_delay_ms[i])); - min_y_axis = std::min(min_y_axis, *playout_delay_ms[i]); - max_y_axis = std::max(max_y_axis, *playout_delay_ms[i]); - } - if (target_delay_ms[i]) { - time_series_target_time[ssrc].points.emplace_back( - TimeSeriesPoint(send_times_s[i], *target_delay_ms[i])); - min_y_axis = std::min(min_y_axis, *target_delay_ms[i]); - max_y_axis = std::max(max_y_axis, *target_delay_ms[i]); - } + TimeSeriesPoint(x, y)); + } + for (const auto& data : playout_delay_ms) { + const float x = ToCallTimeSec(data.first * 1000); // ms to us. + const float y = data.second; + time_series_play_time[ssrc].points.emplace_back(TimeSeriesPoint(x, y)); + } + for (const auto& data : target_delay_ms) { + const float x = ToCallTimeSec(data.first * 1000); // ms to us. + const float y = data.second; + time_series_target_time[ssrc].points.emplace_back(TimeSeriesPoint(x, y)); } // This code is adapted for a single stream. The creation of the streams above @@ -1847,8 +1841,8 @@ void EventLogAnalyzer::CreateAudioJitterBufferGraph( plot->SetXAxis(ToCallTimeSec(begin_time_), call_duration_s_, "Time (s)", kLeftMargin, kRightMargin); - plot->SetYAxis(min_y_axis, max_y_axis, "Relative delay (ms)", kBottomMargin, - kTopMargin); + plot->SetSuggestedYAxis(0, 1, "Relative delay (ms)", kBottomMargin, + kTopMargin); plot->SetTitle("NetEq timing for " + GetStreamName(kIncomingPacket, ssrc)); } @@ -1857,12 +1851,7 @@ void EventLogAnalyzer::CreateNetEqStatsGraph( rtc::FunctionView stats_extractor, const std::string& plot_name, Plot* plot) const { - if (neteq_stats.size() < 1) - return; - std::map time_series; - float min_y_axis = std::numeric_limits::max(); - float max_y_axis = std::numeric_limits::min(); for (const auto& st : neteq_stats) { const uint32_t ssrc = st.first; @@ -1872,8 +1861,6 @@ void EventLogAnalyzer::CreateNetEqStatsGraph( const float time = ToCallTimeSec(stats[i].first * 1000); // ms to us. const float value = stats_extractor(stats[i].second); time_series[ssrc].points.emplace_back(TimeSeriesPoint(time, value)); - min_y_axis = std::min(min_y_axis, value); - max_y_axis = std::max(max_y_axis, value); } } @@ -1885,7 +1872,7 @@ void EventLogAnalyzer::CreateNetEqStatsGraph( plot->SetXAxis(ToCallTimeSec(begin_time_), call_duration_s_, "Time (s)", kLeftMargin, kRightMargin); - plot->SetYAxis(min_y_axis, max_y_axis, plot_name, kBottomMargin, kTopMargin); + plot->SetSuggestedYAxis(0, 1, plot_name, kBottomMargin, kTopMargin); plot->SetTitle(plot_name); } diff --git a/rtc_tools/event_log_visualizer/main.cc b/rtc_tools/event_log_visualizer/main.cc index c82295ea07..7d9d45e4cc 100644 --- a/rtc_tools/event_log_visualizer/main.cc +++ b/rtc_tools/event_log_visualizer/main.cc @@ -335,8 +335,12 @@ int main(int argc, char* argv[]) { "audio_processing/conversational_speech/EN_script2_F_sp2_B1", "wav"); } auto neteq_stats = analyzer.SimulateNetEq(wav_path, 48000); - analyzer.CreateAudioJitterBufferGraph(neteq_stats, - collection->AppendNewPlot()); + + if (!neteq_stats.empty()) { + analyzer.CreateAudioJitterBufferGraph(neteq_stats, + collection->AppendNewPlot()); + } + analyzer.CreateNetEqStatsGraph( neteq_stats, [](const webrtc::NetEqNetworkStatistics& stats) {