From 86f25d3bcd0b7f2a162b216fbe4c0ee588d30350 Mon Sep 17 00:00:00 2001 From: Alessio Bazzica Date: Tue, 10 Oct 2017 13:28:56 +0200 Subject: [PATCH] Create links to single experiments in APM-QA exported results. For each single experiment, a URL is defined by adding a specific anchor. A URL can be copied using the button beneath the score of the experiment one would like to share. This CL also includes a few optimizations and fixes: - JS and CSS are minified - Dialog close event listener added, this fixes a small bug preventing the play out audio to stop when pressing ESC instead of using the close button - Snackbar notifications added - Simple unit test for the export module BUG=webrtc:7218 Change-Id: Iad00ce69094a5968ee0520d105d59656cfafa4e2 TBR= Change-Id: Iad00ce69094a5968ee0520d105d59656cfafa4e2 Reviewed-on: https://webrtc-review.googlesource.com/7960 Commit-Queue: Alessio Bazzica Reviewed-by: Alessio Bazzica Cr-Commit-Position: refs/heads/master@{#20266} --- .../test/py_quality_assessment/BUILD.gn | 1 + .../test/py_quality_assessment/README.md | 8 +- .../quality_assessment/collect_data.py | 1 + .../quality_assessment/export.py | 96 ++++++--- .../quality_assessment/export_unittest.py | 83 ++++++++ .../quality_assessment/results.css | 4 + .../quality_assessment/results.js | 183 +++++++++++++++--- 7 files changed, 322 insertions(+), 54 deletions(-) create mode 100644 modules/audio_processing/test/py_quality_assessment/quality_assessment/export_unittest.py diff --git a/modules/audio_processing/test/py_quality_assessment/BUILD.gn b/modules/audio_processing/test/py_quality_assessment/BUILD.gn index b627dbad41..c5b7bec5d7 100644 --- a/modules/audio_processing/test/py_quality_assessment/BUILD.gn +++ b/modules/audio_processing/test/py_quality_assessment/BUILD.gn @@ -64,6 +64,7 @@ copy("lib") { "quality_assessment/evaluation.py", "quality_assessment/exceptions.py", "quality_assessment/export.py", + "quality_assessment/export_unittest.py", "quality_assessment/input_mixer.py", "quality_assessment/input_signal_creator.py", "quality_assessment/results.css", diff --git a/modules/audio_processing/test/py_quality_assessment/README.md b/modules/audio_processing/test/py_quality_assessment/README.md index 2c116e6be4..6fa0b7004c 100644 --- a/modules/audio_processing/test/py_quality_assessment/README.md +++ b/modules/audio_processing/test/py_quality_assessment/README.md @@ -10,7 +10,8 @@ reference one used for evaluation. ## Dependencies - OS: Linux - Python 2.7 - - Python libraries: enum34, numpy, scipy, pydub (0.17.0+), pandas (0.20.1+) + - Python libraries: enum34, numpy, scipy, pydub (0.17.0+), pandas (0.20.1+), + pyquery (1.2+), jsmin (2.2+), csscompressor (0.9.4) - It is recommended that a dedicated Python environment is used - install `virtualenv` - `$ sudo apt-get install python-virtualenv` @@ -19,7 +20,8 @@ reference one used for evaluation. - activate the new Python environment - `$ source ~/my_env/bin/activate` - add dependcies via `pip` - - `(my_env)$ pip install numpy pydub scipy pandas` + - `(my_env)$ pip install enum34 numpy pydub scipy pandas pyquery jsmin \` + `csscompressor` - PolqaOem64 (see http://www.polqa.info/) - Tested with POLQA Library v1.180 / P863 v2.400 - Aachen Impulse Response (AIR) Database @@ -34,7 +36,7 @@ reference one used for evaluation. ## Unit tests - Compile WebRTC - Go to `out/Default/py_quality_assessment` - - Run `python -m unittest -p "*_unittest.py" discover` + - Run `python -m unittest discover -p "*_unittest.py"` ## First time setup - Deploy PolqaOem64 and set the `POLQA_PATH` environment variable diff --git a/modules/audio_processing/test/py_quality_assessment/quality_assessment/collect_data.py b/modules/audio_processing/test/py_quality_assessment/quality_assessment/collect_data.py index 3147d051c5..fc1f44bcf8 100644 --- a/modules/audio_processing/test/py_quality_assessment/quality_assessment/collect_data.py +++ b/modules/audio_processing/test/py_quality_assessment/quality_assessment/collect_data.py @@ -42,6 +42,7 @@ RE_TEST_DATA_GEN_PARAMS = re.compile( RE_SCORE_NAME = re.compile( sim.ApmModuleSimulator.GetPrefixScore() + r'(.+)(\..+)') + def InstanceArgumentsParser(): """Arguments parser factory. """ diff --git a/modules/audio_processing/test/py_quality_assessment/quality_assessment/export.py b/modules/audio_processing/test/py_quality_assessment/quality_assessment/export.py index b32dfc5066..4c50cea21f 100644 --- a/modules/audio_processing/test/py_quality_assessment/quality_assessment/export.py +++ b/modules/audio_processing/test/py_quality_assessment/quality_assessment/export.py @@ -8,8 +8,22 @@ import functools import hashlib +import logging import os import re +import sys + +try: + import csscompressor +except ImportError: + logging.critical('Cannot import the third-party Python package csscompressor') + sys.exit(1) + +try: + import jsmin +except ImportError: + logging.critical('Cannot import the third-party Python package jsmin') + sys.exit(1) class HtmlExport(object): @@ -20,7 +34,9 @@ class HtmlExport(object): # CSS and JS file paths. _PATH = os.path.dirname(os.path.realpath(__file__)) _CSS_FILEPATH = os.path.join(_PATH, 'results.css') + _CSS_MINIFIED = True _JS_FILEPATH = os.path.join(_PATH, 'results.js') + _JS_MINIFIED = True def __init__(self, output_filepath): self._scores_data_frame = None @@ -35,7 +51,14 @@ class HtmlExport(object): self._scores_data_frame = scores_data_frame html = ['', self._BuildHeader(), - '', + (''), + '', self._BuildBody(), '', ''] @@ -63,15 +86,15 @@ class HtmlExport(object): 'material.min.js">') # Embed custom JavaScript and CSS files. - def EmbedFile(filepath): - with open(filepath) as f: - for l in f: - html.append(l.rstrip()) html.append('') html.append('') html.append('') @@ -112,16 +135,24 @@ class HtmlExport(object): 'id="score-tab-{}">'.format( ' is-active' if is_active else '', tab_index)) html.append('
') - html.append(self._BuildScoreTab(score_name)) + html.append(self._BuildScoreTab(score_name, ('s{}'.format(tab_index),))) html.append('
') html.append('') html.append('') html.append('') + # Add snackbar for notifications. + html.append( + '
' + '
' + '' + '
') + return self._NEW_LINE.join(html) - def _BuildScoreTab(self, score_name): + def _BuildScoreTab(self, score_name, anchor_data): """Builds the content of a tab.""" # Find unique values. scores = self._scores_data_frame[ @@ -150,21 +181,22 @@ class HtmlExport(object): for apm_config in apm_configs: html.append('' + self._FormatName(apm_config[0]) + '') for test_data_gen_info in test_data_gen_configs: - onclick_handler = 'openScoreStatsInspector(\'{}\')'.format( - self._ScoreStatsInspectorDialogId(score_name, apm_config[0], - test_data_gen_info[0], - test_data_gen_info[1])) - html.append('{}'.format( - onclick_handler, self._BuildScoreTableCell( - score_name, test_data_gen_info[0], test_data_gen_info[1], - apm_config[0]))) + dialog_id = self._ScoreStatsInspectorDialogId( + score_name, apm_config[0], test_data_gen_info[0], + test_data_gen_info[1]) + html.append( + '{}'.format( + dialog_id, self._BuildScoreTableCell( + score_name, test_data_gen_info[0], test_data_gen_info[1], + apm_config[0]))) html.append('') html.append('') html.append('
') html.append(self._BuildScoreStatsInspectorDialogs( - score_name, apm_configs, test_data_gen_configs)) + score_name, apm_configs, test_data_gen_configs, + anchor_data)) return self._NEW_LINE.join(html) @@ -198,7 +230,7 @@ class HtmlExport(object): return self._NEW_LINE.join(html) def _BuildScoreStatsInspectorDialogs( - self, score_name, apm_configs, test_data_gen_configs): + self, score_name, apm_configs, test_data_gen_configs, anchor_data): """Builds a set of score stats inspector dialogs.""" html = [] for apm_config in apm_configs: @@ -219,13 +251,13 @@ class HtmlExport(object): test_data_gen_info[1])) html.append(self._BuildScoreStatsInspectorDialog( score_name, apm_config[0], test_data_gen_info[0], - test_data_gen_info[1])) + test_data_gen_info[1], anchor_data + (dialog_id,))) html.append('') # Actions. html.append('
') html.append('') html.append('
') @@ -234,7 +266,8 @@ class HtmlExport(object): return self._NEW_LINE.join(html) def _BuildScoreStatsInspectorDialog( - self, score_name, apm_config, test_data_gen, test_data_gen_params): + self, score_name, apm_config, test_data_gen, test_data_gen_params, + anchor_data): """Builds one score stats inspector dialog.""" scores = self._SliceDataForScoreTableCell( score_name, apm_config, test_data_gen, test_data_gen_params) @@ -253,14 +286,16 @@ class HtmlExport(object): # Body. html.append('') - for capture, render in capture_render_pairs: + for row, (capture, render) in enumerate(capture_render_pairs): html.append('
{}
{}
'.format( capture, render)) - for echo_simulator in echo_simulators: + for col, echo_simulator in enumerate(echo_simulators): score_tuple = self._SliceDataForScoreStatsTableCell( scores, capture, render, echo_simulator[0]) - html.append('{}'.format( - self._BuildScoreStatsInspectorTableCell(score_tuple))) + cell_class = 'r{}c{}'.format(row, col) + html.append('{}'.format( + cell_class, self._BuildScoreStatsInspectorTableCell( + score_tuple, anchor_data + (cell_class,)))) html.append('') html.append('') @@ -271,9 +306,14 @@ class HtmlExport(object): return self._NEW_LINE.join(html) - def _BuildScoreStatsInspectorTableCell(self, score_tuple): + def _BuildScoreStatsInspectorTableCell(self, score_tuple, anchor_data): """Builds the content of a cell of a score stats inspector.""" - html = ['
{}
'.format(score_tuple.score)] + anchor = '&'.join(anchor_data) + html = [('
{}
' + '').format(score_tuple.score, anchor)] # Add all the available file paths as hidden data. for field_name in score_tuple.keys(): diff --git a/modules/audio_processing/test/py_quality_assessment/quality_assessment/export_unittest.py b/modules/audio_processing/test/py_quality_assessment/quality_assessment/export_unittest.py new file mode 100644 index 0000000000..0eab6cb57e --- /dev/null +++ b/modules/audio_processing/test/py_quality_assessment/quality_assessment/export_unittest.py @@ -0,0 +1,83 @@ +# Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. +# +# Use of this source code is governed by a BSD-style license +# that can be found in the LICENSE file in the root of the source +# tree. An additional intellectual property rights grant can be found +# in the file PATENTS. All contributing project authors may +# be found in the AUTHORS file in the root of the source tree. + +"""Unit tests for the export module. +""" + +import logging +import os +import shutil +import tempfile +import unittest + +import pyquery as pq + +from . import audioproc_wrapper +from . import collect_data +from . import eval_scores_factory +from . import evaluation +from . import export +from . import simulation +from . import test_data_generation_factory + + +class TestEchoPathSimulators(unittest.TestCase): + """Unit tests for the export module. + """ + + _CLEAN_TMP_OUTPUT = True + + def setUp(self): + """Creates temporary data to export.""" + self._tmp_path = tempfile.mkdtemp() + + # Run a fake experiment to produce data to export. + simulator = simulation.ApmModuleSimulator( + test_data_generator_factory=( + test_data_generation_factory.TestDataGeneratorFactory( + aechen_ir_database_path='', + noise_tracks_path='')), + evaluation_score_factory=( + eval_scores_factory.EvaluationScoreWorkerFactory( + polqa_tool_bin_path=os.path.join( + os.path.dirname(os.path.abspath(__file__)), 'fake_polqa'))), + ap_wrapper=audioproc_wrapper.AudioProcWrapper( + audioproc_wrapper.AudioProcWrapper.DEFAULT_APM_SIMULATOR_BIN_PATH), + evaluator=evaluation.ApmModuleEvaluator()) + simulator.Run( + config_filepaths=['apm_configs/default.json'], + capture_input_filepaths=[ + 'pure_tone-440_1000.wav', + 'pure_tone-880_1000.wav', + ], + test_data_generator_names=['identity', 'white_noise'], + eval_score_names=['audio_level_peak', 'audio_level_mean'], + output_dir=self._tmp_path) + + # Export results. + p = collect_data.InstanceArgumentsParser() + args = p.parse_args(['--output_dir', self._tmp_path]) + src_path = collect_data.ConstructSrcPath(args) + self._data_to_export = collect_data.FindScores(src_path, args) + + def tearDown(self): + """Recursively deletes temporary folders.""" + if self._CLEAN_TMP_OUTPUT: + shutil.rmtree(self._tmp_path) + else: + logging.warning(self.id() + ' did not clean the temporary path ' + ( + self._tmp_path)) + + def testCreateHtmlReport(self): + fn_out = os.path.join(self._tmp_path, 'results.html') + exporter = export.HtmlExport(fn_out) + exporter.Export(self._data_to_export) + + document = pq.PyQuery(filename=fn_out) + self.assertIsInstance(document, pq.PyQuery) + # TODO(alessiob): Use PyQuery API to check the HTML file. diff --git a/modules/audio_processing/test/py_quality_assessment/quality_assessment/results.css b/modules/audio_processing/test/py_quality_assessment/quality_assessment/results.css index 8fd07519d9..2f406bb002 100644 --- a/modules/audio_processing/test/py_quality_assessment/quality_assessment/results.css +++ b/modules/audio_processing/test/py_quality_assessment/quality_assessment/results.css @@ -11,6 +11,10 @@ td.selected-score { background-color: #DDD; } +td.single-score-cell{ + text-align: center; +} + .audio-inspector { text-align: center; } diff --git a/modules/audio_processing/test/py_quality_assessment/quality_assessment/results.js b/modules/audio_processing/test/py_quality_assessment/quality_assessment/results.js index c0272afd9d..8e47411058 100644 --- a/modules/audio_processing/test/py_quality_assessment/quality_assessment/results.js +++ b/modules/audio_processing/test/py_quality_assessment/quality_assessment/results.js @@ -6,35 +6,25 @@ // in the file PATENTS. All contributing project authors may // be found in the AUTHORS file in the root of the source tree. -var inspector = null; - /** * Opens the score stats inspector dialog. * @param {String} dialogId: identifier of the dialog to show. + * @return {DOMElement} The dialog element that has been opened. */ function openScoreStatsInspector(dialogId) { var dialog = document.getElementById(dialogId); dialog.showModal(); + return dialog; } /** * Closes the score stats inspector dialog. - * @param {String} dialogId: identifier of the dialog to close. */ -function closeScoreStatsInspector(dialogId) { - var dialog = document.getElementById(dialogId); +function closeScoreStatsInspector() { + var dialog = document.querySelector('dialog[open]'); + if (dialog == null) + return; dialog.close(); - if (inspector != null) { - inspector.stopAudio(); - } -} - -/** - * Instance and initialize the audio inspector. - */ -function initialize() { - inspector = new AudioInspector(); - inspector.init(); } /** @@ -42,20 +32,82 @@ function initialize() { * @constructor */ function AudioInspector() { + console.debug('Creating an AudioInspector instance.'); this.audioPlayer_ = new Audio(); this.metadata_ = {}; this.currentScore_ = null; this.audioInspector_ = null; + this.snackbarContainer_ = document.querySelector('#snackbar'); + + // Get base URL without anchors. + this.baseUrl_ = window.location.href; + var index = this.baseUrl_.indexOf('#'); + if (index > 0) + this.baseUrl_ = this.baseUrl_.substr(0, index) + console.info('Base URL set to "' + window.location.href + '".'); + + window.event.stopPropagation(); + this.createTextAreasForCopy_(); + this.createAudioInspector_(); + this.initializeEventHandlers_(); + + // When MDL is ready, parse the anchor (if any) to show the requested + // experiment. + var self = this; + document.querySelectorAll('header a')[0].addEventListener( + 'mdl-componentupgraded', function() { + if (!self.parseWindowAnchor()) { + // If not experiment is requested, open the first section. + console.info('No anchor parsing, opening the first section.'); + document.querySelectorAll('header a > span')[0].click(); + } + }); } /** - * Initialize. + * Parse the anchor in the window URL. + * @return {bool} True if the parsing succeeded. */ -AudioInspector.prototype.init = function() { - window.event.stopPropagation(); - this.createAudioInspector_(); - this.initializeEventHandlers_(); -}; +AudioInspector.prototype.parseWindowAnchor = function() { + var index = location.href.indexOf('#'); + if (index == -1) { + console.debug('No # found in the URL.'); + return false; + } + + var anchor = location.href.substr(index - location.href.length + 1); + console.info('Anchor changed: "' + anchor + '".'); + + var parts = anchor.split('&'); + if (parts.length != 3) { + console.info('Ignoring anchor with invalid number of fields.'); + return false; + } + + var openDialog = document.querySelector('dialog[open]'); + try { + // Open the requested dialog if not already open. + if (!openDialog || openDialog.id != parts[1]) { + !openDialog || openDialog.close(); + document.querySelectorAll('header a > span')[ + parseInt(parts[0].substr(1))].click(); + openDialog = openScoreStatsInspector(parts[1]); + } + + // Trigger click on cell. + var cell = openDialog.querySelector('td.' + parts[2]); + cell.focus(); + cell.click(); + + this.showNotification_('Experiment selected.'); + return true; + } catch (e) { + this.showNotification_('Cannot select experiment :('); + console.error('Exception caught while selecting experiment: "' + e + '".'); + } + + return false; +} /** * Set up the inspector for a new score. @@ -88,9 +140,24 @@ AudioInspector.prototype.selectedScoreChange = function(element) { * Stop playing audio. */ AudioInspector.prototype.stopAudio = function() { + console.info('Pausing audio play out.'); this.audioPlayer_.pause(); }; +/** + * Show a text message using the snackbar. + */ +AudioInspector.prototype.showNotification_ = function(text) { + try { + this.snackbarContainer_.MaterialSnackbar.showSnackbar({ + message: text, timeout: 2000}); + } catch (e) { + // Fallback to an alert. + alert(text); + console.warn('Cannot use snackbar: "' + e + '"'); + } +} + /** * Move the audio inspector DOM node into the given parent. * @param {DOMElement} newParentNode: New parent for the inspector. @@ -111,11 +178,38 @@ AudioInspector.prototype.playAudio = function(metadataFieldName) { } this.stopAudio(); this.audioPlayer_.src = this.metadata_[metadataFieldName]; + console.debug('Audio source URL: "' + this.audioPlayer_.src + '"'); this.audioPlayer_.play(); + console.info('Playing out audio.'); }; /** - * Initialize event handlers. + * Create hidden text areas to copy URLs. + * + * For each dialog, one text area is created since it is not possible to select + * text on a text area outside of the active dialog. + */ +AudioInspector.prototype.createTextAreasForCopy_ = function() { + var self = this; + document.querySelectorAll('dialog.mdl-dialog').forEach(function(element) { + var textArea = document.createElement("textarea"); + textArea.classList.add('url-copy'); + textArea.style.position = 'fixed'; + textArea.style.bottom = 0; + textArea.style.left = 0; + textArea.style.width = '2em'; + textArea.style.height = '2em'; + textArea.style.border = 'none'; + textArea.style.outline = 'none'; + textArea.style.boxShadow = 'none'; + textArea.style.background = 'transparent'; + textArea.style.fontSize = '6px'; + element.appendChild(textArea); + }); +} + +/** + * Create audio inspector. */ AudioInspector.prototype.createAudioInspector_ = function() { var buttonIndex = 0; @@ -136,6 +230,9 @@ AudioInspector.prototype.createAudioInspector_ = function() { return html; } + // TODO(alessiob): Add timeline and highlight current track by changing icon + // color. + this.audioInspector_ = document.createElement('div'); this.audioInspector_.classList.add('audio-inspector'); this.audioInspector_.innerHTML = @@ -204,6 +301,34 @@ AudioInspector.prototype.initializeEventHandlers_ = function() { } }); + // Copy anchor URLs icons. + if (document.queryCommandSupported('copy')) { + document.querySelectorAll('td.single-score-cell button').forEach( + function(element) { + element.onclick = function() { + // Find the text area in the dialog. + var textArea = element.closest('dialog').querySelector( + 'textarea.url-copy'); + + // Copy. + textArea.value = self.baseUrl_ + '#' + element.getAttribute( + 'data-anchor'); + textArea.select(); + try { + if (!document.execCommand('copy')) + throw 'Copy returned false'; + self.showNotification_('Experiment URL copied.'); + } catch (e) { + self.showNotification_('Cannot copy experiment URL :('); + console.error(e); + } + } + }); + } else { + self.showNotification_( + 'The copy command is disabled. URL copy is not enabled.'); + } + // Audio inspector buttons. this.audioInspector_.querySelectorAll('button').forEach(function(element) { var target = element.querySelector('input[type=hidden]'); @@ -217,6 +342,13 @@ AudioInspector.prototype.initializeEventHandlers_ = function() { }; }); + // Dialog close handlers. + var dialogs = document.querySelectorAll('dialog').forEach(function(element) { + element.onclose = function() { + self.stopAudio(); + } + }); + // Keyboard shortcuts. window.onkeyup = function(e) { var key = e.keyCode ? e.keyCode : e.which; @@ -236,4 +368,9 @@ AudioInspector.prototype.initializeEventHandlers_ = function() { break; } }; + + // Hash change. + window.onhashchange = function(e) { + self.parseWindowAnchor(); + } };