Create links to single experiments in APM-QA exported results.

For each single experiment, a URL is defined by adding a specific anchor.
A URL can be copied using the button beneath the score of the experiment
one would like to share.

This CL also includes a few optimizations and fixes:
- JS and CSS are minified
- Dialog close event listener added, this fixes a small bug preventing
  the play out audio to stop when pressing ESC instead of using the close
  button
- Snackbar notifications added
- Simple unit test for the export module

BUG=webrtc:7218
Change-Id: Iad00ce69094a5968ee0520d105d59656cfafa4e2

TBR=

Change-Id: Iad00ce69094a5968ee0520d105d59656cfafa4e2
Reviewed-on: https://webrtc-review.googlesource.com/7960
Commit-Queue: Alessio Bazzica <alessiob@webrtc.org>
Reviewed-by: Alessio Bazzica <alessiob@webrtc.org>
Cr-Commit-Position: refs/heads/master@{#20266}
This commit is contained in:
Alessio Bazzica 2017-10-10 13:28:56 +02:00 committed by Commit Bot
parent dc182a486a
commit 86f25d3bcd
7 changed files with 322 additions and 54 deletions

View File

@ -64,6 +64,7 @@ copy("lib") {
"quality_assessment/evaluation.py",
"quality_assessment/exceptions.py",
"quality_assessment/export.py",
"quality_assessment/export_unittest.py",
"quality_assessment/input_mixer.py",
"quality_assessment/input_signal_creator.py",
"quality_assessment/results.css",

View File

@ -10,7 +10,8 @@ reference one used for evaluation.
## Dependencies
- OS: Linux
- Python 2.7
- Python libraries: enum34, numpy, scipy, pydub (0.17.0+), pandas (0.20.1+)
- Python libraries: enum34, numpy, scipy, pydub (0.17.0+), pandas (0.20.1+),
pyquery (1.2+), jsmin (2.2+), csscompressor (0.9.4)
- It is recommended that a dedicated Python environment is used
- install `virtualenv`
- `$ sudo apt-get install python-virtualenv`
@ -19,7 +20,8 @@ reference one used for evaluation.
- activate the new Python environment
- `$ source ~/my_env/bin/activate`
- add dependcies via `pip`
- `(my_env)$ pip install numpy pydub scipy pandas`
- `(my_env)$ pip install enum34 numpy pydub scipy pandas pyquery jsmin \`
`csscompressor`
- PolqaOem64 (see http://www.polqa.info/)
- Tested with POLQA Library v1.180 / P863 v2.400
- Aachen Impulse Response (AIR) Database
@ -34,7 +36,7 @@ reference one used for evaluation.
## Unit tests
- Compile WebRTC
- Go to `out/Default/py_quality_assessment`
- Run `python -m unittest -p "*_unittest.py" discover`
- Run `python -m unittest discover -p "*_unittest.py"`
## First time setup
- Deploy PolqaOem64 and set the `POLQA_PATH` environment variable

View File

@ -42,6 +42,7 @@ RE_TEST_DATA_GEN_PARAMS = re.compile(
RE_SCORE_NAME = re.compile(
sim.ApmModuleSimulator.GetPrefixScore() + r'(.+)(\..+)')
def InstanceArgumentsParser():
"""Arguments parser factory.
"""

View File

@ -8,8 +8,22 @@
import functools
import hashlib
import logging
import os
import re
import sys
try:
import csscompressor
except ImportError:
logging.critical('Cannot import the third-party Python package csscompressor')
sys.exit(1)
try:
import jsmin
except ImportError:
logging.critical('Cannot import the third-party Python package jsmin')
sys.exit(1)
class HtmlExport(object):
@ -20,7 +34,9 @@ class HtmlExport(object):
# CSS and JS file paths.
_PATH = os.path.dirname(os.path.realpath(__file__))
_CSS_FILEPATH = os.path.join(_PATH, 'results.css')
_CSS_MINIFIED = True
_JS_FILEPATH = os.path.join(_PATH, 'results.js')
_JS_MINIFIED = True
def __init__(self, output_filepath):
self._scores_data_frame = None
@ -35,7 +51,14 @@ class HtmlExport(object):
self._scores_data_frame = scores_data_frame
html = ['<html>',
self._BuildHeader(),
'<body onload="initialize()">',
('<script type="text/javascript">'
'(function () {'
'window.addEventListener(\'load\', function () {'
'var inspector = new AudioInspector();'
'});'
'})();'
'</script>'),
'<body>',
self._BuildBody(),
'</body>',
'</html>']
@ -63,15 +86,15 @@ class HtmlExport(object):
'material.min.js"></script>')
# Embed custom JavaScript and CSS files.
def EmbedFile(filepath):
with open(filepath) as f:
for l in f:
html.append(l.rstrip())
html.append('<script>')
EmbedFile(self._JS_FILEPATH)
with open(self._JS_FILEPATH) as f:
html.append(jsmin.jsmin(f.read()) if self._JS_MINIFIED else (
f.read().rstrip()))
html.append('</script>')
html.append('<style>')
EmbedFile(self._CSS_FILEPATH)
with open(self._CSS_FILEPATH) as f:
html.append(csscompressor.compress(f.read()) if self._CSS_MINIFIED else (
f.read().rstrip()))
html.append('</style>')
html.append('</head>')
@ -112,16 +135,24 @@ class HtmlExport(object):
'id="score-tab-{}">'.format(
' is-active' if is_active else '', tab_index))
html.append('<div class="page-content">')
html.append(self._BuildScoreTab(score_name))
html.append(self._BuildScoreTab(score_name, ('s{}'.format(tab_index),)))
html.append('</div>')
html.append('</section>')
html.append('</main>')
html.append('</div>')
# Add snackbar for notifications.
html.append(
'<div id="snackbar" aria-live="assertive" aria-atomic="true"'
' aria-relevant="text" class="mdl-snackbar mdl-js-snackbar">'
'<div class="mdl-snackbar__text"></div>'
'<button type="button" class="mdl-snackbar__action"></button>'
'</div>')
return self._NEW_LINE.join(html)
def _BuildScoreTab(self, score_name):
def _BuildScoreTab(self, score_name, anchor_data):
"""Builds the content of a tab."""
# Find unique values.
scores = self._scores_data_frame[
@ -150,21 +181,22 @@ class HtmlExport(object):
for apm_config in apm_configs:
html.append('<tr><td>' + self._FormatName(apm_config[0]) + '</td>')
for test_data_gen_info in test_data_gen_configs:
onclick_handler = 'openScoreStatsInspector(\'{}\')'.format(
self._ScoreStatsInspectorDialogId(score_name, apm_config[0],
test_data_gen_info[0],
test_data_gen_info[1]))
html.append('<td onclick="{}">{}</td>'.format(
onclick_handler, self._BuildScoreTableCell(
score_name, test_data_gen_info[0], test_data_gen_info[1],
apm_config[0])))
dialog_id = self._ScoreStatsInspectorDialogId(
score_name, apm_config[0], test_data_gen_info[0],
test_data_gen_info[1])
html.append(
'<td onclick="openScoreStatsInspector(\'{}\')">{}</td>'.format(
dialog_id, self._BuildScoreTableCell(
score_name, test_data_gen_info[0], test_data_gen_info[1],
apm_config[0])))
html.append('</tr>')
html.append('</tbody>')
html.append('</table></div><div class="mdl-layout-spacer"></div></div>')
html.append(self._BuildScoreStatsInspectorDialogs(
score_name, apm_configs, test_data_gen_configs))
score_name, apm_configs, test_data_gen_configs,
anchor_data))
return self._NEW_LINE.join(html)
@ -198,7 +230,7 @@ class HtmlExport(object):
return self._NEW_LINE.join(html)
def _BuildScoreStatsInspectorDialogs(
self, score_name, apm_configs, test_data_gen_configs):
self, score_name, apm_configs, test_data_gen_configs, anchor_data):
"""Builds a set of score stats inspector dialogs."""
html = []
for apm_config in apm_configs:
@ -219,13 +251,13 @@ class HtmlExport(object):
test_data_gen_info[1]))
html.append(self._BuildScoreStatsInspectorDialog(
score_name, apm_config[0], test_data_gen_info[0],
test_data_gen_info[1]))
test_data_gen_info[1], anchor_data + (dialog_id,)))
html.append('</div>')
# Actions.
html.append('<div class="mdl-dialog__actions">')
html.append('<button type="button" class="mdl-button" '
'onclick="closeScoreStatsInspector(\'' + dialog_id + '\')">'
'onclick="closeScoreStatsInspector()">'
'Close</button>')
html.append('</div>')
@ -234,7 +266,8 @@ class HtmlExport(object):
return self._NEW_LINE.join(html)
def _BuildScoreStatsInspectorDialog(
self, score_name, apm_config, test_data_gen, test_data_gen_params):
self, score_name, apm_config, test_data_gen, test_data_gen_params,
anchor_data):
"""Builds one score stats inspector dialog."""
scores = self._SliceDataForScoreTableCell(
score_name, apm_config, test_data_gen, test_data_gen_params)
@ -253,14 +286,16 @@ class HtmlExport(object):
# Body.
html.append('<tbody>')
for capture, render in capture_render_pairs:
for row, (capture, render) in enumerate(capture_render_pairs):
html.append('<tr><td><div>{}</div><div>{}</div></td>'.format(
capture, render))
for echo_simulator in echo_simulators:
for col, echo_simulator in enumerate(echo_simulators):
score_tuple = self._SliceDataForScoreStatsTableCell(
scores, capture, render, echo_simulator[0])
html.append('<td class="single-score-cell">{}</td>'.format(
self._BuildScoreStatsInspectorTableCell(score_tuple)))
cell_class = 'r{}c{}'.format(row, col)
html.append('<td class="single-score-cell {}">{}</td>'.format(
cell_class, self._BuildScoreStatsInspectorTableCell(
score_tuple, anchor_data + (cell_class,))))
html.append('</tr>')
html.append('</tbody>')
@ -271,9 +306,14 @@ class HtmlExport(object):
return self._NEW_LINE.join(html)
def _BuildScoreStatsInspectorTableCell(self, score_tuple):
def _BuildScoreStatsInspectorTableCell(self, score_tuple, anchor_data):
"""Builds the content of a cell of a score stats inspector."""
html = ['<div>{}</div>'.format(score_tuple.score)]
anchor = '&'.join(anchor_data)
html = [('<div class="v">{}</div>'
'<button class="mdl-button mdl-js-button mdl-button--icon"'
' data-anchor="{}">'
'<i class="material-icons mdl-color-text--blue-grey">link</i>'
'</button>').format(score_tuple.score, anchor)]
# Add all the available file paths as hidden data.
for field_name in score_tuple.keys():

View File

@ -0,0 +1,83 @@
# Copyright (c) 2017 The WebRTC project authors. All Rights Reserved.
#
# Use of this source code is governed by a BSD-style license
# that can be found in the LICENSE file in the root of the source
# tree. An additional intellectual property rights grant can be found
# in the file PATENTS. All contributing project authors may
# be found in the AUTHORS file in the root of the source tree.
"""Unit tests for the export module.
"""
import logging
import os
import shutil
import tempfile
import unittest
import pyquery as pq
from . import audioproc_wrapper
from . import collect_data
from . import eval_scores_factory
from . import evaluation
from . import export
from . import simulation
from . import test_data_generation_factory
class TestEchoPathSimulators(unittest.TestCase):
"""Unit tests for the export module.
"""
_CLEAN_TMP_OUTPUT = True
def setUp(self):
"""Creates temporary data to export."""
self._tmp_path = tempfile.mkdtemp()
# Run a fake experiment to produce data to export.
simulator = simulation.ApmModuleSimulator(
test_data_generator_factory=(
test_data_generation_factory.TestDataGeneratorFactory(
aechen_ir_database_path='',
noise_tracks_path='')),
evaluation_score_factory=(
eval_scores_factory.EvaluationScoreWorkerFactory(
polqa_tool_bin_path=os.path.join(
os.path.dirname(os.path.abspath(__file__)), 'fake_polqa'))),
ap_wrapper=audioproc_wrapper.AudioProcWrapper(
audioproc_wrapper.AudioProcWrapper.DEFAULT_APM_SIMULATOR_BIN_PATH),
evaluator=evaluation.ApmModuleEvaluator())
simulator.Run(
config_filepaths=['apm_configs/default.json'],
capture_input_filepaths=[
'pure_tone-440_1000.wav',
'pure_tone-880_1000.wav',
],
test_data_generator_names=['identity', 'white_noise'],
eval_score_names=['audio_level_peak', 'audio_level_mean'],
output_dir=self._tmp_path)
# Export results.
p = collect_data.InstanceArgumentsParser()
args = p.parse_args(['--output_dir', self._tmp_path])
src_path = collect_data.ConstructSrcPath(args)
self._data_to_export = collect_data.FindScores(src_path, args)
def tearDown(self):
"""Recursively deletes temporary folders."""
if self._CLEAN_TMP_OUTPUT:
shutil.rmtree(self._tmp_path)
else:
logging.warning(self.id() + ' did not clean the temporary path ' + (
self._tmp_path))
def testCreateHtmlReport(self):
fn_out = os.path.join(self._tmp_path, 'results.html')
exporter = export.HtmlExport(fn_out)
exporter.Export(self._data_to_export)
document = pq.PyQuery(filename=fn_out)
self.assertIsInstance(document, pq.PyQuery)
# TODO(alessiob): Use PyQuery API to check the HTML file.

View File

@ -11,6 +11,10 @@ td.selected-score {
background-color: #DDD;
}
td.single-score-cell{
text-align: center;
}
.audio-inspector {
text-align: center;
}

View File

@ -6,35 +6,25 @@
// in the file PATENTS. All contributing project authors may
// be found in the AUTHORS file in the root of the source tree.
var inspector = null;
/**
* Opens the score stats inspector dialog.
* @param {String} dialogId: identifier of the dialog to show.
* @return {DOMElement} The dialog element that has been opened.
*/
function openScoreStatsInspector(dialogId) {
var dialog = document.getElementById(dialogId);
dialog.showModal();
return dialog;
}
/**
* Closes the score stats inspector dialog.
* @param {String} dialogId: identifier of the dialog to close.
*/
function closeScoreStatsInspector(dialogId) {
var dialog = document.getElementById(dialogId);
function closeScoreStatsInspector() {
var dialog = document.querySelector('dialog[open]');
if (dialog == null)
return;
dialog.close();
if (inspector != null) {
inspector.stopAudio();
}
}
/**
* Instance and initialize the audio inspector.
*/
function initialize() {
inspector = new AudioInspector();
inspector.init();
}
/**
@ -42,20 +32,82 @@ function initialize() {
* @constructor
*/
function AudioInspector() {
console.debug('Creating an AudioInspector instance.');
this.audioPlayer_ = new Audio();
this.metadata_ = {};
this.currentScore_ = null;
this.audioInspector_ = null;
this.snackbarContainer_ = document.querySelector('#snackbar');
// Get base URL without anchors.
this.baseUrl_ = window.location.href;
var index = this.baseUrl_.indexOf('#');
if (index > 0)
this.baseUrl_ = this.baseUrl_.substr(0, index)
console.info('Base URL set to "' + window.location.href + '".');
window.event.stopPropagation();
this.createTextAreasForCopy_();
this.createAudioInspector_();
this.initializeEventHandlers_();
// When MDL is ready, parse the anchor (if any) to show the requested
// experiment.
var self = this;
document.querySelectorAll('header a')[0].addEventListener(
'mdl-componentupgraded', function() {
if (!self.parseWindowAnchor()) {
// If not experiment is requested, open the first section.
console.info('No anchor parsing, opening the first section.');
document.querySelectorAll('header a > span')[0].click();
}
});
}
/**
* Initialize.
* Parse the anchor in the window URL.
* @return {bool} True if the parsing succeeded.
*/
AudioInspector.prototype.init = function() {
window.event.stopPropagation();
this.createAudioInspector_();
this.initializeEventHandlers_();
};
AudioInspector.prototype.parseWindowAnchor = function() {
var index = location.href.indexOf('#');
if (index == -1) {
console.debug('No # found in the URL.');
return false;
}
var anchor = location.href.substr(index - location.href.length + 1);
console.info('Anchor changed: "' + anchor + '".');
var parts = anchor.split('&');
if (parts.length != 3) {
console.info('Ignoring anchor with invalid number of fields.');
return false;
}
var openDialog = document.querySelector('dialog[open]');
try {
// Open the requested dialog if not already open.
if (!openDialog || openDialog.id != parts[1]) {
!openDialog || openDialog.close();
document.querySelectorAll('header a > span')[
parseInt(parts[0].substr(1))].click();
openDialog = openScoreStatsInspector(parts[1]);
}
// Trigger click on cell.
var cell = openDialog.querySelector('td.' + parts[2]);
cell.focus();
cell.click();
this.showNotification_('Experiment selected.');
return true;
} catch (e) {
this.showNotification_('Cannot select experiment :(');
console.error('Exception caught while selecting experiment: "' + e + '".');
}
return false;
}
/**
* Set up the inspector for a new score.
@ -88,9 +140,24 @@ AudioInspector.prototype.selectedScoreChange = function(element) {
* Stop playing audio.
*/
AudioInspector.prototype.stopAudio = function() {
console.info('Pausing audio play out.');
this.audioPlayer_.pause();
};
/**
* Show a text message using the snackbar.
*/
AudioInspector.prototype.showNotification_ = function(text) {
try {
this.snackbarContainer_.MaterialSnackbar.showSnackbar({
message: text, timeout: 2000});
} catch (e) {
// Fallback to an alert.
alert(text);
console.warn('Cannot use snackbar: "' + e + '"');
}
}
/**
* Move the audio inspector DOM node into the given parent.
* @param {DOMElement} newParentNode: New parent for the inspector.
@ -111,11 +178,38 @@ AudioInspector.prototype.playAudio = function(metadataFieldName) {
}
this.stopAudio();
this.audioPlayer_.src = this.metadata_[metadataFieldName];
console.debug('Audio source URL: "' + this.audioPlayer_.src + '"');
this.audioPlayer_.play();
console.info('Playing out audio.');
};
/**
* Initialize event handlers.
* Create hidden text areas to copy URLs.
*
* For each dialog, one text area is created since it is not possible to select
* text on a text area outside of the active dialog.
*/
AudioInspector.prototype.createTextAreasForCopy_ = function() {
var self = this;
document.querySelectorAll('dialog.mdl-dialog').forEach(function(element) {
var textArea = document.createElement("textarea");
textArea.classList.add('url-copy');
textArea.style.position = 'fixed';
textArea.style.bottom = 0;
textArea.style.left = 0;
textArea.style.width = '2em';
textArea.style.height = '2em';
textArea.style.border = 'none';
textArea.style.outline = 'none';
textArea.style.boxShadow = 'none';
textArea.style.background = 'transparent';
textArea.style.fontSize = '6px';
element.appendChild(textArea);
});
}
/**
* Create audio inspector.
*/
AudioInspector.prototype.createAudioInspector_ = function() {
var buttonIndex = 0;
@ -136,6 +230,9 @@ AudioInspector.prototype.createAudioInspector_ = function() {
return html;
}
// TODO(alessiob): Add timeline and highlight current track by changing icon
// color.
this.audioInspector_ = document.createElement('div');
this.audioInspector_.classList.add('audio-inspector');
this.audioInspector_.innerHTML =
@ -204,6 +301,34 @@ AudioInspector.prototype.initializeEventHandlers_ = function() {
}
});
// Copy anchor URLs icons.
if (document.queryCommandSupported('copy')) {
document.querySelectorAll('td.single-score-cell button').forEach(
function(element) {
element.onclick = function() {
// Find the text area in the dialog.
var textArea = element.closest('dialog').querySelector(
'textarea.url-copy');
// Copy.
textArea.value = self.baseUrl_ + '#' + element.getAttribute(
'data-anchor');
textArea.select();
try {
if (!document.execCommand('copy'))
throw 'Copy returned false';
self.showNotification_('Experiment URL copied.');
} catch (e) {
self.showNotification_('Cannot copy experiment URL :(');
console.error(e);
}
}
});
} else {
self.showNotification_(
'The copy command is disabled. URL copy is not enabled.');
}
// Audio inspector buttons.
this.audioInspector_.querySelectorAll('button').forEach(function(element) {
var target = element.querySelector('input[type=hidden]');
@ -217,6 +342,13 @@ AudioInspector.prototype.initializeEventHandlers_ = function() {
};
});
// Dialog close handlers.
var dialogs = document.querySelectorAll('dialog').forEach(function(element) {
element.onclose = function() {
self.stopAudio();
}
});
// Keyboard shortcuts.
window.onkeyup = function(e) {
var key = e.keyCode ? e.keyCode : e.which;
@ -236,4 +368,9 @@ AudioInspector.prototype.initializeEventHandlers_ = function() {
break;
}
};
// Hash change.
window.onhashchange = function(e) {
self.parseWindowAnchor();
}
};