diff --git a/webrtc/modules/audio_processing/test/py_quality_assessment/BUILD.gn b/webrtc/modules/audio_processing/test/py_quality_assessment/BUILD.gn index f2a685de9c..81ef562837 100644 --- a/webrtc/modules/audio_processing/test/py_quality_assessment/BUILD.gn +++ b/webrtc/modules/audio_processing/test/py_quality_assessment/BUILD.gn @@ -45,6 +45,7 @@ copy("lib") { "quality_assessment/noise_generation.py", "quality_assessment/noise_generation_factory.py", "quality_assessment/noise_generation_unittest.py", + "quality_assessment/results.css", "quality_assessment/signal_processing.py", "quality_assessment/signal_processing_unittest.py", "quality_assessment/simulation.py", diff --git a/webrtc/modules/audio_processing/test/py_quality_assessment/apm_quality_assessment_export.py b/webrtc/modules/audio_processing/test/py_quality_assessment/apm_quality_assessment_export.py index dcfea944a7..1b2fe36c52 100755 --- a/webrtc/modules/audio_processing/test/py_quality_assessment/apm_quality_assessment_export.py +++ b/webrtc/modules/audio_processing/test/py_quality_assessment/apm_quality_assessment_export.py @@ -146,10 +146,10 @@ def main(): } # Export. - exporter = export.HtmlExport( - output_path=args.output_dir, - output_filename=_GetOutputFilename(args.filename_suffix)) - output_filepath = exporter.export(scores) + output_filepath = os.path.join(args.output_dir, _GetOutputFilename( + args.filename_suffix)) + exporter = export.HtmlExport(output_filepath) + exporter.export(scores) logging.info('output file successfully written in %s', output_filepath) sys.exit(0) diff --git a/webrtc/modules/audio_processing/test/py_quality_assessment/quality_assessment/export.py b/webrtc/modules/audio_processing/test/py_quality_assessment/quality_assessment/export.py index 5891b5fe24..4b025023a3 100644 --- a/webrtc/modules/audio_processing/test/py_quality_assessment/quality_assessment/export.py +++ b/webrtc/modules/audio_processing/test/py_quality_assessment/quality_assessment/export.py @@ -8,19 +8,196 @@ import logging import os +import re + class HtmlExport(object): - def __init__(self, output_path, output_filename): - self._output_path = output_path - self._output_filename = output_filename + # Path to CSS and JS files. + _PATH = os.path.dirname(os.path.realpath(__file__)) + + # CSS file parameters. + _CSS_FILEPATH = os.path.join(_PATH, 'results.css') + _INLINE_CSS = True + + _NEW_LINE = '\n' + + def __init__(self, output_filepath): + self._noise_names = None + self._noise_params = None + self._output_filepath = output_filepath def export(self, scores): - logging.debug('%d score names found', len(scores)) - output_filepath = os.path.join(self._output_path, self._output_filename) + """ + Export the scores into an HTML file. - # TODO(alessio): remove once implemented + Args: + scores: nested dictionary containing the scores. + """ + # Generate one table for each evaluation score. + tables = [] + for score_name in sorted(scores.keys()): + tables.append(self._build_score_table(score_name, scores[score_name])) + + # Create the html file. + html = ( + '' + + self._build_header() + + '' + + '

Results from {}

'.format(self._output_filepath) + + self._NEW_LINE.join(tables) + + '' + + '') + + self._save(self._output_filepath, html) + + def _build_header(self): + """ + HTML file header with page title and either embedded or linked CSS and JS + files. + """ + html = ['', 'Results'] + + # CSS. + if self._INLINE_CSS: + # Embed. + html.append('') + else: + # Link. + html.append(''.format(self._CSS_FILEPATH)) + + html.append('') + + return self._NEW_LINE.join(html) + + def _build_score_table(self, score_name, scores): + """ + Generate a table for a specific evaluation score (e.g., POLQA). + """ + config_names = sorted(scores.keys()) + input_names = sorted(scores[config_names[0]].keys()) + rows = [self._table_row( + score_name, config_name, scores[config_name], input_names) for ( + config_name) in config_names] + + html = ( + '' + + '{}'.format( + self._table_header(score_name, input_names)) + + '' + + '' + ''.join(rows) + '' + + '' + + '
' + self._legend()) + + return html + + def _table_header(self, score_name, input_names): + """ + Generate a table header with the name of the evaluation score in the first + column and then one column for each probing signal. + """ + html = ( + '{}'.format(self._format_name(score_name)) + + '' + ''.join( + [self._format_name(name) for name in input_names]) + '') + return html + + def _table_row(self, score_name, config_name, scores, input_names): + """ + Generate a table body row with the name of the APM configuration file in the + first column and then one column for each probing singal. + """ + cells = [self._table_cell( + scores[input_name], score_name, config_name, input_name) for ( + input_name) in input_names] + html = ('{}'.format(self._format_name(config_name)) + + '' + ''.join(cells) + '') + return html + + def _table_cell(self, scores, score_name, config_name, input_name): + """ + Generate a table cell content with all the scores for the current evaluation + score, APM configuration, and probing signal. + """ + # Init noise generator names and noise parameters cache (if not done). + if self._noise_names is None: + self._noise_names = sorted(scores.keys()) + self._noise_params = {noise_name: sorted(scores[noise_name].keys()) for ( + noise_name) in self._noise_names} + + # For each noisy input (that is a pair of noise generator name and noise + # generator parameters), add an item with the score and its metadata. + items = [] + for name_index, noise_name in enumerate(self._noise_names): + for params_index, noise_params in enumerate( + self._noise_params[noise_name]): + + # Init. + score_value = '?' + metadata = '' + + # Extract score value and its metadata. + try: + data = scores[noise_name][noise_params] + score_value = '{0:f}'.format(data['score']) + metadata = ( + '' + '' + '' + '' + '' + ).format( + noise_name, + noise_params, + data['audio_in_filepath'], + data['audio_out_filepath'], + data['audio_ref_filepath']) + except TypeError: + logging.warning( + 'missing score found: ' + ' ', score_name, config_name, input_name, + noise_name, noise_params) + + # Add the score. + items.append( + '
[{0:d}, {1:d}]{2}
' + '
{3}
'.format( + name_index, params_index, metadata, score_value)) + + html = ( + '
' + + '
'.join(items) + + '
') + + return html + + def _legend(self): + """ + Generate the legend for each noise generator name and parameters pair. + """ + items = [] + for name_index, noise_name in enumerate(self._noise_names): + for params_index, noise_params in enumerate( + self._noise_params[noise_name]): + items.append('
[{0:d}, {1:d}]
: {2} noise, ' + '{3}'.format(name_index, params_index, noise_name, + noise_params)) + html = ( + '
' + + '
'.join(items) + '
') + + return html + + @classmethod + def _save(cls, output_filepath, html): with open(output_filepath, 'w') as f: - f.write('APM Quality Assessment scores\n') + f.write(html) - return output_filepath + @classmethod + def _format_name(cls, name): + return re.sub(r'[_\-]', ' ', name) diff --git a/webrtc/modules/audio_processing/test/py_quality_assessment/quality_assessment/noise_generation.py b/webrtc/modules/audio_processing/test/py_quality_assessment/quality_assessment/noise_generation.py index 6af6eb6f1a..9bfce34db2 100644 --- a/webrtc/modules/audio_processing/test/py_quality_assessment/quality_assessment/noise_generation.py +++ b/webrtc/modules/audio_processing/test/py_quality_assessment/quality_assessment/noise_generation.py @@ -35,6 +35,7 @@ from . import data_access from . import exceptions from . import signal_processing + class NoiseGenerator(object): """Abstract class responsible for the generation of noisy signals. diff --git a/webrtc/modules/audio_processing/test/py_quality_assessment/quality_assessment/results.css b/webrtc/modules/audio_processing/test/py_quality_assessment/quality_assessment/results.css new file mode 100644 index 0000000000..313b7111dc --- /dev/null +++ b/webrtc/modules/audio_processing/test/py_quality_assessment/quality_assessment/results.css @@ -0,0 +1,60 @@ +/* Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +body{ + font-family: Arial; + font-size: 75%; +} + +table{ + border-top: 1px solid #000; + border-right: 1px solid #000; + margin: 1em 0 0.2em 0; +} + +table thead tr th{ + font-size: 0.9em; + text-align: center; + border-bottom: 1px solid #000; + border-left: 1px solid #000; + min-width: 8em; +} + +table thead tr th:first-child{ + text-transform: uppercase; +} + +table tbody tr td{ + font-size: 0.8em; + text-align: center; + border-bottom: 1px solid #000; + border-left: 1px solid #000; +} + +table tbody tr td:first-child{ + font-weight: bold; +} + +table tbody tr td .selected{ + background-color: #EE9; +} + +table tbody tr td .value{ + display: inline-block; +} + +.noise-desc{ + display: inline-block; + margin-right: 0.3em; + border: 1px solid #555; + color: #555; + background-color: #EEE; + padding: 1px; + font-size: 0.8em; +}