diff --git a/webrtc/modules/audio_processing/test/py_quality_assessment/BUILD.gn b/webrtc/modules/audio_processing/test/py_quality_assessment/BUILD.gn index f191319bfe..d607d15bfd 100644 --- a/webrtc/modules/audio_processing/test/py_quality_assessment/BUILD.gn +++ b/webrtc/modules/audio_processing/test/py_quality_assessment/BUILD.gn @@ -33,6 +33,10 @@ copy("lib") { testonly = true sources = [ "quality_assessment/__init__.py", + "quality_assessment/data_access.py", + "quality_assessment/eval_scores.py", + "quality_assessment/noise_generation.py", + "quality_assessment/simulation.py", ] visibility = [ ":*" ] # Only targets in this file can depend on this. outputs = [ @@ -46,7 +50,7 @@ copy("lib") { copy("apm_configs") { testonly = true sources = [ - "quality_assessment/apm_configs/default.json", + "apm_configs/default.json", ] visibility = [ ":*" ] # Only targets in this file can depend on this. outputs = [ diff --git a/webrtc/modules/audio_processing/test/py_quality_assessment/quality_assessment/apm_configs/default.json b/webrtc/modules/audio_processing/test/py_quality_assessment/apm_configs/default.json similarity index 100% rename from webrtc/modules/audio_processing/test/py_quality_assessment/quality_assessment/apm_configs/default.json rename to webrtc/modules/audio_processing/test/py_quality_assessment/apm_configs/default.json diff --git a/webrtc/modules/audio_processing/test/py_quality_assessment/apm_quality_assessment-export.py b/webrtc/modules/audio_processing/test/py_quality_assessment/apm_quality_assessment-export.py old mode 100644 new mode 100755 index 1b526c1b05..d3d3a83a6f --- a/webrtc/modules/audio_processing/test/py_quality_assessment/apm_quality_assessment-export.py +++ b/webrtc/modules/audio_processing/test/py_quality_assessment/apm_quality_assessment-export.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python3 +#!/usr/bin/env python # Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. # # Use of this source code is governed by a BSD-style license diff --git a/webrtc/modules/audio_processing/test/py_quality_assessment/apm_quality_assessment-gencfgs.py b/webrtc/modules/audio_processing/test/py_quality_assessment/apm_quality_assessment-gencfgs.py old mode 100644 new mode 100755 index 1b526c1b05..d3d3a83a6f --- a/webrtc/modules/audio_processing/test/py_quality_assessment/apm_quality_assessment-gencfgs.py +++ b/webrtc/modules/audio_processing/test/py_quality_assessment/apm_quality_assessment-gencfgs.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python3 +#!/usr/bin/env python # Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. # # Use of this source code is governed by a BSD-style license diff --git a/webrtc/modules/audio_processing/test/py_quality_assessment/apm_quality_assessment.py b/webrtc/modules/audio_processing/test/py_quality_assessment/apm_quality_assessment.py old mode 100644 new mode 100755 index 1b526c1b05..ca7e2c3e93 --- a/webrtc/modules/audio_processing/test/py_quality_assessment/apm_quality_assessment.py +++ b/webrtc/modules/audio_processing/test/py_quality_assessment/apm_quality_assessment.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python3 +#!/usr/bin/env python # Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. # # Use of this source code is governed by a BSD-style license @@ -6,3 +6,83 @@ # tree. An additional intellectual property rights grant can be found # in the file PATENTS. All contributing project authors may # be found in the AUTHORS file in the root of the source tree. + +"""Perform APM module quality assessment on one or more input files using one or + more audioproc_f configuration files and one or more noise generators. + +Usage: apm_quality_assessment.py -i audio1.wav [audio2.wav ...] + -c cfg1.json [cfg2.json ...] + -n white [echo ...] + -e audio_level [polqa ...] + -o /path/to/output +""" + +import argparse +import logging +import sys + +import quality_assessment.eval_scores as eval_scores +import quality_assessment.noise_generation as noise_generation +import quality_assessment.simulation as simulation + +_NOISE_GENERATOR_CLASSES = noise_generation.NoiseGenerator.REGISTERED_CLASSES +_NOISE_GENERATORS_NAMES = _NOISE_GENERATOR_CLASSES.keys() +_EVAL_SCORE_WORKER_CLASSES = eval_scores.EvaluationScore.REGISTERED_CLASSES +_EVAL_SCORE_WORKER_NAMES = _EVAL_SCORE_WORKER_CLASSES.keys() + +_DEFAULT_CONFIG_FILE = 'apm_configs/default.json' + +def _instance_arguments_parser(): + parser = argparse.ArgumentParser(description=( + 'Perform APM module quality assessment on one or more input files using ' + 'one or more audioproc_f configuration files and one or more noise ' + 'generators.')) + + parser.add_argument('-c', '--config_files', nargs='+', required=False, + help=('path to the configuration files defining the ' + 'arguments with which the audioproc_f tool is ' + 'called'), + default=[_DEFAULT_CONFIG_FILE]) + + parser.add_argument('-i', '--input_files', nargs='+', required=True, + help='path to the input wav files (one or more)') + + parser.add_argument('-n', '--noise_generators', nargs='+', required=False, + help='custom list of noise generators to use', + choices=_NOISE_GENERATORS_NAMES, + default=_NOISE_GENERATORS_NAMES) + + parser.add_argument('-e', '--eval_scores', nargs='+', required=False, + help='custom list of evaluation scores to use', + choices=_EVAL_SCORE_WORKER_NAMES, + default=_EVAL_SCORE_WORKER_NAMES) + + parser.add_argument('-o', '--output_dir', required=False, + help=('base path to the output directory in which the ' + 'output wav files and the evaluation outcomes ' + 'are saved'), + default='output') + + return parser + + +def main(): + # TODO(alessiob): level = logging.INFO once debugged. + logging.basicConfig(level=logging.DEBUG) + + parser = _instance_arguments_parser() + args = parser.parse_args() + + simulator = simulation.ApmModuleSimulator() + simulator.run( + config_filepaths=args.config_files, + input_filepaths=args.input_files, + noise_generator_names=args.noise_generators, + eval_score_names=args.eval_scores, + output_dir=args.output_dir) + + sys.exit(0) + + +if __name__ == '__main__': + main() diff --git a/webrtc/modules/audio_processing/test/py_quality_assessment/apm_quality_assessment.sh b/webrtc/modules/audio_processing/test/py_quality_assessment/apm_quality_assessment.sh old mode 100644 new mode 100755 index 593f03e985..ae76de57b6 --- a/webrtc/modules/audio_processing/test/py_quality_assessment/apm_quality_assessment.sh +++ b/webrtc/modules/audio_processing/test/py_quality_assessment/apm_quality_assessment.sh @@ -8,7 +8,7 @@ # be found in the AUTHORS file in the root of the source tree. # Customize probing signals, noise sources and scores if needed. -PROBING_SIGNALS=(py_quality_assessment/probing_signals/*.wav) +PROBING_SIGNALS=(probing_signals/*.wav) NOISE_SOURCES=( \ "identity" \ "white" \ @@ -26,7 +26,7 @@ chmod +x apm_quality_assessment-gencfgs.py ./apm_quality_assessment-gencfgs.py # Customize APM configurations if needed. -APM_CONFIGS=(py_quality_assessment/apm_configs/*.json) +APM_CONFIGS=(apm_configs/*.json) # Add output path if missing. if [ ! -d ${OUTPUT_PATH} ]; then @@ -56,6 +56,7 @@ done wait # Export results. +chmod +x ./apm_quality_assessment-export.py ./apm_quality_assessment-export.py -o ${OUTPUT_PATH} # Show results in the browser. diff --git a/webrtc/modules/audio_processing/test/py_quality_assessment/quality_assessment/data_access.py b/webrtc/modules/audio_processing/test/py_quality_assessment/quality_assessment/data_access.py new file mode 100644 index 0000000000..d3967c651a --- /dev/null +++ b/webrtc/modules/audio_processing/test/py_quality_assessment/quality_assessment/data_access.py @@ -0,0 +1,17 @@ +# Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. +# +# Use of this source code is governed by a BSD-style license +# that can be found in the LICENSE file in the root of the source +# tree. An additional intellectual property rights grant can be found +# in the file PATENTS. All contributing project authors may +# be found in the AUTHORS file in the root of the source tree. + +import os + +def make_directory(path): + """ + Recursively make a directory without rising exceptions if it already exists. + """ + if os.path.exists(path): + return + os.makedirs(path) diff --git a/webrtc/modules/audio_processing/test/py_quality_assessment/quality_assessment/eval_scores.py b/webrtc/modules/audio_processing/test/py_quality_assessment/quality_assessment/eval_scores.py new file mode 100644 index 0000000000..8e900251e6 --- /dev/null +++ b/webrtc/modules/audio_processing/test/py_quality_assessment/quality_assessment/eval_scores.py @@ -0,0 +1,56 @@ +# Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. +# +# Use of this source code is governed by a BSD-style license +# that can be found in the LICENSE file in the root of the source +# tree. An additional intellectual property rights grant can be found +# in the file PATENTS. All contributing project authors may +# be found in the AUTHORS file in the root of the source tree. + +class EvaluationScore(object): + + NAME = None + REGISTERED_CLASSES = {} + + def __init__(self): + pass + + @classmethod + def register_class(cls, class_to_register): + """ + Decorator to automatically register the classes that extend EvaluationScore. + """ + cls.REGISTERED_CLASSES[class_to_register.NAME] = class_to_register + + +@EvaluationScore.register_class +class AudioLevelScore(EvaluationScore): + """ + Compute the difference between the average audio level of the tested and + the reference signals. + + Unit: dB + Ideal: 0 dB + Worst case: +/-inf dB + """ + + NAME = 'audio_level' + + def __init__(self): + super(AudioLevelScore, self).__init__() + + +@EvaluationScore.register_class +class PolqaScore(EvaluationScore): + """ + Compute the POLQA score. It requires that the POLQA_PATH environment variable + points to the PolqaOem64 executable. + + Unit: MOS + Ideal: 4.5 + Worst case: 1.0 + """ + + NAME = 'polqa' + + def __init__(self): + super(PolqaScore, self).__init__() diff --git a/webrtc/modules/audio_processing/test/py_quality_assessment/quality_assessment/noise_generation.py b/webrtc/modules/audio_processing/test/py_quality_assessment/quality_assessment/noise_generation.py new file mode 100644 index 0000000000..13a6993606 --- /dev/null +++ b/webrtc/modules/audio_processing/test/py_quality_assessment/quality_assessment/noise_generation.py @@ -0,0 +1,84 @@ +# Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. +# +# Use of this source code is governed by a BSD-style license +# that can be found in the LICENSE file in the root of the source +# tree. An additional intellectual property rights grant can be found +# in the file PATENTS. All contributing project authors may +# be found in the AUTHORS file in the root of the source tree. + +class NoiseGenerator(object): + + NAME = None + REGISTERED_CLASSES = {} + + def __init__(self): + pass + + @classmethod + def register_class(cls, class_to_register): + """ + Decorator to automatically register the classes that extend NoiseGenerator. + """ + cls.REGISTERED_CLASSES[class_to_register.NAME] = class_to_register + + +# Identity generator. +@NoiseGenerator.register_class +class IdentityGenerator(NoiseGenerator): + """ + Generator that adds no noise, therefore both the noisy and the reference + signals are the input signal. + """ + + NAME = 'identity' + + def __init__(self): + super(IdentityGenerator, self).__init__() + + +@NoiseGenerator.register_class +class WhiteNoiseGenerator(NoiseGenerator): + """ + Additive white noise generator. + """ + + NAME = 'white' + + def __init__(self): + super(WhiteNoiseGenerator, self).__init__() + + +@NoiseGenerator.register_class +class NarrowBandNoiseGenerator(NoiseGenerator): + """ + Additive narrow-band noise generator. + """ + + NAME = 'narrow_band' + + def __init__(self): + super(NarrowBandNoiseGenerator, self).__init__() + + +@NoiseGenerator.register_class +class EnvironmentalNoiseGenerator(NoiseGenerator): + """ + Additive environmental noise generator. + """ + + NAME = 'environmental' + + def __init__(self): + super(EnvironmentalNoiseGenerator, self).__init__() + + +@NoiseGenerator.register_class +class EchoNoiseGenerator(NoiseGenerator): + """ + Echo noise generator. + """ + + NAME = 'echo' + + def __init__(self): + super(EchoNoiseGenerator, self).__init__() diff --git a/webrtc/modules/audio_processing/test/py_quality_assessment/quality_assessment/simulation.py b/webrtc/modules/audio_processing/test/py_quality_assessment/quality_assessment/simulation.py new file mode 100644 index 0000000000..0132c751c3 --- /dev/null +++ b/webrtc/modules/audio_processing/test/py_quality_assessment/quality_assessment/simulation.py @@ -0,0 +1,115 @@ +# Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. +# +# Use of this source code is governed by a BSD-style license +# that can be found in the LICENSE file in the root of the source +# tree. An additional intellectual property rights grant can be found +# in the file PATENTS. All contributing project authors may +# be found in the AUTHORS file in the root of the source tree. + +import logging +import os + +from . import data_access +from . import eval_scores +from . import noise_generation + +class ApmModuleSimulator(object): + + _NOISE_GENERATOR_CLASSES = noise_generation.NoiseGenerator.REGISTERED_CLASSES + _EVAL_SCORE_WORKER_CLASSES = eval_scores.EvaluationScore.REGISTERED_CLASSES + + def __init__(self): + # TODO(alessio): instance when implementation is ready. + self._audioproc_wrapper = None + self._evaluator = None + + self._base_output_path = None + self._noise_generators = None + self._evaluation_score_workers = None + self._config_filepaths = None + self._input_filepaths = None + + def run(self, config_filepaths, input_filepaths, noise_generator_names, + eval_score_names, output_dir): + """ + Initializes paths and required instances, then runs all the simulations. + """ + self._base_output_path = os.path.abspath(output_dir) + + # Instance noise generators. + self._noise_generators = [ + self._NOISE_GENERATOR_CLASSES[name]() for name in noise_generator_names] + + # Instance evaluation score workers. + self._evaluation_score_workers = [ + self._EVAL_SCORE_WORKER_CLASSES[name]() for name in eval_score_names] + + # Set APM configuration file paths. + self._config_filepaths = self._get_paths_collection(config_filepaths) + + # Set probing signal file paths. + self._input_filepaths = self._get_paths_collection(input_filepaths) + + self._simulate_all() + + def _simulate_all(self): + """ + Iterates over the combinations of APM configurations, probing signals, and + noise generators. + """ + # Try different APM config files. + for config_name in self._config_filepaths: + config_filepath = self._config_filepaths[config_name] + + # Try different probing signal files. + for input_name in self._input_filepaths: + input_filepath = self._input_filepaths[input_name] + + # Try different noise generators. + for noise_generator in self._noise_generators: + logging.info('config: <%s>, input: <%s>, noise: <%s>', + config_name, input_name, noise_generator.NAME) + + # Output path for the input-noise pairs. It is used to cache the noisy + # copies of the probing signals (shared across some simulations). + input_noise_cache_path = os.path.join( + self._base_output_path, + '_cache', + 'input_{}-noise_{}'.format(input_name, noise_generator.NAME)) + data_access.make_directory(input_noise_cache_path) + logging.debug('input-noise cache path: <%s>', input_noise_cache_path) + + # Full output path. + output_path = os.path.join( + self._base_output_path, + 'cfg-{}'.format(config_name), + 'input-{}'.format(input_name), + 'noise-{}'.format(noise_generator.NAME)) + data_access.make_directory(output_path) + logging.debug('output path: <%s>', output_path) + + self._simulate(noise_generator, input_filepath, + input_noise_cache_path, output_path, config_filepath) + + def _simulate(self, noise_generator, input_filepath, input_noise_cache_path, + output_path, config_filepath): + """ + Simulates a given combination of APM configurations, probing signals, and + noise generators. It iterates over the noise generator internal + configurations. + """ + # TODO(alessio): implement. + pass + + @classmethod + def _get_paths_collection(cls, filepaths): + """ + Given a list of file paths, makes a collection with one pair for each item + in the list where the key is the file name without extension and the value + is the path. + """ + filepaths_collection = {} + for filepath in filepaths: + name = os.path.splitext(os.path.split(filepath)[1])[0] + filepaths_collection[name] = os.path.abspath(filepath) + return filepaths_collection diff --git a/webrtc/modules/audio_processing/test/py_quality_assessment/quality_assessment/test_evalscores.py b/webrtc/modules/audio_processing/test/py_quality_assessment/quality_assessment/test_evalscores.py new file mode 100644 index 0000000000..7ac7dd3615 --- /dev/null +++ b/webrtc/modules/audio_processing/test/py_quality_assessment/quality_assessment/test_evalscores.py @@ -0,0 +1,19 @@ +# Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. +# +# Use of this source code is governed by a BSD-style license +# that can be found in the LICENSE file in the root of the source +# tree. An additional intellectual property rights grant can be found +# in the file PATENTS. All contributing project authors may +# be found in the AUTHORS file in the root of the source tree. + +import unittest + +from . import eval_scores + +class TestEvalScores(unittest.TestCase): + + def test_registered_classes(self): + # Check that there is at least one registered evaluation score worker. + classes = eval_scores.EvaluationScore.REGISTERED_CLASSES + self.assertIsInstance(classes, dict) + self.assertGreater(len(classes), 0) diff --git a/webrtc/modules/audio_processing/test/py_quality_assessment/quality_assessment/test_noisegen.py b/webrtc/modules/audio_processing/test/py_quality_assessment/quality_assessment/test_noisegen.py new file mode 100644 index 0000000000..2d80ecbf70 --- /dev/null +++ b/webrtc/modules/audio_processing/test/py_quality_assessment/quality_assessment/test_noisegen.py @@ -0,0 +1,19 @@ +# Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. +# +# Use of this source code is governed by a BSD-style license +# that can be found in the LICENSE file in the root of the source +# tree. An additional intellectual property rights grant can be found +# in the file PATENTS. All contributing project authors may +# be found in the AUTHORS file in the root of the source tree. + +import unittest + +from . import noise_generation + +class TestNoiseGen(unittest.TestCase): + + def test_registered_classes(self): + # Check that there is at least one registered noise generator. + classes = noise_generation.NoiseGenerator.REGISTERED_CLASSES + self.assertIsInstance(classes, dict) + self.assertGreater(len(classes), 0) diff --git a/webrtc/modules/audio_processing/test/py_quality_assessment/test_simulation.py b/webrtc/modules/audio_processing/test/py_quality_assessment/test_simulation.py new file mode 100644 index 0000000000..0f64a6af97 --- /dev/null +++ b/webrtc/modules/audio_processing/test/py_quality_assessment/test_simulation.py @@ -0,0 +1,19 @@ +# Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. +# +# Use of this source code is governed by a BSD-style license +# that can be found in the LICENSE file in the root of the source +# tree. An additional intellectual property rights grant can be found +# in the file PATENTS. All contributing project authors may +# be found in the AUTHORS file in the root of the source tree. + +import unittest + +import apm_quality_assessment + +class TestSimulationScript(unittest.TestCase): + + def test_main(self): + # Exit with error code if no arguments are passed. + with self.assertRaises(SystemExit) as cm: + apm_quality_assessment.main() + self.assertGreater(cm.exception.code, 0)