From 5bc022929c8d65d389171d3385186b44e0431b61 Mon Sep 17 00:00:00 2001 From: Alessio Bazzica Date: Wed, 27 Sep 2017 13:22:36 +0200 Subject: [PATCH] Injectable APM simulator binary in APM-QA Allow a custom version of audioproc_f in APM-QA. Bug: webrtc:7494 Change-Id: Id9adffd63927202d868bc2fc8b6a54c8e6b07039 Reviewed-on: https://webrtc-review.googlesource.com/4060 Reviewed-by: Alex Loiko Commit-Queue: Alessio Bazzica Cr-Commit-Position: refs/heads/master@{#20033} --- .../test/py_quality_assessment/README.md | 13 ++------- .../apm_quality_assessment.py | 12 +++++--- .../apm_quality_assessment_gencfgs.py | 2 +- .../quality_assessment/audioproc_wrapper.py | 28 +++++++++++-------- .../quality_assessment/data_access.py | 6 ++-- .../quality_assessment/simulation_unittest.py | 10 ++++--- 6 files changed, 38 insertions(+), 33 deletions(-) diff --git a/modules/audio_processing/test/py_quality_assessment/README.md b/modules/audio_processing/test/py_quality_assessment/README.md index 79e1650f08..64f4f0a732 100644 --- a/modules/audio_processing/test/py_quality_assessment/README.md +++ b/modules/audio_processing/test/py_quality_assessment/README.md @@ -1,14 +1,13 @@ # APM Quality Assessment tool -Python wrapper of `audioproc_f` with which quality assessment can be -automatized. The tool allows to simulate different noise conditions, input -signals, APM configurations and it computes different scores. +Python wrapper of APM simulators (e.g., `audioproc_f`) with which quality +assessment can be automatized. The tool allows to simulate different noise +conditions, input signals, APM configurations and it computes different scores. Once the scores are computed, the results can be easily exported to an HTML page which allows to listen to the APM input and output signals and also the reference one used for evaluation. ## Dependencies - - OS: Linux - Python 2.7 - Python libraries: numpy, scipy, pydub (0.17.0+), pandas (0.20.1+) @@ -28,19 +27,16 @@ reference one used for evaluation. - Input probing signals and noise tracks (you can make your own dataset - *1) ## Build - - Compile WebRTC - Go to `out/Default/py_quality_assessment` and check that `apm_quality_assessment.py` exists ## Unit tests - - Compile WebRTC - Go to `out/Default/py_quality_assessment` - Run `python -m unittest -p "*_unittest.py" discover` ## First time setup - - Deploy PolqaOem64 and set the `POLQA_PATH` environment variable - e.g., `$ export POLQA_PATH=/var/opt/PolqaOem64` - Deploy the AIR Database and set the `AECHEN_IR_DATABASE_PATH` environment @@ -59,7 +55,6 @@ converted and exported with Audacity). `out/Default/py_quality_assessment/quality_assessment/test_data_generation.py`. ## Usage (scores computation) - - Go to `out/Default/py_quality_assessment` - Check the `apm_quality_assessment.sh` as an example script to parallelize the experiments @@ -69,7 +64,6 @@ converted and exported with Audacity). scores ## Usage (export reports) - Showing all the results at once can be confusing. You therefore may want to export separate reports. In this case, you can use the `apm_quality_assessment_export.py` script as follows: @@ -96,7 +90,6 @@ $ ./apm_quality_assessment_export.py \ ``` ## Troubleshooting - The input wav file must be: - sampled at a sample rate that is a multiple of 100 (required by POLQA) - in the 16 bit format (required by `audioproc_f`) diff --git a/modules/audio_processing/test/py_quality_assessment/apm_quality_assessment.py b/modules/audio_processing/test/py_quality_assessment/apm_quality_assessment.py index ee08ff7b7f..1e4ecc07ca 100755 --- a/modules/audio_processing/test/py_quality_assessment/apm_quality_assessment.py +++ b/modules/audio_processing/test/py_quality_assessment/apm_quality_assessment.py @@ -8,7 +8,7 @@ # be found in the AUTHORS file in the root of the source tree. """Perform APM module quality assessment on one or more input files using one or - more audioproc_f configuration files and one or more test data generators. + more APM simulator configuration files and one or more test data generators. Usage: apm_quality_assessment.py -i audio1.wav [audio2.wav ...] -c cfg1.json [cfg2.json ...] @@ -47,12 +47,12 @@ def _InstanceArgumentsParser(): """ parser = argparse.ArgumentParser(description=( 'Perform APM module quality assessment on one or more input files using ' - 'one or more audioproc_f configuration files and one or more ' + 'one or more APM simulator configuration files and one or more ' 'test data generators.')) parser.add_argument('-c', '--config_files', nargs='+', required=False, help=('path to the configuration files defining the ' - 'arguments with which the audioproc_f tool is ' + 'arguments with which the APM simulator tool is ' 'called'), default=[_DEFAULT_CONFIG_FILE]) @@ -93,6 +93,10 @@ def _InstanceArgumentsParser(): parser.add_argument('--air_db_path', required=True, help='path to the Aechen IR database') + parser.add_argument( + '--apm_sim_path', required=False, help='path to the APM simulator tool', + default=audioproc_wrapper.AudioProcWrapper.DEFAULT_APM_SIMULATOR_BIN_PATH) + return parser @@ -115,7 +119,7 @@ def main(): simulator = simulation.ApmModuleSimulator( aechen_ir_database_path=args.air_db_path, polqa_tool_bin_path=os.path.join(args.polqa_path, _POLQA_BIN_NAME), - ap_wrapper=audioproc_wrapper.AudioProcWrapper(), + ap_wrapper=audioproc_wrapper.AudioProcWrapper(args.apm_sim_path), evaluator=evaluation.ApmModuleEvaluator()) simulator.Run( config_filepaths=args.config_files, diff --git a/modules/audio_processing/test/py_quality_assessment/apm_quality_assessment_gencfgs.py b/modules/audio_processing/test/py_quality_assessment/apm_quality_assessment_gencfgs.py index 5b567fc643..4017747cc2 100755 --- a/modules/audio_processing/test/py_quality_assessment/apm_quality_assessment_gencfgs.py +++ b/modules/audio_processing/test/py_quality_assessment/apm_quality_assessment_gencfgs.py @@ -8,7 +8,7 @@ # be found in the AUTHORS file in the root of the source tree. """Generate .json files with which the APM module can be tested using the - apm_quality_assessment.py script. + apm_quality_assessment.py script and audioproc_f as APM simulator. """ import logging diff --git a/modules/audio_processing/test/py_quality_assessment/quality_assessment/audioproc_wrapper.py b/modules/audio_processing/test/py_quality_assessment/quality_assessment/audioproc_wrapper.py index d0dd51c34f..399e3864dc 100644 --- a/modules/audio_processing/test/py_quality_assessment/quality_assessment/audioproc_wrapper.py +++ b/modules/audio_processing/test/py_quality_assessment/quality_assessment/audioproc_wrapper.py @@ -6,7 +6,7 @@ # in the file PATENTS. All contributing project authors may # be found in the AUTHORS file in the root of the source tree. -"""Class implementing a wrapper for audioproc_f. +"""Class implementing a wrapper for APM simulators. """ import cProfile @@ -19,18 +19,24 @@ from . import exceptions class AudioProcWrapper(object): - """Wrapper for audioproc_f. + """Wrapper for APM simulators. """ - OUTPUT_FILENAME = 'output.wav' - _AUDIOPROC_F_BIN_PATH = os.path.abspath(os.path.join( + DEFAULT_APM_SIMULATOR_BIN_PATH = os.path.abspath(os.path.join( os.pardir, 'audioproc_f')) + OUTPUT_FILENAME = 'output.wav' - def __init__(self): + def __init__(self, simulator_bin_path): + """Ctor. + + Args: + simulator_bin_path: path to the APM simulator binary. + """ + self._simulator_bin_path = simulator_bin_path self._config = None self._output_signal_filepath = None - # Profiler instance to measure audioproc_f running time. + # Profiler instance to measure running time. self._profiler = cProfile.Profile() @property @@ -39,11 +45,11 @@ class AudioProcWrapper(object): def Run(self, config_filepath, capture_input_filepath, output_path, render_input_filepath=None): - """Run audioproc_f. + """Runs APM simulator. Args: - config_filepath: path to the configuration file specifing the arguments - for audioproc_f. + config_filepath: path to the configuration file specifying the arguments + for the APM simulator. capture_input_filepath: path to the capture audio track input file (aka forward or near-end). output_path: path of the audio track output file. @@ -63,7 +69,7 @@ class AudioProcWrapper(object): # Load configuration. self._config = data_access.AudioProcConfigFile.Load(config_filepath) - # Set remaining parametrs. + # Set remaining parameters. if not os.path.exists(capture_input_filepath): raise exceptions.FileNotFoundError('cannot find capture input file') self._config['-i'] = capture_input_filepath @@ -74,7 +80,7 @@ class AudioProcWrapper(object): self._config['-ri'] = render_input_filepath # Build arguments list. - args = [self._AUDIOPROC_F_BIN_PATH] + args = [self._simulator_bin_path] for param_name in self._config: args.append(param_name) if self._config[param_name] is not None: diff --git a/modules/audio_processing/test/py_quality_assessment/quality_assessment/data_access.py b/modules/audio_processing/test/py_quality_assessment/quality_assessment/data_access.py index c488859b89..17aa7e2b67 100644 --- a/modules/audio_processing/test/py_quality_assessment/quality_assessment/data_access.py +++ b/modules/audio_processing/test/py_quality_assessment/quality_assessment/data_access.py @@ -89,7 +89,7 @@ class Metadata(object): class AudioProcConfigFile(object): - """Data access to load/save audioproc_f argument lists. + """Data access to load/save APM simulator argument lists. The arguments stored in the config files are used to control the APM flags. """ @@ -99,7 +99,7 @@ class AudioProcConfigFile(object): @classmethod def Load(cls, filepath): - """Loads a configuration file for audioproc_f. + """Loads a configuration file for an APM simulator. Args: filepath: path to the configuration file. @@ -112,7 +112,7 @@ class AudioProcConfigFile(object): @classmethod def Save(cls, filepath, config): - """Saves a configuration file for audioproc_f. + """Saves a configuration file for an APM simulator. Args: filepath: path to the configuration file. diff --git a/modules/audio_processing/test/py_quality_assessment/quality_assessment/simulation_unittest.py b/modules/audio_processing/test/py_quality_assessment/quality_assessment/simulation_unittest.py index 33ee92190c..017a316665 100644 --- a/modules/audio_processing/test/py_quality_assessment/quality_assessment/simulation_unittest.py +++ b/modules/audio_processing/test/py_quality_assessment/quality_assessment/simulation_unittest.py @@ -52,7 +52,8 @@ class TestApmModuleSimulator(unittest.TestCase): def testSimulation(self): # Instance dependencies to inject and mock. - ap_wrapper = audioproc_wrapper.AudioProcWrapper() + ap_wrapper = audioproc_wrapper.AudioProcWrapper( + audioproc_wrapper.AudioProcWrapper.DEFAULT_APM_SIMULATOR_BIN_PATH) evaluator = evaluation.ApmModuleEvaluator() ap_wrapper.Run = mock.MagicMock(name='Run') evaluator.Run = mock.MagicMock(name='Run') @@ -81,8 +82,8 @@ class TestApmModuleSimulator(unittest.TestCase): # Check. # TODO(alessiob): Once the TestDataGenerator classes can be configured by - # the client code (e.g., number of SNR pairs for the white noise teste data - # gnerator), the exact number of calls to ap_wrapper.Run and evaluator.Run + # the client code (e.g., number of SNR pairs for the white noise test data + # generator), the exact number of calls to ap_wrapper.Run and evaluator.Run # is known; use that with assertEqual. min_number_of_simulations = len(config_files) * len(input_files) * len( test_data_generators) @@ -99,7 +100,8 @@ class TestApmModuleSimulator(unittest.TestCase): aechen_ir_database_path='', polqa_tool_bin_path=os.path.join( os.path.dirname(__file__), 'fake_polqa'), - ap_wrapper=audioproc_wrapper.AudioProcWrapper(), + ap_wrapper=audioproc_wrapper.AudioProcWrapper( + audioproc_wrapper.AudioProcWrapper.DEFAULT_APM_SIMULATOR_BIN_PATH), evaluator=evaluation.ApmModuleEvaluator()) # What to simulate.