admin/webrtc_m130
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

APM-QA tool, renaming noise generators into input-reference generators.

Browse Source 
This CL changes the name of classes, methods and variables making using "noise generator".
This naming is replaced with "input-reference generator" which is more descriptive of the actual role.
Comments, CSS class and HTML item names have also been changed.
Consistency for variable names has been verified and the style checked with pylint.

BUG=webrtc:7218

Review-Url: https://codereview.webrtc.org/2805653002
Cr-Commit-Position: refs/heads/master@{#17639}
This commit is contained in:
alessiob 2017-04-11 01:06:28 -07:00 committed by Commit bot
parent 9765370416
commit 93cda2ebde
15 changed files with 307 additions and 285 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
webrtc/modules/audio_processing/test/py_quality_assessment/BUILD.gn
View File

@ -42,14 +42,14 @@ copy("lib") {
"quality_assessment/evaluation.py",
"quality_assessment/exceptions.py",
"quality_assessment/export.py",
"quality_assessment/noise_generation.py",
"quality_assessment/noise_generation_factory.py",
"quality_assessment/noise_generation_unittest.py",
"quality_assessment/results.css",
"quality_assessment/results.js",
"quality_assessment/signal_processing.py",
"quality_assessment/signal_processing_unittest.py",
"quality_assessment/simulation.py",
"quality_assessment/test_data_generation.py",
"quality_assessment/test_data_generation_factory.py",
"quality_assessment/test_data_generation_unittest.py",
]
visibility = [ ":*" ] # Only targets in this file can depend on this.
outputs = [

10
webrtc/modules/audio_processing/test/py_quality_assessment/README.md
View File

@ -46,11 +46,11 @@ reference one used for evaluation.
- `out/Default/py_quality_assessment/noise_tracks` (*1, *2)
(*1) You can use custom files as long as they are mono tracks sampled at 48kHz
encoded in the 16 bit signed format (it is recommended that the tracks are
converted and exported with Audacity).
encoded in the 16 bit signed format (it is recommended that the tracks are
converted and exported with Audacity).
(*2) Adapt `EnvironmentalNoiseGenerator._NOISE_TRACKS` accordingly in
`out/Default/py_quality_assessment/quality_assessment/noise_generation.py`.
(*2) Adapt `EnvironmentalNoiseTestDataGenerator._NOISE_TRACKS` accordingly in
`out/Default/py_quality_assessment/quality_assessment/test_data_generation.py`.
## Usage (scores computation)
@ -72,7 +72,7 @@ export separate reports. In this case, you can use the
- Use regular expressions to select/filter out scores by
- APM configurations: `--config_names, -c`
- probing signals: `--input_names, -i`
- noise generators: `--noise_generators, -n`
- test data generators: `--test_data_generators, -t`
- scores: `--eval_scores, -e`
- Assign a suffix to the report name using `-f <suffix>`

23
webrtc/modules/audio_processing/test/py_quality_assessment/apm_quality_assessment.py
View File

@ -8,7 +8,7 @@
# be found in the AUTHORS file in the root of the source tree.
"""Perform APM module quality assessment on one or more input files using one or
more audioproc_f configuration files and one or more noise generators.
more audioproc_f configuration files and one or more test data generators.
Usage: apm_quality_assessment.py -i audio1.wav [audio2.wav ...]
-c cfg1.json [cfg2.json ...]
@ -22,11 +22,12 @@ import logging
import sys
import quality_assessment.eval_scores as eval_scores
import quality_assessment.noise_generation as noise_generation
import quality_assessment.test_data_generation as test_data_generation
import quality_assessment.simulation as simulation
_NOISE_GENERATOR_CLASSES = noise_generation.NoiseGenerator.REGISTERED_CLASSES
_NOISE_GENERATORS_NAMES = _NOISE_GENERATOR_CLASSES.keys()
_TEST_DATA_GENERATOR_CLASSES = (
test_data_generation.TestDataGenerator.REGISTERED_CLASSES)
_TEST_DATA_GENERATORS_NAMES = _TEST_DATA_GENERATOR_CLASSES.keys()
_EVAL_SCORE_WORKER_CLASSES = eval_scores.EvaluationScore.REGISTERED_CLASSES
_EVAL_SCORE_WORKER_NAMES = _EVAL_SCORE_WORKER_CLASSES.keys()
@ -38,8 +39,8 @@ def _InstanceArgumentsParser():
"""
parser = argparse.ArgumentParser(description=(
'Perform APM module quality assessment on one or more input files using '
'one or more audioproc_f configuration files and one or more noise '
'generators.'))
'one or more audioproc_f configuration files and one or more '
'test data generators.'))
parser.add_argument('-c', '--config_files', nargs='+', required=False,
help=('path to the configuration files defining the '
@ -50,10 +51,10 @@ def _InstanceArgumentsParser():
parser.add_argument('-i', '--input_files', nargs='+', required=True,
help='path to the input wav files (one or more)')
parser.add_argument('-n', '--noise_generators', nargs='+', required=False,
help='custom list of noise generators to use',
choices=_NOISE_GENERATORS_NAMES,
default=_NOISE_GENERATORS_NAMES)
parser.add_argument('-t', '--test_data_generators', nargs='+', required=False,
help='custom list of test data generators to use',
choices=_TEST_DATA_GENERATORS_NAMES,
default=_TEST_DATA_GENERATORS_NAMES)
parser.add_argument('-e', '--eval_scores', nargs='+', required=False,
help='custom list of evaluation scores to use',
@ -88,7 +89,7 @@ def main():
simulator.Run(
config_filepaths=args.config_files,
input_filepaths=args.input_files,
noise_generator_names=args.noise_generators,
test_data_generator_names=args.test_data_generators,
eval_score_names=args.eval_scores,
output_dir=args.output_dir)

20
webrtc/modules/audio_processing/test/py_quality_assessment/apm_quality_assessment.sh
View File

@ -31,13 +31,13 @@ else
exit 1
fi
# Customize probing signals, noise sources and scores if needed.
# Customize probing signals, test data generators and scores if needed.
PROBING_SIGNALS=(probing_signals/*.wav)
NOISE_SOURCES=( \
TEST_DATA_GENERATORS=( \
"identity" \
"white" \
"environmental" \
"echo" \
"white_noise" \
"environmental_noise" \
"reverberation" \
)
SCORES=( \
"polqa" \
@ -57,22 +57,22 @@ if [ ! -d ${OUTPUT_PATH} ]; then
mkdir ${OUTPUT_PATH}
fi
# Start one process for each "probing signal"-"noise source" pair.
# Start one process for each "probing signal"-"test data source" pair.
chmod +x apm_quality_assessment.py
for probing_signal_filepath in "${PROBING_SIGNALS[@]}" ; do
probing_signal_name="$(basename $probing_signal_filepath)"
probing_signal_name="${probing_signal_name%.*}"
for noise_source_name in "${NOISE_SOURCES[@]}" ; do
for test_data_gen_name in "${TEST_DATA_GENERATORS[@]}" ; do
LOG_FILE="${OUTPUT_PATH}/apm_qa-${probing_signal_name}-"`
`"${noise_source_name}.log"
echo "Starting ${probing_signal_name} ${noise_source_name} "`
`"${test_data_gen_name}.log"
echo "Starting ${probing_signal_name} ${test_data_gen_name} "`
`"(see ${LOG_FILE})"
./apm_quality_assessment.py \
--polqa_path ${POLQA_PATH}\
--air_db_path ${AECHEN_IR_DATABASE_PATH}\
-i ${probing_signal_filepath} \
-o ${OUTPUT_PATH} \
-n ${noise_source_name} \
-t ${test_data_gen_name} \
-c "${APM_CONFIGS[@]}" \
-e "${SCORES[@]}" > $LOG_FILE 2>&1 &
done

59
webrtc/modules/audio_processing/test/py_quality_assessment/apm_quality_assessment_export.py
View File

@ -26,7 +26,7 @@ import quality_assessment.export as export
# Regular expressions used to derive score descriptors from file paths.
RE_CONFIG_NAME = re.compile(r'cfg-(.+)')
RE_INPUT_NAME = re.compile(r'input-(.+)')
RE_NOISE_NAME = re.compile(r'noise-(.+)')
RE_TEST_DATA_GEN_NAME = re.compile(r'gen-(.+)')
RE_SCORE_NAME = re.compile(r'score-(.+)\.txt')
@ -52,9 +52,9 @@ def _InstanceArgumentsParser():
help=('regular expression to filter the probing signal '
'names'))
parser.add_argument('-n', '--noise_generators', type=re.compile,
help=('regular expression to filter the noise generator '
'names'))
parser.add_argument('-t', '--test_data_generators', type=re.compile,
help=('regular expression to filter the test data '
'generator names'))
parser.add_argument('-e', '--eval_scores', type=re.compile,
help=('regular expression to filter the evaluation score '
@ -71,29 +71,32 @@ def _GetScoreDescriptors(score_filepath):
Returns:
A tuple of strings (APM configuration name, input audio track name,
noise generator name, noise generator parameters name, evaluation score
name).
test data generator name, test data generator parameters name,
evaluation score name).
"""
config_name, input_name, noise_name, noise_params, score_name = (
score_filepath.split(os.sep)[-5:])
(config_name, input_name, test_data_gen_name, test_data_gen_params,
score_name) = score_filepath.split(os.sep)[-5:]
config_name = RE_CONFIG_NAME.match(config_name).groups(0)[0]
input_name = RE_INPUT_NAME.match(input_name).groups(0)[0]
noise_name = RE_NOISE_NAME.match(noise_name).groups(0)[0]
test_data_gen_name = RE_TEST_DATA_GEN_NAME.match(
test_data_gen_name).groups(0)[0]
score_name = RE_SCORE_NAME.match(score_name).groups(0)[0]
return config_name, input_name, noise_name, noise_params, score_name
return (config_name, input_name, test_data_gen_name, test_data_gen_params,
score_name)
def _ExcludeScore(config_name, input_name, noise_name, score_name, args):
def _ExcludeScore(config_name, input_name, test_data_gen_name, score_name,
args):
"""Decides whether excluding a score.
Given a score descriptor, encoded in config_name, input_name, noise_name, and
score_name, use the corresponding regular expressions to determine if the
score should be excluded.
Given a score descriptor, encoded in config_name, input_name,
test_data_gen_name and score_name, use the corresponding regular expressions
to determine if the score should be excluded.
Args:
config_name: APM configuration name.
input_name: input audio track name.
noise_name: noise generator name.
test_data_gen_name: test data generator name.
score_name: evaluation score name.
args: parsed arguments.
@ -103,7 +106,7 @@ def _ExcludeScore(config_name, input_name, noise_name, score_name, args):
value_regexpr_pairs = [
(config_name, args.config_names),
(input_name, args.input_names),
(noise_name, args.noise_generators),
(test_data_gen_name, args.test_data_generators),
(score_name, args.eval_scores),
]
@ -143,32 +146,34 @@ def main():
# Find score files in the output path.
src_path = os.path.join(
args.output_dir, 'cfg-*', 'input-*', 'noise-*', '*', 'score-*.txt')
args.output_dir, 'cfg-*', 'input-*', 'gen-*', '*', 'score-*.txt')
logging.debug(src_path)
for score_filepath in glob.iglob(src_path):
# Extract score descriptors from the path.
config_name, input_name, noise_name, noise_params, score_name = (
_GetScoreDescriptors(score_filepath))
(config_name, input_name, test_data_gen_name, test_data_gen_params,
score_name) = _GetScoreDescriptors(score_filepath)
# Ignore the score if required.
if _ExcludeScore(config_name, input_name, noise_name, score_name, args):
if _ExcludeScore(
config_name, input_name, test_data_gen_name, score_name, args):
logging.info('ignored score: %s %s %s %s',
config_name, input_name, noise_name, score_name)
config_name, input_name, test_data_gen_name, score_name)
continue
# Get metadata.
score_path, _ = os.path.split(score_filepath)
audio_in_filepath, audio_ref_filepath = (
data_access.Metadata.LoadAudioInRefPaths(score_path))
data_access.Metadata.LoadAudioTestDataPaths(score_path))
audio_out_filepath = os.path.abspath(os.path.join(
score_path, audioproc_wrapper.AudioProcWrapper.OUTPUT_FILENAME))
# Add the score to the nested dictionary.
scores[score_name][config_name][input_name][noise_name][noise_params] = {
'score': data_access.ScoreFile.Load(score_filepath),
'audio_in_filepath': audio_in_filepath,
'audio_out_filepath': audio_out_filepath,
'audio_ref_filepath': audio_ref_filepath,
scores[score_name][config_name][input_name][test_data_gen_name][
test_data_gen_params] = {
'score': data_access.ScoreFile.Load(score_filepath),
'audio_in_filepath': audio_in_filepath,
'audio_out_filepath': audio_out_filepath,
'audio_ref_filepath': audio_ref_filepath,
}
# Export.

13
webrtc/modules/audio_processing/test/py_quality_assessment/quality_assessment/data_access.py
View File

@ -31,26 +31,27 @@ class Metadata(object):
def __init__(self):
pass
_AUDIO_IN_REF_FILENAME = 'audio_in_ref.txt'
_AUDIO_TEST_DATA_FILENAME = 'audio_test_data.txt'
@classmethod
def LoadAudioInRefPaths(cls, metadata_path):
def LoadAudioTestDataPaths(cls, metadata_path):
"""Loads the input and the reference audio track paths.
Args:
metadata_path: path to the directory containing the metadata file.
Returns:
Pair of metadata file paths for the input and output audio tracks.
Tuple with the paths to the input and output audio tracks.
"""
metadata_filepath = os.path.join(metadata_path, cls._AUDIO_IN_REF_FILENAME)
metadata_filepath = os.path.join(
metadata_path, cls._AUDIO_TEST_DATA_FILENAME)
with open(metadata_filepath) as f:
audio_in_filepath = f.readline().strip()
audio_ref_filepath = f.readline().strip()
return audio_in_filepath, audio_ref_filepath
@classmethod
def SaveAudioInRefPaths(cls, output_path, audio_in_filepath,
def SaveAudioTestDataPaths(cls, output_path, audio_in_filepath,
audio_ref_filepath):
"""Saves the input and the reference audio track paths.
@ -59,7 +60,7 @@ class Metadata(object):
audio_in_filepath: path to the input audio track file.
audio_ref_filepath: path to the reference audio track file.
"""
output_filepath = os.path.join(output_path, cls._AUDIO_IN_REF_FILENAME)
output_filepath = os.path.join(output_path, cls._AUDIO_TEST_DATA_FILENAME)
with open(output_filepath, 'w') as f:
f.write('{}\n{}\n'.format(audio_in_filepath, audio_ref_filepath))

2
webrtc/modules/audio_processing/test/py_quality_assessment/quality_assessment/eval_scores.py
View File

@ -71,7 +71,7 @@ class EvaluationScore(object):
self._tested_signal_filepath = filepath
def Run(self, output_path):
"""Extracts the score for the set input-reference pair.
"""Extracts the score for the set test data pair.
Args:
output_path: path to the directory where the output is written.

56
webrtc/modules/audio_processing/test/py_quality_assessment/quality_assessment/export.py
View File

@ -29,8 +29,8 @@ class HtmlExport(object):
_NEW_LINE = '\n'
def __init__(self, output_filepath):
self._noise_names = None
self._noise_params = None
self._test_data_generator_names = None
self._test_data_generator_params = None
self._output_filepath = output_filepath
def Export(self, scores):
@ -182,18 +182,20 @@ class HtmlExport(object):
Returns:
A string with the HTML of a table body cell.
"""
# Init noise generator names and noise parameters cache (if not done).
if self._noise_names is None:
self._noise_names = sorted(scores.keys())
self._noise_params = {noise_name: sorted(scores[noise_name].keys()) for (
noise_name) in self._noise_names}
# Init test data generator names and parameters cache (if not done).
if self._test_data_generator_names is None:
self._test_data_generator_names = sorted(scores.keys())
self._test_data_generator_params = {test_data_generator_name: sorted(
scores[test_data_generator_name].keys()) for (
test_data_generator_name) in self._test_data_generator_names}
# For each noisy input (that is a pair of noise generator name and noise
# For each noisy input (that is a pair of test data generator and
# generator parameters), add an item with the score and its metadata.
items = []
for name_index, noise_name in enumerate(self._noise_names):
for params_index, noise_params in enumerate(
self._noise_params[noise_name]):
for name_index, test_data_generator_name in enumerate(
self._test_data_generator_names):
for params_index, test_data_generator_params in enumerate(
self._test_data_generator_params[test_data_generator_name]):
# Init.
score_value = '?'
@ -201,29 +203,29 @@ class HtmlExport(object):
# Extract score value and its metadata.
try:
data = scores[noise_name][noise_params]
data = scores[test_data_generator_name][test_data_generator_params]
score_value = '{0:f}'.format(data['score'])
metadata = (
'<input type="hidden" name="noise_name" value="{}"/>'
'<input type="hidden" name="noise_params" value="{}"/>'
'<input type="hidden" name="gen_name" value="{}"/>'
'<input type="hidden" name="gen_params" value="{}"/>'
'<input type="hidden" name="audio_in" value="file://{}"/>'
'<input type="hidden" name="audio_out" value="file://{}"/>'
'<input type="hidden" name="audio_ref" value="file://{}"/>'
).format(
noise_name,
noise_params,
test_data_generator_name,
test_data_generator_params,
data['audio_in_filepath'],
data['audio_out_filepath'],
data['audio_ref_filepath'])
except TypeError:
logging.warning(
'missing score found: <score:%s> <config:%s> <input:%s> '
'<noise:%s> <params:%s>', score_name, config_name, input_name,
noise_name, noise_params)
'<generator:%s> <params:%s>', score_name, config_name, input_name,
test_data_generator_name, test_data_generator_params)
# Add the score.
items.append(
'<div class="noise-desc">[{0:d}, {1:d}]{2}</div>'
'<div class="test-data-gen-desc">[{0:d}, {1:d}]{2}</div>'
'<div class="value">{3}</div>'.format(
name_index, params_index, metadata, score_value))
@ -237,18 +239,20 @@ class HtmlExport(object):
def _BuildLegend(self):
"""Builds the legend.
The legend details noise generator name and parameter pairs.
The legend details test data generator name and parameter pairs.
Returns:
A string with a <div class="legend">...</div> HTML element.
"""
items = []
for name_index, noise_name in enumerate(self._noise_names):
for params_index, noise_params in enumerate(
self._noise_params[noise_name]):
items.append('<div class="noise-desc">[{0:d}, {1:d}]</div>: {2} noise, '
'{3}'.format(name_index, params_index, noise_name,
noise_params))
for name_index, test_data_generator_name in enumerate(
self._test_data_generator_names):
for params_index, test_data_generator_params in enumerate(
self._test_data_generator_params[test_data_generator_name]):
items.append(
'<div class="test-data-gen-desc">[{0:d}, {1:d}]</div>: {2}, '
'{3}'.format(name_index, params_index, test_data_generator_name,
test_data_generator_params))
html = (
'<div class="legend"><div>' +
'</div><div>'.join(items) + '</div></div>')

40
webrtc/modules/audio_processing/test/py_quality_assessment/quality_assessment/noise_generation_factory.py
View File

@ -1,40 +0,0 @@
# Copyright (c) 2017 The WebRTC project authors. All Rights Reserved.
#
# Use of this source code is governed by a BSD-style license
# that can be found in the LICENSE file in the root of the source
# tree. An additional intellectual property rights grant can be found
# in the file PATENTS. All contributing project authors may
# be found in the AUTHORS file in the root of the source tree.
"""NoiseGenerator factory class.
"""
import logging
from . import noise_generation
class NoiseGeneratorFactory(object):
"""Factory class used to instantiate noise generator workers.
It can be used by instanciating a factory, passing parameters to the
constructor. These parameters are used to instantiate noise generator
workers.
"""
def __init__(self, aechen_ir_database_path):
self._aechen_ir_database_path = aechen_ir_database_path
def GetInstance(self, noise_generator_class):
"""Creates an NoiseGenerator instance given a class object.
Args:
noise_generator_class: NoiseGenerator class object (not an instance).
"""
logging.debug(
'factory producing a %s noise generator', noise_generator_class)
if noise_generator_class == noise_generation.EchoNoiseGenerator:
return noise_generation.EchoNoiseGenerator(self._aechen_ir_database_path)
else:
# By default, no arguments in the constructor.
return noise_generator_class()

2
webrtc/modules/audio_processing/test/py_quality_assessment/quality_assessment/results.css
View File

@ -49,7 +49,7 @@ table tbody tr td .value{
display: inline-block;
}
.noise-desc{
.test-data-gen-desc{
display: inline-block;
margin-right: 0.3em;
border: 1px solid #555;

33
webrtc/modules/audio_processing/test/py_quality_assessment/quality_assessment/results.js
View File

@ -13,8 +13,8 @@
function Inspector() {
this.audioPlayer_ = new Audio();
this.inspectorNode_ = document.createElement('div');
this.divNoiseGenerator_ = document.createElement('div');
this.divNoiseParameters_ = document.createElement('div');
this.divTestDataGeneratorName_ = document.createElement('div');
this.divTestDataGenParameters_ = document.createElement('div');
this.buttonPlayAudioIn_ = document.createElement('button');
this.buttonPlayAudioOut_ = document.createElement('button');
this.buttonPlayAudioRef_ = document.createElement('button');
@ -79,15 +79,15 @@ Inspector.prototype.openInspector = function(target) {
this.selectedItem_ = target;
this.selectedItem_.classList.add('selected');
var target = this.selectedItem_.querySelector('.noise-desc');
var noiseName = target.querySelector('input[name=noise_name]').value;
var noiseParams = target.querySelector('input[name=noise_params]').value;
var target = this.selectedItem_.querySelector('.test-data-gen-desc');
var testDataGenName = target.querySelector('input[name=gen_name]').value;
var testDataGenParams = target.querySelector('input[name=gen_params]').value;
var audioIn = target.querySelector('input[name=audio_in]').value;
var audioOut = target.querySelector('input[name=audio_out]').value;
var audioRef = target.querySelector('input[name=audio_ref]').value;
this.divNoiseGenerator_.innerHTML = noiseName;
this.divNoiseParameters_.innerHTML = noiseParams;
this.divTestDataGeneratorName_.innerHTML = testDataGenName;
this.divTestDataGenParameters_.innerHTML = testDataGenParams;
this.audioInUrl_ = audioIn;
this.audioOutUrl_ = audioOut;
@ -143,13 +143,14 @@ Inspector.prototype.buildInspector_ = function() {
var self = this;
this.inspectorNode_.setAttribute('class', 'inspector');
this.inspectorNode_.innerHTML = '<div class="property noise-generator">' +
'<div class="name">noise generator</div>' +
'</div>' +
'<div class="property noise-parmas">' +
'<div class="name">parameters</div>' +
'</div>' +
'<div class="buttons"></div>';
this.inspectorNode_.innerHTML =
'<div class="property test-data-gen-name">' +
'<div class="name">test data generator</div>' +
'</div>' +
'<div class="property test-data-gen-parmas">' +
'<div class="name">parameters</div>' +
'</div>' +
'<div class="buttons"></div>';
// Add value nodes.
function addValueNode(node, parent_selector) {
@ -158,8 +159,8 @@ Inspector.prototype.buildInspector_ = function() {
var parentNode = self.inspectorNode_.querySelector(parent_selector);
parentNode.appendChild(node);
}
addValueNode(this.divNoiseGenerator_, 'div.noise-generator');
addValueNode(this.divNoiseParameters_, 'div.noise-parmas');
addValueNode(this.divTestDataGeneratorName_, 'div.test-data-gen-name');
addValueNode(this.divTestDataGenParameters_, 'div.test-data-gen-parmas');
// Add buttons.
var buttonsNode = this.inspectorNode_.querySelector('div.buttons');

74
webrtc/modules/audio_processing/test/py_quality_assessment/quality_assessment/simulation.py
View File

@ -17,15 +17,16 @@ from . import data_access
from . import eval_scores
from . import eval_scores_factory
from . import evaluation
from . import noise_generation
from . import noise_generation_factory
from . import test_data_generation
from . import test_data_generation_factory
class ApmModuleSimulator(object):
"""APM module simulator class.
"""
_NOISE_GENERATOR_CLASSES = noise_generation.NoiseGenerator.REGISTERED_CLASSES
_TEST_DATA_GENERATOR_CLASSES = (
test_data_generation.TestDataGenerator.REGISTERED_CLASSES)
_EVAL_SCORE_WORKER_CLASSES = eval_scores.EvaluationScore.REGISTERED_CLASSES
def __init__(self, aechen_ir_database_path, polqa_tool_path):
@ -34,8 +35,8 @@ class ApmModuleSimulator(object):
self._evaluator = evaluation.ApmModuleEvaluator()
# Instance factory objects.
self._noise_generator_factory = (
noise_generation_factory.NoiseGeneratorFactory(
self._test_data_generator_factory = (
test_data_generation_factory.TestDataGeneratorFactory(
aechen_ir_database_path=aechen_ir_database_path))
self._evaluation_score_factory = (
eval_scores_factory.EvaluationScoreWorkerFactory(
@ -43,12 +44,12 @@ class ApmModuleSimulator(object):
# Properties for each run.
self._base_output_path = None
self._noise_generators = None
self._test_data_generators = None
self._evaluation_score_workers = None
self._config_filepaths = None
self._input_filepaths = None
def Run(self, config_filepaths, input_filepaths, noise_generator_names,
def Run(self, config_filepaths, input_filepaths, test_data_generator_names,
eval_score_names, output_dir):
"""Runs the APM simulation.
@ -57,16 +58,17 @@ class ApmModuleSimulator(object):
Args:
config_filepaths: set of APM configuration files to test.
input_filepaths: set of input audio track files to test.
noise_generator_names: set of noise generator names to test.
test_data_generator_names: set of test data generator names to test.
eval_score_names: set of evaluation score names to test.
output_dir: base path to the output directory for wav files and outcomes.
"""
self._base_output_path = os.path.abspath(output_dir)
# Instance noise generators.
self._noise_generators = [self._noise_generator_factory.GetInstance(
noise_generator_class=self._NOISE_GENERATOR_CLASSES[name]) for name in (
noise_generator_names)]
# Instance test data generators.
self._test_data_generators = [self._test_data_generator_factory.GetInstance(
test_data_generators_class=(
self._TEST_DATA_GENERATOR_CLASSES[name])) for name in (
test_data_generator_names)]
# Instance evaluation score workers.
self._evaluation_score_workers = [
@ -86,7 +88,7 @@ class ApmModuleSimulator(object):
"""Runs all the simulations.
Iterates over the combinations of APM configurations, probing signals, and
noise generators.
test data generators.
"""
# Try different APM config files.
for config_name in self._config_filepaths:
@ -96,17 +98,17 @@ class ApmModuleSimulator(object):
for input_name in self._input_filepaths:
input_filepath = self._input_filepaths[input_name]
# Try different noise generators.
for noise_generator in self._noise_generators:
# Try different test data generators.
for test_data_generators in self._test_data_generators:
logging.info('config: <%s>, input: <%s>, noise: <%s>',
config_name, input_name, noise_generator.NAME)
config_name, input_name, test_data_generators.NAME)
# Output path for the input-noise pairs. It is used to cache the noisy
# copies of the probing signals (shared across some simulations).
input_noise_cache_path = os.path.join(
self._base_output_path,
'_cache',
'input_{}-noise_{}'.format(input_name, noise_generator.NAME))
'input_{}-noise_{}'.format(input_name, test_data_generators.NAME))
data_access.MakeDirectory(input_noise_cache_path)
logging.debug('input-noise cache path: <%s>', input_noise_cache_path)
@ -115,43 +117,44 @@ class ApmModuleSimulator(object):
self._base_output_path,
'cfg-{}'.format(config_name),
'input-{}'.format(input_name),
'noise-{}'.format(noise_generator.NAME))
'gen-{}'.format(test_data_generators.NAME))
data_access.MakeDirectory(output_path)
logging.debug('output path: <%s>', output_path)
self._Simulate(noise_generator, input_filepath,
self._Simulate(test_data_generators, input_filepath,
input_noise_cache_path, output_path, config_filepath)
def _Simulate(self, noise_generator, input_filepath, input_noise_cache_path,
output_path, config_filepath):
def _Simulate(self, test_data_generators, input_filepath,
input_noise_cache_path, output_path, config_filepath):
"""Runs a single set of simulation.
Simulates a given combination of APM configuration, probing signal, and
noise generator. It iterates over the noise generator internal
configurations.
test data generator. It iterates over the test data generator
internal configurations.
Args:
noise_generator: NoiseGenerator instance.
test_data_generators: TestDataGenerator instance.
input_filepath: input audio track file to test.
input_noise_cache_path: path for the noisy audio track files.
output_path: base output path for the noise generator.
output_path: base output path for the test data generator.
config_filepath: APM configuration file to test.
"""
# Generate pairs of noisy input and reference signal files.
noise_generator.Generate(
test_data_generators.Generate(
input_signal_filepath=input_filepath,
input_noise_cache_path=input_noise_cache_path,
base_output_path=output_path)
# For each input-reference pair, simulate a call and evaluate.
for noise_generator_config_name in noise_generator.config_names:
logging.info(' - noise config: <%s>', noise_generator_config_name)
# For each test data pair, simulate a call and evaluate.
for test_data_generators_config_name in test_data_generators.config_names:
logging.info(' - test data generator config: <%s>',
test_data_generators_config_name)
# APM input and output signal paths.
noisy_signal_filepath = noise_generator.noisy_signal_filepaths[
noise_generator_config_name]
evaluation_output_path = noise_generator.apm_output_paths[
noise_generator_config_name]
noisy_signal_filepath = test_data_generators.noisy_signal_filepaths[
test_data_generators_config_name]
evaluation_output_path = test_data_generators.apm_output_paths[
test_data_generators_config_name]
# Simulate a call using the audio processing module.
self._audioproc_wrapper.Run(
@ -160,8 +163,9 @@ class ApmModuleSimulator(object):
output_path=evaluation_output_path)
# Reference signal path for the evaluation step.
reference_signal_filepath = noise_generator.reference_signal_filepaths[
noise_generator_config_name]
reference_signal_filepath = (
test_data_generators.reference_signal_filepaths[
test_data_generators_config_name])
# Evaluate.
self._evaluator.Run(

102
webrtc/modules/audio_processing/test/py_quality_assessment/quality_assessment/noise_generation.py → webrtc/modules/audio_processing/test/py_quality_assessment/quality_assessment/test_data_generation.py
View File

@ -6,19 +6,19 @@
# in the file PATENTS. All contributing project authors may
# be found in the AUTHORS file in the root of the source tree.
"""Noise generators producing pairs of signals intended to be used to test the
APM module. Each pair consists of a noisy and a reference signal. The former
is used as input for APM, and it is generated by adding noise to a signal.
The reference is the expected APM output when using the generated input.
"""Test data generators producing signals pairs intended to be used to
test the APM module. Each pair consists of a noisy input and a reference signal.
The former is used as APM input and it is generated by adding noise to a
clean audio track. The reference is the expected APM output.
Throughout this file, the following naming convention is used:
Throughout this file, the following naming convention is used:
- input signal: the clean signal (e.g., speech),
- noise signal: the noise to be summed up to the input signal (e.g., white
noise, Gaussian noise),
- noisy signal: input + noise.
The noise signal may or may not be a function of the clean signal. For
instance, white noise is independently generated, whereas reverberation is
obtained by convolving the input signal with an impulse response.
The noise signal may or may not be a function of the clean signal. For
instance, white noise is independently generated, whereas reverberation is
obtained by convolving the input signal with an impulse response.
"""
import logging
@ -36,7 +36,7 @@ from . import exceptions
from . import signal_processing
class NoiseGenerator(object):
class TestDataGenerator(object):
"""Abstract class responsible for the generation of noisy signals.
Given a clean signal, it generates two streams named noisy signal and
@ -46,17 +46,15 @@ class NoiseGenerator(object):
expected at the output of the APM module when the latter is fed with the nosiy
signal.
A noise generator generates one or more input-reference pairs.
TODO(alessiob): Rename from NoiseGenerator to InputReferencePairGenerator.
An test data generator generates one or more pairs.
"""
NAME = None
REGISTERED_CLASSES = {}
def __init__(self):
# Init dictionaries with one entry for each noise generator configuration
# (e.g., different SNRs).
# Init dictionaries with one entry for each test data generator
# configuration (e.g., different SNRs).
# Noisy audio track files (stored separately in a cache folder).
self._noisy_signal_filepaths = None
# Path to be used for the APM simulation output files.
@ -67,13 +65,14 @@ class NoiseGenerator(object):
@classmethod
def RegisterClass(cls, class_to_register):
"""Registers an NoiseGenerator implementation.
"""Registers an TestDataGenerator implementation.
Decorator to automatically register the classes that extend NoiseGenerator.
Decorator to automatically register the classes that extend
TestDataGenerator.
Example usage:
@NoiseGenerator.RegisterClass
class IdentityGenerator(NoiseGenerator):
@TestDataGenerator.RegisterClass
class IdentityGenerator(TestDataGenerator):
pass
"""
cls.REGISTERED_CLASSES[class_to_register.NAME] = class_to_register
@ -165,21 +164,21 @@ class NoiseGenerator(object):
reference_signal_filepath)
# Save noisy and reference file paths.
data_access.Metadata.SaveAudioInRefPaths(
data_access.Metadata.SaveAudioTestDataPaths(
output_path=output_path,
audio_in_filepath=self._noisy_signal_filepaths[config_name],
audio_ref_filepath=self._reference_signal_filepaths[config_name])
@classmethod
def _MakeDir(cls, base_output_path, noise_generator_config_name):
output_path = os.path.join(base_output_path, noise_generator_config_name)
def _MakeDir(cls, base_output_path, test_data_generator_config_name):
output_path = os.path.join(
base_output_path, test_data_generator_config_name)
data_access.MakeDirectory(output_path)
return output_path
# Identity generator.
@NoiseGenerator.RegisterClass
class IdentityGenerator(NoiseGenerator):
@TestDataGenerator.RegisterClass
class IdentityTestDataGenerator(TestDataGenerator):
"""Generator that adds no noise.
Both the noisy and the reference signals are the input signal.
@ -188,7 +187,7 @@ class IdentityGenerator(NoiseGenerator):
NAME = 'identity'
def __init__(self):
NoiseGenerator.__init__(self)
TestDataGenerator.__init__(self)
def _Generate(
self, input_signal_filepath, input_noise_cache_path, base_output_path):
@ -201,12 +200,12 @@ class IdentityGenerator(NoiseGenerator):
output_path=output_path)
@NoiseGenerator.RegisterClass
class WhiteNoiseGenerator(NoiseGenerator):
"""Additive white noise generator.
@TestDataGenerator.RegisterClass
class WhiteNoiseTestDataGenerator(TestDataGenerator):
"""Generator that adds white noise.
"""
NAME = 'white'
NAME = 'white_noise'
# Each pair indicates the clean vs. noisy and reference vs. noisy SNRs.
# The reference (second value of each pair) always has a lower amount of noise
@ -221,7 +220,7 @@ class WhiteNoiseGenerator(NoiseGenerator):
_NOISY_SIGNAL_FILENAME_TEMPLATE = 'noise_{0:d}_SNR.wav'
def __init__(self):
NoiseGenerator.__init__(self)
TestDataGenerator.__init__(self)
def _Generate(
self, input_signal_filepath, input_noise_cache_path, base_output_path):
@ -270,15 +269,15 @@ class WhiteNoiseGenerator(NoiseGenerator):
# TODO(alessiob): remove comment when class implemented.
# @NoiseGenerator.RegisterClass
class NarrowBandNoiseGenerator(NoiseGenerator):
"""Additive narrow-band noise generator.
# @TestDataGenerator.RegisterClass
class NarrowBandNoiseTestDataGenerator(TestDataGenerator):
"""Generator that adds narrow-band noise.
"""
NAME = 'narrow_band'
NAME = 'narrow_band_noise'
def __init__(self):
NoiseGenerator.__init__(self)
TestDataGenerator.__init__(self)
def _Generate(
self, input_signal_filepath, input_noise_cache_path, base_output_path):
@ -286,19 +285,22 @@ class NarrowBandNoiseGenerator(NoiseGenerator):
pass
@NoiseGenerator.RegisterClass
class EnvironmentalNoiseGenerator(NoiseGenerator):
"""Additive environmental noise generator.
@TestDataGenerator.RegisterClass
class EnvironmentalNoiseTestDataGenerator(TestDataGenerator):
"""Generator that adds environmental noise.
TODO(alessiob): Make the class more generic e.g.,
MixNoiseTrackTestDataGenerator.
"""
NAME = 'environmental'
NAME = 'environmental_noise'
_NOISY_SIGNAL_FILENAME_TEMPLATE = '{0}_{1:d}_SNR.wav'
# TODO(alessiob): allow the user to store the noise tracks in a custom path.
_NOISE_TRACKS_PATH = os.path.join(os.getcwd(), 'noise_tracks')
# TODO(alessiob): allow the user to have custom noise tracks.
# TODO(alessiob): exploit NoiseGeneratorFactory.GetInstance().
# TODO(alessiob): exploit TestDataGeneratorFactory.GetInstance().
_NOISE_TRACKS = [
'city.wav'
]
@ -314,11 +316,11 @@ class EnvironmentalNoiseGenerator(NoiseGenerator):
]
def __init__(self):
NoiseGenerator.__init__(self)
TestDataGenerator.__init__(self)
def _Generate(
self, input_signal_filepath, input_noise_cache_path, base_output_path):
"""Generates environmental noise.
"""Generates test data pairs using environmental noise.
For each noise track and pair of SNR values, the following two audio tracks
are created: the noisy signal and the reference signal. The former is
@ -374,14 +376,16 @@ class EnvironmentalNoiseGenerator(NoiseGenerator):
base_output_path, noisy_mix_filepaths, self._SNR_VALUE_PAIRS)
@NoiseGenerator.RegisterClass
class EchoNoiseGenerator(NoiseGenerator):
"""Echo noise generator.
@TestDataGenerator.RegisterClass
class ReverberationTestDataGenerator(TestDataGenerator):
"""Generator that adds reverberation noise.
TODO(alessiob): Rename from echo to reverberation.
TODO(alessiob): Make this class more generic since the impulse response can be
anything (not just reverberation); call it e.g.,
ConvolutionalNoiseTestDataGenerator.
"""
NAME = 'echo'
NAME = 'reverberation'
_IMPULSE_RESPONSES = {
'lecture': 'air_binaural_lecture_0_0_1.mat', # Long echo.
@ -401,12 +405,12 @@ class EchoNoiseGenerator(NoiseGenerator):
_NOISY_SIGNAL_FILENAME_TEMPLATE = '{0}_{1:d}_SNR.wav'
def __init__(self, aechen_ir_database_path):
NoiseGenerator.__init__(self)
TestDataGenerator.__init__(self)
self._aechen_ir_database_path = aechen_ir_database_path
def _Generate(
self, input_signal_filepath, input_noise_cache_path, base_output_path):
"""Generates echo noise.
"""Generates test data pairs using reverberation noise.
For each impulse response, one noise track is created. For each impulse
response and pair of SNR values, the following 2 audio tracks are

41
webrtc/modules/audio_processing/test/py_quality_assessment/quality_assessment/test_data_generation_factory.py Normal file
View File

@ -0,0 +1,41 @@
# Copyright (c) 2017 The WebRTC project authors. All Rights Reserved.
#
# Use of this source code is governed by a BSD-style license
# that can be found in the LICENSE file in the root of the source
# tree. An additional intellectual property rights grant can be found
# in the file PATENTS. All contributing project authors may
# be found in the AUTHORS file in the root of the source tree.
"""TestDataGenerator factory class.
"""
import logging
from . import test_data_generation
class TestDataGeneratorFactory(object):
"""Factory class used to create test data generators.
Usage: Create a factory passing parameters to the ctor with which the
generators will be produced.
"""
def __init__(self, aechen_ir_database_path):
self._aechen_ir_database_path = aechen_ir_database_path
def GetInstance(self, test_data_generators_class):
"""Creates an TestDataGenerator instance given a class object.
Args:
test_data_generators_class: TestDataGenerator class object (not an
instance).
"""
logging.debug('factory producing %s', test_data_generators_class)
if test_data_generators_class == (
test_data_generation.ReverberationTestDataGenerator):
return test_data_generation.ReverberationTestDataGenerator(
self._aechen_ir_database_path)
else:
# By default, no arguments in the constructor.
return test_data_generators_class()

111
webrtc/modules/audio_processing/test/py_quality_assessment/quality_assessment/noise_generation_unittest.py → webrtc/modules/audio_processing/test/py_quality_assessment/quality_assessment/test_data_generation_unittest.py
View File

@ -6,7 +6,7 @@
# in the file PATENTS. All contributing project authors may
# be found in the AUTHORS file in the root of the source tree.
"""Unit tests for the noise_generation module.
"""Unit tests for the test_data_generation module.
"""
import os
@ -14,13 +14,13 @@ import shutil
import tempfile
import unittest
from . import noise_generation
from . import noise_generation_factory
from . import test_data_generation
from . import test_data_generation_factory
from . import signal_processing
class TestNoiseGen(unittest.TestCase):
"""Unit tests for the noise_generation module.
class TestTestDataGenerators(unittest.TestCase):
"""Unit tests for the test_data_generation module.
"""
def setUp(self):
@ -33,22 +33,25 @@ class TestNoiseGen(unittest.TestCase):
shutil.rmtree(self._base_output_path)
shutil.rmtree(self._input_noise_cache_path)
def testNoiseGenerators(self):
def testTestDataGenerators(self):
# Preliminary check.
self.assertTrue(os.path.exists(self._base_output_path))
self.assertTrue(os.path.exists(self._input_noise_cache_path))
# Check that there is at least one registered noise generator.
registered_classes = noise_generation.NoiseGenerator.REGISTERED_CLASSES
# Check that there is at least one registered test data generator.
registered_classes = (
test_data_generation.TestDataGenerator.REGISTERED_CLASSES)
self.assertIsInstance(registered_classes, dict)
self.assertGreater(len(registered_classes), 0)
# Instance noise generator factory.
noise_generator_factory = noise_generation_factory.NoiseGeneratorFactory(
aechen_ir_database_path='')
# TODO(alessiob): Replace with a mock of NoiseGeneratorFactory that takes
# no arguments in the ctor. For those generators that need parameters, it
# will return a mock generator (see the first comment in the next for loop).
# Instance generators factory.
generators_factory = (
test_data_generation_factory.TestDataGeneratorFactory(
aechen_ir_database_path=''))
# TODO(alessiob): Replace with a mock of TestDataGeneratorFactory that
# takes no arguments in the ctor. For those generators that need parameters,
# it will return a mock generator (see the first comment in the next for
# loop).
# Use a sample input file as clean input signal.
input_signal_filepath = os.path.join(
@ -59,64 +62,62 @@ class TestNoiseGen(unittest.TestCase):
input_signal = signal_processing.SignalProcessingUtils.LoadWav(
input_signal_filepath)
# Try each registered noise generator.
for noise_generator_name in registered_classes:
# Exclude EchoNoiseGenerator.
# TODO(alessiob): Mock EchoNoiseGenerator, the mock should rely on
# hard-coded impulse responses. This requires a mock for
# NoiseGeneratorFactory. The latter knows whether returning the actual
# generator or a mock object (as in the case of EchoNoiseGenerator).
if noise_generator_name == 'echo':
# Try each registered test data generator.
for generator_name in registered_classes:
# Exclude ReverberationTestDataGenerator.
# TODO(alessiob): Mock ReverberationTestDataGenerator, the mock
# should rely on hard-coded impulse responses. This requires a mock for
# TestDataGeneratorFactory. The latter knows whether returning the
# actual generator or a mock object (as in the case of
# ReverberationTestDataGenerator).
if generator_name == (
test_data_generation.ReverberationTestDataGenerator.NAME):
continue
# Instance noise generator.
noise_generator = noise_generator_factory.GetInstance(
registered_classes[noise_generator_name])
# Instance test data generator.
generator = generators_factory.GetInstance(
registered_classes[generator_name])
# Generate the noisy input - reference pairs.
noise_generator.Generate(
generator.Generate(
input_signal_filepath=input_signal_filepath,
input_noise_cache_path=self._input_noise_cache_path,
base_output_path=self._base_output_path)
# Perform checks.
self._CheckNoiseGeneratorPairsListSizes(noise_generator)
self._CheckNoiseGeneratorPairsSignalDurations(
noise_generator, input_signal)
self._CheckNoiseGeneratorPairsOutputPaths(noise_generator)
self._CheckGeneratedPairsListSizes(generator)
self._CheckGeneratedPairsSignalDurations(generator, input_signal)
self._CheckGeneratedPairsOutputPaths(generator)
def _CheckNoiseGeneratorPairsListSizes(self, noise_generator):
# Noise configuration names.
noise_config_names = noise_generator.config_names
number_of_pairs = len(noise_config_names)
def _CheckGeneratedPairsListSizes(self, generator):
config_names = generator.config_names
number_of_pairs = len(config_names)
self.assertEqual(number_of_pairs,
len(generator.noisy_signal_filepaths))
self.assertEqual(number_of_pairs,
len(generator.apm_output_paths))
self.assertEqual(number_of_pairs,
len(generator.reference_signal_filepaths))
# Check.
self.assertEqual(number_of_pairs,
len(noise_generator.noisy_signal_filepaths))
self.assertEqual(number_of_pairs,
len(noise_generator.apm_output_paths))
self.assertEqual(number_of_pairs,
len(noise_generator.reference_signal_filepaths))
def _CheckNoiseGeneratorPairsSignalDurations(
self, noise_generator, input_signal):
"""Check duration of the signals generated by a noise generator.
def _CheckGeneratedPairsSignalDurations(
self, generator, input_signal):
"""Checks duration of the generated signals.
Checks that the noisy input and the reference tracks are audio files
with duration equal to or greater than that of the input signal.
Args:
noise_generator: NoiseGenerator instance.
generator: TestDataGenerator instance.
input_signal: AudioSegment instance.
"""
input_signal_length = (
signal_processing.SignalProcessingUtils.CountSamples(input_signal))
# Iterate over the noisy signal - reference pairs.
for noise_config_name in noise_generator.config_names:
for config_name in generator.config_names:
# Load the noisy input file.
noisy_signal_filepath = noise_generator.noisy_signal_filepaths[
noise_config_name]
noisy_signal_filepath = generator.noisy_signal_filepaths[
config_name]
noisy_signal = signal_processing.SignalProcessingUtils.LoadWav(
noisy_signal_filepath)
@ -126,8 +127,8 @@ class TestNoiseGen(unittest.TestCase):
self.assertGreaterEqual(noisy_signal_length, input_signal_length)
# Load the reference file.
reference_signal_filepath = (
noise_generator.reference_signal_filepaths[noise_config_name])
reference_signal_filepath = generator.reference_signal_filepaths[
config_name]
reference_signal = signal_processing.SignalProcessingUtils.LoadWav(
reference_signal_filepath)
@ -137,13 +138,13 @@ class TestNoiseGen(unittest.TestCase):
reference_signal))
self.assertGreaterEqual(reference_signal_length, input_signal_length)
def _CheckNoiseGeneratorPairsOutputPaths(self, noise_generator):
def _CheckGeneratedPairsOutputPaths(self, generator):
"""Checks that the output path created by the generator exists.
Args:
noise_generator: NoiseGenerator instance.
generator: TestDataGenerator instance.
"""
# Iterate over the noisy signal - reference pairs.
for noise_config_name in noise_generator.config_names:
output_path = noise_generator.apm_output_paths[noise_config_name]
for config_name in generator.config_names:
output_path = generator.apm_output_paths[config_name]
self.assertTrue(os.path.exists(output_path))