diff --git a/webrtc/modules/audio_processing/test/py_quality_assessment/BUILD.gn b/webrtc/modules/audio_processing/test/py_quality_assessment/BUILD.gn index d5dda54a1a..557a5982df 100644 --- a/webrtc/modules/audio_processing/test/py_quality_assessment/BUILD.gn +++ b/webrtc/modules/audio_processing/test/py_quality_assessment/BUILD.gn @@ -36,6 +36,7 @@ copy("lib") { "quality_assessment/audioproc_wrapper.py", "quality_assessment/data_access.py", "quality_assessment/eval_scores.py", + "quality_assessment/evaluation.py", "quality_assessment/noise_generation.py", "quality_assessment/simulation.py", ] diff --git a/webrtc/modules/audio_processing/test/py_quality_assessment/quality_assessment/data_access.py b/webrtc/modules/audio_processing/test/py_quality_assessment/quality_assessment/data_access.py index 90956442e2..acbe03f9af 100644 --- a/webrtc/modules/audio_processing/test/py_quality_assessment/quality_assessment/data_access.py +++ b/webrtc/modules/audio_processing/test/py_quality_assessment/quality_assessment/data_access.py @@ -68,3 +68,22 @@ class AudioProcConfigFile(object): def save(cls, filepath, config): with open(filepath, 'w') as f: json.dump(config, f) + + +class ScoreFile(object): + """ + Data access class to save and load float scalar scores. + """ + + def __init__(self): + pass + + @classmethod + def load(cls, filepath): + with open(filepath) as f: + return float(f.readline().strip()) + + @classmethod + def save(cls, filepath, score): + with open(filepath, 'w') as f: + f.write('{0:f}\n'.format(score)) diff --git a/webrtc/modules/audio_processing/test/py_quality_assessment/quality_assessment/eval_scores.py b/webrtc/modules/audio_processing/test/py_quality_assessment/quality_assessment/eval_scores.py index 8e900251e6..3a1c4bf911 100644 --- a/webrtc/modules/audio_processing/test/py_quality_assessment/quality_assessment/eval_scores.py +++ b/webrtc/modules/audio_processing/test/py_quality_assessment/quality_assessment/eval_scores.py @@ -6,13 +6,23 @@ # in the file PATENTS. All contributing project authors may # be found in the AUTHORS file in the root of the source tree. +import logging +import os + +from .data_access import ScoreFile + class EvaluationScore(object): NAME = None REGISTERED_CLASSES = {} def __init__(self): - pass + self._reference_signal = None + self._reference_signal_filepath = None + self._tested_signal = None + self._tested_signal_filepath = None + self._output_filepath = None + self._score = None @classmethod def register_class(cls, class_to_register): @@ -21,6 +31,58 @@ class EvaluationScore(object): """ cls.REGISTERED_CLASSES[class_to_register.NAME] = class_to_register + @property + def output_filepath(self): + return self._output_filepath + + @property + def score(self): + return self._score + + def set_reference_signal_filepath(self, filepath): + """ + Set the path to the audio track used as reference signal. + """ + self._reference_signal_filepath = filepath + + def set_tested_signal_filepath(self, filepath): + """ + Set the path to the audio track used as test signal. + """ + self._tested_signal_filepath = filepath + + def _load_reference_signal(self): + assert self._reference_signal_filepath is not None + # TODO(alessio): load signal. + self._reference_signal = None + + def _load_tested_signal(self): + assert self._tested_signal_filepath is not None + # TODO(alessio): load signal. + self._tested_signal = None + + def run(self, output_path): + self._output_filepath = os.path.join(output_path, 'score-{}.txt'.format( + self.NAME)) + try: + # If the score has already been computed, load. + self._load_score() + logging.debug('score found and loaded') + except IOError: + # Compute the score. + logging.debug('score not found, compute') + self._run(output_path) + + def _run(self, output_path): + # Abstract method. + raise NotImplementedError() + + def _load_score(self): + return ScoreFile.load(self._output_filepath) + + def _save_score(self): + return ScoreFile.save(self._output_filepath, self._score) + @EvaluationScore.register_class class AudioLevelScore(EvaluationScore): @@ -36,14 +98,18 @@ class AudioLevelScore(EvaluationScore): NAME = 'audio_level' def __init__(self): - super(AudioLevelScore, self).__init__() + EvaluationScore.__init__(self) + + def _run(self, output_path): + # TODO(alessio): implement. + self._score = 0.0 + self._save_score() @EvaluationScore.register_class class PolqaScore(EvaluationScore): """ - Compute the POLQA score. It requires that the POLQA_PATH environment variable - points to the PolqaOem64 executable. + Compute the POLQA score. Unit: MOS Ideal: 4.5 @@ -53,4 +119,9 @@ class PolqaScore(EvaluationScore): NAME = 'polqa' def __init__(self): - super(PolqaScore, self).__init__() + EvaluationScore.__init__(self) + + def _run(self, output_path): + # TODO(alessio): implement. + self._score = 0.0 + self._save_score() diff --git a/webrtc/modules/audio_processing/test/py_quality_assessment/quality_assessment/evaluation.py b/webrtc/modules/audio_processing/test/py_quality_assessment/quality_assessment/evaluation.py new file mode 100644 index 0000000000..78446a1e18 --- /dev/null +++ b/webrtc/modules/audio_processing/test/py_quality_assessment/quality_assessment/evaluation.py @@ -0,0 +1,32 @@ +# Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. +# +# Use of this source code is governed by a BSD-style license +# that can be found in the LICENSE file in the root of the source +# tree. An additional intellectual property rights grant can be found +# in the file PATENTS. All contributing project authors may +# be found in the AUTHORS file in the root of the source tree. + +import logging + +class ApmModuleEvaluator(object): + + def __init__(self): + pass + + @classmethod + def run(cls, evaluation_score_workers, apm_output_filepath, + reference_input_filepath, output_path): + # Init. + scores = {} + + for evaluation_score_worker in evaluation_score_workers: + logging.info(' computing <%s> score', evaluation_score_worker.NAME) + evaluation_score_worker.set_reference_signal_filepath( + reference_input_filepath) + evaluation_score_worker.set_tested_signal_filepath( + apm_output_filepath) + + evaluation_score_worker.run(output_path) + scores[evaluation_score_worker.NAME] = evaluation_score_worker.score + + return scores diff --git a/webrtc/modules/audio_processing/test/py_quality_assessment/quality_assessment/noise_generation.py b/webrtc/modules/audio_processing/test/py_quality_assessment/quality_assessment/noise_generation.py index 13a6993606..5c8126bb28 100644 --- a/webrtc/modules/audio_processing/test/py_quality_assessment/quality_assessment/noise_generation.py +++ b/webrtc/modules/audio_processing/test/py_quality_assessment/quality_assessment/noise_generation.py @@ -33,7 +33,7 @@ class IdentityGenerator(NoiseGenerator): NAME = 'identity' def __init__(self): - super(IdentityGenerator, self).__init__() + NoiseGenerator.__init__(self) @NoiseGenerator.register_class @@ -45,7 +45,7 @@ class WhiteNoiseGenerator(NoiseGenerator): NAME = 'white' def __init__(self): - super(WhiteNoiseGenerator, self).__init__() + NoiseGenerator.__init__(self) @NoiseGenerator.register_class @@ -57,7 +57,7 @@ class NarrowBandNoiseGenerator(NoiseGenerator): NAME = 'narrow_band' def __init__(self): - super(NarrowBandNoiseGenerator, self).__init__() + NoiseGenerator.__init__(self) @NoiseGenerator.register_class @@ -69,7 +69,7 @@ class EnvironmentalNoiseGenerator(NoiseGenerator): NAME = 'environmental' def __init__(self): - super(EnvironmentalNoiseGenerator, self).__init__() + NoiseGenerator.__init__(self) @NoiseGenerator.register_class @@ -81,4 +81,4 @@ class EchoNoiseGenerator(NoiseGenerator): NAME = 'echo' def __init__(self): - super(EchoNoiseGenerator, self).__init__() + NoiseGenerator.__init__(self) diff --git a/webrtc/modules/audio_processing/test/py_quality_assessment/quality_assessment/simulation.py b/webrtc/modules/audio_processing/test/py_quality_assessment/quality_assessment/simulation.py index f6cecd3a40..ac20edc9d7 100644 --- a/webrtc/modules/audio_processing/test/py_quality_assessment/quality_assessment/simulation.py +++ b/webrtc/modules/audio_processing/test/py_quality_assessment/quality_assessment/simulation.py @@ -12,6 +12,7 @@ import os from . import audioproc_wrapper from . import data_access from . import eval_scores +from . import evaluation from . import noise_generation class ApmModuleSimulator(object): @@ -21,9 +22,7 @@ class ApmModuleSimulator(object): def __init__(self): self._audioproc_wrapper = audioproc_wrapper.AudioProcWrapper() - - # TODO(alessio): instance when implementation is ready. - self._evaluator = None + self._evaluator = evaluation.ApmModuleEvaluator() self._base_output_path = None self._noise_generators = None