Evaluation controller, evaluation score abstract class, and score data access.

BUG=webrtc:7218

Review-Url: https://codereview.webrtc.org/2714543005
Cr-Commit-Position: refs/heads/master@{#17012}

webrtc/modules/audio_processing/test/py_quality_assessment/BUILD.gn

@@ -36,6 +36,7 @@ copy("lib") {
     "quality_assessment/audioproc_wrapper.py",
     "quality_assessment/data_access.py",
     "quality_assessment/eval_scores.py",
+    "quality_assessment/evaluation.py",
     "quality_assessment/noise_generation.py",
     "quality_assessment/simulation.py",
   ]

webrtc/modules/audio_processing/test/py_quality_assessment/quality_assessment/data_access.py

@@ -68,3 +68,22 @@ class AudioProcConfigFile(object):
   def save(cls, filepath, config):
     with open(filepath, 'w') as f:
       json.dump(config, f)
+
+
+class ScoreFile(object):
+  """
+  Data access class to save and load float scalar scores.
+  """
+
+  def __init__(self):
+    pass
+
+  @classmethod
+  def load(cls, filepath):
+    with open(filepath) as f:
+      return float(f.readline().strip())
+
+  @classmethod
+  def save(cls, filepath, score):
+    with open(filepath, 'w') as f:
+      f.write('{0:f}\n'.format(score))

webrtc/modules/audio_processing/test/py_quality_assessment/quality_assessment/eval_scores.py

@@ -6,13 +6,23 @@
 # in the file PATENTS.  All contributing project authors may
 # be found in the AUTHORS file in the root of the source tree.
 
+import logging
+import os
+
+from .data_access import ScoreFile
+
 class EvaluationScore(object):
 
   NAME = None
   REGISTERED_CLASSES = {}
 
   def __init__(self):
-    pass
+    self._reference_signal = None
+    self._reference_signal_filepath = None
+    self._tested_signal = None
+    self._tested_signal_filepath = None
+    self._output_filepath = None
+    self._score = None
 
   @classmethod
   def register_class(cls, class_to_register):
@@ -21,6 +31,58 @@ class EvaluationScore(object):
     """
     cls.REGISTERED_CLASSES[class_to_register.NAME] = class_to_register
 
+  @property
+  def output_filepath(self):
+    return self._output_filepath
+
+  @property
+  def score(self):
+    return self._score
+
+  def set_reference_signal_filepath(self, filepath):
+    """
+    Set the path to the audio track used as reference signal.
+    """
+    self._reference_signal_filepath = filepath
+
+  def set_tested_signal_filepath(self, filepath):
+    """
+    Set the path to the audio track used as test signal.
+    """
+    self._tested_signal_filepath = filepath
+
+  def _load_reference_signal(self):
+    assert self._reference_signal_filepath is not None
+    # TODO(alessio): load signal.
+    self._reference_signal = None
+
+  def _load_tested_signal(self):
+    assert self._tested_signal_filepath is not None
+    # TODO(alessio): load signal.
+    self._tested_signal = None
+
+  def run(self, output_path):
+    self._output_filepath = os.path.join(output_path, 'score-{}.txt'.format(
+        self.NAME))
+    try:
+      # If the score has already been computed, load.
+      self._load_score()
+      logging.debug('score found and loaded')
+    except IOError:
+      # Compute the score.
+      logging.debug('score not found, compute')
+      self._run(output_path)
+
+  def _run(self, output_path):
+    # Abstract method.
+    raise NotImplementedError()
+
+  def _load_score(self):
+    return ScoreFile.load(self._output_filepath)
+
+  def _save_score(self):
+    return ScoreFile.save(self._output_filepath, self._score)
+
 
 @EvaluationScore.register_class
 class AudioLevelScore(EvaluationScore):
@@ -36,14 +98,18 @@ class AudioLevelScore(EvaluationScore):
   NAME = 'audio_level'
 
   def __init__(self):
-    super(AudioLevelScore, self).__init__()
+    EvaluationScore.__init__(self)
+
+  def _run(self, output_path):
+    # TODO(alessio): implement.
+    self._score = 0.0
+    self._save_score()
 
 
 @EvaluationScore.register_class
 class PolqaScore(EvaluationScore):
   """
-  Compute the POLQA score. It requires that the POLQA_PATH environment variable
-  points to the PolqaOem64 executable.
+  Compute the POLQA score.
 
   Unit: MOS
   Ideal: 4.5
@@ -53,4 +119,9 @@ class PolqaScore(EvaluationScore):
   NAME = 'polqa'
 
   def __init__(self):
-    super(PolqaScore, self).__init__()
+    EvaluationScore.__init__(self)
+
+  def _run(self, output_path):
+    # TODO(alessio): implement.
+    self._score = 0.0
+    self._save_score()

webrtc/modules/audio_processing/test/py_quality_assessment/quality_assessment/evaluation.py

@@ -0,0 +1,32 @@
+# Copyright (c) 2017 The WebRTC project authors. All Rights Reserved.
+#
+# Use of this source code is governed by a BSD-style license
+# that can be found in the LICENSE file in the root of the source
+# tree. An additional intellectual property rights grant can be found
+# in the file PATENTS.  All contributing project authors may
+# be found in the AUTHORS file in the root of the source tree.
+
+import logging
+
+class ApmModuleEvaluator(object):
+
+  def __init__(self):
+    pass
+
+  @classmethod
+  def run(cls, evaluation_score_workers, apm_output_filepath,
+          reference_input_filepath, output_path):
+    # Init.
+    scores = {}
+
+    for evaluation_score_worker in evaluation_score_workers:
+      logging.info('   computing <%s> score', evaluation_score_worker.NAME)
+      evaluation_score_worker.set_reference_signal_filepath(
+          reference_input_filepath)
+      evaluation_score_worker.set_tested_signal_filepath(
+          apm_output_filepath)
+
+      evaluation_score_worker.run(output_path)
+      scores[evaluation_score_worker.NAME] = evaluation_score_worker.score
+
+    return scores

webrtc/modules/audio_processing/test/py_quality_assessment/quality_assessment/noise_generation.py

@@ -33,7 +33,7 @@ class IdentityGenerator(NoiseGenerator):
   NAME = 'identity'
 
   def __init__(self):
-    super(IdentityGenerator, self).__init__()
+    NoiseGenerator.__init__(self)
 
 
 @NoiseGenerator.register_class
@@ -45,7 +45,7 @@ class WhiteNoiseGenerator(NoiseGenerator):
   NAME = 'white'
 
   def __init__(self):
-    super(WhiteNoiseGenerator, self).__init__()
+    NoiseGenerator.__init__(self)
 
 
 @NoiseGenerator.register_class
@@ -57,7 +57,7 @@ class NarrowBandNoiseGenerator(NoiseGenerator):
   NAME = 'narrow_band'
 
   def __init__(self):
-    super(NarrowBandNoiseGenerator, self).__init__()
+    NoiseGenerator.__init__(self)
 
 
 @NoiseGenerator.register_class
@@ -69,7 +69,7 @@ class EnvironmentalNoiseGenerator(NoiseGenerator):
   NAME = 'environmental'
 
   def __init__(self):
-    super(EnvironmentalNoiseGenerator, self).__init__()
+    NoiseGenerator.__init__(self)
 
 
 @NoiseGenerator.register_class
@@ -81,4 +81,4 @@ class EchoNoiseGenerator(NoiseGenerator):
   NAME = 'echo'
 
   def __init__(self):
-    super(EchoNoiseGenerator, self).__init__()
+    NoiseGenerator.__init__(self)

webrtc/modules/audio_processing/test/py_quality_assessment/quality_assessment/simulation.py

@@ -12,6 +12,7 @@ import os
 from . import audioproc_wrapper
 from . import data_access
 from . import eval_scores
+from . import evaluation
 from . import noise_generation
 
 class ApmModuleSimulator(object):
@@ -21,9 +22,7 @@ class ApmModuleSimulator(object):
 
   def __init__(self):
     self._audioproc_wrapper = audioproc_wrapper.AudioProcWrapper()
-
-    # TODO(alessio): instance when implementation is ready.
-    self._evaluator = None
+    self._evaluator = evaluation.ApmModuleEvaluator()
 
     self._base_output_path = None
     self._noise_generators = None