Echo metric support for the APM-QA.

This was done by
* adding an EchoMetric class to EvaluationScore
* passing an echo metric binary path from the cmd arguments to the
  EvaluationScoreWorkerFactory
* passing the render input filepath to the Evaluator.

The echo score is supposed to be computed by the provided binary. It
should print the echo score in [0.0, 1.0] to a text file. It should
satisfy the cmd flags in its invocation in EchoMetric._Run()


Bug: webrtc:7494
Change-Id: I397013d6ed17659ea01d0623d98a14d4fcdcc161
Reviewed-on: https://webrtc-review.googlesource.com/97022
Commit-Queue: Alex Loiko <aleloi@webrtc.org>
Reviewed-by: Alessio Bazzica <alessiob@webrtc.org>
Cr-Commit-Position: refs/heads/master@{#24537}

modules/audio_processing/test/py_quality_assessment/apm_quality_assessment.py

@@ -109,6 +109,11 @@ def _InstanceArgumentsParser():
                               AudioProcWrapper.  \
                               DEFAULT_APM_SIMULATOR_BIN_PATH)
 
+  parser.add_argument('--echo_metric_tool_bin_path', required=False,
+                      help=('path to the echo metric binary '
+                           '(required for the echo eval score)'),
+                      default=None)
+
   parser.add_argument('--copy_with_identity_generator', required=False,
                       help=('If true, the identity test data generator makes a '
                             'copy of the clean speech input file.'),
@@ -158,7 +163,9 @@ def main():
               noise_tracks_path=args.additive_noise_tracks_path,
               copy_with_identity=args.copy_with_identity_generator)),
       evaluation_score_factory=eval_scores_factory.EvaluationScoreWorkerFactory(
-          polqa_tool_bin_path=os.path.join(args.polqa_path, _POLQA_BIN_NAME)),
+          polqa_tool_bin_path=os.path.join(args.polqa_path, _POLQA_BIN_NAME),
+          echo_metric_tool_bin_path=args.echo_metric_tool_bin_path
+      ),
       ap_wrapper=audioproc_wrapper.AudioProcWrapper(args.apm_sim_path),
       evaluator=evaluation.ApmModuleEvaluator(),
       external_vads=external_vad.ExternalVad.ConstructVadDict(

modules/audio_processing/test/py_quality_assessment/quality_assessment/echo_path_simulation_factory.py

@@ -16,9 +16,9 @@ from . import echo_path_simulation
 
 class EchoPathSimulatorFactory(object):
 
-  # TODO(alessiob): Replace 5 ms delay (at 48 kHz sample rate) with a more
+  # TODO(alessiob): Replace 20 ms delay (at 48 kHz sample rate) with a more
   # realistic impulse response.
-  _LINEAR_ECHO_IMPULSE_RESPONSE = np.array([0.0]*(5 * 48) + [0.15])
+  _LINEAR_ECHO_IMPULSE_RESPONSE = np.array([0.0]*(20 * 48) + [0.15])
 
   def __init__(self):
     pass

modules/audio_processing/test/py_quality_assessment/quality_assessment/eval_scores.py

@@ -41,6 +41,7 @@ class EvaluationScore(object):
     self._tested_signal_filepath = None
     self._output_filepath = None
     self._score = None
+    self._render_signal_filepath = None
 
   @classmethod
   def RegisterClass(cls, class_to_register):
@@ -88,6 +89,14 @@ class EvaluationScore(object):
     """
     self._tested_signal_filepath = filepath
 
+  def SetRenderSignalFilepath(self, filepath):
+    """Sets the path to the audio track used as render signal.
+
+    Args:
+      filepath: path to the test audio track.
+    """
+    self._render_signal_filepath = filepath
+
   def Run(self, output_path):
     """Extracts the score for the set test data pair.
 
@@ -183,6 +192,68 @@ class MeanAudioLevelScore(EvaluationScore):
     self._SaveScore()
 
 
+@EvaluationScore.RegisterClass
+class EchoMetric(EvaluationScore):
+  """Echo score.
+
+  Proportion of detected echo.
+
+  Unit: ratio
+  Ideal: 0
+  Worst case: 1
+  """
+
+  NAME = 'echo_metric'
+
+  def __init__(self, score_filename_prefix, echo_detector_bin_filepath):
+    EvaluationScore.__init__(self, score_filename_prefix)
+
+    # POLQA binary file path.
+    self._echo_detector_bin_filepath = echo_detector_bin_filepath
+    if not os.path.exists(self._echo_detector_bin_filepath):
+      logging.error('cannot find EchoMetric tool binary file')
+      raise exceptions.FileNotFoundError()
+
+    self._echo_detector_bin_path, _ = os.path.split(
+        self._echo_detector_bin_filepath)
+
+  def _Run(self, output_path):
+    echo_detector_out_filepath = os.path.join(output_path, 'echo_detector.out')
+    if os.path.exists(echo_detector_out_filepath):
+      os.unlink(echo_detector_out_filepath)
+
+    logging.debug("Render signal filepath: %s", self._render_signal_filepath)
+    if not os.path.exists(self._render_signal_filepath):
+      logging.error("Render input required for evaluating the echo metric.")
+
+    args = [
+        self._echo_detector_bin_filepath,
+        '--output_file', echo_detector_out_filepath,
+        '--',
+        '-i', self._tested_signal_filepath,
+        '-ri', self._render_signal_filepath
+    ]
+    logging.debug(' '.join(args))
+    subprocess.call(args, cwd=self._echo_detector_bin_path)
+
+    # Parse Echo detector tool output and extract the score.
+    self._score = self._ParseOutputFile(echo_detector_out_filepath)
+    self._SaveScore()
+
+  @classmethod
+  def _ParseOutputFile(cls, echo_metric_file_path):
+    """
+    Parses the POLQA tool output formatted as a table ('-t' option).
+
+    Args:
+      polqa_out_filepath: path to the POLQA tool output file.
+
+    Returns:
+      The score as a number in [0, 1].
+    """
+    with open(echo_metric_file_path) as f:
+      return float(f.read())
+
 @EvaluationScore.RegisterClass
 class PolqaScore(EvaluationScore):
   """POLQA score.

modules/audio_processing/test/py_quality_assessment/quality_assessment/eval_scores_factory.py

@@ -22,9 +22,10 @@ class EvaluationScoreWorkerFactory(object):
   workers.
   """
 
-  def __init__(self, polqa_tool_bin_path):
+  def __init__(self, polqa_tool_bin_path, echo_metric_tool_bin_path):
     self._score_filename_prefix = None
     self._polqa_tool_bin_path = polqa_tool_bin_path
+    self._echo_metric_tool_bin_path = echo_metric_tool_bin_path
 
   def SetScoreFilenamePrefix(self, prefix):
     self._score_filename_prefix = prefix
@@ -47,5 +48,8 @@ class EvaluationScoreWorkerFactory(object):
     if evaluation_score_class == eval_scores.PolqaScore:
       return eval_scores.PolqaScore(
           self._score_filename_prefix, self._polqa_tool_bin_path)
+    elif evaluation_score_class == eval_scores.EchoMetric:
+      return eval_scores.EchoMetric(
+          self._score_filename_prefix, self._echo_metric_tool_bin_path)
     else:
       return evaluation_score_class(self._score_filename_prefix)

modules/audio_processing/test/py_quality_assessment/quality_assessment/eval_scores_unittest.py

@@ -53,7 +53,7 @@ class TestEvalScores(unittest.TestCase):
 
   def testRegisteredClasses(self):
     # Evaluation score names to exclude (tested separately).
-    exceptions = ['thd']
+    exceptions = ['thd', 'echo_metric']
 
     # Preliminary check.
     self.assertTrue(os.path.exists(self._output_path))
@@ -67,7 +67,9 @@ class TestEvalScores(unittest.TestCase):
     eval_score_workers_factory = (
         eval_scores_factory.EvaluationScoreWorkerFactory(
             polqa_tool_bin_path=os.path.join(
-                os.path.dirname(os.path.abspath(__file__)), 'fake_polqa')))
+                os.path.dirname(os.path.abspath(__file__)), 'fake_polqa'),
+            echo_metric_tool_bin_path=None
+        ))
     eval_score_workers_factory.SetScoreFilenamePrefix('scores-')
 
     # Try each registered evaluation score worker.

modules/audio_processing/test/py_quality_assessment/quality_assessment/evaluation.py

@@ -21,7 +21,8 @@ class ApmModuleEvaluator(object):
 
   @classmethod
   def Run(cls, evaluation_score_workers, apm_input_metadata,
-          apm_output_filepath, reference_input_filepath, output_path):
+          apm_output_filepath, reference_input_filepath,
+          render_input_filepath, output_path):
     """Runs the evaluation.
 
     Iterates over the given evaluation score workers.
@@ -46,6 +47,8 @@ class ApmModuleEvaluator(object):
           reference_input_filepath)
       evaluation_score_worker.SetTestedSignalFilepath(
           apm_output_filepath)
+      evaluation_score_worker.SetRenderSignalFilepath(
+          render_input_filepath)
 
       evaluation_score_worker.Run(output_path)
       scores[evaluation_score_worker.NAME] = evaluation_score_worker.score

modules/audio_processing/test/py_quality_assessment/quality_assessment/export_unittest.py

@@ -46,7 +46,9 @@ class TestExport(unittest.TestCase):
         evaluation_score_factory=(
           eval_scores_factory.EvaluationScoreWorkerFactory(
               polqa_tool_bin_path=os.path.join(
-                  os.path.dirname(os.path.abspath(__file__)), 'fake_polqa'))),
+                  os.path.dirname(os.path.abspath(__file__)), 'fake_polqa'),
+              echo_metric_tool_bin_path=None
+          )),
         ap_wrapper=audioproc_wrapper.AudioProcWrapper(
             audioproc_wrapper.AudioProcWrapper.DEFAULT_APM_SIMULATOR_BIN_PATH),
         evaluator=evaluation.ApmModuleEvaluator())

modules/audio_processing/test/py_quality_assessment/quality_assessment/simulation.py

@@ -359,7 +359,9 @@ class ApmModuleSimulator(object):
             apm_input_metadata=apm_input_metadata,
             apm_output_filepath=self._audioproc_wrapper.output_filepath,
             reference_input_filepath=reference_signal_filepath,
-            output_path=evaluation_output_path)
+            render_input_filepath=render_input_filepath,
+            output_path=evaluation_output_path,
+        )
 
         # Save simulation metadata.
         data_access.Metadata.SaveAudioTestDataPaths(
@@ -370,7 +372,9 @@ class ApmModuleSimulator(object):
             render_filepath=render_input_filepath,
             capture_filepath=apm_input_filepath,
             apm_output_filepath=self._audioproc_wrapper.output_filepath,
-            apm_reference_filepath=reference_signal_filepath)
+            apm_reference_filepath=reference_signal_filepath,
+            apm_config_filepath=config_filepath,
+        )
       except exceptions.EvaluationScoreException as e:
         logging.warning('the evaluation failed: %s', e.message)
         continue

modules/audio_processing/test/py_quality_assessment/quality_assessment/simulation_unittest.py

@@ -69,7 +69,9 @@ class TestApmModuleSimulator(unittest.TestCase):
             copy_with_identity=False))
     evaluation_score_factory = eval_scores_factory.EvaluationScoreWorkerFactory(
         polqa_tool_bin_path=os.path.join(
-            os.path.dirname(__file__), 'fake_polqa'))
+            os.path.dirname(__file__), 'fake_polqa'),
+        echo_metric_tool_bin_path=None
+    )
 
     # Instance simulator.
     simulator = simulation.ApmModuleSimulator(
@@ -118,7 +120,9 @@ class TestApmModuleSimulator(unittest.TestCase):
         evaluation_score_factory=(
             eval_scores_factory.EvaluationScoreWorkerFactory(
                 polqa_tool_bin_path=os.path.join(
-                    os.path.dirname(__file__), 'fake_polqa'))),
+                    os.path.dirname(__file__), 'fake_polqa'),
+                echo_metric_tool_bin_path=None
+            )),
         ap_wrapper=audioproc_wrapper.AudioProcWrapper(
             audioproc_wrapper.AudioProcWrapper.DEFAULT_APM_SIMULATOR_BIN_PATH),
         evaluator=evaluation.ApmModuleEvaluator())
@@ -154,7 +158,9 @@ class TestApmModuleSimulator(unittest.TestCase):
         evaluation_score_factory=(
             eval_scores_factory.EvaluationScoreWorkerFactory(
                 polqa_tool_bin_path=os.path.join(
-                    os.path.dirname(__file__), 'fake_polqa'))),
+                    os.path.dirname(__file__), 'fake_polqa'),
+                echo_metric_tool_bin_path=None
+            )),
         ap_wrapper=audioproc_wrapper.AudioProcWrapper(
             audioproc_wrapper.AudioProcWrapper.DEFAULT_APM_SIMULATOR_BIN_PATH),
         evaluator=evaluation.ApmModuleEvaluator())