admin/webrtc_m130
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Remove loopback video quality analysis test.

Browse Source 
This test has become flaky and is not important enough to keep.

Bug: webrtc:10030
Change-Id: Ie60dc73136397d376e308d95a52eb042daf18142
Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/113260
Commit-Queue: Oleh Prypin <oprypin@webrtc.org>
Reviewed-by: Magnus Jedvert <magjed@webrtc.org>
Cr-Commit-Position: refs/heads/master@{#27591}
This commit is contained in:
Sami Kalliomäki 2018-12-05 13:48:32 +01:00 committed by Commit Bot
parent 5a000165d2
commit 753741fe53
3 changed files with 0 additions and 370 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

46
examples/BUILD.gn
View File

@ -25,7 +25,6 @@ group("examples") {
if (is_android) {
deps += [
":AppRTCMobile",
":AppRTCMobile_stubbed_video_io_test_apk",
":AppRTCMobile_test_apk",
":libwebrtc_unity",
]
@ -180,51 +179,6 @@ if (is_android) {
"//third_party/junit",
]
}
group("video_quality_loopback_test") {
testonly = true
deps = [
":AppRTCMobile_stubbed_video_io_test_apk",
"../rtc_tools:frame_analyzer_host",
]
data = [
"../build/android/adb_reverse_forwarder.py",
"../examples/androidtests/video_quality_loopback_test.py",
"../resources/reference_video_640x360_30fps.y4m",
"../rtc_tools/compare_videos.py",
"../rtc_tools/testing/prebuilt_apprtc.zip",
"../rtc_tools/testing/golang/linux/go.tar.gz",
"../rtc_tools/testing/build_apprtc.py",
"../rtc_tools/testing/utils.py",
"../tools_webrtc/video_quality_toolchain/linux/ffmpeg",
"${root_out_dir}/frame_analyzer_host",
]
}
rtc_instrumentation_test_apk("AppRTCMobile_stubbed_video_io_test_apk") {
apk_name = "AppRTCMobileTestStubbedVideoIO"
android_manifest = "androidtests/AndroidManifest.xml"
java_files = [ "androidtests/src/org/appspot/apprtc/test/CallActivityStubbedInputOutputTest.java" ]
apk_under_test = ":AppRTCMobile"
deps = [
":AppRTCMobile_javalib",
"../sdk/android:libjingle_peerconnection_java",
"//third_party/android_support_test_runner:rules_java",
"//third_party/android_support_test_runner:runner_java",
"//third_party/espresso:espresso_all_java",
"//third_party/hamcrest:hamcrest_java",
"//third_party/junit",
]
data = [
"../resources/reference_video_640x360_30fps.y4m",
]
}
}
if (is_ios || (is_mac && target_cpu != "x86")) {

87
examples/androidtests/src/org/appspot/apprtc/test/CallActivityStubbedInputOutputTest.java
View File

@ -1,87 +0,0 @@
/*
* Copyright 2016 The WebRTC Project Authors. All rights reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
package org.appspot.apprtc.test;
import static android.support.test.espresso.action.ViewActions.click;
import static android.support.test.espresso.matcher.ViewMatchers.withId;
import static android.support.test.espresso.Espresso.onView;
import android.content.Context;
import android.content.Intent;
import android.net.Uri;
import android.os.Environment;
import android.support.test.espresso.IdlingPolicies;
import android.support.test.filters.LargeTest;
import android.support.test.rule.ActivityTestRule;
import android.support.test.runner.AndroidJUnit4;
import android.support.test.InstrumentationRegistry;
import java.util.UUID;
import java.util.concurrent.TimeUnit;
import org.appspot.apprtc.CallActivity;
import org.appspot.apprtc.R;
import org.junit.Rule;
import org.junit.Test;
import org.junit.runner.RunWith;
/**
* Used to start a loopback call with video input from file and video output also to file.
* The test case is a building block in other testing for video quality.
*/
@RunWith(AndroidJUnit4.class)
@LargeTest
public class CallActivityStubbedInputOutputTest {
private static final String TAG = "CallActivityStubbedInputOutputTest";
@Rule
public ActivityTestRule<CallActivity> rule = new ActivityTestRule<CallActivity>(
CallActivity.class) {
@Override
protected Intent getActivityIntent() {
Context context = InstrumentationRegistry.getContext();
Intent intent = new Intent("android.intent.action.VIEW", Uri.parse("http://localhost:9999"));
intent.putExtra(CallActivity.EXTRA_USE_VALUES_FROM_INTENT, true);
intent.putExtra(CallActivity.EXTRA_LOOPBACK, true);
intent.putExtra(CallActivity.EXTRA_AUDIOCODEC, "OPUS");
intent.putExtra(CallActivity.EXTRA_VIDEOCODEC, "VP8");
intent.putExtra(CallActivity.EXTRA_CAPTURETOTEXTURE_ENABLED, false);
intent.putExtra(CallActivity.EXTRA_CAMERA2, false);
intent.putExtra(CallActivity.EXTRA_ROOMID, UUID.randomUUID().toString().substring(0, 8));
// TODO false for wstls to disable https, should be option later or if URL is http
intent.putExtra(CallActivity.EXTRA_URLPARAMETERS,
"debug=loopback&ts=&wshpp=localhost:8089&wstls=false");
intent.putExtra(CallActivity.EXTRA_VIDEO_FILE_AS_CAMERA,
Environment.getExternalStorageDirectory().getAbsolutePath()
+ "/chromium_tests_root/resources/reference_video_640x360_30fps.y4m");
intent.putExtra(CallActivity.EXTRA_SAVE_REMOTE_VIDEO_TO_FILE,
Environment.getExternalStorageDirectory().getAbsolutePath() + "/output.y4m");
intent.putExtra(CallActivity.EXTRA_SAVE_REMOTE_VIDEO_TO_FILE_WIDTH, 640);
intent.putExtra(CallActivity.EXTRA_SAVE_REMOTE_VIDEO_TO_FILE_HEIGHT, 360);
return intent;
}
};
@Test
public void testLoopback() throws InterruptedException {
// The time to write down the data during closing of the program can take a while.
IdlingPolicies.setMasterPolicyTimeout(240000, TimeUnit.MILLISECONDS);
// During the time we sleep it will record video.
Thread.sleep(8000);
// Click on hang-up button.
onView(withId(R.id.button_call_disconnect)).perform(click());
}
}

237
examples/androidtests/video_quality_loopback_test.py
View File

@ -1,237 +0,0 @@
#!/usr/bin/env python
# Copyright (c) 2017 The WebRTC project authors. All Rights Reserved.
#
# Use of this source code is governed by a BSD-style license
# that can be found in the LICENSE file in the root of the source
# tree. An additional intellectual property rights grant can be found
# in the file PATENTS. All contributing project authors may
# be found in the AUTHORS file in the root of the source tree.
"""
This script is the wrapper that starts a loopback call with stubbed video in
and out. It then analyses the video quality of the output video against the
reference input video.
It expect to be given the webrtc output build directory as the first argument
all other arguments are optional.
It assumes you have a Android device plugged in.
"""
import argparse
import json
import logging
import os
import subprocess
import sys
import tempfile
import time
SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__))
SRC_DIR = os.path.normpath(os.path.join(SCRIPT_DIR, os.pardir, os.pardir))
RTC_TOOLS_DIR = os.path.join(SRC_DIR, 'rtc_tools', 'testing')
TOOLCHAIN_DIR = os.path.join(SRC_DIR, 'tools_webrtc', 'video_quality_toolchain',
'linux')
BAD_DEVICES_JSON = os.path.join(SRC_DIR,
os.environ.get('CHROMIUM_OUT_DIR', 'out'),
'bad_devices.json')
sys.path.append(RTC_TOOLS_DIR)
import utils
class Error(Exception):
pass
class VideoQualityTestError(Error):
pass
def _RunCommand(argv, **kwargs):
logging.info('Running %r', argv)
subprocess.check_call(argv, **kwargs)
def _RunCommandWithOutput(argv, **kwargs):
logging.info('Running %r', argv)
return subprocess.check_output(argv, **kwargs)
def _RunBackgroundCommand(argv):
logging.info('Running %r', argv)
process = subprocess.Popen(argv)
time.sleep(0.5)
status = process.poll()
if status: # is not None or 0
raise subprocess.CalledProcessError(status, argv)
return process
def CreateEmptyDir(suggested_dir):
if not suggested_dir:
return tempfile.mkdtemp()
utils.RemoveDirectory(suggested_dir)
os.makedirs(suggested_dir)
return suggested_dir
def _ParseArgs():
parser = argparse.ArgumentParser(description='Start loopback video analysis.')
parser.add_argument('build_dir_android',
help='The path to the build directory for Android.')
parser.add_argument('--temp_dir',
help='A temporary directory to put the output.')
parser.add_argument('--adb-path', help='Path to adb binary.', default='adb')
parser.add_argument('--num-retries', default='0',
help='Number of times to retry the test on Android.')
parser.add_argument('--isolated-script-test-perf-output',
help='Where to store perf results in chartjson format.', default=None)
parser.add_argument('--isolated-script-test-output', default=None,
help='Path to output an empty JSON file which Chromium infra requires.')
args, unknown_args = parser.parse_known_args()
# Ignore Chromium-specific flags
parser = argparse.ArgumentParser()
parser.add_argument('--test-launcher-summary-output',
type=str, default=None)
parser.parse_args(unknown_args)
return args
def SelectAndroidDevice(adb_path):
# Select an Android device in case multiple are connected.
try:
with open(BAD_DEVICES_JSON) as bad_devices_file:
bad_devices = json.load(bad_devices_file)
except IOError:
if os.environ.get('CHROME_HEADLESS'):
logging.warning('Cannot read %r', BAD_DEVICES_JSON)
bad_devices = {}
for line in _RunCommandWithOutput([adb_path, 'devices']).splitlines():
if line.endswith('\tdevice'):
android_device = line.split('\t')[0]
if android_device not in bad_devices:
return android_device
raise VideoQualityTestError('Cannot find any connected Android device.')
def SetUpTools(android_device, temp_dir, processes):
# Extract AppRTC.
apprtc_archive = os.path.join(RTC_TOOLS_DIR, 'prebuilt_apprtc.zip')
golang_archive = os.path.join(RTC_TOOLS_DIR, 'golang', 'linux', 'go.tar.gz')
utils.UnpackArchiveTo(apprtc_archive, temp_dir)
utils.UnpackArchiveTo(golang_archive, temp_dir)
# Build AppRTC.
build_apprtc_script = os.path.join(RTC_TOOLS_DIR, 'build_apprtc.py')
apprtc_dir = os.path.join(temp_dir, 'apprtc')
go_dir = os.path.join(temp_dir, 'go')
collider_dir = os.path.join(temp_dir, 'collider')
_RunCommand([sys.executable, build_apprtc_script, apprtc_dir, go_dir,
collider_dir])
# Start AppRTC Server.
dev_appserver = os.path.join(temp_dir, 'apprtc', 'temp', 'google-cloud-sdk',
'bin', 'dev_appserver.py')
appengine_dir = os.path.join(temp_dir, 'apprtc', 'out', 'app_engine')
processes.append(_RunBackgroundCommand([
sys.executable, dev_appserver, appengine_dir, '--port=9999',
'--admin_port=9998', '--skip_sdk_update_check', '--clear_datastore=yes']))
# Start Collider.
collider_path = os.path.join(temp_dir, 'collider', 'collidermain')
processes.append(_RunBackgroundCommand([
collider_path, '-tls=false', '-port=8089',
'-room-server=http://localhost:9999']))
# Start adb reverse forwarder.
reverseforwarder_path = os.path.join(
SRC_DIR, 'build', 'android', 'adb_reverse_forwarder.py')
processes.append(_RunBackgroundCommand([
reverseforwarder_path, '--device', android_device, '9999', '9999', '8089',
'8089']))
def RunTest(android_device, adb_path, build_dir, temp_dir, num_retries,
chartjson_result_file):
ffmpeg_path = os.path.join(TOOLCHAIN_DIR, 'ffmpeg')
def ConvertVideo(input_video, output_video):
_RunCommand([ffmpeg_path, '-y', '-i', input_video, output_video])
# Start loopback call and record video.
test_script = os.path.join(
build_dir, 'bin', 'run_AppRTCMobile_stubbed_video_io_test_apk')
_RunCommand([test_script, '--device', android_device,
'--num-retries', num_retries])
# Pull the recorded video.
test_video = os.path.join(temp_dir, 'test_video.y4m')
_RunCommand([adb_path, '-s', android_device,
'pull', '/sdcard/output.y4m', test_video])
# Convert the recorded and reference videos to YUV.
reference_video = os.path.join(SRC_DIR,
'resources', 'reference_video_640x360_30fps.y4m')
test_video_yuv = os.path.join(temp_dir, 'test_video.yuv')
reference_video_yuv = os.path.join(
temp_dir, 'reference_video_640x360_30fps.yuv')
ConvertVideo(test_video, test_video_yuv)
ConvertVideo(reference_video, reference_video_yuv)
# Run comparison script.
compare_script = os.path.join(SRC_DIR, 'rtc_tools', 'compare_videos.py')
frame_analyzer = os.path.join(build_dir, 'frame_analyzer_host')
args = [
'--ref_video', reference_video_yuv,
'--test_video', test_video_yuv,
'--yuv_frame_width', '640',
'--yuv_frame_height', '360',
'--frame_analyzer', frame_analyzer,
]
if chartjson_result_file:
args.extend(['--chartjson_result_file', chartjson_result_file])
_RunCommand([sys.executable, compare_script] + args)
def main():
logging.basicConfig(level=logging.INFO)
args = _ParseArgs()
temp_dir = args.temp_dir
build_dir = args.build_dir_android
adb_path = args.adb_path
processes = []
temp_dir = CreateEmptyDir(temp_dir)
try:
android_device = SelectAndroidDevice(adb_path)
SetUpTools(android_device, temp_dir, processes)
RunTest(android_device, adb_path, build_dir, temp_dir, args.num_retries,
args.isolated_script_test_perf_output)
finally:
for process in processes:
if process:
process.terminate()
process.wait()
utils.RemoveDirectory(temp_dir)
if args.isolated_script_test_output:
with open(args.isolated_script_test_output, 'w') as f:
f.write('{"version": 3}')
if __name__ == '__main__':
sys.exit(main())