diff --git a/.gitignore b/.gitignore index 55b2781ade..726febc0ee 100644 --- a/.gitignore +++ b/.gitignore @@ -57,5 +57,9 @@ /tools_webrtc/video_quality_toolchain/mac/zxing /tools_webrtc/video_quality_toolchain/win/*.dll /tools_webrtc/video_quality_toolchain/win/*.exe +/webrtc/tools/testing/*.zip +/webrtc/tools/testing/*.gz +/webrtc/tools/testing/golang/*/*.gz +/webrtc/tools/testing/golang/*/*.zip /x86-generic_out/ /xcodebuild diff --git a/webrtc/tools/testing/build_apprtc_collider.py b/webrtc/tools/testing/build_apprtc_collider.py new file mode 100755 index 0000000000..5ff09bbeab --- /dev/null +++ b/webrtc/tools/testing/build_apprtc_collider.py @@ -0,0 +1,62 @@ +#!/usr/bin/env python +# Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. +# +# Use of this source code is governed by a BSD-style license +# that can be found in the LICENSE file in the root of the source +# tree. An additional intellectual property rights grant can be found +# in the file PATENTS. All contributing project authors may +# be found in the AUTHORS file in the root of the source tree. + +"""Builds the AppRTC collider using the golang toolchain. + +The golang toolchain is downloaded by download_golang.py. We use that here +to build the AppRTC collider server. + +This script needs to know the path to the 'src' directory in apprtc, the +root directory of 'go', the root directory of 'hg' and the output_dir. +""" + +import os +import shutil +import subprocess +import sys + +import utils + + +USAGE_STR = "Usage: {} " + + +def main(argv): + if len(argv) != 5: + return USAGE_STR.format(argv[0]) + + apprtc_dir = argv[1] + go_root_dir = argv[2] + mercurial_dir = argv[3] + golang_workspace = argv[4] + + utils.RemoveDirectory(golang_workspace) + + golang_workspace_src = os.path.join(golang_workspace, 'src') + + collider_dir = os.path.join(apprtc_dir, 'collider') + shutil.copytree(collider_dir, golang_workspace_src, + ignore=shutil.ignore_patterns('.svn', '.git')) + + golang_binary = 'go%s' % ('.exe' if utils.GetPlatform() == 'win' else '') + golang_path = os.path.join(go_root_dir, 'bin', golang_binary) + + golang_env = os.environ.copy() + golang_env['GOROOT'] = go_root_dir + golang_env['GOPATH'] = os.path.abspath(golang_workspace) + golang_env['PATH'] += os.pathsep + mercurial_dir + subprocess.check_call([golang_path, 'get', 'collidermain'], + env=golang_env) + collider_exec = os.path.join(golang_workspace, 'collidermain') + subprocess.check_call([golang_path, 'build', '-o', collider_exec, + 'collidermain'], env=golang_env) + + +if __name__ == '__main__': + sys.exit(main(sys.argv)) diff --git a/webrtc/tools/testing/build_mercurial_local.py b/webrtc/tools/testing/build_mercurial_local.py new file mode 100755 index 0000000000..512ca78d3e --- /dev/null +++ b/webrtc/tools/testing/build_mercurial_local.py @@ -0,0 +1,47 @@ +#!/usr/bin/env python +# Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. +# +# Use of this source code is governed by a BSD-style license +# that can be found in the LICENSE file in the root of the source +# tree. An additional intellectual property rights grant can be found +# in the file PATENTS. All contributing project authors may +# be found in the AUTHORS file in the root of the source tree. + +"""Builds a local mercurial (hg) copy. + +This is used by the go toolchain. +""" + +import os +import subprocess +import sys + +import utils + + +def main(argv): + if len(argv) != 2: + return 'Usage: %s ' % argv[0] + + mercurial_dir = argv[1] + if not os.path.exists(mercurial_dir): + return 'Expected mercurial at {}.'.format(mercurial_dir) + + os.chdir(mercurial_dir) + + if utils.GetPlatform() == 'win': + subprocess.check_call(['python', 'setup.py', '--pure', 'build_py', '-c', + '-d', '.', 'build_ext', + '-i', 'build_mo', '--force']) + with open('hg.bat', 'w') as put_hg_in_path: + # Write a hg.bat since the go toolchain expects to find something called + # 'hg' in the path, but Windows only recognizes executables ending with + # an extension in PATHEXT. Writing hg.bat effectively makes 'hg' callable + # if the mercurial folder is in PATH. + mercurial_path = os.path.abspath('hg') + put_hg_in_path.write('python %s %%*' % mercurial_path) + else: + subprocess.check_call(['make', 'local']) + +if __name__ == '__main__': + sys.exit(main(sys.argv)) diff --git a/webrtc/tools/testing/download_apprtc_appengine_and_mercurial.py b/webrtc/tools/testing/download_apprtc_appengine_and_mercurial.py new file mode 100755 index 0000000000..2d5b620561 --- /dev/null +++ b/webrtc/tools/testing/download_apprtc_appengine_and_mercurial.py @@ -0,0 +1,75 @@ +#!/usr/bin/env python +# Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. +# +# Use of this source code is governed by a BSD-style license +# that can be found in the LICENSE file in the root of the source +# tree. An additional intellectual property rights grant can be found +# in the file PATENTS. All contributing project authors may +# be found in the AUTHORS file in the root of the source tree. + +"""Downloads the appengine SDK from WebRTC storage and unpacks it. + +Requires that depot_tools is installed and in the PATH. + +It downloads compressed files in the directory where the script lives. +This is because the precondition is that the script lives in the same +directory of the .sha1 files. +""" + +import glob +import os +import sys +import subprocess + +import utils + + +SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__)) + +def _DownloadResources(dir_to_scan_for_sha1): + print 'Downloading files in %s...' % dir_to_scan_for_sha1 + + extension = 'bat' if 'win32' in sys.platform else 'py' + cmd = ['download_from_google_storage.%s' % extension, + '--bucket=chromium-webrtc-resources', + '--directory', dir_to_scan_for_sha1] + subprocess.check_call(cmd) + + +def _StripVersionNumberFromMercurialFolder(output_dir): + unpacked_name = glob.glob(os.path.join(output_dir, 'mercurial*')) + assert len(unpacked_name) == 1, 'Should have precisely one mercurial!' + os.rename(unpacked_name[0], os.path.join(output_dir, 'mercurial')) + + +def main(argv): + if len(argv) == 1: + return 'Usage: %s ' % argv[0] + + output_dir = argv[1] + appengine_zip_path = os.path.join(SCRIPT_DIR, 'google-appengine.zip') + old_appengine_sha1 = utils.ComputeSHA1(appengine_zip_path) + + mercurial_tar_path = os.path.join(SCRIPT_DIR, 'mercurial-src.tar.gz') + old_mercurial_sha1 = utils.ComputeSHA1(mercurial_tar_path) + + apprtc_zip_path = os.path.join(SCRIPT_DIR, 'prebuilt_apprtc.zip') + old_apprtc_sha1 = utils.ComputeSHA1(apprtc_zip_path) + + _DownloadResources(SCRIPT_DIR) + + if old_appengine_sha1 != utils.ComputeSHA1(appengine_zip_path): + utils.RemoveDirectory(os.path.join(output_dir, 'google_appengine')) + utils.UnpackArchiveTo(appengine_zip_path, output_dir) + + if old_mercurial_sha1 != utils.ComputeSHA1(mercurial_tar_path): + utils.RemoveDirectory(os.path.join(output_dir, 'mercurial')) + utils.UnpackArchiveTo(mercurial_tar_path, output_dir) + _StripVersionNumberFromMercurialFolder(output_dir) + + if old_apprtc_sha1 != utils.ComputeSHA1(apprtc_zip_path): + utils.RemoveDirectory(os.path.join(output_dir, 'apprtc')) + utils.UnpackArchiveTo(apprtc_zip_path, output_dir) + +if __name__ == '__main__': + sys.exit(main(sys.argv)) diff --git a/webrtc/tools/testing/download_golang.py b/webrtc/tools/testing/download_golang.py new file mode 100755 index 0000000000..a6665bb719 --- /dev/null +++ b/webrtc/tools/testing/download_golang.py @@ -0,0 +1,57 @@ +#!/usr/bin/env python +# Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. +# +# Use of this source code is governed by a BSD-style license +# that can be found in the LICENSE file in the root of the source +# tree. An additional intellectual property rights grant can be found +# in the file PATENTS. All contributing project authors may +# be found in the AUTHORS file in the root of the source tree. + +"""Downloads the golang SDK from WebRTC storage and unpacks it. + +Requires that depot_tools is installed and in the PATH. + +The precondition is that a directory 'golang' lives in the same directory +as the script. +This 'golang' directory has the following structure: + +/golang + | + |-----/linux/go.tar.gz.sha1 + |-----/win/go.zip.sha1 + |-----/mac/go.tar.gz.sha1 +""" + +import os +import sys + +import utils + + +SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__)) + + +def _GetGoArchivePathForPlatform(): + archive_extension = 'zip' if utils.GetPlatform() == 'win' else 'tar.gz' + return os.path.join(utils.GetPlatform(), 'go.%s' % archive_extension) + + +def main(argv): + if len(argv) == 1: + return 'Usage: %s ' % argv[0] + + output_dir = os.path.join(argv[1]) + golang_path = os.path.join(SCRIPT_DIR, 'golang') + archive_path = os.path.join(golang_path, _GetGoArchivePathForPlatform()) + old_archive_sha1 = utils.ComputeSHA1(archive_path) + + utils.DownloadFilesFromGoogleStorage(golang_path) + + if (old_archive_sha1 != utils.ComputeSHA1(archive_path) + or not os.path.exists('go')): + utils.RemoveDirectory(os.path.join(output_dir, 'go')) + utils.UnpackArchiveTo(archive_path, output_dir) + + +if __name__ == '__main__': + sys.exit(main(sys.argv)) diff --git a/webrtc/tools/testing/golang/linux/go.tar.gz.sha1 b/webrtc/tools/testing/golang/linux/go.tar.gz.sha1 new file mode 100644 index 0000000000..f6554894af --- /dev/null +++ b/webrtc/tools/testing/golang/linux/go.tar.gz.sha1 @@ -0,0 +1 @@ +14068fbe349db34b838853a7878621bbd2b24646 diff --git a/webrtc/tools/testing/golang/mac/go.tar.gz.sha1 b/webrtc/tools/testing/golang/mac/go.tar.gz.sha1 new file mode 100644 index 0000000000..c76e704527 --- /dev/null +++ b/webrtc/tools/testing/golang/mac/go.tar.gz.sha1 @@ -0,0 +1 @@ +be686ec7ba68d588735cc2094ccab8bdd651de9e diff --git a/webrtc/tools/testing/golang/win/go.zip.sha1 b/webrtc/tools/testing/golang/win/go.zip.sha1 new file mode 100644 index 0000000000..a6bcbab4a1 --- /dev/null +++ b/webrtc/tools/testing/golang/win/go.zip.sha1 @@ -0,0 +1 @@ +ba99083b22e0b22b560bb2d28b9b99b405d01b6b diff --git a/webrtc/tools/testing/google-appengine.zip.sha1 b/webrtc/tools/testing/google-appengine.zip.sha1 new file mode 100644 index 0000000000..cf13e4c6c8 --- /dev/null +++ b/webrtc/tools/testing/google-appengine.zip.sha1 @@ -0,0 +1 @@ +827468f89c78f292c28ceac50e6109c8d649fa61 diff --git a/webrtc/tools/testing/mercurial-src.tar.gz.sha1 b/webrtc/tools/testing/mercurial-src.tar.gz.sha1 new file mode 100644 index 0000000000..5ef3e9d738 --- /dev/null +++ b/webrtc/tools/testing/mercurial-src.tar.gz.sha1 @@ -0,0 +1 @@ +a8a51aa412abd5155c7de29fd39c9774decb4d3f \ No newline at end of file diff --git a/webrtc/tools/testing/prebuilt_apprtc.zip.sha1 b/webrtc/tools/testing/prebuilt_apprtc.zip.sha1 new file mode 100644 index 0000000000..7e2833ef3d --- /dev/null +++ b/webrtc/tools/testing/prebuilt_apprtc.zip.sha1 @@ -0,0 +1 @@ +6a96fc234cfa92ffaa79b0aeb5f4bdc1f31c6340 \ No newline at end of file diff --git a/webrtc/tools/testing/setup_apprtc.py b/webrtc/tools/testing/setup_apprtc.py new file mode 100644 index 0000000000..801da73ae0 --- /dev/null +++ b/webrtc/tools/testing/setup_apprtc.py @@ -0,0 +1,52 @@ +#!/usr/bin/env python +# Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. +# +# Use of this source code is governed by a BSD-style license +# that can be found in the LICENSE file in the root of the source +# tree. An additional intellectual property rights grant can be found +# in the file PATENTS. All contributing project authors may +# be found in the AUTHORS file in the root of the source tree. + +"""This script sets up AppRTC and its dependencies. + +Requires that depot_tools is installed and in the PATH. + +It will put the result under /collider. +""" + +import os +import sys + +import utils + + +SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__)) + +def main(argv): + if len(argv) == 1: + return 'Usage %s ' % argv[0] + + output_dir = argv[1] + apprtc_appengine_mercurial_path = os.path.join( + SCRIPT_DIR, 'download_apprtc_appengine_and_mercurial.py') + utils.RunSubprocessWithRetry([apprtc_appengine_mercurial_path, + output_dir]) + + download_golang_path = os.path.join(SCRIPT_DIR, 'download_golang.py') + utils.RunSubprocessWithRetry([download_golang_path, output_dir]) + + build_mercurial_path = os.path.join(SCRIPT_DIR, 'build_mercurial_local.py') + hg_dir = os.path.join(output_dir, 'mercurial') + utils.RunSubprocessWithRetry([build_mercurial_path, hg_dir]) + + build_apprtc_collider_path = os.path.join(SCRIPT_DIR, + 'build_apprtc_collider.py') + apprtc_src_dir = os.path.join(output_dir, 'apprtc', 'src') + go_dir = os.path.join(output_dir, 'go') + collider_dir = os.path.join(output_dir, 'collider') + utils.RunSubprocessWithRetry([build_apprtc_collider_path, apprtc_src_dir, + go_dir, hg_dir, collider_dir]) + + +if __name__ == '__main__': + sys.exit(main(sys.argv)) diff --git a/webrtc/tools/testing/utils.py b/webrtc/tools/testing/utils.py new file mode 100755 index 0000000000..7e38b42a71 --- /dev/null +++ b/webrtc/tools/testing/utils.py @@ -0,0 +1,135 @@ +#!/usr/bin/env python +# Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. +# +# Use of this source code is governed by a BSD-style license +# that can be found in the LICENSE file in the root of the source +# tree. An additional intellectual property rights grant can be found +# in the file PATENTS. All contributing project authors may +# be found in the AUTHORS file in the root of the source tree. + +"""Utilities for all our deps-management stuff.""" + +import hashlib +import os +import shutil +import sys +import subprocess +import tarfile +import time +import zipfile + + +def RunSubprocessWithRetry(cmd): + """Invokes the subprocess and backs off exponentially on fail.""" + for i in range(5): + try: + subprocess.check_call(cmd) + return + except subprocess.CalledProcessError as exception: + backoff = pow(2, i) + print 'Got %s, retrying in %d seconds...' % (exception, backoff) + time.sleep(backoff) + + print 'Giving up.' + raise exception + + +def DownloadFilesFromGoogleStorage(path): + print 'Downloading files in %s...' % path + + extension = 'bat' if 'win32' in sys.platform else 'py' + cmd = ['download_from_google_storage.%s' % extension, + '--bucket=chromium-webrtc-resources', + '--auto_platform', + '--recursive', + '--directory', path] + subprocess.check_call(cmd) + + +def ComputeSHA1(path): + if not os.path.exists(path): + return 0 + + sha1 = hashlib.sha1() + file_to_hash = open(path, 'rb') + try: + sha1.update(file_to_hash.read()) + finally: + file_to_hash.close() + + return sha1.hexdigest() + + +# Code partially copied from +# https://cs.chromium.org#chromium/build/scripts/common/chromium_utils.py +def RemoveDirectory(*path): + """Recursively removes a directory, even if it's marked read-only. + + Remove the directory located at *path, if it exists. + + shutil.rmtree() doesn't work on Windows if any of the files or directories + are read-only, which svn repositories and some .svn files are. We need to + be able to force the files to be writable (i.e., deletable) as we traverse + the tree. + + Even with all this, Windows still sometimes fails to delete a file, citing + a permission error (maybe something to do with antivirus scans or disk + indexing). The best suggestion any of the user forums had was to wait a + bit and try again, so we do that too. It's hand-waving, but sometimes it + works. :/ + """ + file_path = os.path.join(*path) + print 'Deleting `{}`.'.format(file_path) + if not os.path.exists(file_path): + print '`{}` does not exist.'.format(file_path) + return + + if sys.platform == 'win32': + # Give up and use cmd.exe's rd command. + file_path = os.path.normcase(file_path) + for _ in xrange(3): + print 'RemoveDirectory running %s' % (' '.join( + ['cmd.exe', '/c', 'rd', '/q', '/s', file_path])) + if not subprocess.call(['cmd.exe', '/c', 'rd', '/q', '/s', file_path]): + break + print ' Failed' + time.sleep(3) + return + else: + shutil.rmtree(file_path, ignore_errors=True) + + +def UnpackArchiveTo(archive_path, output_dir): + extension = os.path.splitext(archive_path)[1] + if extension == '.zip': + _UnzipArchiveTo(archive_path, output_dir) + else: + _UntarArchiveTo(archive_path, output_dir) + + +def _UnzipArchiveTo(archive_path, output_dir): + print 'Unzipping {} in {}.'.format(archive_path, output_dir) + zip_file = zipfile.ZipFile(archive_path) + try: + zip_file.extractall(output_dir) + finally: + zip_file.close() + + +def _UntarArchiveTo(archive_path, output_dir): + print 'Untarring {} in {}.'.format(archive_path, output_dir) + tar_file = tarfile.open(archive_path, 'r:gz') + try: + tar_file.extractall(output_dir) + finally: + tar_file.close() + + +def GetPlatform(): + if sys.platform.startswith('win'): + return 'win' + if sys.platform.startswith('linux'): + return 'linux' + if sys.platform.startswith('darwin'): + return 'mac' + raise Exception("Can't run on platform %s." % sys.platform)