Moving scripts to download and build apprtc/collider.

We are starting to move code to setup our tests from [1].
We are moving only scripts to download and build apprtc, collider and node
(scripts that work on firefox have not been copied by this CL).

[1] - https://chromium.googlesource.com/chromium/deps/webrtc/webrtc.DEPS/

BUG=webrtc:7602
NOTRY=True

Review-Url: https://codereview.webrtc.org/2861653005
Cr-Commit-Position: refs/heads/master@{#18060}

.gitignore

@@ -57,5 +57,9 @@
 /tools_webrtc/video_quality_toolchain/mac/zxing
 /tools_webrtc/video_quality_toolchain/win/*.dll
 /tools_webrtc/video_quality_toolchain/win/*.exe
+/webrtc/tools/testing/*.zip
+/webrtc/tools/testing/*.gz
+/webrtc/tools/testing/golang/*/*.gz
+/webrtc/tools/testing/golang/*/*.zip
 /x86-generic_out/
 /xcodebuild

webrtc/tools/testing/build_apprtc_collider.py

@@ -0,0 +1,62 @@
+#!/usr/bin/env python
+# Copyright (c) 2017 The WebRTC project authors. All Rights Reserved.
+#
+# Use of this source code is governed by a BSD-style license
+# that can be found in the LICENSE file in the root of the source
+# tree. An additional intellectual property rights grant can be found
+# in the file PATENTS.  All contributing project authors may
+# be found in the AUTHORS file in the root of the source tree.
+
+"""Builds the AppRTC collider using the golang toolchain.
+
+The golang toolchain is downloaded by download_golang.py. We use that here
+to build the AppRTC collider server.
+
+This script needs to know the path to the 'src' directory in apprtc, the
+root directory of 'go', the root directory of 'hg' and the output_dir.
+"""
+
+import os
+import shutil
+import subprocess
+import sys
+
+import utils
+
+
+USAGE_STR = "Usage: {} <apprtc_src_dir> <go_dir> <hg_dir> <output_dir>"
+
+
+def main(argv):
+  if len(argv) != 5:
+    return USAGE_STR.format(argv[0])
+
+  apprtc_dir = argv[1]
+  go_root_dir = argv[2]
+  mercurial_dir = argv[3]
+  golang_workspace = argv[4]
+
+  utils.RemoveDirectory(golang_workspace)
+
+  golang_workspace_src = os.path.join(golang_workspace, 'src')
+
+  collider_dir = os.path.join(apprtc_dir, 'collider')
+  shutil.copytree(collider_dir, golang_workspace_src,
+                  ignore=shutil.ignore_patterns('.svn', '.git'))
+
+  golang_binary = 'go%s' % ('.exe' if utils.GetPlatform() == 'win' else '')
+  golang_path = os.path.join(go_root_dir, 'bin', golang_binary)
+
+  golang_env = os.environ.copy()
+  golang_env['GOROOT'] = go_root_dir
+  golang_env['GOPATH'] = os.path.abspath(golang_workspace)
+  golang_env['PATH'] += os.pathsep + mercurial_dir
+  subprocess.check_call([golang_path, 'get', 'collidermain'],
+                        env=golang_env)
+  collider_exec = os.path.join(golang_workspace, 'collidermain')
+  subprocess.check_call([golang_path, 'build', '-o', collider_exec,
+                         'collidermain'], env=golang_env)
+
+
+if __name__ == '__main__':
+  sys.exit(main(sys.argv))

webrtc/tools/testing/build_mercurial_local.py

@@ -0,0 +1,47 @@
+#!/usr/bin/env python
+# Copyright (c) 2017 The WebRTC project authors. All Rights Reserved.
+#
+# Use of this source code is governed by a BSD-style license
+# that can be found in the LICENSE file in the root of the source
+# tree. An additional intellectual property rights grant can be found
+# in the file PATENTS.  All contributing project authors may
+# be found in the AUTHORS file in the root of the source tree.
+
+"""Builds a local mercurial (hg) copy.
+
+This is used by the go toolchain.
+"""
+
+import os
+import subprocess
+import sys
+
+import utils
+
+
+def main(argv):
+  if len(argv) != 2:
+    return 'Usage: %s <mercurial_dir>' % argv[0]
+
+  mercurial_dir = argv[1]
+  if not os.path.exists(mercurial_dir):
+    return 'Expected mercurial at {}.'.format(mercurial_dir)
+
+  os.chdir(mercurial_dir)
+
+  if utils.GetPlatform() == 'win':
+    subprocess.check_call(['python', 'setup.py', '--pure', 'build_py', '-c',
+                           '-d', '.', 'build_ext',
+                           '-i', 'build_mo', '--force'])
+    with open('hg.bat', 'w') as put_hg_in_path:
+      # Write a hg.bat since the go toolchain expects to find something called
+      # 'hg' in the path, but Windows only recognizes executables ending with
+      # an extension in PATHEXT. Writing hg.bat effectively makes 'hg' callable
+      # if the mercurial folder is in PATH.
+      mercurial_path = os.path.abspath('hg')
+      put_hg_in_path.write('python %s %%*' % mercurial_path)
+  else:
+    subprocess.check_call(['make', 'local'])
+
+if __name__ == '__main__':
+  sys.exit(main(sys.argv))

webrtc/tools/testing/download_apprtc_appengine_and_mercurial.py

@@ -0,0 +1,75 @@
+#!/usr/bin/env python
+# Copyright (c) 2017 The WebRTC project authors. All Rights Reserved.
+#
+# Use of this source code is governed by a BSD-style license
+# that can be found in the LICENSE file in the root of the source
+# tree. An additional intellectual property rights grant can be found
+# in the file PATENTS.  All contributing project authors may
+# be found in the AUTHORS file in the root of the source tree.
+
+"""Downloads the appengine SDK from WebRTC storage and unpacks it.
+
+Requires that depot_tools is installed and in the PATH.
+
+It downloads compressed files in the directory where the script lives.
+This is because the precondition is that the script lives in the same
+directory of the .sha1 files.
+"""
+
+import glob
+import os
+import sys
+import subprocess
+
+import utils
+
+
+SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__))
+
+def _DownloadResources(dir_to_scan_for_sha1):
+  print 'Downloading files in %s...' % dir_to_scan_for_sha1
+
+  extension = 'bat' if 'win32' in sys.platform else 'py'
+  cmd = ['download_from_google_storage.%s' % extension,
+         '--bucket=chromium-webrtc-resources',
+         '--directory', dir_to_scan_for_sha1]
+  subprocess.check_call(cmd)
+
+
+def _StripVersionNumberFromMercurialFolder(output_dir):
+  unpacked_name = glob.glob(os.path.join(output_dir, 'mercurial*'))
+  assert len(unpacked_name) == 1, 'Should have precisely one mercurial!'
+  os.rename(unpacked_name[0], os.path.join(output_dir, 'mercurial'))
+
+
+def main(argv):
+  if len(argv) == 1:
+    return 'Usage: %s <output_dir>' % argv[0]
+
+  output_dir = argv[1]
+  appengine_zip_path = os.path.join(SCRIPT_DIR, 'google-appengine.zip')
+  old_appengine_sha1 = utils.ComputeSHA1(appengine_zip_path)
+
+  mercurial_tar_path = os.path.join(SCRIPT_DIR, 'mercurial-src.tar.gz')
+  old_mercurial_sha1 = utils.ComputeSHA1(mercurial_tar_path)
+
+  apprtc_zip_path = os.path.join(SCRIPT_DIR, 'prebuilt_apprtc.zip')
+  old_apprtc_sha1 = utils.ComputeSHA1(apprtc_zip_path)
+
+  _DownloadResources(SCRIPT_DIR)
+
+  if old_appengine_sha1 != utils.ComputeSHA1(appengine_zip_path):
+    utils.RemoveDirectory(os.path.join(output_dir, 'google_appengine'))
+    utils.UnpackArchiveTo(appengine_zip_path, output_dir)
+
+  if old_mercurial_sha1 != utils.ComputeSHA1(mercurial_tar_path):
+    utils.RemoveDirectory(os.path.join(output_dir, 'mercurial'))
+    utils.UnpackArchiveTo(mercurial_tar_path, output_dir)
+    _StripVersionNumberFromMercurialFolder(output_dir)
+
+  if old_apprtc_sha1 != utils.ComputeSHA1(apprtc_zip_path):
+    utils.RemoveDirectory(os.path.join(output_dir, 'apprtc'))
+    utils.UnpackArchiveTo(apprtc_zip_path, output_dir)
+
+if __name__ == '__main__':
+  sys.exit(main(sys.argv))

webrtc/tools/testing/download_golang.py

@@ -0,0 +1,57 @@
+#!/usr/bin/env python
+# Copyright (c) 2017 The WebRTC project authors. All Rights Reserved.
+#
+# Use of this source code is governed by a BSD-style license
+# that can be found in the LICENSE file in the root of the source
+# tree. An additional intellectual property rights grant can be found
+# in the file PATENTS.  All contributing project authors may
+# be found in the AUTHORS file in the root of the source tree.
+
+"""Downloads the golang SDK from WebRTC storage and unpacks it.
+
+Requires that depot_tools is installed and in the PATH.
+
+The precondition is that a directory 'golang' lives in the same directory
+as the script.
+This 'golang' directory has the following structure:
+
+/golang
+ |
+ |-----/linux/go.tar.gz.sha1
+ |-----/win/go.zip.sha1
+ |-----/mac/go.tar.gz.sha1
+"""
+
+import os
+import sys
+
+import utils
+
+
+SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__))
+
+
+def _GetGoArchivePathForPlatform():
+  archive_extension = 'zip' if utils.GetPlatform() == 'win' else 'tar.gz'
+  return os.path.join(utils.GetPlatform(), 'go.%s' % archive_extension)
+
+
+def main(argv):
+  if len(argv) == 1:
+    return 'Usage: %s <output_dir>' % argv[0]
+
+  output_dir = os.path.join(argv[1])
+  golang_path = os.path.join(SCRIPT_DIR, 'golang')
+  archive_path = os.path.join(golang_path, _GetGoArchivePathForPlatform())
+  old_archive_sha1 = utils.ComputeSHA1(archive_path)
+
+  utils.DownloadFilesFromGoogleStorage(golang_path)
+
+  if (old_archive_sha1 != utils.ComputeSHA1(archive_path)
+      or not os.path.exists('go')):
+    utils.RemoveDirectory(os.path.join(output_dir, 'go'))
+    utils.UnpackArchiveTo(archive_path, output_dir)
+
+
+if __name__ == '__main__':
+  sys.exit(main(sys.argv))

webrtc/tools/testing/golang/linux/go.tar.gz.sha1

@@ -0,0 +1 @@
+14068fbe349db34b838853a7878621bbd2b24646

webrtc/tools/testing/golang/mac/go.tar.gz.sha1

@@ -0,0 +1 @@
+be686ec7ba68d588735cc2094ccab8bdd651de9e

webrtc/tools/testing/golang/win/go.zip.sha1

@@ -0,0 +1 @@
+ba99083b22e0b22b560bb2d28b9b99b405d01b6b

webrtc/tools/testing/google-appengine.zip.sha1

@@ -0,0 +1 @@
+827468f89c78f292c28ceac50e6109c8d649fa61

webrtc/tools/testing/mercurial-src.tar.gz.sha1

@@ -0,0 +1 @@
+a8a51aa412abd5155c7de29fd39c9774decb4d3f

webrtc/tools/testing/prebuilt_apprtc.zip.sha1

@@ -0,0 +1 @@
+6a96fc234cfa92ffaa79b0aeb5f4bdc1f31c6340

webrtc/tools/testing/setup_apprtc.py

@@ -0,0 +1,52 @@
+#!/usr/bin/env python
+# Copyright (c) 2017 The WebRTC project authors. All Rights Reserved.
+#
+# Use of this source code is governed by a BSD-style license
+# that can be found in the LICENSE file in the root of the source
+# tree. An additional intellectual property rights grant can be found
+# in the file PATENTS.  All contributing project authors may
+# be found in the AUTHORS file in the root of the source tree.
+
+"""This script sets up AppRTC and its dependencies.
+
+Requires that depot_tools is installed and in the PATH.
+
+It will put the result under <output_dir>/collider.
+"""
+
+import os
+import sys
+
+import utils
+
+
+SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__))
+
+def main(argv):
+  if len(argv) == 1:
+    return 'Usage %s <output_dir>' % argv[0]
+
+  output_dir = argv[1]
+  apprtc_appengine_mercurial_path = os.path.join(
+      SCRIPT_DIR, 'download_apprtc_appengine_and_mercurial.py')
+  utils.RunSubprocessWithRetry([apprtc_appengine_mercurial_path,
+                                output_dir])
+
+  download_golang_path = os.path.join(SCRIPT_DIR, 'download_golang.py')
+  utils.RunSubprocessWithRetry([download_golang_path, output_dir])
+
+  build_mercurial_path = os.path.join(SCRIPT_DIR, 'build_mercurial_local.py')
+  hg_dir = os.path.join(output_dir, 'mercurial')
+  utils.RunSubprocessWithRetry([build_mercurial_path, hg_dir])
+
+  build_apprtc_collider_path = os.path.join(SCRIPT_DIR,
+                                            'build_apprtc_collider.py')
+  apprtc_src_dir = os.path.join(output_dir, 'apprtc', 'src')
+  go_dir = os.path.join(output_dir, 'go')
+  collider_dir = os.path.join(output_dir, 'collider')
+  utils.RunSubprocessWithRetry([build_apprtc_collider_path, apprtc_src_dir,
+                                go_dir, hg_dir, collider_dir])
+
+
+if __name__ == '__main__':
+  sys.exit(main(sys.argv))

webrtc/tools/testing/utils.py

@@ -0,0 +1,135 @@
+#!/usr/bin/env python
+# Copyright (c) 2017 The WebRTC project authors. All Rights Reserved.
+#
+# Use of this source code is governed by a BSD-style license
+# that can be found in the LICENSE file in the root of the source
+# tree. An additional intellectual property rights grant can be found
+# in the file PATENTS.  All contributing project authors may
+# be found in the AUTHORS file in the root of the source tree.
+
+"""Utilities for all our deps-management stuff."""
+
+import hashlib
+import os
+import shutil
+import sys
+import subprocess
+import tarfile
+import time
+import zipfile
+
+
+def RunSubprocessWithRetry(cmd):
+  """Invokes the subprocess and backs off exponentially on fail."""
+  for i in range(5):
+    try:
+      subprocess.check_call(cmd)
+      return
+    except subprocess.CalledProcessError as exception:
+      backoff = pow(2, i)
+      print 'Got %s, retrying in %d seconds...' % (exception, backoff)
+      time.sleep(backoff)
+
+  print 'Giving up.'
+  raise exception
+
+
+def DownloadFilesFromGoogleStorage(path):
+  print 'Downloading files in %s...' % path
+
+  extension = 'bat' if 'win32' in sys.platform else 'py'
+  cmd = ['download_from_google_storage.%s' % extension,
+         '--bucket=chromium-webrtc-resources',
+         '--auto_platform',
+         '--recursive',
+         '--directory', path]
+  subprocess.check_call(cmd)
+
+
+def ComputeSHA1(path):
+  if not os.path.exists(path):
+    return 0
+
+  sha1 = hashlib.sha1()
+  file_to_hash = open(path, 'rb')
+  try:
+    sha1.update(file_to_hash.read())
+  finally:
+    file_to_hash.close()
+
+  return sha1.hexdigest()
+
+
+# Code partially copied from
+# https://cs.chromium.org#chromium/build/scripts/common/chromium_utils.py
+def RemoveDirectory(*path):
+  """Recursively removes a directory, even if it's marked read-only.
+
+  Remove the directory located at *path, if it exists.
+
+  shutil.rmtree() doesn't work on Windows if any of the files or directories
+  are read-only, which svn repositories and some .svn files are.  We need to
+  be able to force the files to be writable (i.e., deletable) as we traverse
+  the tree.
+
+  Even with all this, Windows still sometimes fails to delete a file, citing
+  a permission error (maybe something to do with antivirus scans or disk
+  indexing).  The best suggestion any of the user forums had was to wait a
+  bit and try again, so we do that too.  It's hand-waving, but sometimes it
+  works. :/
+  """
+  file_path = os.path.join(*path)
+  print 'Deleting `{}`.'.format(file_path)
+  if not os.path.exists(file_path):
+    print '`{}` does not exist.'.format(file_path)
+    return
+
+  if sys.platform == 'win32':
+    # Give up and use cmd.exe's rd command.
+    file_path = os.path.normcase(file_path)
+    for _ in xrange(3):
+      print 'RemoveDirectory running %s' % (' '.join(
+          ['cmd.exe', '/c', 'rd', '/q', '/s', file_path]))
+      if not subprocess.call(['cmd.exe', '/c', 'rd', '/q', '/s', file_path]):
+        break
+      print '  Failed'
+      time.sleep(3)
+    return
+  else:
+    shutil.rmtree(file_path, ignore_errors=True)
+
+
+def UnpackArchiveTo(archive_path, output_dir):
+  extension = os.path.splitext(archive_path)[1]
+  if extension == '.zip':
+    _UnzipArchiveTo(archive_path, output_dir)
+  else:
+    _UntarArchiveTo(archive_path, output_dir)
+
+
+def _UnzipArchiveTo(archive_path, output_dir):
+  print 'Unzipping {} in {}.'.format(archive_path, output_dir)
+  zip_file = zipfile.ZipFile(archive_path)
+  try:
+    zip_file.extractall(output_dir)
+  finally:
+    zip_file.close()
+
+
+def _UntarArchiveTo(archive_path, output_dir):
+  print 'Untarring {} in {}.'.format(archive_path, output_dir)
+  tar_file = tarfile.open(archive_path, 'r:gz')
+  try:
+    tar_file.extractall(output_dir)
+  finally:
+    tar_file.close()
+
+
+def GetPlatform():
+  if sys.platform.startswith('win'):
+    return 'win'
+  if sys.platform.startswith('linux'):
+    return 'linux'
+  if sys.platform.startswith('darwin'):
+    return 'mac'
+  raise Exception("Can't run on platform %s." % sys.platform)