From 2de346e89f53921d09f4e45332db3c5cb2a3a2fb Mon Sep 17 00:00:00 2001
From: Junji Watanabe <jwata@google.com>
Date: Fri, 26 Aug 2022 11:05:17 +0900
Subject: [PATCH] Add Linux64 Release (reclient) CI builder

- this builder is excluded from tree closers.

Since this is the very first builder that uses reclient, it may fail for misconfiguration.
Feel free to revert if it blocks critical tasks.

Bug: b:243628179
Change-Id: Ia7535f181d3cb195cd030670899e1730f06500c7
Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/272869
Reviewed-by: Mirko Bonadei <mbonadei@webrtc.org>
Reviewed-by: Takuto Ikuta <tikuta@google.com>
Commit-Queue: Mirko Bonadei <mbonadei@webrtc.org>
Auto-Submit: Junji Watanabe <jwata@google.com>
Cr-Commit-Position: refs/heads/main@{#37909}
---
 infra/config/config.star        |  28 +++++++-
 infra/config/cr-buildbucket.cfg |  48 +++++++++++++
 infra/config/luci-milo.cfg      |   5 ++
 infra/config/luci-notify.cfg    | 115 ++++++++++++++++++++------------
 infra/config/luci-scheduler.cfg |  11 +++
 5 files changed, 161 insertions(+), 46 deletions(-)

diff --git a/infra/config/config.star b/infra/config/config.star
index 2f41ff49bb..0fd1868c9d 100755
--- a/infra/config/config.star
+++ b/infra/config/config.star
@@ -37,6 +37,23 @@ def make_goma_properties(enable_ats = True, jobs = None):
         goma_properties["jobs"] = jobs
     return {"$build/goma": goma_properties}
 
+def make_reclient_properties(instance, jobs = None):
+    """Makes a default reclient property with the specified argument.
+
+    Args:
+      instance: RBE insatnce name.
+      jobs: Number of jobs to be used by the builder.
+    Returns:
+      A dictonary with the reclient properties.
+    """
+    reclient_props = {
+        "instance": instance,
+        "metrics_project": "chromium-reclient-metrics",
+    }
+    if jobs:
+        reclient_props["jobs"] = jobs
+    return {"$build/reclient": reclient_props}
+
 # Add names of builders to remove from LKGR finder to this list. This is
 # useful when a failure can be safely ignored while fixing it without
 # blocking the LKGR finder on it.
@@ -388,7 +405,8 @@ luci.tree_closer(
         "compile",
         "gn",
     ],
-    failed_step_regexp_exclude = ".*\\(experimental\\).*",
+    # TODO(b/239908030, b/243594984): remove reclient builders after migration.
+    failed_step_regexp_exclude = ".*(\\(experimental\\)|\\(reclient\\)).*",
 )
 
 # Recipe definitions:
@@ -505,6 +523,7 @@ def ci_builder(
         perf_cat = None,
         prioritized = False,
         enabled = True,
+        use_reclient = False,
         **kwargs):
     """Add a post-submit builder.
 
@@ -516,6 +535,7 @@ def ci_builder(
       perf_cat: the category + name for the /perf/ console, or None to omit from the console.
       prioritized: True to make this builder have a higher priority and never batch builds.
       enabled: False to exclude this builder from consoles and failure notifications.
+      use_reclient: True to use reclient to build instead of goma.
       **kwargs: Pass on to webrtc_builder / luci.builder.
     Returns:
       A luci.builder.
@@ -536,7 +556,10 @@ def ci_builder(
     dimensions.update({"pool": "luci.webrtc.ci", "cpu": kwargs.pop("cpu", DEFAULT_CPU)})
     properties = properties or {}
     properties["builder_group"] = "client.webrtc"
-    properties.update(make_goma_properties())
+    if use_reclient:
+        properties.update(make_reclient_properties("rbe-webrtc-trusted"))
+    else:
+        properties.update(make_goma_properties())
     notifies = ["post_submit_failure_notifier", "infra_failure_notifier"]
     notifies += ["webrtc_tree_closer"] if name not in skipped_lkgr_bots else []
     return webrtc_builder(
@@ -724,6 +747,7 @@ linux_builder("Linux64 Debug", "Linux|x64|dbg")
 linux_try_job("linux_dbg", cq = None)
 linux_try_job("linux_compile_dbg")
 linux_builder("Linux64 Release", "Linux|x64|rel")
+linux_builder("Linux64 Release (reclient)", "Linux|x64|re", use_reclient = True)
 linux_try_job("linux_rel")
 linux_builder("Linux64 Builder", "Linux|x64|size", perf_cat = "Linux|x64|Builder|", prioritized = True)
 linux_try_job("linux_compile_rel")
diff --git a/infra/config/cr-buildbucket.cfg b/infra/config/cr-buildbucket.cfg
index 5553276f58..e15ddb804c 100644
--- a/infra/config/cr-buildbucket.cfg
+++ b/infra/config/cr-buildbucket.cfg
@@ -1041,6 +1041,53 @@ buckets {
         }
       }
     }
+    builders {
+      name: "Linux64 Release (reclient)"
+      swarming_host: "chromium-swarm.appspot.com"
+      swarming_tags: "vpython:native-python-wrapper"
+      dimensions: "cpu:x86-64"
+      dimensions: "inside_docker:0"
+      dimensions: "os:Linux"
+      dimensions: "pool:luci.webrtc.ci"
+      exe {
+        cipd_package: "infra/recipe_bundles/chromium.googlesource.com/chromium/tools/build"
+        cipd_version: "refs/heads/main"
+        cmd: "luciexe"
+      }
+      properties:
+        '{'
+        '  "$build/reclient": {'
+        '    "instance": "rbe-webrtc-trusted",'
+        '    "metrics_project": "chromium-reclient-metrics"'
+        '  },'
+        '  "$recipe_engine/resultdb/test_presentation": {'
+        '    "column_keys": [],'
+        '    "grouping_keys": ['
+        '      "status",'
+        '      "v.test_suite"'
+        '    ]'
+        '  },'
+        '  "builder_group": "client.webrtc",'
+        '  "recipe": "webrtc/standalone"'
+        '}'
+      priority: 30
+      execution_timeout_secs: 7200
+      build_numbers: YES
+      service_account: "webrtc-ci-builder@chops-service-accounts.iam.gserviceaccount.com"
+      experiments {
+        key: "luci.recipes.use_python3"
+        value: 100
+      }
+      resultdb {
+        enable: true
+        bq_exports {
+          project: "webrtc-ci"
+          dataset: "resultdb"
+          table: "ci_test_results"
+          test_results {}
+        }
+      }
+    }
     builders {
       name: "Mac Asan"
       swarming_host: "chromium-swarm.appspot.com"
@@ -1924,6 +1971,7 @@ buckets {
         '          "Linux64 Release",'
         '          "Linux64 Release (ARM)",'
         '          "Linux64 Release (Libfuzzer)",'
+        '          "Linux64 Release (reclient)",'
         '          "Mac Asan",'
         '          "Mac64 Debug",'
         '          "Mac64 Release",'
diff --git a/infra/config/luci-milo.cfg b/infra/config/luci-milo.cfg
index f226fa4d94..53aa5345d7 100644
--- a/infra/config/luci-milo.cfg
+++ b/infra/config/luci-milo.cfg
@@ -106,6 +106,11 @@ consoles {
     category: "Linux|x64"
     short_name: "rel"
   }
+  builders {
+    name: "buildbucket/luci.webrtc.ci/Linux64 Release (reclient)"
+    category: "Linux|x64"
+    short_name: "re"
+  }
   builders {
     name: "buildbucket/luci.webrtc.perf/Linux64 Builder"
     category: "Linux|x64"
diff --git a/infra/config/luci-notify.cfg b/infra/config/luci-notify.cfg
index 3ed6d1be3e..3e76f169d4 100644
--- a/infra/config/luci-notify.cfg
+++ b/infra/config/luci-notify.cfg
@@ -28,7 +28,7 @@ notifiers {
   tree_closers {
     tree_status_host: "webrtc-status.appspot.com"
     failed_step_regexp: "bot_update|compile|gclient runhooks|runhooks|update|extract build|cleanup_temp|taskkill|compile|gn"
-    failed_step_regexp_exclude: ".*\\(experimental\\).*"
+    failed_step_regexp_exclude: ".*(\\(experimental\\)|\\(reclient\\)).*"
   }
 }
 notifiers {
@@ -55,7 +55,7 @@ notifiers {
   tree_closers {
     tree_status_host: "webrtc-status.appspot.com"
     failed_step_regexp: "bot_update|compile|gclient runhooks|runhooks|update|extract build|cleanup_temp|taskkill|compile|gn"
-    failed_step_regexp_exclude: ".*\\(experimental\\).*"
+    failed_step_regexp_exclude: ".*(\\(experimental\\)|\\(reclient\\)).*"
   }
 }
 notifiers {
@@ -82,7 +82,7 @@ notifiers {
   tree_closers {
     tree_status_host: "webrtc-status.appspot.com"
     failed_step_regexp: "bot_update|compile|gclient runhooks|runhooks|update|extract build|cleanup_temp|taskkill|compile|gn"
-    failed_step_regexp_exclude: ".*\\(experimental\\).*"
+    failed_step_regexp_exclude: ".*(\\(experimental\\)|\\(reclient\\)).*"
   }
 }
 notifiers {
@@ -109,7 +109,7 @@ notifiers {
   tree_closers {
     tree_status_host: "webrtc-status.appspot.com"
     failed_step_regexp: "bot_update|compile|gclient runhooks|runhooks|update|extract build|cleanup_temp|taskkill|compile|gn"
-    failed_step_regexp_exclude: ".*\\(experimental\\).*"
+    failed_step_regexp_exclude: ".*(\\(experimental\\)|\\(reclient\\)).*"
   }
 }
 notifiers {
@@ -136,7 +136,7 @@ notifiers {
   tree_closers {
     tree_status_host: "webrtc-status.appspot.com"
     failed_step_regexp: "bot_update|compile|gclient runhooks|runhooks|update|extract build|cleanup_temp|taskkill|compile|gn"
-    failed_step_regexp_exclude: ".*\\(experimental\\).*"
+    failed_step_regexp_exclude: ".*(\\(experimental\\)|\\(reclient\\)).*"
   }
 }
 notifiers {
@@ -163,7 +163,7 @@ notifiers {
   tree_closers {
     tree_status_host: "webrtc-status.appspot.com"
     failed_step_regexp: "bot_update|compile|gclient runhooks|runhooks|update|extract build|cleanup_temp|taskkill|compile|gn"
-    failed_step_regexp_exclude: ".*\\(experimental\\).*"
+    failed_step_regexp_exclude: ".*(\\(experimental\\)|\\(reclient\\)).*"
   }
 }
 notifiers {
@@ -190,7 +190,7 @@ notifiers {
   tree_closers {
     tree_status_host: "webrtc-status.appspot.com"
     failed_step_regexp: "bot_update|compile|gclient runhooks|runhooks|update|extract build|cleanup_temp|taskkill|compile|gn"
-    failed_step_regexp_exclude: ".*\\(experimental\\).*"
+    failed_step_regexp_exclude: ".*(\\(experimental\\)|\\(reclient\\)).*"
   }
 }
 notifiers {
@@ -217,7 +217,7 @@ notifiers {
   tree_closers {
     tree_status_host: "webrtc-status.appspot.com"
     failed_step_regexp: "bot_update|compile|gclient runhooks|runhooks|update|extract build|cleanup_temp|taskkill|compile|gn"
-    failed_step_regexp_exclude: ".*\\(experimental\\).*"
+    failed_step_regexp_exclude: ".*(\\(experimental\\)|\\(reclient\\)).*"
   }
 }
 notifiers {
@@ -244,7 +244,7 @@ notifiers {
   tree_closers {
     tree_status_host: "webrtc-status.appspot.com"
     failed_step_regexp: "bot_update|compile|gclient runhooks|runhooks|update|extract build|cleanup_temp|taskkill|compile|gn"
-    failed_step_regexp_exclude: ".*\\(experimental\\).*"
+    failed_step_regexp_exclude: ".*(\\(experimental\\)|\\(reclient\\)).*"
   }
 }
 notifiers {
@@ -271,7 +271,7 @@ notifiers {
   tree_closers {
     tree_status_host: "webrtc-status.appspot.com"
     failed_step_regexp: "bot_update|compile|gclient runhooks|runhooks|update|extract build|cleanup_temp|taskkill|compile|gn"
-    failed_step_regexp_exclude: ".*\\(experimental\\).*"
+    failed_step_regexp_exclude: ".*(\\(experimental\\)|\\(reclient\\)).*"
   }
 }
 notifiers {
@@ -298,7 +298,7 @@ notifiers {
   tree_closers {
     tree_status_host: "webrtc-status.appspot.com"
     failed_step_regexp: "bot_update|compile|gclient runhooks|runhooks|update|extract build|cleanup_temp|taskkill|compile|gn"
-    failed_step_regexp_exclude: ".*\\(experimental\\).*"
+    failed_step_regexp_exclude: ".*(\\(experimental\\)|\\(reclient\\)).*"
   }
 }
 notifiers {
@@ -325,7 +325,7 @@ notifiers {
   tree_closers {
     tree_status_host: "webrtc-status.appspot.com"
     failed_step_regexp: "bot_update|compile|gclient runhooks|runhooks|update|extract build|cleanup_temp|taskkill|compile|gn"
-    failed_step_regexp_exclude: ".*\\(experimental\\).*"
+    failed_step_regexp_exclude: ".*(\\(experimental\\)|\\(reclient\\)).*"
   }
 }
 notifiers {
@@ -352,7 +352,7 @@ notifiers {
   tree_closers {
     tree_status_host: "webrtc-status.appspot.com"
     failed_step_regexp: "bot_update|compile|gclient runhooks|runhooks|update|extract build|cleanup_temp|taskkill|compile|gn"
-    failed_step_regexp_exclude: ".*\\(experimental\\).*"
+    failed_step_regexp_exclude: ".*(\\(experimental\\)|\\(reclient\\)).*"
   }
 }
 notifiers {
@@ -379,7 +379,7 @@ notifiers {
   tree_closers {
     tree_status_host: "webrtc-status.appspot.com"
     failed_step_regexp: "bot_update|compile|gclient runhooks|runhooks|update|extract build|cleanup_temp|taskkill|compile|gn"
-    failed_step_regexp_exclude: ".*\\(experimental\\).*"
+    failed_step_regexp_exclude: ".*(\\(experimental\\)|\\(reclient\\)).*"
   }
 }
 notifiers {
@@ -406,7 +406,7 @@ notifiers {
   tree_closers {
     tree_status_host: "webrtc-status.appspot.com"
     failed_step_regexp: "bot_update|compile|gclient runhooks|runhooks|update|extract build|cleanup_temp|taskkill|compile|gn"
-    failed_step_regexp_exclude: ".*\\(experimental\\).*"
+    failed_step_regexp_exclude: ".*(\\(experimental\\)|\\(reclient\\)).*"
   }
 }
 notifiers {
@@ -433,7 +433,7 @@ notifiers {
   tree_closers {
     tree_status_host: "webrtc-status.appspot.com"
     failed_step_regexp: "bot_update|compile|gclient runhooks|runhooks|update|extract build|cleanup_temp|taskkill|compile|gn"
-    failed_step_regexp_exclude: ".*\\(experimental\\).*"
+    failed_step_regexp_exclude: ".*(\\(experimental\\)|\\(reclient\\)).*"
   }
 }
 notifiers {
@@ -460,7 +460,7 @@ notifiers {
   tree_closers {
     tree_status_host: "webrtc-status.appspot.com"
     failed_step_regexp: "bot_update|compile|gclient runhooks|runhooks|update|extract build|cleanup_temp|taskkill|compile|gn"
-    failed_step_regexp_exclude: ".*\\(experimental\\).*"
+    failed_step_regexp_exclude: ".*(\\(experimental\\)|\\(reclient\\)).*"
   }
 }
 notifiers {
@@ -487,7 +487,7 @@ notifiers {
   tree_closers {
     tree_status_host: "webrtc-status.appspot.com"
     failed_step_regexp: "bot_update|compile|gclient runhooks|runhooks|update|extract build|cleanup_temp|taskkill|compile|gn"
-    failed_step_regexp_exclude: ".*\\(experimental\\).*"
+    failed_step_regexp_exclude: ".*(\\(experimental\\)|\\(reclient\\)).*"
   }
 }
 notifiers {
@@ -514,7 +514,7 @@ notifiers {
   tree_closers {
     tree_status_host: "webrtc-status.appspot.com"
     failed_step_regexp: "bot_update|compile|gclient runhooks|runhooks|update|extract build|cleanup_temp|taskkill|compile|gn"
-    failed_step_regexp_exclude: ".*\\(experimental\\).*"
+    failed_step_regexp_exclude: ".*(\\(experimental\\)|\\(reclient\\)).*"
   }
 }
 notifiers {
@@ -541,7 +541,7 @@ notifiers {
   tree_closers {
     tree_status_host: "webrtc-status.appspot.com"
     failed_step_regexp: "bot_update|compile|gclient runhooks|runhooks|update|extract build|cleanup_temp|taskkill|compile|gn"
-    failed_step_regexp_exclude: ".*\\(experimental\\).*"
+    failed_step_regexp_exclude: ".*(\\(experimental\\)|\\(reclient\\)).*"
   }
 }
 notifiers {
@@ -568,7 +568,7 @@ notifiers {
   tree_closers {
     tree_status_host: "webrtc-status.appspot.com"
     failed_step_regexp: "bot_update|compile|gclient runhooks|runhooks|update|extract build|cleanup_temp|taskkill|compile|gn"
-    failed_step_regexp_exclude: ".*\\(experimental\\).*"
+    failed_step_regexp_exclude: ".*(\\(experimental\\)|\\(reclient\\)).*"
   }
 }
 notifiers {
@@ -595,7 +595,34 @@ notifiers {
   tree_closers {
     tree_status_host: "webrtc-status.appspot.com"
     failed_step_regexp: "bot_update|compile|gclient runhooks|runhooks|update|extract build|cleanup_temp|taskkill|compile|gn"
-    failed_step_regexp_exclude: ".*\\(experimental\\).*"
+    failed_step_regexp_exclude: ".*(\\(experimental\\)|\\(reclient\\)).*"
+  }
+}
+notifiers {
+  notifications {
+    on_new_status: INFRA_FAILURE
+    email {
+      recipients: "webrtc-troopers-robots@google.com"
+    }
+    template: "infra_failure"
+  }
+  notifications {
+    on_new_status: FAILURE
+    email {
+      recipients: "webrtc-troopers-robots@google.com"
+    }
+    template: "build_failure"
+    notify_blamelist {}
+  }
+  builders {
+    bucket: "ci"
+    name: "Linux64 Release (reclient)"
+    repository: "https://webrtc.googlesource.com/src"
+  }
+  tree_closers {
+    tree_status_host: "webrtc-status.appspot.com"
+    failed_step_regexp: "bot_update|compile|gclient runhooks|runhooks|update|extract build|cleanup_temp|taskkill|compile|gn"
+    failed_step_regexp_exclude: ".*(\\(experimental\\)|\\(reclient\\)).*"
   }
 }
 notifiers {
@@ -622,7 +649,7 @@ notifiers {
   tree_closers {
     tree_status_host: "webrtc-status.appspot.com"
     failed_step_regexp: "bot_update|compile|gclient runhooks|runhooks|update|extract build|cleanup_temp|taskkill|compile|gn"
-    failed_step_regexp_exclude: ".*\\(experimental\\).*"
+    failed_step_regexp_exclude: ".*(\\(experimental\\)|\\(reclient\\)).*"
   }
 }
 notifiers {
@@ -649,7 +676,7 @@ notifiers {
   tree_closers {
     tree_status_host: "webrtc-status.appspot.com"
     failed_step_regexp: "bot_update|compile|gclient runhooks|runhooks|update|extract build|cleanup_temp|taskkill|compile|gn"
-    failed_step_regexp_exclude: ".*\\(experimental\\).*"
+    failed_step_regexp_exclude: ".*(\\(experimental\\)|\\(reclient\\)).*"
   }
 }
 notifiers {
@@ -676,7 +703,7 @@ notifiers {
   tree_closers {
     tree_status_host: "webrtc-status.appspot.com"
     failed_step_regexp: "bot_update|compile|gclient runhooks|runhooks|update|extract build|cleanup_temp|taskkill|compile|gn"
-    failed_step_regexp_exclude: ".*\\(experimental\\).*"
+    failed_step_regexp_exclude: ".*(\\(experimental\\)|\\(reclient\\)).*"
   }
 }
 notifiers {
@@ -703,7 +730,7 @@ notifiers {
   tree_closers {
     tree_status_host: "webrtc-status.appspot.com"
     failed_step_regexp: "bot_update|compile|gclient runhooks|runhooks|update|extract build|cleanup_temp|taskkill|compile|gn"
-    failed_step_regexp_exclude: ".*\\(experimental\\).*"
+    failed_step_regexp_exclude: ".*(\\(experimental\\)|\\(reclient\\)).*"
   }
 }
 notifiers {
@@ -730,7 +757,7 @@ notifiers {
   tree_closers {
     tree_status_host: "webrtc-status.appspot.com"
     failed_step_regexp: "bot_update|compile|gclient runhooks|runhooks|update|extract build|cleanup_temp|taskkill|compile|gn"
-    failed_step_regexp_exclude: ".*\\(experimental\\).*"
+    failed_step_regexp_exclude: ".*(\\(experimental\\)|\\(reclient\\)).*"
   }
 }
 notifiers {
@@ -757,7 +784,7 @@ notifiers {
   tree_closers {
     tree_status_host: "webrtc-status.appspot.com"
     failed_step_regexp: "bot_update|compile|gclient runhooks|runhooks|update|extract build|cleanup_temp|taskkill|compile|gn"
-    failed_step_regexp_exclude: ".*\\(experimental\\).*"
+    failed_step_regexp_exclude: ".*(\\(experimental\\)|\\(reclient\\)).*"
   }
 }
 notifiers {
@@ -784,7 +811,7 @@ notifiers {
   tree_closers {
     tree_status_host: "webrtc-status.appspot.com"
     failed_step_regexp: "bot_update|compile|gclient runhooks|runhooks|update|extract build|cleanup_temp|taskkill|compile|gn"
-    failed_step_regexp_exclude: ".*\\(experimental\\).*"
+    failed_step_regexp_exclude: ".*(\\(experimental\\)|\\(reclient\\)).*"
   }
 }
 notifiers {
@@ -811,7 +838,7 @@ notifiers {
   tree_closers {
     tree_status_host: "webrtc-status.appspot.com"
     failed_step_regexp: "bot_update|compile|gclient runhooks|runhooks|update|extract build|cleanup_temp|taskkill|compile|gn"
-    failed_step_regexp_exclude: ".*\\(experimental\\).*"
+    failed_step_regexp_exclude: ".*(\\(experimental\\)|\\(reclient\\)).*"
   }
 }
 notifiers {
@@ -838,7 +865,7 @@ notifiers {
   tree_closers {
     tree_status_host: "webrtc-status.appspot.com"
     failed_step_regexp: "bot_update|compile|gclient runhooks|runhooks|update|extract build|cleanup_temp|taskkill|compile|gn"
-    failed_step_regexp_exclude: ".*\\(experimental\\).*"
+    failed_step_regexp_exclude: ".*(\\(experimental\\)|\\(reclient\\)).*"
   }
 }
 notifiers {
@@ -865,7 +892,7 @@ notifiers {
   tree_closers {
     tree_status_host: "webrtc-status.appspot.com"
     failed_step_regexp: "bot_update|compile|gclient runhooks|runhooks|update|extract build|cleanup_temp|taskkill|compile|gn"
-    failed_step_regexp_exclude: ".*\\(experimental\\).*"
+    failed_step_regexp_exclude: ".*(\\(experimental\\)|\\(reclient\\)).*"
   }
 }
 notifiers {
@@ -892,7 +919,7 @@ notifiers {
   tree_closers {
     tree_status_host: "webrtc-status.appspot.com"
     failed_step_regexp: "bot_update|compile|gclient runhooks|runhooks|update|extract build|cleanup_temp|taskkill|compile|gn"
-    failed_step_regexp_exclude: ".*\\(experimental\\).*"
+    failed_step_regexp_exclude: ".*(\\(experimental\\)|\\(reclient\\)).*"
   }
 }
 notifiers {
@@ -919,7 +946,7 @@ notifiers {
   tree_closers {
     tree_status_host: "webrtc-status.appspot.com"
     failed_step_regexp: "bot_update|compile|gclient runhooks|runhooks|update|extract build|cleanup_temp|taskkill|compile|gn"
-    failed_step_regexp_exclude: ".*\\(experimental\\).*"
+    failed_step_regexp_exclude: ".*(\\(experimental\\)|\\(reclient\\)).*"
   }
 }
 notifiers {
@@ -946,7 +973,7 @@ notifiers {
   tree_closers {
     tree_status_host: "webrtc-status.appspot.com"
     failed_step_regexp: "bot_update|compile|gclient runhooks|runhooks|update|extract build|cleanup_temp|taskkill|compile|gn"
-    failed_step_regexp_exclude: ".*\\(experimental\\).*"
+    failed_step_regexp_exclude: ".*(\\(experimental\\)|\\(reclient\\)).*"
   }
 }
 notifiers {
@@ -973,7 +1000,7 @@ notifiers {
   tree_closers {
     tree_status_host: "webrtc-status.appspot.com"
     failed_step_regexp: "bot_update|compile|gclient runhooks|runhooks|update|extract build|cleanup_temp|taskkill|compile|gn"
-    failed_step_regexp_exclude: ".*\\(experimental\\).*"
+    failed_step_regexp_exclude: ".*(\\(experimental\\)|\\(reclient\\)).*"
   }
 }
 notifiers {
@@ -1000,7 +1027,7 @@ notifiers {
   tree_closers {
     tree_status_host: "webrtc-status.appspot.com"
     failed_step_regexp: "bot_update|compile|gclient runhooks|runhooks|update|extract build|cleanup_temp|taskkill|compile|gn"
-    failed_step_regexp_exclude: ".*\\(experimental\\).*"
+    failed_step_regexp_exclude: ".*(\\(experimental\\)|\\(reclient\\)).*"
   }
 }
 notifiers {
@@ -1027,7 +1054,7 @@ notifiers {
   tree_closers {
     tree_status_host: "webrtc-status.appspot.com"
     failed_step_regexp: "bot_update|compile|gclient runhooks|runhooks|update|extract build|cleanup_temp|taskkill|compile|gn"
-    failed_step_regexp_exclude: ".*\\(experimental\\).*"
+    failed_step_regexp_exclude: ".*(\\(experimental\\)|\\(reclient\\)).*"
   }
 }
 notifiers {
@@ -1096,7 +1123,7 @@ notifiers {
   tree_closers {
     tree_status_host: "webrtc-status.appspot.com"
     failed_step_regexp: "bot_update|compile|gclient runhooks|runhooks|update|extract build|cleanup_temp|taskkill|compile|gn"
-    failed_step_regexp_exclude: ".*\\(experimental\\).*"
+    failed_step_regexp_exclude: ".*(\\(experimental\\)|\\(reclient\\)).*"
   }
 }
 notifiers {
@@ -1123,7 +1150,7 @@ notifiers {
   tree_closers {
     tree_status_host: "webrtc-status.appspot.com"
     failed_step_regexp: "bot_update|compile|gclient runhooks|runhooks|update|extract build|cleanup_temp|taskkill|compile|gn"
-    failed_step_regexp_exclude: ".*\\(experimental\\).*"
+    failed_step_regexp_exclude: ".*(\\(experimental\\)|\\(reclient\\)).*"
   }
 }
 notifiers {
@@ -1150,7 +1177,7 @@ notifiers {
   tree_closers {
     tree_status_host: "webrtc-status.appspot.com"
     failed_step_regexp: "bot_update|compile|gclient runhooks|runhooks|update|extract build|cleanup_temp|taskkill|compile|gn"
-    failed_step_regexp_exclude: ".*\\(experimental\\).*"
+    failed_step_regexp_exclude: ".*(\\(experimental\\)|\\(reclient\\)).*"
   }
 }
 notifiers {
@@ -1177,7 +1204,7 @@ notifiers {
   tree_closers {
     tree_status_host: "webrtc-status.appspot.com"
     failed_step_regexp: "bot_update|compile|gclient runhooks|runhooks|update|extract build|cleanup_temp|taskkill|compile|gn"
-    failed_step_regexp_exclude: ".*\\(experimental\\).*"
+    failed_step_regexp_exclude: ".*(\\(experimental\\)|\\(reclient\\)).*"
   }
 }
 notifiers {
@@ -1204,7 +1231,7 @@ notifiers {
   tree_closers {
     tree_status_host: "webrtc-status.appspot.com"
     failed_step_regexp: "bot_update|compile|gclient runhooks|runhooks|update|extract build|cleanup_temp|taskkill|compile|gn"
-    failed_step_regexp_exclude: ".*\\(experimental\\).*"
+    failed_step_regexp_exclude: ".*(\\(experimental\\)|\\(reclient\\)).*"
   }
 }
 notifiers {
@@ -1407,7 +1434,7 @@ notifiers {
   tree_closers {
     tree_status_host: "webrtc-status.appspot.com"
     failed_step_regexp: "bot_update|compile|gclient runhooks|runhooks|update|extract build|cleanup_temp|taskkill|compile|gn"
-    failed_step_regexp_exclude: ".*\\(experimental\\).*"
+    failed_step_regexp_exclude: ".*(\\(experimental\\)|\\(reclient\\)).*"
   }
 }
 notifiers {
diff --git a/infra/config/luci-scheduler.cfg b/infra/config/luci-scheduler.cfg
index f25a399c83..e388ba1122 100644
--- a/infra/config/luci-scheduler.cfg
+++ b/infra/config/luci-scheduler.cfg
@@ -280,6 +280,16 @@ job {
     builder: "Linux64 Release (Libfuzzer)"
   }
 }
+job {
+  id: "Linux64 Release (reclient)"
+  realm: "ci"
+  acl_sets: "ci"
+  buildbucket {
+    server: "cr-buildbucket.appspot.com"
+    bucket: "ci"
+    builder: "Linux64 Release (reclient)"
+  }
+}
 job {
   id: "Mac Asan"
   realm: "ci"
@@ -667,6 +677,7 @@ trigger {
   triggers: "Linux64 Release"
   triggers: "Linux64 Release (ARM)"
   triggers: "Linux64 Release (Libfuzzer)"
+  triggers: "Linux64 Release (reclient)"
   triggers: "Mac Asan"
   triggers: "Mac64 Debug"
   triggers: "Mac64 Release"