From fc402760e9b2c2d946ab75c0560c3e8954c27ec8 Mon Sep 17 00:00:00 2001 From: "phoglund@webrtc.org" Date: Mon, 12 Mar 2012 09:12:32 +0000 Subject: [PATCH] Implemented branch coverage and integration bot coverage on the dashboard. BUG= TEST= Review URL: https://webrtc-codereview.appspot.com/434002 git-svn-id: http://webrtc.googlecode.com/svn/trunk@1873 4adac7df-926f-26a2-2b94-8c16560cd09d --- .../dashboard/add_coverage_data.py | 56 +++++++++---- tools/quality_tracking/dashboard/dashboard.py | 5 +- .../dashboard/load_coverage.py | 18 ++-- .../templates/dashboard_template.html | 34 +++++--- tools/quality_tracking/track_coverage.py | 84 ++++++++++++++----- 5 files changed, 142 insertions(+), 55 deletions(-) diff --git a/tools/quality_tracking/dashboard/add_coverage_data.py b/tools/quality_tracking/dashboard/add_coverage_data.py index 52a388adc7..7f3a1a54cc 100644 --- a/tools/quality_tracking/dashboard/add_coverage_data.py +++ b/tools/quality_tracking/dashboard/add_coverage_data.py @@ -12,25 +12,29 @@ __author__ = 'phoglund@webrtc.org (Patrik Höglund)' -import datetime +from datetime import datetime import logging from google.appengine.ext import db import oauth_post_request_handler +REPORT_CATEGORIES = ('small_medium_tests', 'large_tests') + + class CoverageData(db.Model): """This represents one coverage report from the build bot.""" + + # The date the report was made. date = db.DateTimeProperty(required=True) + + # Coverage percentages. line_coverage = db.FloatProperty(required=True) function_coverage = db.FloatProperty(required=True) + branch_coverage = db.FloatProperty() - -def _parse_percentage(string_value): - percentage = float(string_value) - if percentage < 0.0 or percentage > 100.0: - raise ValueError('%s is not a valid percentage.' % string_value) - return percentage + # The report category must be one of the REPORT_CATEGORIES. + report_category = db.CategoryProperty() class AddCoverageData(oauth_post_request_handler.OAuthPostRequestHandler): @@ -40,19 +44,24 @@ class AddCoverageData(oauth_post_request_handler.OAuthPostRequestHandler): the regular oauth_* parameters, these values: date: The POSIX timestamp for when the coverage observation was made. - line_coverage: A float percentage in the interval 0-100.0. - function_coverage: A float percentage in the interval 0-100.0. + report_category: A value in REPORT_CATEGORIES which characterizes the + coverage information (e.g. is the coverage from small / medium tests + or large tests?) + + line_coverage: Line coverage percentage. + function_coverage: Function coverage percentage. + branch_coverage: Branch coverage percentage. """ def _parse_and_store_data(self): try: - posix_time = int(self.request.get('date')) - parsed_date = datetime.datetime.fromtimestamp(posix_time) + request_posix_timestamp = float(self.request.get('oauth_timestamp')) + parsed_date = datetime.fromtimestamp(request_posix_timestamp) - line_coverage_string = self.request.get('line_coverage') - line_coverage = _parse_percentage(line_coverage_string) - function_coverage_string = self.request.get('function_coverage') - function_coverage = _parse_percentage(function_coverage_string) + line_coverage = self._parse_percentage('line_coverage') + function_coverage = self._parse_percentage('function_coverage') + branch_coverage = self._parse_percentage('branch_coverage') + report_category = self._parse_category('report_category') except ValueError as error: logging.warn('Invalid parameter in request: %s.' % error) @@ -61,6 +70,21 @@ class AddCoverageData(oauth_post_request_handler.OAuthPostRequestHandler): item = CoverageData(date=parsed_date, line_coverage=line_coverage, - function_coverage=function_coverage) + function_coverage=function_coverage, + branch_coverage=branch_coverage, + report_category=report_category) item.put() + def _parse_percentage(self, key): + """Parses out a percentage value from the request.""" + value = float(self.request.get(key)) + if percentage < 0.0 or percentage > 100.0: + raise ValueError('%s is not a valid percentage.' % string_value) + return percentage + + def _parse_category(self, key): + value = self.request.get(key) + if value in REPORT_CATEGORIES: + return value + else: + raise ValueError("Invalid category %s." % value) diff --git a/tools/quality_tracking/dashboard/dashboard.py b/tools/quality_tracking/dashboard/dashboard.py index 5cc5a61181..08e4877f98 100644 --- a/tools/quality_tracking/dashboard/dashboard.py +++ b/tools/quality_tracking/dashboard/dashboard.py @@ -40,7 +40,10 @@ class ShowDashboard(webapp2.RequestHandler): lkgr = build_status_loader.compute_lkgr() coverage_loader = load_coverage.CoverageDataLoader() - coverage_json_data = coverage_loader.load_coverage_json_data() + small_medium_coverage_json_data = ( + coverage_loader.load_coverage_json_data('small_medium_tests')) + large_coverage_json_data = ( + coverage_loader.load_coverage_json_data('large_tests')) page_template_filename = 'templates/dashboard_template.html' self.response.write(template.render(page_template_filename, vars())) diff --git a/tools/quality_tracking/dashboard/load_coverage.py b/tools/quality_tracking/dashboard/load_coverage.py index eafed3b330..f7b79d796c 100644 --- a/tools/quality_tracking/dashboard/load_coverage.py +++ b/tools/quality_tracking/dashboard/load_coverage.py @@ -12,6 +12,8 @@ __author__ = 'phoglund@webrtc.org (Patrik Höglund)' +import logging + from google.appengine.ext import db import gviz_api @@ -19,21 +21,27 @@ import gviz_api class CoverageDataLoader: """ Loads coverage data from the database.""" - def load_coverage_json_data(self): + def load_coverage_json_data(self, report_category): coverage_entries = db.GqlQuery('SELECT * ' 'FROM CoverageData ' - 'ORDER BY date ASC') + 'WHERE report_category = :1 ' + 'ORDER BY date ASC', report_category) data = [] for coverage_entry in coverage_entries: - data.append({'date': coverage_entry.date, + # Note: The date column must be first in alphabetical order since it is + # the primary column. This is a bug in the gviz api (or at least it + # doesn't make much sense). + data.append({'aa_date': coverage_entry.date, 'line_coverage': coverage_entry.line_coverage, 'function_coverage': coverage_entry.function_coverage, + 'branch_coverage': coverage_entry.branch_coverage, }) description = { - 'date': ('datetime', 'Date'), + 'aa_date': ('datetime', 'Date'), 'line_coverage': ('number', 'Line Coverage'), - 'function_coverage': ('number', 'Function Coverage') + 'function_coverage': ('number', 'Function Coverage'), + 'branch_coverage': ('number', 'Branch Coverage'), } coverage_data = gviz_api.DataTable(description, data) return coverage_data.ToJSon(order_by='date') diff --git a/tools/quality_tracking/dashboard/templates/dashboard_template.html b/tools/quality_tracking/dashboard/templates/dashboard_template.html index ab019d5605..70cf6e5832 100644 --- a/tools/quality_tracking/dashboard/templates/dashboard_template.html +++ b/tools/quality_tracking/dashboard/templates/dashboard_template.html @@ -32,16 +32,29 @@ coverage table JSON data otherwise. {% endcomment %} {% autoescape off %} - var coverage_data_table = - new google.visualization.DataTable({{ coverage_json_data }}); + var small_medium_coverage_data_table = + new google.visualization.DataTable( + {{ small_medium_coverage_json_data }}); + var large_coverage_data_table = + new google.visualization.DataTable( + {{ large_coverage_json_data }}); {% endautoescape %} /* Display tables and charts */ - var coverage_chart = new google.visualization.LineChart( - document.getElementById('table_div_coverage')); - coverage_chart.draw(coverage_data_table, { - colors: ['blue', 'red'], - vAxis: {title: 'Coverage'}, + var small_medium_coverage_chart = new google.visualization.LineChart( + document.getElementById('table_div_small_medium_coverage')); + small_medium_coverage_chart.draw(small_medium_coverage_data_table, { + colors: ['blue', 'red', 'black'], + vAxis: {title: 'Coverage (%)'}, + hAxis: {title: 'Date'}, + width: 1200, height: 300, + }); + + var large_coverage_chart = new google.visualization.LineChart( + document.getElementById('table_div_large_coverage')); + large_coverage_chart.draw(large_coverage_data_table, { + colors: ['blue', 'red', 'black'], + vAxis: {title: 'Coverage (%)'}, hAxis: {title: 'Date'}, width: 1200, height: 300, }); @@ -49,7 +62,6 @@ -

WebRTC Quality Dashboard

Current Build Status

(as of {{ last_updated_at }} UTC)
@@ -80,7 +92,9 @@ {% endif %} -

Code Coverage History

-
+

Code Coverage History (Small / Medium Tests)

+
+

Code Coverage History (Large Tests)

+
diff --git a/tools/quality_tracking/track_coverage.py b/tools/quality_tracking/track_coverage.py index 9d4ed6184d..248fad2e82 100755 --- a/tools/quality_tracking/track_coverage.py +++ b/tools/quality_tracking/track_coverage.py @@ -28,6 +28,7 @@ __author__ = 'phoglund@webrtc.org (Patrik Höglund)' import os import re +import sys import time import constants @@ -42,28 +43,39 @@ class CouldNotFindCoverageDirectory(Exception): pass -def _find_latest_32bit_debug_build(www_directory_contents, coverage_www_dir): - """Finds the latest 32-bit coverage directory in the directory listing. +def _find_latest_build_coverage(www_directory_contents, coverage_www_dir, + directory_prefix): + """Finds the most recent coverage directory in the directory listing. - Coverage directories have the form Linux32bitDBG_. There may be - other directories in the list though, for instance for other build - configurations. We assume here that build numbers keep rising and never - wrap around or anything like that. + We assume here that build numbers keep rising and never wrap around. + + Args: + www_directory_contents: A list of entries in the coverage directory. + coverage_www_dir: The coverage directory on the bot. + directory_prefix: Coverage directories have the form , + and the prefix is different on different bots. The prefix is + generally the builder name, such as Linux32DBG. + + Returns: + The most recent directory name. + + Raises: + CouldNotFindCoverageDirectory: if we failed to find coverage data. """ found_build_numbers = [] for entry in www_directory_contents: - match = re.match('Linux32DBG_(\d+)', entry) + match = re.match(directory_prefix + '(\d+)', entry) if match is not None: found_build_numbers.append(int(match.group(1))) if not found_build_numbers: - raise CouldNotFindCoverageDirectory('Error: Found no 32-bit ' - 'debug build in directory %s.' % - coverage_www_dir) + raise CouldNotFindCoverageDirectory('Error: Found no directories %s* ' + 'in directory %s.' % + (directory_prefix, coverage_www_dir)) most_recent = max(found_build_numbers) - return 'Linux32DBG_' + str(most_recent) + return directory_prefix + str(most_recent) def _grab_coverage_percentage(label, index_html_contents): @@ -84,25 +96,37 @@ def _grab_coverage_percentage(label, index_html_contents): raise FailedToParseCoverageHtml('%s is not a float.' % match.group(1)) -def _report_coverage_to_dashboard(dashboard, now, line_coverage, - function_coverage): - parameters = {'date': '%d' % now, - 'line_coverage': '%f' % line_coverage, - 'function_coverage': '%f' % function_coverage +def _report_coverage_to_dashboard(dashboard, line_coverage, function_coverage, + branch_coverage, report_category): + parameters = {'line_coverage': '%f' % line_coverage, + 'function_coverage': '%f' % function_coverage, + 'branch_coverage': '%f' % branch_coverage, + 'report_category': report_category, } dashboard.send_post_request(constants.ADD_COVERAGE_DATA_URL, parameters) -def _main(): +def _main(report_category, directory_prefix): + """Grabs coverage data from disk on a bot and publishes it. + + Args: + report_category: The kind of coverage to report. The dashboard + application decides what is acceptable here (see + dashboard/add_coverage_data.py for more information). + directory_prefix: This bot's coverage directory prefix. Generally a bot's + coverage directories will have the form , + like Linux32DBG_345. + """ dashboard = dashboard_connection.DashboardConnection(constants.CONSUMER_KEY) dashboard.read_required_files(constants.CONSUMER_SECRET_FILE, constants.ACCESS_TOKEN_FILE) coverage_www_dir = constants.BUILD_BOT_COVERAGE_WWW_DIRECTORY www_dir_contents = os.listdir(coverage_www_dir) - latest_build_directory = _find_latest_32bit_debug_build(www_dir_contents, - coverage_www_dir) + latest_build_directory = _find_latest_build_coverage(www_dir_contents, + coverage_www_dir, + directory_prefix) index_html_path = os.path.join(coverage_www_dir, latest_build_directory, 'index.html') @@ -111,12 +135,26 @@ def _main(): line_coverage = _grab_coverage_percentage('Lines:', whole_file) function_coverage = _grab_coverage_percentage('Functions:', whole_file) - now = int(time.time()) + branch_coverage = _grab_coverage_percentage('Branches:', whole_file) - _report_coverage_to_dashboard(dashboard, now, line_coverage, - function_coverage) + _report_coverage_to_dashboard(dashboard, line_coverage, function_coverage, + branch_coverage, report_category) + + +def _parse_args(): + if len(sys.argv) != 3: + print ('Usage: %s \n\n' + 'The coverage category describes the kind of coverage you are ' + 'uploading. Known acceptable values are small_medium_tests and' + 'large_tests. The directory prefix is what the directories in %s ' + 'are prefixed on this bot (such as Linux32DBG_).' % + (sys.argv[0], constants.BUILD_BOT_COVERAGE_WWW_DIRECTORY)) + return (None, None) + return (sys.argv[1], sys.argv[2]) if __name__ == '__main__': - _main() + report_category, directory_prefix = _parse_args() + if report_category: + _main(report_category, directory_prefix)