Implemented branch coverage and integration bot coverage on the dashboard.
BUG=
TEST=
Review URL: https://webrtc-codereview.appspot.com/434002
git-svn-id: http://webrtc.googlecode.com/svn/trunk@1873 4adac7df-926f-26a2-2b94-8c16560cd09d
diff --git a/tools/quality_tracking/dashboard/add_coverage_data.py b/tools/quality_tracking/dashboard/add_coverage_data.py
index 52a388a..7f3a1a5 100644
--- a/tools/quality_tracking/dashboard/add_coverage_data.py
+++ b/tools/quality_tracking/dashboard/add_coverage_data.py
@@ -12,25 +12,29 @@
__author__ = 'phoglund@webrtc.org (Patrik Höglund)'
-import datetime
+from datetime import datetime
import logging
from google.appengine.ext import db
import oauth_post_request_handler
+REPORT_CATEGORIES = ('small_medium_tests', 'large_tests')
+
+
class CoverageData(db.Model):
"""This represents one coverage report from the build bot."""
+
+ # The date the report was made.
date = db.DateTimeProperty(required=True)
+
+ # Coverage percentages.
line_coverage = db.FloatProperty(required=True)
function_coverage = db.FloatProperty(required=True)
+ branch_coverage = db.FloatProperty()
-
-def _parse_percentage(string_value):
- percentage = float(string_value)
- if percentage < 0.0 or percentage > 100.0:
- raise ValueError('%s is not a valid percentage.' % string_value)
- return percentage
+ # The report category must be one of the REPORT_CATEGORIES.
+ report_category = db.CategoryProperty()
class AddCoverageData(oauth_post_request_handler.OAuthPostRequestHandler):
@@ -40,19 +44,24 @@
the regular oauth_* parameters, these values:
date: The POSIX timestamp for when the coverage observation was made.
- line_coverage: A float percentage in the interval 0-100.0.
- function_coverage: A float percentage in the interval 0-100.0.
+ report_category: A value in REPORT_CATEGORIES which characterizes the
+ coverage information (e.g. is the coverage from small / medium tests
+ or large tests?)
+
+ line_coverage: Line coverage percentage.
+ function_coverage: Function coverage percentage.
+ branch_coverage: Branch coverage percentage.
"""
def _parse_and_store_data(self):
try:
- posix_time = int(self.request.get('date'))
- parsed_date = datetime.datetime.fromtimestamp(posix_time)
+ request_posix_timestamp = float(self.request.get('oauth_timestamp'))
+ parsed_date = datetime.fromtimestamp(request_posix_timestamp)
- line_coverage_string = self.request.get('line_coverage')
- line_coverage = _parse_percentage(line_coverage_string)
- function_coverage_string = self.request.get('function_coverage')
- function_coverage = _parse_percentage(function_coverage_string)
+ line_coverage = self._parse_percentage('line_coverage')
+ function_coverage = self._parse_percentage('function_coverage')
+ branch_coverage = self._parse_percentage('branch_coverage')
+ report_category = self._parse_category('report_category')
except ValueError as error:
logging.warn('Invalid parameter in request: %s.' % error)
@@ -61,6 +70,21 @@
item = CoverageData(date=parsed_date,
line_coverage=line_coverage,
- function_coverage=function_coverage)
+ function_coverage=function_coverage,
+ branch_coverage=branch_coverage,
+ report_category=report_category)
item.put()
+ def _parse_percentage(self, key):
+ """Parses out a percentage value from the request."""
+ value = float(self.request.get(key))
+ if percentage < 0.0 or percentage > 100.0:
+ raise ValueError('%s is not a valid percentage.' % string_value)
+ return percentage
+
+ def _parse_category(self, key):
+ value = self.request.get(key)
+ if value in REPORT_CATEGORIES:
+ return value
+ else:
+ raise ValueError("Invalid category %s." % value)
diff --git a/tools/quality_tracking/dashboard/dashboard.py b/tools/quality_tracking/dashboard/dashboard.py
index 5cc5a61..08e4877 100644
--- a/tools/quality_tracking/dashboard/dashboard.py
+++ b/tools/quality_tracking/dashboard/dashboard.py
@@ -40,7 +40,10 @@
lkgr = build_status_loader.compute_lkgr()
coverage_loader = load_coverage.CoverageDataLoader()
- coverage_json_data = coverage_loader.load_coverage_json_data()
+ small_medium_coverage_json_data = (
+ coverage_loader.load_coverage_json_data('small_medium_tests'))
+ large_coverage_json_data = (
+ coverage_loader.load_coverage_json_data('large_tests'))
page_template_filename = 'templates/dashboard_template.html'
self.response.write(template.render(page_template_filename, vars()))
diff --git a/tools/quality_tracking/dashboard/load_coverage.py b/tools/quality_tracking/dashboard/load_coverage.py
index eafed3b..f7b79d7 100644
--- a/tools/quality_tracking/dashboard/load_coverage.py
+++ b/tools/quality_tracking/dashboard/load_coverage.py
@@ -12,6 +12,8 @@
__author__ = 'phoglund@webrtc.org (Patrik Höglund)'
+import logging
+
from google.appengine.ext import db
import gviz_api
@@ -19,21 +21,27 @@
class CoverageDataLoader:
""" Loads coverage data from the database."""
- def load_coverage_json_data(self):
+ def load_coverage_json_data(self, report_category):
coverage_entries = db.GqlQuery('SELECT * '
'FROM CoverageData '
- 'ORDER BY date ASC')
+ 'WHERE report_category = :1 '
+ 'ORDER BY date ASC', report_category)
data = []
for coverage_entry in coverage_entries:
- data.append({'date': coverage_entry.date,
+ # Note: The date column must be first in alphabetical order since it is
+ # the primary column. This is a bug in the gviz api (or at least it
+ # doesn't make much sense).
+ data.append({'aa_date': coverage_entry.date,
'line_coverage': coverage_entry.line_coverage,
'function_coverage': coverage_entry.function_coverage,
+ 'branch_coverage': coverage_entry.branch_coverage,
})
description = {
- 'date': ('datetime', 'Date'),
+ 'aa_date': ('datetime', 'Date'),
'line_coverage': ('number', 'Line Coverage'),
- 'function_coverage': ('number', 'Function Coverage')
+ 'function_coverage': ('number', 'Function Coverage'),
+ 'branch_coverage': ('number', 'Branch Coverage'),
}
coverage_data = gviz_api.DataTable(description, data)
return coverage_data.ToJSon(order_by='date')
diff --git a/tools/quality_tracking/dashboard/templates/dashboard_template.html b/tools/quality_tracking/dashboard/templates/dashboard_template.html
index ab019d5..70cf6e5 100644
--- a/tools/quality_tracking/dashboard/templates/dashboard_template.html
+++ b/tools/quality_tracking/dashboard/templates/dashboard_template.html
@@ -32,16 +32,29 @@
coverage table JSON data otherwise.
{% endcomment %}
{% autoescape off %}
- var coverage_data_table =
- new google.visualization.DataTable({{ coverage_json_data }});
+ var small_medium_coverage_data_table =
+ new google.visualization.DataTable(
+ {{ small_medium_coverage_json_data }});
+ var large_coverage_data_table =
+ new google.visualization.DataTable(
+ {{ large_coverage_json_data }});
{% endautoescape %}
/* Display tables and charts */
- var coverage_chart = new google.visualization.LineChart(
- document.getElementById('table_div_coverage'));
- coverage_chart.draw(coverage_data_table, {
- colors: ['blue', 'red'],
- vAxis: {title: 'Coverage'},
+ var small_medium_coverage_chart = new google.visualization.LineChart(
+ document.getElementById('table_div_small_medium_coverage'));
+ small_medium_coverage_chart.draw(small_medium_coverage_data_table, {
+ colors: ['blue', 'red', 'black'],
+ vAxis: {title: 'Coverage (%)'},
+ hAxis: {title: 'Date'},
+ width: 1200, height: 300,
+ });
+
+ var large_coverage_chart = new google.visualization.LineChart(
+ document.getElementById('table_div_large_coverage'));
+ large_coverage_chart.draw(large_coverage_data_table, {
+ colors: ['blue', 'red', 'black'],
+ vAxis: {title: 'Coverage (%)'},
hAxis: {title: 'Date'},
width: 1200, height: 300,
});
@@ -49,7 +62,6 @@
</script>
</head>
<body>
-
<h1>WebRTC Quality Dashboard</h1>
<h2>Current Build Status</h2>
<div>(as of {{ last_updated_at }} UTC)</div>
@@ -80,7 +92,9 @@
{% endif %}
</div>
- <h2>Code Coverage History</h2>
- <div id="table_div_coverage"></div>
+ <h2>Code Coverage History (Small / Medium Tests)</h2>
+ <div id="table_div_small_medium_coverage"></div>
+ <h2>Code Coverage History (Large Tests)</h2>
+ <div id="table_div_large_coverage"></div>
</body>
</html>
diff --git a/tools/quality_tracking/track_coverage.py b/tools/quality_tracking/track_coverage.py
index 9d4ed61..248fad2 100755
--- a/tools/quality_tracking/track_coverage.py
+++ b/tools/quality_tracking/track_coverage.py
@@ -28,6 +28,7 @@
import os
import re
+import sys
import time
import constants
@@ -42,28 +43,39 @@
pass
-def _find_latest_32bit_debug_build(www_directory_contents, coverage_www_dir):
- """Finds the latest 32-bit coverage directory in the directory listing.
+def _find_latest_build_coverage(www_directory_contents, coverage_www_dir,
+ directory_prefix):
+ """Finds the most recent coverage directory in the directory listing.
- Coverage directories have the form Linux32bitDBG_<number>. There may be
- other directories in the list though, for instance for other build
- configurations. We assume here that build numbers keep rising and never
- wrap around or anything like that.
+ We assume here that build numbers keep rising and never wrap around.
+
+ Args:
+ www_directory_contents: A list of entries in the coverage directory.
+ coverage_www_dir: The coverage directory on the bot.
+ directory_prefix: Coverage directories have the form <prefix><number>,
+ and the prefix is different on different bots. The prefix is
+ generally the builder name, such as Linux32DBG.
+
+ Returns:
+ The most recent directory name.
+
+ Raises:
+ CouldNotFindCoverageDirectory: if we failed to find coverage data.
"""
found_build_numbers = []
for entry in www_directory_contents:
- match = re.match('Linux32DBG_(\d+)', entry)
+ match = re.match(directory_prefix + '(\d+)', entry)
if match is not None:
found_build_numbers.append(int(match.group(1)))
if not found_build_numbers:
- raise CouldNotFindCoverageDirectory('Error: Found no 32-bit '
- 'debug build in directory %s.' %
- coverage_www_dir)
+ raise CouldNotFindCoverageDirectory('Error: Found no directories %s* '
+ 'in directory %s.' %
+ (directory_prefix, coverage_www_dir))
most_recent = max(found_build_numbers)
- return 'Linux32DBG_' + str(most_recent)
+ return directory_prefix + str(most_recent)
def _grab_coverage_percentage(label, index_html_contents):
@@ -84,25 +96,37 @@
raise FailedToParseCoverageHtml('%s is not a float.' % match.group(1))
-def _report_coverage_to_dashboard(dashboard, now, line_coverage,
- function_coverage):
- parameters = {'date': '%d' % now,
- 'line_coverage': '%f' % line_coverage,
- 'function_coverage': '%f' % function_coverage
+def _report_coverage_to_dashboard(dashboard, line_coverage, function_coverage,
+ branch_coverage, report_category):
+ parameters = {'line_coverage': '%f' % line_coverage,
+ 'function_coverage': '%f' % function_coverage,
+ 'branch_coverage': '%f' % branch_coverage,
+ 'report_category': report_category,
}
dashboard.send_post_request(constants.ADD_COVERAGE_DATA_URL, parameters)
-def _main():
+def _main(report_category, directory_prefix):
+ """Grabs coverage data from disk on a bot and publishes it.
+
+ Args:
+ report_category: The kind of coverage to report. The dashboard
+ application decides what is acceptable here (see
+ dashboard/add_coverage_data.py for more information).
+ directory_prefix: This bot's coverage directory prefix. Generally a bot's
+ coverage directories will have the form <prefix><build number>,
+ like Linux32DBG_345.
+ """
dashboard = dashboard_connection.DashboardConnection(constants.CONSUMER_KEY)
dashboard.read_required_files(constants.CONSUMER_SECRET_FILE,
constants.ACCESS_TOKEN_FILE)
coverage_www_dir = constants.BUILD_BOT_COVERAGE_WWW_DIRECTORY
www_dir_contents = os.listdir(coverage_www_dir)
- latest_build_directory = _find_latest_32bit_debug_build(www_dir_contents,
- coverage_www_dir)
+ latest_build_directory = _find_latest_build_coverage(www_dir_contents,
+ coverage_www_dir,
+ directory_prefix)
index_html_path = os.path.join(coverage_www_dir, latest_build_directory,
'index.html')
@@ -111,12 +135,26 @@
line_coverage = _grab_coverage_percentage('Lines:', whole_file)
function_coverage = _grab_coverage_percentage('Functions:', whole_file)
- now = int(time.time())
+ branch_coverage = _grab_coverage_percentage('Branches:', whole_file)
- _report_coverage_to_dashboard(dashboard, now, line_coverage,
- function_coverage)
+ _report_coverage_to_dashboard(dashboard, line_coverage, function_coverage,
+ branch_coverage, report_category)
+
+
+def _parse_args():
+ if len(sys.argv) != 3:
+ print ('Usage: %s <coverage category> <directory prefix>\n\n'
+ 'The coverage category describes the kind of coverage you are '
+ 'uploading. Known acceptable values are small_medium_tests and'
+ 'large_tests. The directory prefix is what the directories in %s '
+ 'are prefixed on this bot (such as Linux32DBG_).' %
+ (sys.argv[0], constants.BUILD_BOT_COVERAGE_WWW_DIRECTORY))
+ return (None, None)
+ return (sys.argv[1], sys.argv[2])
if __name__ == '__main__':
- _main()
+ report_category, directory_prefix = _parse_args()
+ if report_category:
+ _main(report_category, directory_prefix)
