Completed implementation of oauth in coverage scripts.
The access token is now transferred between the request and track scripts.
Almost completed the implementation, using oauth2.
Initial, incomplete implementation of the permission-request script.
The coverage tracking is broken temporarily, but it can now make OAuth requests.
BUG=
TEST=
Review URL: https://webrtc-codereview.appspot.com/366002
git-svn-id: http://webrtc.googlecode.com/svn/trunk@1530 4adac7df-926f-26a2-2b94-8c16560cd09d
diff --git a/.gitignore b/.gitignore
index 23651e6..7fc2525 100644
--- a/.gitignore
+++ b/.gitignore
@@ -49,6 +49,7 @@
/third_party/libvpx
/third_party/libyuv
/third_party/llvm-build
+/third_party/oauth2
/third_party/protobuf
/third_party/valgrind
/third_party/yasm
diff --git a/DEPS b/DEPS
index 6b7c07e..31fdfd0 100644
--- a/DEPS
+++ b/DEPS
@@ -73,6 +73,7 @@
"trunk/third_party/yasm/source/patched-yasm":
Var("chromium_trunk") + "/deps/third_party/yasm/patched-yasm@73761",
+
# Used by libjpeg-turbo
"trunk/third_party/yasm/binaries":
Var("chromium_trunk") + "/deps/third_party/yasm/binaries@74228",
@@ -82,9 +83,14 @@
"trunk/third_party/libyuv":
(Var("googlecode_url") % "libyuv") + "/trunk@121",
-
+
+ # Used by tools/coverage/dashboard and tools/python_charts
"trunk/third_party/google-visualization-python":
(Var("googlecode_url") % "google-visualization-python") + "/trunk@15",
+
+ # Used by tools/coverage
+ "trunk/third_party/oauth2":
+ "https://github.com/simplegeo/python-oauth2.git@a83f4a297336b631e75cba102910c19231518159"
}
deps_os = {
diff --git a/tools/coverage/README b/tools/coverage/README
index 12843fb..faf3e7a 100644
--- a/tools/coverage/README
+++ b/tools/coverage/README
@@ -7,10 +7,18 @@
and extract the data from there. The dashboard doesn't care how often this
script runs, but running each hour should be more than enough.
-The track_coverage.py script communicates with the dashboard using plain GET
-requests (that, and POST, are basically the only way to get data into a
-appengine application such as the dashboard). The dashboard is intented to
-run on the Google appengine.
+The track_coverage.py script uses OAuth to authenticate itself. In order to do
+this, it needs two files: consumer.secret and access.token. The consumer secret
+is known within the organization and is stored in a plain file on the bot
+running the scripts (we don't want to check in this secret in the code in the
+public repository). The consumer secret is a plain file with a single line
+containing the secret string.
+
+The access.token file is generated by request_oauth_permission.py. It does this
+by going through the three-legged OAuth authorization process. An administrator
+of the dashboard must approve the request from the script. Once that is done,
+access.token will be written and track_coverage.py will be able to report
+results.
HOW TO RUN LOCALLY:
Follow the following instructions:
diff --git a/tools/coverage/dashboard/dashboard.py b/tools/coverage/dashboard/dashboard.py
index 3283438..3d82e23 100644
--- a/tools/coverage/dashboard/dashboard.py
+++ b/tools/coverage/dashboard/dashboard.py
@@ -12,11 +12,18 @@
__author__ = 'phoglund@webrtc.org (Patrik Hoglund)'
import datetime
+from google.appengine.api import oauth
+from google.appengine.api import users
from google.appengine.ext import db
import webapp2
import gviz_api
+class UserNotAuthenticatedException(Exception):
+ """Gets thrown if a user is not permitted to store coverage data."""
+ pass
+
+
class CoverageData(db.Model):
"""This represents one coverage report from the build bot."""
date = db.DateTimeProperty(required=True)
@@ -40,8 +47,8 @@
page_template = template_file.read()
template_file.close()
except IOError as exception:
- self.ShowErrorPage('Cannot open page template file: %s<br>Details: %s' %
- (page_template_filename, exception))
+ self._show_error_page('Cannot open page template file: %s<br>Details: %s'
+ % (page_template_filename, exception))
return
coverage_entries = db.GqlQuery('SELECT * '
@@ -65,18 +72,24 @@
# Fill in the template with the data and respond:
self.response.write(page_template % vars())
- def ShowErrorPage(self, error_message):
+ def _show_error_page(self, error_message):
self.response.write('<html><body>%s</body></html>' % error_message)
class AddCoverageData(webapp2.RequestHandler):
"""Used to report coverage data.
- It will verify the data, but not the sender. Thus, it should be secured
- more properly if accessible from an outside network.
+ The user is required to have obtained an OAuth access token from an
+ administrator for this application earlier.
"""
- def get(self):
+ def post(self):
+ try:
+ self._authenticate_user()
+ except UserNotAuthenticatedException as exception:
+ self._show_error_page('Failed to authenticate user: %s' % exception)
+ return
+
try:
posix_time = int(self.request.get('date'))
parsed_date = datetime.datetime.fromtimestamp(posix_time)
@@ -84,8 +97,8 @@
line_coverage = float(self.request.get('line_coverage'))
function_coverage = float(self.request.get('function_coverage'))
except ValueError as exception:
- self.ShowErrorPage('Invalid parameter in request. Details: %s' %
- exception)
+ self._show_error_page('Invalid parameter in request. Details: %s' %
+ exception)
return
item = CoverageData(date=parsed_date,
@@ -93,7 +106,22 @@
function_coverage=function_coverage)
item.put()
- def ShowErrorPage(self, error_message):
+ def _authenticate_user(self):
+ try:
+ if oauth.is_current_user_admin():
+ # The user on whose behalf we are acting is indeed an administrator
+ # of this application, so we're good to go.
+ return
+ else:
+ raise UserNotAuthenticatedException('We are acting on behalf of '
+ 'user %s, but that user is not '
+ 'an administrator.' %
+ oauth.get_current_user())
+ except oauth.OAuthRequestError as exception:
+ raise UserNotAuthenticatedException('Invalid OAuth request: %s' %
+ exception)
+
+ def _show_error_page(self, error_message):
self.response.write('<html><body>%s</body></html>' % error_message)
app = webapp2.WSGIApplication([('/', ShowDashboard),
diff --git a/tools/coverage/oauth2 b/tools/coverage/oauth2
new file mode 120000
index 0000000..a4d1dde
--- /dev/null
+++ b/tools/coverage/oauth2
@@ -0,0 +1 @@
+../../third_party/oauth2/oauth2/
\ No newline at end of file
diff --git a/tools/coverage/request_oauth_permission.py b/tools/coverage/request_oauth_permission.py
new file mode 100755
index 0000000..bad0902
--- /dev/null
+++ b/tools/coverage/request_oauth_permission.py
@@ -0,0 +1,135 @@
+#!/usr/bin/env python
+#-*- coding: utf-8 -*-
+# Copyright (c) 2012 The WebRTC project authors. All Rights Reserved.
+#
+# Use of this source code is governed by a BSD-style license
+# that can be found in the LICENSE file in the root of the source
+# tree. An additional intellectual property rights grant can be found
+# in the file PATENTS. All contributing project authors may
+# be found in the AUTHORS file in the root of the source tree.
+
+"""This script request an access token from the appengine running the dashboard.
+
+ The script is intended to be run manually whenever we wish to change which
+ dashboard administrator we act on behalf of when running the
+ track_coverage.py script. For example, this will be useful if the current
+ dashboard administrator leaves the project.
+
+ This script should be run on the build bot which runs the track_coverage.py
+ script. This script will present a link during its execution, which the new
+ administrator should follow and then click approve on the web page that
+ appears. The new administrator should have admin rights on the coverage
+ dashboard, otherwise the track_coverage.py will not work.
+
+ If successful, this script will write the access token to a file access.token
+ in the current directory, which later can be read by track_coverage.py.
+ The token is stored in string form (as reported by the web server) using the
+ shelve module.
+"""
+
+__author__ = 'phoglund@webrtc.org (Patrik Höglund)'
+
+import shelve
+import sys
+import urlparse
+import oauth2 as oauth
+
+
+class FailedToRequestPermissionException(Exception):
+ pass
+
+
+# This identifies our application using the information we got when we
+# registered the application on Google appengine.
+# TODO(phoglund): update to the right value when we have registered the app.
+DASHBOARD_SERVER = 'http://127.0.0.1:8080'
+CONSUMER_KEY = DASHBOARD_SERVER
+
+REQUEST_TOKEN_URL = DASHBOARD_SERVER + '/_ah/OAuthGetRequestToken'
+AUTHORIZE_TOKEN_URL = DASHBOARD_SERVER + '/_ah/OAuthAuthorizeToken'
+ACCESS_TOKEN_URL = DASHBOARD_SERVER + '/_ah/OAuthGetAccessToken'
+
+
+def _ensure_token_response_is_200(response, queried_url, token_type):
+ if response.status != 200:
+ raise FailedToRequestPermissionException('Failed to request %s from %s: '
+ 'received status %d, reason %s.' %
+ (token_type,
+ queried_url,
+ response.status,
+ response.reason))
+
+def _request_unauthorized_token(consumer, request_token_url):
+ """Requests the initial token from the dashboard service.
+
+ Given that the response from the server is correct, we will return a
+ dictionary containing oauth_token and oauth_token_secret mapped to the
+ token and secret value, respectively.
+ """
+ client = oauth.Client(consumer)
+
+ try:
+ response, content = client.request(request_token_url, 'POST')
+ except AttributeError as error:
+ # This catch handler is here since we'll get very confusing messages
+ # if the target server is down for some reason.
+ raise FailedToRequestPermissionException("Failed to request token: "
+ "the dashboard is likely down.",
+ error)
+
+ _ensure_token_response_is_200(response, request_token_url,
+ "unauthorized token")
+
+ return dict(urlparse.parse_qsl(content))
+
+
+def _ask_user_to_authorize_us(unauthorized_token):
+ """This function will block until the user enters y + newline."""
+ print 'Go to the following link in your browser:'
+ print '%s?oauth_token=%s' % (AUTHORIZE_TOKEN_URL,
+ unauthorized_token['oauth_token'])
+
+ accepted = 'n'
+ while accepted.lower() != 'y':
+ accepted = raw_input('Have you authorized me yet? (y/n) ')
+
+
+def _request_access_token(consumer, unauthorized_token):
+ token = oauth.Token(unauthorized_token['oauth_token'],
+ unauthorized_token['oauth_token_secret'])
+ client = oauth.Client(consumer, token)
+ response, content = client.request(ACCESS_TOKEN_URL, 'POST')
+
+ _ensure_token_response_is_200(response, ACCESS_TOKEN_URL, "access token")
+
+ return content
+
+
+def _write_access_token_to_file(access_token, filename):
+ output = shelve.open(filename)
+ output['access_token'] = access_token
+ output.close()
+
+ print 'Wrote the access token to the file %s.' % filename
+
+
+def _main():
+ if len(sys.argv) != 2:
+ print ('Usage: %s <consumer secret>.\n\nThe consumer secret is an OAuth '
+ 'concept and is obtained from the appengine running the dashboard.' %
+ sys.argv[0])
+ return
+
+ consumer_secret = sys.argv[1]
+ consumer = oauth.Consumer(CONSUMER_KEY, consumer_secret)
+
+ unauthorized_token = _request_unauthorized_token(consumer, REQUEST_TOKEN_URL)
+
+ _ask_user_to_authorize_us(unauthorized_token)
+
+ access_token_string = _request_access_token(consumer, unauthorized_token)
+
+ _write_access_token_to_file(access_token_string, 'access.token')
+
+if __name__ == '__main__':
+ _main()
diff --git a/tools/coverage/track_coverage.py b/tools/coverage/track_coverage.py
index 34cf125..8bc9f37 100755
--- a/tools/coverage/track_coverage.py
+++ b/tools/coverage/track_coverage.py
@@ -13,6 +13,15 @@
It grabs coverage information from the latest Linux 32-bit build and
pushes it to the coverage tracker, enabling us to track code coverage
over time. This script is intended to run on the 32-bit Linux slave.
+
+ This script requires an access.token file in the current directory, as
+ generated by the request_oauth_permission.py script. It also expects a file
+ customer.secret with a single line containing the customer secret. The
+ customer secret is an OAuth concept and is received when one registers the
+ application with the appengine running the dashboard.
+
+ The script assumes that all coverage data is stored under
+ /home/<build bot user>/www.
"""
__author__ = 'phoglund@webrtc.org (Patrik Höglund)'
@@ -20,8 +29,10 @@
import httplib
import os
import re
+import shelve
import sys
import time
+import oauth.oauth as oauth
# The build-bot user which runs build bot jobs.
BUILD_BOT_USER = 'phoglund'
@@ -29,6 +40,8 @@
# The server to send coverage data to.
# TODO(phoglund): replace with real server once we get it up.
DASHBOARD_SERVER = 'localhost:8080'
+CONSUMER_KEY = DASHBOARD_SERVER
+ADD_COVERAGE_DATA_URL = '/add_coverage_data'
class FailedToParseCoverageHtml(Exception):
@@ -39,6 +52,35 @@
pass
+class FailedToReadRequiredInputFile(Exception):
+ pass
+
+
+def _read_access_token_from_file(filename):
+ input_file = shelve.open(filename)
+
+ if not input_file.has_key('access_token'):
+ raise FailedToReadRequiredInputFile('Missing %s file in current directory. '
+ 'You may have to run '
+ 'request_oauth_permission.py.')
+
+ return oauth.OAuthToken.from_string(input_file['access_token'])
+
+
+def _read_customer_secret_from_file(filename):
+ try:
+ input_file = open(filename, 'r')
+ except IOError as error:
+ raise FailedToReadRequiredInputFile(error)
+
+ whole_file = input_file.read()
+ if '\n' in whole_file or not whole_file.strip():
+ raise FailedToReadRequiredInputFile('Expected a single line with the '
+ 'customer secret in file %s.' %
+ filename)
+ return whole_file
+
+
def _find_latest_32bit_debug_build(www_directory_contents):
# Build directories have the form Linux32bitDebug_<number>. There may be other
# directories in the list though, for instance for other build configurations.
@@ -73,13 +115,30 @@
raise FailedToParseCoverageHtml('%s is not a float.' % match.group(1))
-def _report_coverage_to_dashboard(now, line_coverage, function_coverage):
- request_string = ('/add_coverage_data?'
- 'date=%d&line_coverage=%f&function_coverage=%f' %
- (now, line_coverage, function_coverage))
+def _report_coverage_to_dashboard(now, line_coverage, function_coverage,
+ access_token, consumer_key, consumer_secret):
+ parameters = {'date': '%d' % now,
+ 'line_coverage': '%f' % line_coverage,
+ 'function_coverage': '%f' % function_coverage
+ }
+ consumer = oauth.OAuthConsumer(consumer_key, consumer_secret)
+ create_oauth_request = oauth.OAuthRequest.from_consumer_and_token
+ oauth_request = create_oauth_request(consumer,
+ token=access_token,
+ http_method='POST',
+ http_url=ADD_COVERAGE_DATA_URL,
+ parameters=parameters)
+
+ signature_method_hmac_sha1 = oauth.OAuthSignatureMethod_HMAC_SHA1()
+ oauth_request.sign_request(signature_method_hmac_sha1, consumer, access_token)
connection = httplib.HTTPConnection(DASHBOARD_SERVER)
- connection.request('GET', request_string)
+
+ headers = {'Content-Type': 'application/x-www-form-urlencoded'}
+ connection.request('POST', ADD_COVERAGE_DATA_URL,
+ body=oauth_request.to_postdata(),
+ headers=headers)
+
response = connection.getresponse()
if response.status != 200:
message = ('Error: Failed to report to %s%s: got response %d (%s)' %
@@ -96,6 +155,9 @@
def _main():
+ access_token = _read_access_token_from_file('access.token')
+ customer_secret = _read_customer_secret_from_file('customer.secret')
+
coverage_www_dir = os.path.join('/home', BUILD_BOT_USER, 'www')
www_dir_contents = os.listdir(coverage_www_dir)
@@ -114,7 +176,8 @@
function_coverage = _grab_coverage_percentage('Functions:', whole_file)
now = int(time.time())
- _report_coverage_to_dashboard(now, line_coverage, function_coverage)
+ _report_coverage_to_dashboard(now, line_coverage, function_coverage,
+ access_token, CONSUMER_KEY, customer_secret)
if __name__ == '__main__':
_main()
