| #!/usr/bin/env python |
| # |
| # Copyright 2007 Google Inc. |
| # |
| # Licensed under the Apache License, Version 2.0 (the "License"); |
| # you may not use this file except in compliance with the License. |
| # You may obtain a copy of the License at |
| # |
| # http://www.apache.org/licenses/LICENSE-2.0 |
| # |
| # Unless required by applicable law or agreed to in writing, software |
| # distributed under the License is distributed on an "AS IS" BASIS, |
| # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| # See the License for the specific language governing permissions and |
| # limitations under the License. |
| # |
| |
| """Tool for performing authenticated RPCs against App Engine.""" |
| |
| |
| import cookielib |
| import logging |
| import os |
| import re |
| import socket |
| import sys |
| import urllib |
| import urllib2 |
| |
| |
| https_handler = urllib2.HTTPSHandler |
| uses_cert_verification = False |
| certpath = os.path.join(os.path.dirname(__file__), "cacerts.txt") |
| cert_file_available = os.path.exists(certpath) |
| try: |
| import https_wrapper |
| if cert_file_available: |
| https_handler = lambda: https_wrapper.CertValidatingHTTPSHandler( |
| ca_certs=certpath) |
| uses_cert_verification = True |
| except ImportError: |
| pass |
| |
| logger = logging.getLogger('google.appengine.tools.appengine_rpc') |
| |
| def GetPlatformToken(os_module=os, sys_module=sys, platform=sys.platform): |
| """Returns a 'User-agent' token for the host system platform. |
| |
| Args: |
| os_module, sys_module, platform: Used for testing. |
| |
| Returns: |
| String containing the platform token for the host system. |
| """ |
| if hasattr(sys_module, "getwindowsversion"): |
| windows_version = sys_module.getwindowsversion() |
| version_info = ".".join(str(i) for i in windows_version[:4]) |
| return platform + "/" + version_info |
| elif hasattr(os_module, "uname"): |
| uname = os_module.uname() |
| return "%s/%s" % (uname[0], uname[2]) |
| else: |
| return "unknown" |
| |
| def HttpRequestToString(req, include_data=True): |
| """Converts a urllib2.Request to a string. |
| |
| Args: |
| req: urllib2.Request |
| Returns: |
| Multi-line string representing the request. |
| """ |
| |
| headers = "" |
| for header in req.header_items(): |
| headers += "%s: %s\n" % (header[0], header[1]) |
| |
| template = ("%(method)s %(selector)s %(type)s/1.1\n" |
| "Host: %(host)s\n" |
| "%(headers)s") |
| if include_data: |
| template = template + "\n%(data)s" |
| |
| return template % { |
| 'method' : req.get_method(), |
| 'selector' : req.get_selector(), |
| 'type' : req.get_type().upper(), |
| 'host' : req.get_host(), |
| 'headers': headers, |
| 'data': req.get_data(), |
| } |
| |
| class ClientLoginError(urllib2.HTTPError): |
| """Raised to indicate there was an error authenticating with ClientLogin.""" |
| |
| def __init__(self, url, code, msg, headers, args): |
| urllib2.HTTPError.__init__(self, url, code, msg, headers, None) |
| self.args = args |
| self.reason = args["Error"] |
| |
| def read(self): |
| return '%d %s: %s' % (self.code, self.msg, self.reason) |
| |
| |
| class AbstractRpcServer(object): |
| """Provides a common interface for a simple RPC server.""" |
| |
| def __init__(self, host, auth_function, user_agent, source, |
| host_override=None, extra_headers=None, save_cookies=False, |
| auth_tries=3, account_type=None, debug_data=True, secure=False): |
| """Creates a new HttpRpcServer. |
| |
| Args: |
| host: The host to send requests to. |
| auth_function: A function that takes no arguments and returns an |
| (email, password) tuple when called. Will be called if authentication |
| is required. |
| user_agent: The user-agent string to send to the server. Specify None to |
| omit the user-agent header. |
| source: The source to specify in authentication requests. |
| host_override: The host header to send to the server (defaults to host). |
| extra_headers: A dict of extra headers to append to every request. Values |
| supplied here will override other default headers that are supplied. |
| save_cookies: If True, save the authentication cookies to local disk. |
| If False, use an in-memory cookiejar instead. Subclasses must |
| implement this functionality. Defaults to False. |
| auth_tries: The number of times to attempt auth_function before failing. |
| account_type: One of GOOGLE, HOSTED_OR_GOOGLE, or None for automatic. |
| debug_data: Whether debugging output should include data contents. |
| """ |
| if secure: |
| self.scheme = "https" |
| else: |
| self.scheme = "http" |
| self.host = host |
| self.host_override = host_override |
| self.auth_function = auth_function |
| self.source = source |
| self.authenticated = False |
| self.auth_tries = auth_tries |
| self.debug_data = debug_data |
| |
| self.account_type = account_type |
| |
| self.extra_headers = {} |
| if user_agent: |
| self.extra_headers["User-Agent"] = user_agent |
| if extra_headers: |
| self.extra_headers.update(extra_headers) |
| |
| self.save_cookies = save_cookies |
| self.cookie_jar = cookielib.MozillaCookieJar() |
| self.opener = self._GetOpener() |
| if self.host_override: |
| logger.info("Server: %s; Host: %s", self.host, self.host_override) |
| else: |
| logger.info("Server: %s", self.host) |
| |
| if ((self.host_override and self.host_override == "localhost") or |
| self.host == "localhost" or self.host.startswith("localhost:")): |
| self._DevAppServerAuthenticate() |
| |
| def _GetOpener(self): |
| """Returns an OpenerDirector for making HTTP requests. |
| |
| Returns: |
| A urllib2.OpenerDirector object. |
| """ |
| raise NotImplemented() |
| |
| def _CreateRequest(self, url, data=None): |
| """Creates a new urllib request.""" |
| req = urllib2.Request(url, data=data) |
| if self.host_override: |
| req.add_header("Host", self.host_override) |
| for key, value in self.extra_headers.iteritems(): |
| req.add_header(key, value) |
| return req |
| |
| def _GetAuthToken(self, email, password): |
| """Uses ClientLogin to authenticate the user, returning an auth token. |
| |
| Args: |
| email: The user's email address |
| password: The user's password |
| |
| Raises: |
| ClientLoginError: If there was an error authenticating with ClientLogin. |
| HTTPError: If there was some other form of HTTP error. |
| |
| Returns: |
| The authentication token returned by ClientLogin. |
| """ |
| account_type = self.account_type |
| if not account_type: |
| if (self.host.split(':')[0].endswith(".google.com") |
| or (self.host_override |
| and self.host_override.split(':')[0].endswith(".google.com"))): |
| account_type = "HOSTED_OR_GOOGLE" |
| else: |
| account_type = "GOOGLE" |
| data = { |
| "Email": email, |
| "Passwd": password, |
| "service": "ah", |
| "source": self.source, |
| "accountType": account_type |
| } |
| |
| req = self._CreateRequest( |
| url="https://www.google.com/accounts/ClientLogin", |
| data=urllib.urlencode(data)) |
| try: |
| response = self.opener.open(req) |
| response_body = response.read() |
| response_dict = dict(x.split("=") |
| for x in response_body.split("\n") if x) |
| return response_dict["Auth"] |
| except urllib2.HTTPError, e: |
| if e.code == 403: |
| body = e.read() |
| response_dict = dict(x.split("=", 1) for x in body.split("\n") if x) |
| raise ClientLoginError(req.get_full_url(), e.code, e.msg, |
| e.headers, response_dict) |
| else: |
| raise |
| |
| def _GetAuthCookie(self, auth_token): |
| """Fetches authentication cookies for an authentication token. |
| |
| Args: |
| auth_token: The authentication token returned by ClientLogin. |
| |
| Raises: |
| HTTPError: If there was an error fetching the authentication cookies. |
| """ |
| continue_location = "http://localhost/" |
| args = {"continue": continue_location, "auth": auth_token} |
| login_path = os.environ.get("APPCFG_LOGIN_PATH", "/_ah") |
| req = self._CreateRequest("%s://%s%s/login?%s" % |
| (self.scheme, self.host, login_path, |
| urllib.urlencode(args))) |
| try: |
| response = self.opener.open(req) |
| except urllib2.HTTPError, e: |
| response = e |
| if (response.code != 302 or |
| response.info()["location"] != continue_location): |
| raise urllib2.HTTPError(req.get_full_url(), response.code, response.msg, |
| response.headers, response.fp) |
| self.authenticated = True |
| |
| def _Authenticate(self): |
| """Authenticates the user. |
| |
| The authentication process works as follows: |
| 1) We get a username and password from the user |
| 2) We use ClientLogin to obtain an AUTH token for the user |
| (see http://code.google.com/apis/accounts/AuthForInstalledApps.html). |
| 3) We pass the auth token to /_ah/login on the server to obtain an |
| authentication cookie. If login was successful, it tries to redirect |
| us to the URL we provided. |
| |
| If we attempt to access the upload API without first obtaining an |
| authentication cookie, it returns a 401 response and directs us to |
| authenticate ourselves with ClientLogin. |
| """ |
| for unused_i in range(self.auth_tries): |
| credentials = self.auth_function() |
| try: |
| auth_token = self._GetAuthToken(credentials[0], credentials[1]) |
| except ClientLoginError, e: |
| if e.reason == "BadAuthentication": |
| print >>sys.stderr, "Invalid username or password." |
| continue |
| if e.reason == "CaptchaRequired": |
| print >>sys.stderr, ( |
| "Please go to\n" |
| "https://www.google.com/accounts/DisplayUnlockCaptcha\n" |
| "and verify you are a human. Then try again.") |
| break |
| if e.reason == "NotVerified": |
| print >>sys.stderr, "Account not verified." |
| break |
| if e.reason == "TermsNotAgreed": |
| print >>sys.stderr, "User has not agreed to TOS." |
| break |
| if e.reason == "AccountDeleted": |
| print >>sys.stderr, "The user account has been deleted." |
| break |
| if e.reason == "AccountDisabled": |
| print >>sys.stderr, "The user account has been disabled." |
| break |
| if e.reason == "ServiceDisabled": |
| print >>sys.stderr, ("The user's access to the service has been " |
| "disabled.") |
| break |
| if e.reason == "ServiceUnavailable": |
| print >>sys.stderr, "The service is not available; try again later." |
| break |
| raise |
| self._GetAuthCookie(auth_token) |
| return |
| |
| def _DevAppServerAuthenticate(self): |
| """Authenticates the user on the dev_appserver.""" |
| credentials = self.auth_function() |
| self.extra_headers["Cookie"] = ('dev_appserver_login="%s:True"; Path=/;' % |
| (credentials[0],)) |
| |
| def Send(self, request_path, payload="", |
| content_type="application/octet-stream", |
| timeout=None, |
| **kwargs): |
| """Sends an RPC and returns the response. |
| |
| Args: |
| request_path: The path to send the request to, eg /api/appversion/create. |
| payload: The body of the request, or None to send an empty request. |
| content_type: The Content-Type header to use. |
| timeout: timeout in seconds; default None i.e. no timeout. |
| (Note: for large requests on OS X, the timeout doesn't work right.) |
| kwargs: Any keyword arguments are converted into query string parameters. |
| |
| Returns: |
| The response body, as a string. |
| """ |
| old_timeout = socket.getdefaulttimeout() |
| socket.setdefaulttimeout(timeout) |
| try: |
| tries = 0 |
| auth_tried = False |
| while True: |
| tries += 1 |
| args = dict(kwargs) |
| url = "%s://%s%s?%s" % (self.scheme, self.host, request_path, |
| urllib.urlencode(args)) |
| req = self._CreateRequest(url=url, data=payload) |
| req.add_header("Content-Type", content_type) |
| req.add_header("X-appcfg-api-version", "1") |
| try: |
| logger.debug('Sending HTTP request:\n%s' % |
| HttpRequestToString(req, include_data=self.debug_data)) |
| f = self.opener.open(req) |
| response = f.read() |
| f.close() |
| return response |
| except urllib2.HTTPError, e: |
| logger.debug("Got http error, this is try #%s" % tries) |
| if tries > self.auth_tries: |
| raise |
| elif e.code == 401: |
| if auth_tried: |
| raise |
| auth_tried = True |
| self._Authenticate() |
| elif e.code >= 500 and e.code < 600: |
| continue |
| elif e.code == 302: |
| if auth_tried: |
| raise |
| auth_tried = True |
| loc = e.info()["location"] |
| logger.debug("Got 302 redirect. Location: %s" % loc) |
| if loc.startswith("https://www.google.com/accounts/ServiceLogin"): |
| self._Authenticate() |
| elif re.match(r"https://www.google.com/a/[a-z0-9.-]+/ServiceLogin", |
| loc): |
| self.account_type = "HOSTED" |
| self._Authenticate() |
| elif loc.startswith("http://%s/_ah/login" % (self.host,)): |
| self._DevAppServerAuthenticate() |
| else: |
| raise |
| finally: |
| socket.setdefaulttimeout(old_timeout) |
| |
| |
| class HttpRpcServer(AbstractRpcServer): |
| """Provides a simplified RPC-style interface for HTTP requests.""" |
| |
| DEFAULT_COOKIE_FILE_PATH = "~/.appcfg_cookies" |
| |
| def _Authenticate(self): |
| """Save the cookie jar after authentication.""" |
| if cert_file_available and not uses_cert_verification: |
| logger.warn("ssl module not found. Without this the identity of the " |
| "remote host cannot be verified, and connections are NOT " |
| "secure. To fix this, please install the ssl module from " |
| "http://pypi.python.org/pypi/ssl") |
| super(HttpRpcServer, self)._Authenticate() |
| if self.cookie_jar.filename is not None and self.save_cookies: |
| logger.info("Saving authentication cookies to %s" % |
| self.cookie_jar.filename) |
| self.cookie_jar.save() |
| |
| def _GetOpener(self): |
| """Returns an OpenerDirector that supports cookies and ignores redirects. |
| |
| Returns: |
| A urllib2.OpenerDirector object. |
| """ |
| opener = urllib2.OpenerDirector() |
| opener.add_handler(urllib2.ProxyHandler()) |
| opener.add_handler(urllib2.UnknownHandler()) |
| opener.add_handler(urllib2.HTTPHandler()) |
| opener.add_handler(urllib2.HTTPDefaultErrorHandler()) |
| opener.add_handler(https_handler()) |
| opener.add_handler(urllib2.HTTPErrorProcessor()) |
| |
| if self.save_cookies: |
| self.cookie_jar.filename = os.path.expanduser( |
| HttpRpcServer.DEFAULT_COOKIE_FILE_PATH) |
| |
| if os.path.exists(self.cookie_jar.filename): |
| try: |
| self.cookie_jar.load() |
| self.authenticated = True |
| logger.info("Loaded authentication cookies from %s" % |
| self.cookie_jar.filename) |
| except (OSError, IOError, cookielib.LoadError), e: |
| logger.debug("Could not load authentication cookies; %s: %s", |
| e.__class__.__name__, e) |
| self.cookie_jar.filename = None |
| else: |
| try: |
| fd = os.open(self.cookie_jar.filename, os.O_CREAT, 0600) |
| os.close(fd) |
| except (OSError, IOError), e: |
| logger.debug("Could not create authentication cookies file; %s: %s", |
| e.__class__.__name__, e) |
| self.cookie_jar.filename = None |
| |
| opener.add_handler(urllib2.HTTPCookieProcessor(self.cookie_jar)) |
| return opener |
