Adding server mode

moto/core/models.py

@@ -2,6 +2,7 @@ import functools
 import re
 
 from moto.packages.httpretty import HTTPretty
+from .utils import convert_regex_to_flask_path
 
 
 class MockAWS(object):
@@ -48,13 +49,56 @@ class BaseBackend(object):
         self.__init__()
 
     @property
-    def urls(self):
+    def _url_module(self):
         backend_module = self.__class__.__module__
         backend_urls_module_name = backend_module.replace("models", "urls")
-        backend_urls_module = __import__(backend_urls_module_name, fromlist=['urls'])
-        urls = backend_urls_module.urls
+        backend_urls_module = __import__(backend_urls_module_name, fromlist=['url_bases', 'url_paths'])
+        return backend_urls_module
+
+    @property
+    def urls(self):
+        """
+        A dictionary of the urls to be mocked with this service and the handlers
+        that should be called in their place
+        """
+        url_bases = self._url_module.url_bases
+        unformatted_paths = self._url_module.url_paths
+
+        urls = {}
+        for url_base in url_bases:
+            for url_path, handler in unformatted_paths.iteritems():
+                url = url_path.format(url_base)
+                urls[url] = handler
+
         return urls
 
+    @property
+    def url_paths(self):
+        """
+        A dictionary of the paths of the urls to be mocked with this service and
+        the handlers that should be called in their place
+        """
+        unformatted_paths = self._url_module.url_paths
+
+        paths = {}
+        for unformatted_path, handler in unformatted_paths.iteritems():
+            path = unformatted_path.format("")
+            paths[path] = handler
+
+        return paths
+
+    @property
+    def flask_paths(self):
+        """
+        The url paths that will be used for the flask server
+        """
+        paths = {}
+        for url_path, handler in self.url_paths.iteritems():
+            url_path = convert_regex_to_flask_path(url_path)
+            paths[url_path] = handler
+
+        return paths
+
     def decorator(self, func=None):
         if func:
             return MockAWS(self)(func)

moto/core/responses.py

@@ -4,6 +4,9 @@ from moto.core.utils import headers_to_dict, camelcase_to_underscores, method_na
 
 
 class BaseResponse(object):
+    def dispatch2(self, uri, body, headers):
+        return self.dispatch(uri, body, headers)
+
     def dispatch(self, uri, body, headers):
         if body:
             querystring = parse_qs(body)
@@ -13,7 +16,7 @@ class BaseResponse(object):
         self.path = uri.path
         self.querystring = querystring
 
-        action = querystring['Action'][0]
+        action = querystring.get('Action', [""])[0]
         action = camelcase_to_underscores(action)
 
         method_names = method_names_from_class(self.__class__)

moto/core/utils.py

@@ -1,9 +1,17 @@
+from collections import namedtuple
 import inspect
 import random
+import re
 from urlparse import parse_qs
 
+from flask import request
+
 
 def headers_to_dict(headers):
+    if isinstance(headers, dict):
+        # If already dict, return
+        return headers
+
     result = {}
     for index, header in enumerate(headers.split("\r\n")):
         if not header:
@@ -51,3 +59,55 @@ def get_random_hex(length=8):
 
 def get_random_message_id():
     return '{}-{}-{}-{}-{}'.format(get_random_hex(8), get_random_hex(4), get_random_hex(4), get_random_hex(4), get_random_hex(12))
+
+
+def convert_regex_to_flask_path(url_path):
+    """
+    Converts a regex matching url to one that can be used with flask
+    """
+    for token in ["$"]:
+        url_path = url_path.replace(token, "")
+
+    def caller(reg):
+        match_name, match_pattern = reg.groups()
+        return '<regex("{0}"):{1}>'.format(match_pattern, match_name)
+
+    url_path = re.sub("\(\?P<(.*?)>(.*?)\)", caller, url_path)
+    return url_path
+
+
+class convert_flask_to_httpretty_response(object):
+    def __init__(self, callback):
+        self.callback = callback
+
+    @property
+    def __name__(self):
+        # For instance methods, use class and method names. Otherwise
+        # use module and method name
+        if inspect.ismethod(self.callback):
+            outer = self.callback.im_class.__name__
+        else:
+            outer = self.callback.__module__
+        return "{}.{}".format(outer, self.callback.__name__)
+
+    def __call__(self, args=None, **kwargs):
+        hostname = request.host_url
+        method = request.method
+        path = request.path
+        query = request.query_string
+
+        # Mimic the HTTPretty URIInfo class
+        URI = namedtuple('URI', 'hostname method path query')
+        uri = URI(hostname, method, path, query)
+
+        body = request.data or query
+        headers = dict(request.headers)
+        result = self.callback(uri, body, headers)
+        if isinstance(result, basestring):
+            # result is just the response
+            return result
+        else:
+            # result is a responce, headers tuple
+            response, headers = result
+            status = headers.pop('status', None)
+            return response, status, headers

moto/dynamodb/responses.py

@@ -1,6 +1,6 @@
-import re
 import json
 
+from moto.core.utils import headers_to_dict
 from .models import dynamodb_backend
 
 
@@ -17,12 +17,16 @@ class DynamoHandler(object):
 
         ie: X-Amz-Target: DynamoDB_20111205.ListTables -> ListTables
         """
-        match = re.search(r'X-Amz-Target: \w+\.(\w+)', headers)
-        return match.groups()[0]
+        match = headers.get('X-Amz-Target')
+        if match:
+            return match.split(".")[1]
 
     def dispatch(self):
         method = self.get_method_name(self.headers)
-        return getattr(self, method)(self.uri, self.body, self.headers)
+        if method:
+            return getattr(self, method)(self.uri, self.body, self.headers)
+        else:
+            return "", dict(status=404)
 
     def ListTables(self, uri, body, headers):
         tables = dynamodb_backend.tables.keys()
@@ -36,4 +40,4 @@ class DynamoHandler(object):
 
 
 def handler(uri, body, headers):
-    return DynamoHandler(uri, body, headers).dispatch()
+    return DynamoHandler(uri, body, headers_to_dict(headers)).dispatch()

moto/dynamodb/urls.py

@@ -4,9 +4,13 @@ from .responses import handler
 def sts_handler(uri, body, headers):
     return GET_SESSION_TOKEN_RESULT
 
-urls = {
-    "https?://dynamodb.us-east-1.amazonaws.com/": handler,
-    "https?://sts.amazonaws.com/": sts_handler,
+url_bases = [
+    "https?://dynamodb.us-east-1.amazonaws.com",
+    "https?://sts.amazonaws.com",
+]
+
+url_paths = {
+    "{0}/": handler,
 }
 
 

moto/ec2/models.py

@@ -31,12 +31,13 @@ class InstanceBackend(object):
             if instance.id == instance_id:
                 return instance
 
-    def add_instances(self, count):
+    def add_instances(self, image_id, count):
         new_reservation = Reservation()
         new_reservation.id = random_reservation_id()
         for index in range(count):
             new_instance = Instance()
             new_instance.id = random_instance_id()
+            new_instance.image_id = image_id
             new_instance._state_name = "pending"
             new_instance._state_code = 0
             new_reservation.instances.append(new_instance)
@@ -226,11 +227,11 @@ class SecurityRule(object):
     @property
     def unique_representation(self):
         return "{}-{}-{}-{}-{}".format(
-                self.ip_protocol,
-                self.from_port,
-                self.to_port,
-                self.ip_ranges,
-                self.source_groups
+               self.ip_protocol,
+               self.from_port,
+               self.to_port,
+               self.ip_ranges,
+               self.source_groups
         )
 
     def __eq__(self, other):

moto/ec2/responses/__init__.py

@@ -71,8 +71,9 @@ class EC2Response(object):
         else:
             querystring = parse_qs(headers)
 
-        action = querystring['Action'][0]
-        action = camelcase_to_underscores(action)
+        action = querystring.get('Action', [None])[0]
+        if action:
+            action = camelcase_to_underscores(action)
 
         for sub_response in self.sub_responses:
             method_names = method_names_from_class(sub_response)

moto/ec2/responses/instances.py

@@ -16,7 +16,8 @@ class InstanceResponse(object):
 
     def run_instances(self):
         min_count = int(self.querystring.get('MinCount', ['1'])[0])
-        new_reservation = ec2_backend.add_instances(min_count)
+        image_id = self.querystring.get('ImageId')[0]
+        new_reservation = ec2_backend.add_instances(image_id, min_count)
         template = Template(EC2_RUN_INSTANCES)
         return template.render(reservation=new_reservation)
 
@@ -75,7 +76,7 @@ EC2_RUN_INSTANCES = """<RunInstancesResponse xmlns="http://ec2.amazonaws.com/doc
     {% for instance in reservation.instances %}
         <item>
           <instanceId>{{ instance.id }}</instanceId>
-          <imageId>ami-60a54009</imageId>
+          <imageId>{{ instance.image_id }}</imageId>
           <instanceState>
             <code>{{ instance._state_code }}</code>
             <name>{{ instance._state_name }}</name>
@@ -127,7 +128,7 @@ EC2_DESCRIBE_INSTANCES = """<DescribeInstancesResponse xmlns='http://ec2.amazona
                 {% for instance in reservation.instances %}
                   <item>
                     <instanceId>{{ instance.id }}</instanceId>
-                    <imageId>ami-1a2b3c4d</imageId>
+                    <imageId>{{ instance.image_id }}</imageId>
                     <instanceState>
                       <code>{{ instance._state_code }}</code>
                       <name>{{ instance._state_name }}</name>

moto/ec2/urls.py

@@ -1,5 +1,10 @@
 from .responses import EC2Response
 
-urls = {
-    "https?://ec2.us-east-1.amazonaws.com/": EC2Response().dispatch,
+
+url_bases = [
+    "https?://ec2.us-east-1.amazonaws.com",
+]
+
+url_paths = {
+    '{0}/': EC2Response().dispatch,
 }

moto/s3/models.py

@@ -61,8 +61,9 @@ class S3Backend(BaseBackend):
         return new_key
 
     def get_key(self, bucket_name, key_name):
-        bucket = self.buckets[bucket_name]
-        return bucket.keys.get(key_name)
+        bucket = self.get_bucket(bucket_name)
+        if bucket:
+            return bucket.keys.get(key_name)
 
     def prefix_query(self, bucket, prefix):
         key_results = set()

moto/s3/responses.py

@@ -7,7 +7,7 @@ from moto.core.utils import headers_to_dict
 from .utils import bucket_name_from_hostname
 
 
-def all_buckets(uri, body, method):
+def all_buckets():
     # No bucket specified. Listing all buckets
     all_buckets = s3_backend.get_all_buckets()
     template = Template(S3_ALL_BUCKETS)
@@ -20,6 +20,9 @@ def bucket_response(uri, body, headers):
     querystring = parse_qs(uri.query)
 
     bucket_name = bucket_name_from_hostname(hostname)
+    if not bucket_name:
+        # If no bucket specified, list all buckets
+        return all_buckets()
 
     if method == 'GET':
         bucket = s3_backend.get_bucket(bucket_name)
@@ -27,8 +30,12 @@ def bucket_response(uri, body, headers):
             prefix = querystring.get('prefix', [None])[0]
             result_keys, result_folders = s3_backend.prefix_query(bucket, prefix)
             template = Template(S3_BUCKET_GET_RESPONSE)
-            return template.render(bucket=bucket, prefix=prefix,
-                  result_keys=result_keys, result_folders=result_folders)
+            return template.render(
+                bucket=bucket,
+                prefix=prefix,
+                result_keys=result_keys,
+                result_folders=result_folders
+            )
         else:
             return "", dict(status=404)
     elif method == 'PUT':

moto/s3/urls.py

@@ -1,9 +1,10 @@
-from .responses import all_buckets, bucket_response, key_response
+from .responses import bucket_response, key_response
 
-base_url = "https?://(.*).s3.amazonaws.com"
+url_bases = [
+    "https?://(?P<bucket_name>\w*)\.?s3.amazonaws.com"
+]
 
-urls = {
-    'https?://s3.amazonaws.com/$': all_buckets,
-    '{0}/$'.format(base_url): bucket_response,
-    '{}/(.+)'.format(base_url): key_response,
+url_paths = {
+    '{0}/$': bucket_response,
+    '{0}/(?P<key_name>\w+)': key_response,
 }

moto/s3/utils.py

@@ -1,8 +1,23 @@
 import re
+import urlparse
 
 bucket_name_regex = re.compile("(.+).s3.amazonaws.com")
 
 
 def bucket_name_from_hostname(hostname):
-    bucket_result = bucket_name_regex.search(hostname)
-    return bucket_result.groups()[0]
+    if 'amazonaws.com' in hostname:
+        bucket_result = bucket_name_regex.search(hostname)
+        if bucket_result:
+            return bucket_result.groups()[0]
+    else:
+        # In server mode. Use left-most part of subdomain for bucket name
+        split_url = urlparse.urlparse(hostname)
+
+        # If 'www' prefixed, strip it.
+        clean_hostname = split_url.netloc.lstrip("www.")
+
+        if '.' in clean_hostname:
+            return clean_hostname.split(".")[0]
+        else:
+            # No subdomain found.
+            return None

moto/server.py

@@ -0,0 +1,45 @@
+import sys
+
+from flask import Flask
+from werkzeug.routing import BaseConverter
+
+from moto.dynamodb import dynamodb_backend  # flake8: noqa
+from moto.ec2 import ec2_backend  # flake8: noqa
+from moto.s3 import s3_backend  # flake8: noqa
+from moto.ses import ses_backend  # flake8: noqa
+from moto.sqs import sqs_backend  # flake8: noqa
+
+from moto.core.utils import convert_flask_to_httpretty_response
+
+app = Flask(__name__)
+HTTP_METHODS = ["GET", "POST", "PUT", "DELETE", "HEAD"]
+
+
+class RegexConverter(BaseConverter):
+    # http://werkzeug.pocoo.org/docs/routing/#custom-converters
+    def __init__(self, url_map, *items):
+        super(RegexConverter, self).__init__(url_map)
+        self.regex = items[0]
+
+
+def configure_urls(service):
+    backend = globals()["{}_backend".format(service)]
+    from werkzeug.routing import Map
+    app.url_map = Map()
+    app.url_map.converters['regex'] = RegexConverter
+    for url_path, handler in backend.flask_paths.iteritems():
+        app.route(url_path, methods=HTTP_METHODS)(convert_flask_to_httpretty_response(handler))
+
+
+def main(args=sys.argv):
+    if len(args) != 2:
+        print("Usage: moto_server <service>")
+        sys.exit(1)
+    service_name = args[1]
+    configure_urls(service_name)
+
+    app.testing = True
+    app.run()
+
+if __name__ == '__main__':
+    main()

moto/ses/urls.py

@@ -1,7 +1,9 @@
 from .responses import EmailResponse
 
-base_url = "https?://email.us-east-1.amazonaws.com"
+url_bases = [
+    "https?://email.us-east-1.amazonaws.com"
+]
 
-urls = {
-    '{0}/$'.format(base_url): EmailResponse().dispatch,
+url_paths = {
+    '{0}/$': EmailResponse().dispatch,
 }

moto/sqs/urls.py

@@ -1,8 +1,10 @@
 from .responses import QueueResponse, QueuesResponse
 
-base_url = "https?://(.*).amazonaws.com"
+url_bases = [
+    "https?://(.*).amazonaws.com"
+]
 
-urls = {
-    '{0}/$'.format(base_url): QueuesResponse().dispatch,
-    '{0}/(\d+)/(.*)$'.format(base_url): QueueResponse().dispatch,
+url_paths = {
+    '{0}/$': QueuesResponse().dispatch2,
+    '{0}/(?P<account_id>\d+)/(?P<queue_name>\w+)': QueueResponse().dispatch,
 }