From 24e942b50e4ea9b4913f48777041551316f0fe22 Mon Sep 17 00:00:00 2001 From: Gary Donovan Date: Mon, 23 Jul 2018 14:38:52 +1000 Subject: [PATCH 01/40] Update list of implemented endpoints for cognito-idp --- IMPLEMENTATION_COVERAGE.md | 48 +++++++++++++++++++------------------- 1 file changed, 24 insertions(+), 24 deletions(-) diff --git a/IMPLEMENTATION_COVERAGE.md b/IMPLEMENTATION_COVERAGE.md index 17b864dc..b8c8dd1b 100644 --- a/IMPLEMENTATION_COVERAGE.md +++ b/IMPLEMENTATION_COVERAGE.md @@ -831,16 +831,16 @@ - [ ] add_custom_attributes - [ ] admin_add_user_to_group - [ ] admin_confirm_sign_up -- [ ] admin_create_user -- [ ] admin_delete_user +- [X] admin_create_user +- [X] admin_delete_user - [ ] admin_delete_user_attributes - [ ] admin_disable_provider_for_user - [ ] admin_disable_user - [ ] admin_enable_user - [ ] admin_forget_device - [ ] admin_get_device -- [ ] admin_get_user -- [ ] admin_initiate_auth +- [X] admin_get_user +- [X] admin_initiate_auth - [ ] admin_link_provider_for_user - [ ] admin_list_devices - [ ] admin_list_groups_for_user @@ -855,32 +855,32 @@ - [ ] admin_update_user_attributes - [ ] admin_user_global_sign_out - [ ] associate_software_token -- [ ] change_password +- [X] change_password - [ ] confirm_device -- [ ] confirm_forgot_password +- [X] confirm_forgot_password - [ ] confirm_sign_up - [ ] create_group -- [ ] create_identity_provider +- [X] create_identity_provider - [ ] create_resource_server - [ ] create_user_import_job -- [ ] create_user_pool -- [ ] create_user_pool_client -- [ ] create_user_pool_domain +- [X] create_user_pool +- [X] create_user_pool_client +- [X] create_user_pool_domain - [ ] delete_group -- [ ] delete_identity_provider +- [X] delete_identity_provider - [ ] delete_resource_server - [ ] delete_user - [ ] delete_user_attributes -- [ ] delete_user_pool -- [ ] delete_user_pool_client -- [ ] delete_user_pool_domain -- [ ] describe_identity_provider +- [X] delete_user_pool +- [X] delete_user_pool_client +- [X] delete_user_pool_domain +- [X] describe_identity_provider - [ ] describe_resource_server - [ ] describe_risk_configuration - [ ] describe_user_import_job -- [ ] describe_user_pool -- [ ] describe_user_pool_client -- [ ] describe_user_pool_domain +- [X] describe_user_pool +- [X] describe_user_pool_client +- [X] describe_user_pool_domain - [ ] forget_device - [ ] forgot_password - [ ] get_csv_header @@ -896,15 +896,15 @@ - [ ] initiate_auth - [ ] list_devices - [ ] list_groups -- [ ] list_identity_providers +- [X] list_identity_providers - [ ] list_resource_servers - [ ] list_user_import_jobs -- [ ] list_user_pool_clients -- [ ] list_user_pools -- [ ] list_users +- [X] list_user_pool_clients +- [X] list_user_pools +- [X] list_users - [ ] list_users_in_group - [ ] resend_confirmation_code -- [ ] respond_to_auth_challenge +- [X] respond_to_auth_challenge - [ ] set_risk_configuration - [ ] set_ui_customization - [ ] set_user_mfa_preference @@ -920,7 +920,7 @@ - [ ] update_resource_server - [ ] update_user_attributes - [ ] update_user_pool -- [ ] update_user_pool_client +- [X] update_user_pool_client - [ ] verify_software_token - [ ] verify_user_attribute From 04fdd5617a2131dc68d2f7461718b70e62215e93 Mon Sep 17 00:00:00 2001 From: Gary Donovan Date: Mon, 23 Jul 2018 16:26:54 +1000 Subject: [PATCH 02/40] Implement group management for cognito-idp --- IMPLEMENTATION_COVERAGE.md | 8 +- moto/cognitoidp/exceptions.py | 10 +++ moto/cognitoidp/models.py | 67 ++++++++++++++++- moto/cognitoidp/responses.py | 41 ++++++++++ tests/test_cognitoidp/test_cognitoidp.py | 96 +++++++++++++++++++++++- 5 files changed, 214 insertions(+), 8 deletions(-) diff --git a/IMPLEMENTATION_COVERAGE.md b/IMPLEMENTATION_COVERAGE.md index b8c8dd1b..cceaa3c8 100644 --- a/IMPLEMENTATION_COVERAGE.md +++ b/IMPLEMENTATION_COVERAGE.md @@ -859,14 +859,14 @@ - [ ] confirm_device - [X] confirm_forgot_password - [ ] confirm_sign_up -- [ ] create_group +- [X] create_group - [X] create_identity_provider - [ ] create_resource_server - [ ] create_user_import_job - [X] create_user_pool - [X] create_user_pool_client - [X] create_user_pool_domain -- [ ] delete_group +- [X] delete_group - [X] delete_identity_provider - [ ] delete_resource_server - [ ] delete_user @@ -885,7 +885,7 @@ - [ ] forgot_password - [ ] get_csv_header - [ ] get_device -- [ ] get_group +- [X] get_group - [ ] get_identity_provider_by_identifier - [ ] get_signing_certificate - [ ] get_ui_customization @@ -895,7 +895,7 @@ - [ ] global_sign_out - [ ] initiate_auth - [ ] list_devices -- [ ] list_groups +- [X] list_groups - [X] list_identity_providers - [ ] list_resource_servers - [ ] list_user_import_jobs diff --git a/moto/cognitoidp/exceptions.py b/moto/cognitoidp/exceptions.py index 1f1ec230..45267021 100644 --- a/moto/cognitoidp/exceptions.py +++ b/moto/cognitoidp/exceptions.py @@ -24,6 +24,16 @@ class UserNotFoundError(BadRequest): }) +class GroupExistsException(BadRequest): + + def __init__(self, message): + super(GroupExistsException, self).__init__() + self.description = json.dumps({ + "message": message, + '__type': 'GroupExistsException', + }) + + class NotAuthorizedError(BadRequest): def __init__(self, message): diff --git a/moto/cognitoidp/models.py b/moto/cognitoidp/models.py index 10da0c6f..a11eb435 100644 --- a/moto/cognitoidp/models.py +++ b/moto/cognitoidp/models.py @@ -11,8 +11,7 @@ from jose import jws from moto.compat import OrderedDict from moto.core import BaseBackend, BaseModel -from .exceptions import NotAuthorizedError, ResourceNotFoundError, UserNotFoundError - +from .exceptions import GroupExistsException, NotAuthorizedError, ResourceNotFoundError, UserNotFoundError UserStatus = { "FORCE_CHANGE_PASSWORD": "FORCE_CHANGE_PASSWORD", @@ -33,6 +32,7 @@ class CognitoIdpUserPool(BaseModel): self.clients = OrderedDict() self.identity_providers = OrderedDict() + self.groups = OrderedDict() self.users = OrderedDict() self.refresh_tokens = {} self.access_tokens = {} @@ -185,6 +185,29 @@ class CognitoIdpIdentityProvider(BaseModel): return identity_provider_json +class CognitoIdpGroup(BaseModel): + + def __init__(self, user_pool_id, group_name, description, role_arn, precedence): + self.user_pool_id = user_pool_id + self.group_name = group_name + self.description = description or "" + self.role_arn = role_arn + self.precedence = precedence + self.last_modified_date = datetime.datetime.now() + self.creation_date = self.last_modified_date + + def to_json(self): + return { + "GroupName": self.group_name, + "UserPoolId": self.user_pool_id, + "Description": self.description, + "RoleArn": self.role_arn, + "Precedence": self.precedence, + "LastModifiedDate": time.mktime(self.last_modified_date.timetuple()), + "CreationDate": time.mktime(self.creation_date.timetuple()), + } + + class CognitoIdpUser(BaseModel): def __init__(self, user_pool_id, username, password, status, attributes): @@ -367,6 +390,46 @@ class CognitoIdpBackend(BaseBackend): del user_pool.identity_providers[name] + # Group + def create_group(self, user_pool_id, group_name, description, role_arn, precedence): + user_pool = self.user_pools.get(user_pool_id) + if not user_pool: + raise ResourceNotFoundError(user_pool_id) + + group = CognitoIdpGroup(user_pool_id, group_name, description, role_arn, precedence) + if group.group_name in user_pool.groups: + raise GroupExistsException("A group with the name already exists") + user_pool.groups[group.group_name] = group + + return group + + def get_group(self, user_pool_id, group_name): + user_pool = self.user_pools.get(user_pool_id) + if not user_pool: + raise ResourceNotFoundError(user_pool_id) + + if group_name not in user_pool.groups: + raise ResourceNotFoundError(group_name) + + return user_pool.groups[group_name] + + def list_groups(self, user_pool_id): + user_pool = self.user_pools.get(user_pool_id) + if not user_pool: + raise ResourceNotFoundError(user_pool_id) + + return user_pool.groups.values() + + def delete_group(self, user_pool_id, group_name): + user_pool = self.user_pools.get(user_pool_id) + if not user_pool: + raise ResourceNotFoundError(user_pool_id) + + if group_name not in user_pool.groups: + raise ResourceNotFoundError(group_name) + + del user_pool.groups[group_name] + # User def admin_create_user(self, user_pool_id, username, temporary_password, attributes): user_pool = self.user_pools.get(user_pool_id) diff --git a/moto/cognitoidp/responses.py b/moto/cognitoidp/responses.py index e6f20367..a2f67ba8 100644 --- a/moto/cognitoidp/responses.py +++ b/moto/cognitoidp/responses.py @@ -129,6 +129,47 @@ class CognitoIdpResponse(BaseResponse): cognitoidp_backends[self.region].delete_identity_provider(user_pool_id, name) return "" + # Group + def create_group(self): + group_name = self._get_param("GroupName") + user_pool_id = self._get_param("UserPoolId") + description = self._get_param("Description") + role_arn = self._get_param("RoleArn") + precedence = self._get_param("Precedence") + + group = cognitoidp_backends[self.region].create_group( + user_pool_id, + group_name, + description, + role_arn, + precedence, + ) + + return json.dumps({ + "Group": group.to_json(), + }) + + def get_group(self): + group_name = self._get_param("GroupName") + user_pool_id = self._get_param("UserPoolId") + group = cognitoidp_backends[self.region].get_group(user_pool_id, group_name) + return json.dumps({ + "Group": group.to_json(), + }) + + def list_groups(self): + user_pool_id = self._get_param("UserPoolId") + groups = cognitoidp_backends[self.region].list_groups(user_pool_id) + return json.dumps({ + "Groups": [group.to_json() for group in groups], + }) + + def delete_group(self): + group_name = self._get_param("GroupName") + user_pool_id = self._get_param("UserPoolId") + cognitoidp_backends[self.region].delete_group(user_pool_id, group_name) + return "" + # User def admin_create_user(self): user_pool_id = self._get_param("UserPoolId") diff --git a/tests/test_cognitoidp/test_cognitoidp.py b/tests/test_cognitoidp/test_cognitoidp.py index 56d7c08a..c226f91b 100644 --- a/tests/test_cognitoidp/test_cognitoidp.py +++ b/tests/test_cognitoidp/test_cognitoidp.py @@ -1,14 +1,18 @@ from __future__ import unicode_literals -import boto3 import json import os +import random import uuid +import boto3 +# noinspection PyUnresolvedReferences +import sure # noqa +from botocore.exceptions import ClientError from jose import jws +from nose.tools import assert_raises from moto import mock_cognitoidp -import sure # noqa @mock_cognitoidp @@ -323,6 +327,94 @@ def test_delete_identity_providers(): caught.should.be.true +@mock_cognitoidp +def test_create_group(): + conn = boto3.client("cognito-idp", "us-west-2") + + user_pool_id = conn.create_user_pool(PoolName=str(uuid.uuid4()))["UserPool"]["Id"] + group_name = str(uuid.uuid4()) + description = str(uuid.uuid4()) + role_arn = "arn:aws:iam:::role/my-iam-role" + precedence = random.randint(0, 100000) + + result = conn.create_group( + GroupName=group_name, + UserPoolId=user_pool_id, + Description=description, + RoleArn=role_arn, + Precedence=precedence, + ) + + result["Group"]["GroupName"].should.equal(group_name) + result["Group"]["UserPoolId"].should.equal(user_pool_id) + result["Group"]["Description"].should.equal(description) + result["Group"]["RoleArn"].should.equal(role_arn) + result["Group"]["Precedence"].should.equal(precedence) + result["Group"]["LastModifiedDate"].should.be.a("datetime.datetime") + result["Group"]["CreationDate"].should.be.a("datetime.datetime") + + +@mock_cognitoidp +def test_create_group_with_duplicate_name_raises_error(): + conn = boto3.client("cognito-idp", "us-west-2") + + user_pool_id = conn.create_user_pool(PoolName=str(uuid.uuid4()))["UserPool"]["Id"] + group_name = str(uuid.uuid4()) + + conn.create_group(GroupName=group_name, UserPoolId=user_pool_id) + + with assert_raises(ClientError) as cm: + conn.create_group(GroupName=group_name, UserPoolId=user_pool_id) + cm.exception.operation_name.should.equal('CreateGroup') + cm.exception.response['Error']['Code'].should.equal('GroupExistsException') + cm.exception.response['ResponseMetadata']['HTTPStatusCode'].should.equal(400) + + +@mock_cognitoidp +def test_get_group(): + conn = boto3.client("cognito-idp", "us-west-2") + + user_pool_id = conn.create_user_pool(PoolName=str(uuid.uuid4()))["UserPool"]["Id"] + group_name = str(uuid.uuid4()) + conn.create_group(GroupName=group_name, UserPoolId=user_pool_id) + + result = conn.get_group(GroupName=group_name, UserPoolId=user_pool_id) + + result["Group"]["GroupName"].should.equal(group_name) + result["Group"]["UserPoolId"].should.equal(user_pool_id) + result["Group"]["LastModifiedDate"].should.be.a("datetime.datetime") + result["Group"]["CreationDate"].should.be.a("datetime.datetime") + + +@mock_cognitoidp +def test_list_groups(): + conn = boto3.client("cognito-idp", "us-west-2") + + user_pool_id = conn.create_user_pool(PoolName=str(uuid.uuid4()))["UserPool"]["Id"] + group_name = str(uuid.uuid4()) + conn.create_group(GroupName=group_name, UserPoolId=user_pool_id) + + result = conn.list_groups(UserPoolId=user_pool_id) + + result["Groups"].should.have.length_of(1) + result["Groups"][0]["GroupName"].should.equal(group_name) + + +@mock_cognitoidp +def test_delete_group(): + conn = boto3.client("cognito-idp", "us-west-2") + + user_pool_id = conn.create_user_pool(PoolName=str(uuid.uuid4()))["UserPool"]["Id"] + group_name = str(uuid.uuid4()) + conn.create_group(GroupName=group_name, UserPoolId=user_pool_id) + + conn.delete_group(GroupName=group_name, UserPoolId=user_pool_id) + + with assert_raises(ClientError) as cm: + conn.get_group(GroupName=group_name, UserPoolId=user_pool_id) + cm.exception.response['Error']['Code'].should.equal('ResourceNotFoundException') + + @mock_cognitoidp def test_admin_create_user(): conn = boto3.client("cognito-idp", "us-west-2") From ffa7560d026b9d0a3380b3c7cb3006d9e89af76a Mon Sep 17 00:00:00 2001 From: Gary Donovan Date: Mon, 23 Jul 2018 20:54:51 +1000 Subject: [PATCH 03/40] Implement user-group relationships for cognito-idp --- IMPLEMENTATION_COVERAGE.md | 10 +- moto/cognitoidp/models.py | 38 ++++++ moto/cognitoidp/responses.py | 42 +++++++ tests/test_cognitoidp/test_cognitoidp.py | 153 ++++++++++++++++++++++- 4 files changed, 237 insertions(+), 6 deletions(-) diff --git a/IMPLEMENTATION_COVERAGE.md b/IMPLEMENTATION_COVERAGE.md index cceaa3c8..47420e8a 100644 --- a/IMPLEMENTATION_COVERAGE.md +++ b/IMPLEMENTATION_COVERAGE.md @@ -827,9 +827,9 @@ - [ ] unlink_identity - [ ] update_identity_pool -## cognito-idp - 0% implemented +## cognito-idp - 34% implemented - [ ] add_custom_attributes -- [ ] admin_add_user_to_group +- [X] admin_add_user_to_group - [ ] admin_confirm_sign_up - [X] admin_create_user - [X] admin_delete_user @@ -843,9 +843,9 @@ - [X] admin_initiate_auth - [ ] admin_link_provider_for_user - [ ] admin_list_devices -- [ ] admin_list_groups_for_user +- [X] admin_list_groups_for_user - [ ] admin_list_user_auth_events -- [ ] admin_remove_user_from_group +- [X] admin_remove_user_from_group - [ ] admin_reset_user_password - [ ] admin_respond_to_auth_challenge - [ ] admin_set_user_mfa_preference @@ -902,7 +902,7 @@ - [X] list_user_pool_clients - [X] list_user_pools - [X] list_users -- [ ] list_users_in_group +- [X] list_users_in_group - [ ] resend_confirmation_code - [X] respond_to_auth_challenge - [ ] set_risk_configuration diff --git a/moto/cognitoidp/models.py b/moto/cognitoidp/models.py index a11eb435..349d1e31 100644 --- a/moto/cognitoidp/models.py +++ b/moto/cognitoidp/models.py @@ -196,6 +196,10 @@ class CognitoIdpGroup(BaseModel): self.last_modified_date = datetime.datetime.now() self.creation_date = self.last_modified_date + # Users who are members of this group. + # Note that these links are bidirectional. + self.users = set() + def to_json(self): return { "GroupName": self.group_name, @@ -221,6 +225,10 @@ class CognitoIdpUser(BaseModel): self.create_date = datetime.datetime.utcnow() self.last_modified_date = datetime.datetime.utcnow() + # Groups this user is a member of. + # Note that these links are bidirectional. + self.groups = set() + def _base_json(self): return { "UserPoolId": self.user_pool_id, @@ -428,8 +436,34 @@ class CognitoIdpBackend(BaseBackend): if group_name not in user_pool.groups: raise ResourceNotFoundError(group_name) + group = user_pool.groups[group_name] + for user in group.users: + user.groups.remove(group) + del user_pool.groups[group_name] + def admin_add_user_to_group(self, user_pool_id, group_name, username): + group = self.get_group(user_pool_id, group_name) + user = self.admin_get_user(user_pool_id, username) + + group.users.add(user) + user.groups.add(group) + + def list_users_in_group(self, user_pool_id, group_name): + group = self.get_group(user_pool_id, group_name) + return list(group.users) + + def admin_list_groups_for_user(self, user_pool_id, username): + user = self.admin_get_user(user_pool_id, username) + return list(user.groups) + + def admin_remove_user_from_group(self, user_pool_id, group_name, username): + group = self.get_group(user_pool_id, group_name) + user = self.admin_get_user(user_pool_id, username) + + group.users.discard(user) + user.groups.discard(group) + # User def admin_create_user(self, user_pool_id, username, temporary_password, attributes): user_pool = self.user_pools.get(user_pool_id) @@ -465,6 +499,10 @@ class CognitoIdpBackend(BaseBackend): if username not in user_pool.users: raise ResourceNotFoundError(username) + user = user_pool.users[username] + for group in user.groups: + group.users.remove(user) + del user_pool.users[username] def _log_user_in(self, user_pool, client, username): diff --git a/moto/cognitoidp/responses.py b/moto/cognitoidp/responses.py index a2f67ba8..7d19fa3e 100644 --- a/moto/cognitoidp/responses.py +++ b/moto/cognitoidp/responses.py @@ -170,6 +170,48 @@ class CognitoIdpResponse(BaseResponse): cognitoidp_backends[self.region].delete_group(user_pool_id, group_name) return "" + def admin_add_user_to_group(self): + user_pool_id = self._get_param("UserPoolId") + username = self._get_param("Username") + group_name = self._get_param("GroupName") + + cognitoidp_backends[self.region].admin_add_user_to_group( + user_pool_id, + group_name, + username, + ) + + return "" + + def list_users_in_group(self): + user_pool_id = self._get_param("UserPoolId") + group_name = self._get_param("GroupName") + users = cognitoidp_backends[self.region].list_users_in_group(user_pool_id, group_name) + return json.dumps({ + "Users": [user.to_json(extended=True) for user in users], + }) + + def admin_list_groups_for_user(self): + username = self._get_param("Username") + user_pool_id = self._get_param("UserPoolId") + groups = cognitoidp_backends[self.region].admin_list_groups_for_user(user_pool_id, username) + return json.dumps({ + "Groups": [group.to_json() for group in groups], + }) + + def admin_remove_user_from_group(self): + user_pool_id = self._get_param("UserPoolId") + username = self._get_param("Username") + group_name = self._get_param("GroupName") + + cognitoidp_backends[self.region].admin_remove_user_from_group( + user_pool_id, + group_name, + username, + ) + + return "" + # User def admin_create_user(self): user_pool_id = self._get_param("UserPoolId") diff --git a/tests/test_cognitoidp/test_cognitoidp.py b/tests/test_cognitoidp/test_cognitoidp.py index c226f91b..07edc990 100644 --- a/tests/test_cognitoidp/test_cognitoidp.py +++ b/tests/test_cognitoidp/test_cognitoidp.py @@ -408,13 +408,164 @@ def test_delete_group(): group_name = str(uuid.uuid4()) conn.create_group(GroupName=group_name, UserPoolId=user_pool_id) - conn.delete_group(GroupName=group_name, UserPoolId=user_pool_id) + result = conn.delete_group(GroupName=group_name, UserPoolId=user_pool_id) + list(result.keys()).should.equal(["ResponseMetadata"]) # No response expected with assert_raises(ClientError) as cm: conn.get_group(GroupName=group_name, UserPoolId=user_pool_id) cm.exception.response['Error']['Code'].should.equal('ResourceNotFoundException') +@mock_cognitoidp +def test_admin_add_user_to_group(): + conn = boto3.client("cognito-idp", "us-west-2") + + user_pool_id = conn.create_user_pool(PoolName=str(uuid.uuid4()))["UserPool"]["Id"] + group_name = str(uuid.uuid4()) + conn.create_group(GroupName=group_name, UserPoolId=user_pool_id) + + username = str(uuid.uuid4()) + conn.admin_create_user(UserPoolId=user_pool_id, Username=username) + + result = conn.admin_add_user_to_group(UserPoolId=user_pool_id, Username=username, GroupName=group_name) + list(result.keys()).should.equal(["ResponseMetadata"]) # No response expected + + +@mock_cognitoidp +def test_admin_add_user_to_group_again_is_noop(): + conn = boto3.client("cognito-idp", "us-west-2") + + user_pool_id = conn.create_user_pool(PoolName=str(uuid.uuid4()))["UserPool"]["Id"] + group_name = str(uuid.uuid4()) + conn.create_group(GroupName=group_name, UserPoolId=user_pool_id) + + username = str(uuid.uuid4()) + conn.admin_create_user(UserPoolId=user_pool_id, Username=username) + + conn.admin_add_user_to_group(UserPoolId=user_pool_id, Username=username, GroupName=group_name) + conn.admin_add_user_to_group(UserPoolId=user_pool_id, Username=username, GroupName=group_name) + + +@mock_cognitoidp +def test_list_users_in_group(): + conn = boto3.client("cognito-idp", "us-west-2") + + user_pool_id = conn.create_user_pool(PoolName=str(uuid.uuid4()))["UserPool"]["Id"] + group_name = str(uuid.uuid4()) + conn.create_group(GroupName=group_name, UserPoolId=user_pool_id) + + username = str(uuid.uuid4()) + conn.admin_create_user(UserPoolId=user_pool_id, Username=username) + + conn.admin_add_user_to_group(UserPoolId=user_pool_id, Username=username, GroupName=group_name) + + result = conn.list_users_in_group(UserPoolId=user_pool_id, GroupName=group_name) + + result["Users"].should.have.length_of(1) + result["Users"][0]["Username"].should.equal(username) + + +@mock_cognitoidp +def test_list_users_in_group_ignores_deleted_user(): + conn = boto3.client("cognito-idp", "us-west-2") + + user_pool_id = conn.create_user_pool(PoolName=str(uuid.uuid4()))["UserPool"]["Id"] + group_name = str(uuid.uuid4()) + conn.create_group(GroupName=group_name, UserPoolId=user_pool_id) + + username = str(uuid.uuid4()) + conn.admin_create_user(UserPoolId=user_pool_id, Username=username) + username2 = str(uuid.uuid4()) + conn.admin_create_user(UserPoolId=user_pool_id, Username=username2) + + conn.admin_add_user_to_group(UserPoolId=user_pool_id, Username=username, GroupName=group_name) + conn.admin_add_user_to_group(UserPoolId=user_pool_id, Username=username2, GroupName=group_name) + conn.admin_delete_user(UserPoolId=user_pool_id, Username=username) + + result = conn.list_users_in_group(UserPoolId=user_pool_id, GroupName=group_name) + + result["Users"].should.have.length_of(1) + result["Users"][0]["Username"].should.equal(username2) + + +@mock_cognitoidp +def test_admin_list_groups_for_user(): + conn = boto3.client("cognito-idp", "us-west-2") + + user_pool_id = conn.create_user_pool(PoolName=str(uuid.uuid4()))["UserPool"]["Id"] + group_name = str(uuid.uuid4()) + conn.create_group(GroupName=group_name, UserPoolId=user_pool_id) + + username = str(uuid.uuid4()) + conn.admin_create_user(UserPoolId=user_pool_id, Username=username) + + conn.admin_add_user_to_group(UserPoolId=user_pool_id, Username=username, GroupName=group_name) + + result = conn.admin_list_groups_for_user(Username=username, UserPoolId=user_pool_id) + + result["Groups"].should.have.length_of(1) + result["Groups"][0]["GroupName"].should.equal(group_name) + + +@mock_cognitoidp +def test_admin_list_groups_for_user_ignores_deleted_group(): + conn = boto3.client("cognito-idp", "us-west-2") + + user_pool_id = conn.create_user_pool(PoolName=str(uuid.uuid4()))["UserPool"]["Id"] + group_name = str(uuid.uuid4()) + conn.create_group(GroupName=group_name, UserPoolId=user_pool_id) + group_name2 = str(uuid.uuid4()) + conn.create_group(GroupName=group_name2, UserPoolId=user_pool_id) + + username = str(uuid.uuid4()) + conn.admin_create_user(UserPoolId=user_pool_id, Username=username) + + conn.admin_add_user_to_group(UserPoolId=user_pool_id, Username=username, GroupName=group_name) + conn.admin_add_user_to_group(UserPoolId=user_pool_id, Username=username, GroupName=group_name2) + conn.delete_group(GroupName=group_name, UserPoolId=user_pool_id) + + result = conn.admin_list_groups_for_user(Username=username, UserPoolId=user_pool_id) + + result["Groups"].should.have.length_of(1) + result["Groups"][0]["GroupName"].should.equal(group_name2) + + +@mock_cognitoidp +def test_admin_remove_user_from_group(): + conn = boto3.client("cognito-idp", "us-west-2") + + user_pool_id = conn.create_user_pool(PoolName=str(uuid.uuid4()))["UserPool"]["Id"] + group_name = str(uuid.uuid4()) + conn.create_group(GroupName=group_name, UserPoolId=user_pool_id) + + username = str(uuid.uuid4()) + conn.admin_create_user(UserPoolId=user_pool_id, Username=username) + + conn.admin_add_user_to_group(UserPoolId=user_pool_id, Username=username, GroupName=group_name) + + result = conn.admin_remove_user_from_group(UserPoolId=user_pool_id, Username=username, GroupName=group_name) + list(result.keys()).should.equal(["ResponseMetadata"]) # No response expected + conn.list_users_in_group(UserPoolId=user_pool_id, GroupName=group_name) \ + ["Users"].should.have.length_of(0) + conn.admin_list_groups_for_user(Username=username, UserPoolId=user_pool_id) \ + ["Groups"].should.have.length_of(0) + + +@mock_cognitoidp +def test_admin_remove_user_from_group_again_is_noop(): + conn = boto3.client("cognito-idp", "us-west-2") + + user_pool_id = conn.create_user_pool(PoolName=str(uuid.uuid4()))["UserPool"]["Id"] + group_name = str(uuid.uuid4()) + conn.create_group(GroupName=group_name, UserPoolId=user_pool_id) + + username = str(uuid.uuid4()) + conn.admin_create_user(UserPoolId=user_pool_id, Username=username) + + conn.admin_add_user_to_group(UserPoolId=user_pool_id, Username=username, GroupName=group_name) + conn.admin_add_user_to_group(UserPoolId=user_pool_id, Username=username, GroupName=group_name) + + @mock_cognitoidp def test_admin_create_user(): conn = boto3.client("cognito-idp", "us-west-2") From b4c44a820f48d22090e131e27ae8f58738649ecb Mon Sep 17 00:00:00 2001 From: Jon Banafato Date: Fri, 19 Oct 2018 17:09:19 -0400 Subject: [PATCH 04/40] Add a long description to setup Include the readme as the `long_description` argument to `setup` and set its content type appropriately. This allows PyPI to render the content correctly. --- setup.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/setup.py b/setup.py index 98780dd5..17e58591 100755 --- a/setup.py +++ b/setup.py @@ -1,10 +1,23 @@ #!/usr/bin/env python from __future__ import unicode_literals +import codecs +import os +import re import setuptools from setuptools import setup, find_packages import sys +# Borrowed from pip at https://github.com/pypa/pip/blob/62c27dee45625e1b63d1e023b0656310f276e050/setup.py#L11-L15 +here = os.path.abspath(os.path.dirname(__file__)) + +def read(*parts): + # intentionally *not* adding an encoding option to open, See: + # https://github.com/pypa/virtualenv/issues/201#issuecomment-3145690 + with codecs.open(os.path.join(here, *parts), 'r') as fp: + return fp.read() + + install_requires = [ "Jinja2>=2.7.3", "boto>=2.36.0", @@ -43,6 +56,8 @@ setup( version='1.3.6', description='A library that allows your python tests to easily' ' mock out the boto library', + long_description=read('README.md'), + long_description_content_type='text/markdown', author='Steve Pulec', author_email='spulec@gmail.com', url='https://github.com/spulec/moto', From ede02e2c2a319fbbe5921800af5aae680b01ca15 Mon Sep 17 00:00:00 2001 From: Adam Davis Date: Mon, 22 Oct 2018 13:37:52 -0700 Subject: [PATCH 05/40] Without double quotes, pip would not install --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 791226d6..d6e9f30a 100644 --- a/README.md +++ b/README.md @@ -259,7 +259,7 @@ It uses flask, which isn't a default dependency. You can install the server 'extra' package with: ```python -pip install moto[server] +pip install "moto[server]" ``` You can then start it running a service: From aa4be6fcad087139f2087c9794def59de1c118d3 Mon Sep 17 00:00:00 2001 From: George Alton Date: Wed, 24 Oct 2018 15:10:28 +0100 Subject: [PATCH 06/40] Adds limiting/pagination to cognitoidp list_* functions --- moto/cognitoidp/models.py | 50 ++++- moto/cognitoidp/responses.py | 53 +++-- tests/test_cognitoidp/test_cognitoidp.py | 244 +++++++++++++++++++++++ 3 files changed, 329 insertions(+), 18 deletions(-) diff --git a/moto/cognitoidp/models.py b/moto/cognitoidp/models.py index 476d470b..521701de 100644 --- a/moto/cognitoidp/models.py +++ b/moto/cognitoidp/models.py @@ -1,6 +1,7 @@ from __future__ import unicode_literals import datetime +import functools import json import os import time @@ -20,6 +21,43 @@ UserStatus = { } +def paginate(limit, start_arg="next_token", limit_arg="max_results"): + """Returns a limited result list, and an offset into list of remaining items + + Takes the next_token, and max_results kwargs given to a function and handles + the slicing of the results. The kwarg `next_token` is the offset into the + list to begin slicing from. `max_results` is the size of the result required + + If the max_results is not supplied then the `limit` parameter is used as a + default + + :param limit_arg: the name of argument in the decorated function that + controls amount of items returned + :param start_arg: the name of the argument in the decorated that provides + the starting offset + :param limit: A default maximum items to return + :return: a tuple containing a list of items, and the offset into the list + """ + default_start = 0 + + def outer_wrapper(func): + @functools.wraps(func) + def wrapper(*args, **kwargs): + # setup + start = int(default_start if kwargs.get(start_arg) is None else kwargs[start_arg]) + stop = int(limit if kwargs.get(limit_arg) is None else kwargs[limit_arg]) + end = start + stop + # call + result = func(*args, **kwargs) + # modify + results = list(result) + limited_results = results[start: end] + next_token = end if end < len(results) else None + return limited_results, next_token + return wrapper + return outer_wrapper + + class CognitoIdpUserPool(BaseModel): def __init__(self, region, name, extended_config): @@ -242,7 +280,8 @@ class CognitoIdpBackend(BaseBackend): self.user_pools[user_pool.id] = user_pool return user_pool - def list_user_pools(self): + @paginate(60) + def list_user_pools(self, max_results=None, next_token=None): return self.user_pools.values() def describe_user_pool(self, user_pool_id): @@ -289,7 +328,8 @@ class CognitoIdpBackend(BaseBackend): user_pool.clients[user_pool_client.id] = user_pool_client return user_pool_client - def list_user_pool_clients(self, user_pool_id): + @paginate(60) + def list_user_pool_clients(self, user_pool_id, max_results=None, next_token=None): user_pool = self.user_pools.get(user_pool_id) if not user_pool: raise ResourceNotFoundError(user_pool_id) @@ -339,7 +379,8 @@ class CognitoIdpBackend(BaseBackend): user_pool.identity_providers[name] = identity_provider return identity_provider - def list_identity_providers(self, user_pool_id): + @paginate(60) + def list_identity_providers(self, user_pool_id, max_results=None, next_token=None): user_pool = self.user_pools.get(user_pool_id) if not user_pool: raise ResourceNotFoundError(user_pool_id) @@ -387,7 +428,8 @@ class CognitoIdpBackend(BaseBackend): return user_pool.users[username] - def list_users(self, user_pool_id): + @paginate(60, "pagination_token", "limit") + def list_users(self, user_pool_id, pagination_token=None, limit=None): user_pool = self.user_pools.get(user_pool_id) if not user_pool: raise ResourceNotFoundError(user_pool_id) diff --git a/moto/cognitoidp/responses.py b/moto/cognitoidp/responses.py index 50939786..cb5374a6 100644 --- a/moto/cognitoidp/responses.py +++ b/moto/cognitoidp/responses.py @@ -22,10 +22,17 @@ class CognitoIdpResponse(BaseResponse): }) def list_user_pools(self): - user_pools = cognitoidp_backends[self.region].list_user_pools() - return json.dumps({ - "UserPools": [user_pool.to_json() for user_pool in user_pools] - }) + max_results = self._get_param("MaxResults") + next_token = self._get_param("NextToken", "0") + user_pools, next_token = cognitoidp_backends[self.region].list_user_pools( + max_results=max_results, next_token=next_token + ) + response = { + "UserPools": [user_pool.to_json() for user_pool in user_pools], + } + if next_token: + response["NextToken"] = str(next_token) + return json.dumps(response) def describe_user_pool(self): user_pool_id = self._get_param("UserPoolId") @@ -72,10 +79,16 @@ class CognitoIdpResponse(BaseResponse): def list_user_pool_clients(self): user_pool_id = self._get_param("UserPoolId") - user_pool_clients = cognitoidp_backends[self.region].list_user_pool_clients(user_pool_id) - return json.dumps({ + max_results = self._get_param("MaxResults") + next_token = self._get_param("NextToken", "0") + user_pool_clients, next_token = cognitoidp_backends[self.region].list_user_pool_clients(user_pool_id, + max_results=max_results, next_token=next_token) + response = { "UserPoolClients": [user_pool_client.to_json() for user_pool_client in user_pool_clients] - }) + } + if next_token: + response["NextToken"] = str(next_token) + return json.dumps(response) def describe_user_pool_client(self): user_pool_id = self._get_param("UserPoolId") @@ -110,10 +123,17 @@ class CognitoIdpResponse(BaseResponse): def list_identity_providers(self): user_pool_id = self._get_param("UserPoolId") - identity_providers = cognitoidp_backends[self.region].list_identity_providers(user_pool_id) - return json.dumps({ + max_results = self._get_param("MaxResults") + next_token = self._get_param("NextToken", "0") + identity_providers, next_token = cognitoidp_backends[self.region].list_identity_providers( + user_pool_id, max_results=max_results, next_token=next_token + ) + response = { "Providers": [identity_provider.to_json() for identity_provider in identity_providers] - }) + } + if next_token: + response["NextToken"] = str(next_token) + return json.dumps(response) def describe_identity_provider(self): user_pool_id = self._get_param("UserPoolId") @@ -155,10 +175,15 @@ class CognitoIdpResponse(BaseResponse): def list_users(self): user_pool_id = self._get_param("UserPoolId") - users = cognitoidp_backends[self.region].list_users(user_pool_id) - return json.dumps({ - "Users": [user.to_json(extended=True) for user in users] - }) + limit = self._get_param("Limit") + token = self._get_param("PaginationToken") + users, token = cognitoidp_backends[self.region].list_users(user_pool_id, + limit=limit, + pagination_token=token) + response = {"Users": [user.to_json(extended=True) for user in users]} + if token: + response["PaginationToken"] = str(token) + return json.dumps(response) def admin_disable_user(self): user_pool_id = self._get_param("UserPoolId") diff --git a/tests/test_cognitoidp/test_cognitoidp.py b/tests/test_cognitoidp/test_cognitoidp.py index f72a4476..362740ce 100644 --- a/tests/test_cognitoidp/test_cognitoidp.py +++ b/tests/test_cognitoidp/test_cognitoidp.py @@ -41,6 +41,56 @@ def test_list_user_pools(): result["UserPools"][0]["Name"].should.equal(name) +@mock_cognitoidp +def test_list_user_pools_returns_max_items(): + conn = boto3.client("cognito-idp", "us-west-2") + + # Given 10 user pools + pool_count = 10 + for i in range(pool_count): + conn.create_user_pool(PoolName=str(uuid.uuid4())) + + max_results = 5 + result = conn.list_user_pools(MaxResults=max_results) + result["UserPools"].should.have.length_of(max_results) + result.should.have.key("NextToken") + + +@mock_cognitoidp +def test_list_user_pools_returns_next_tokens(): + conn = boto3.client("cognito-idp", "us-west-2") + + # Given 10 user pool clients + pool_count = 10 + for i in range(pool_count): + conn.create_user_pool(PoolName=str(uuid.uuid4())) + + max_results = 5 + result = conn.list_user_pools(MaxResults=max_results) + result["UserPools"].should.have.length_of(max_results) + result.should.have.key("NextToken") + + next_token = result["NextToken"] + result_2 = conn.list_user_pools(MaxResults=max_results, NextToken=next_token) + result_2["UserPools"].should.have.length_of(max_results) + result_2.shouldnt.have.key("NextToken") + + +@mock_cognitoidp +def test_list_user_pools_when_max_items_more_than_total_items(): + conn = boto3.client("cognito-idp", "us-west-2") + + # Given 10 user pool clients + pool_count = 10 + for i in range(pool_count): + conn.create_user_pool(PoolName=str(uuid.uuid4())) + + max_results = pool_count + 5 + result = conn.list_user_pools(MaxResults=max_results) + result["UserPools"].should.have.length_of(pool_count) + result.shouldnt.have.key("NextToken") + + @mock_cognitoidp def test_describe_user_pool(): conn = boto3.client("cognito-idp", "us-west-2") @@ -140,6 +190,67 @@ def test_list_user_pool_clients(): result["UserPoolClients"][0]["ClientName"].should.equal(client_name) +@mock_cognitoidp +def test_list_user_pool_clients_returns_max_items(): + conn = boto3.client("cognito-idp", "us-west-2") + user_pool_id = conn.create_user_pool(PoolName=str(uuid.uuid4()))["UserPool"]["Id"] + + # Given 10 user pool clients + client_count = 10 + for i in range(client_count): + client_name = str(uuid.uuid4()) + conn.create_user_pool_client(UserPoolId=user_pool_id, + ClientName=client_name) + max_results = 5 + result = conn.list_user_pool_clients(UserPoolId=user_pool_id, + MaxResults=max_results) + result["UserPoolClients"].should.have.length_of(max_results) + result.should.have.key("NextToken") + + +@mock_cognitoidp +def test_list_user_pool_clients_returns_next_tokens(): + conn = boto3.client("cognito-idp", "us-west-2") + user_pool_id = conn.create_user_pool(PoolName=str(uuid.uuid4()))["UserPool"]["Id"] + + # Given 10 user pool clients + client_count = 10 + for i in range(client_count): + client_name = str(uuid.uuid4()) + conn.create_user_pool_client(UserPoolId=user_pool_id, + ClientName=client_name) + max_results = 5 + result = conn.list_user_pool_clients(UserPoolId=user_pool_id, + MaxResults=max_results) + result["UserPoolClients"].should.have.length_of(max_results) + result.should.have.key("NextToken") + + next_token = result["NextToken"] + result_2 = conn.list_user_pool_clients(UserPoolId=user_pool_id, + MaxResults=max_results, + NextToken=next_token) + result_2["UserPoolClients"].should.have.length_of(max_results) + result_2.shouldnt.have.key("NextToken") + + +@mock_cognitoidp +def test_list_user_pool_clients_when_max_items_more_than_total_items(): + conn = boto3.client("cognito-idp", "us-west-2") + user_pool_id = conn.create_user_pool(PoolName=str(uuid.uuid4()))["UserPool"]["Id"] + + # Given 10 user pool clients + client_count = 10 + for i in range(client_count): + client_name = str(uuid.uuid4()) + conn.create_user_pool_client(UserPoolId=user_pool_id, + ClientName=client_name) + max_results = client_count + 5 + result = conn.list_user_pool_clients(UserPoolId=user_pool_id, + MaxResults=max_results) + result["UserPoolClients"].should.have.length_of(client_count) + result.shouldnt.have.key("NextToken") + + @mock_cognitoidp def test_describe_user_pool_client(): conn = boto3.client("cognito-idp", "us-west-2") @@ -264,6 +375,83 @@ def test_list_identity_providers(): result["Providers"][0]["ProviderType"].should.equal(provider_type) +@mock_cognitoidp +def test_list_identity_providers_returns_max_items(): + conn = boto3.client("cognito-idp", "us-west-2") + user_pool_id = conn.create_user_pool(PoolName=str(uuid.uuid4()))["UserPool"]["Id"] + + # Given 10 identity providers linked to a user pool + identity_provider_count = 10 + for i in range(identity_provider_count): + provider_name = str(uuid.uuid4()) + provider_type = "Facebook" + conn.create_identity_provider( + UserPoolId=user_pool_id, + ProviderName=provider_name, + ProviderType=provider_type, + ProviderDetails={}, + ) + + max_results = 5 + result = conn.list_identity_providers(UserPoolId=user_pool_id, + MaxResults=max_results) + result["Providers"].should.have.length_of(max_results) + result.should.have.key("NextToken") + + +@mock_cognitoidp +def test_list_identity_providers_returns_next_tokens(): + conn = boto3.client("cognito-idp", "us-west-2") + user_pool_id = conn.create_user_pool(PoolName=str(uuid.uuid4()))["UserPool"]["Id"] + + # Given 10 identity providers linked to a user pool + identity_provider_count = 10 + for i in range(identity_provider_count): + provider_name = str(uuid.uuid4()) + provider_type = "Facebook" + conn.create_identity_provider( + UserPoolId=user_pool_id, + ProviderName=provider_name, + ProviderType=provider_type, + ProviderDetails={}, + ) + + max_results = 5 + result = conn.list_identity_providers(UserPoolId=user_pool_id, MaxResults=max_results) + result["Providers"].should.have.length_of(max_results) + result.should.have.key("NextToken") + + next_token = result["NextToken"] + result_2 = conn.list_identity_providers(UserPoolId=user_pool_id, + MaxResults=max_results, + NextToken=next_token) + result_2["Providers"].should.have.length_of(max_results) + result_2.shouldnt.have.key("NextToken") + + +@mock_cognitoidp +def test_list_identity_providers_when_max_items_more_than_total_items(): + conn = boto3.client("cognito-idp", "us-west-2") + user_pool_id = conn.create_user_pool(PoolName=str(uuid.uuid4()))["UserPool"]["Id"] + + # Given 10 identity providers linked to a user pool + identity_provider_count = 10 + for i in range(identity_provider_count): + provider_name = str(uuid.uuid4()) + provider_type = "Facebook" + conn.create_identity_provider( + UserPoolId=user_pool_id, + ProviderName=provider_name, + ProviderType=provider_type, + ProviderDetails={}, + ) + + max_results = identity_provider_count + 5 + result = conn.list_identity_providers(UserPoolId=user_pool_id, MaxResults=max_results) + result["Providers"].should.have.length_of(identity_provider_count) + result.shouldnt.have.key("NextToken") + + @mock_cognitoidp def test_describe_identity_providers(): conn = boto3.client("cognito-idp", "us-west-2") @@ -396,6 +584,62 @@ def test_list_users(): result["Users"][0]["Username"].should.equal(username) +@mock_cognitoidp +def test_list_users_returns_limit_items(): + conn = boto3.client("cognito-idp", "us-west-2") + user_pool_id = conn.create_user_pool(PoolName=str(uuid.uuid4()))["UserPool"]["Id"] + + # Given 10 users + user_count = 10 + for i in range(user_count): + conn.admin_create_user(UserPoolId=user_pool_id, + Username=str(uuid.uuid4())) + max_results = 5 + result = conn.list_users(UserPoolId=user_pool_id, Limit=max_results) + result["Users"].should.have.length_of(max_results) + result.should.have.key("PaginationToken") + + +@mock_cognitoidp +def test_list_users_returns_pagination_tokens(): + conn = boto3.client("cognito-idp", "us-west-2") + user_pool_id = conn.create_user_pool(PoolName=str(uuid.uuid4()))["UserPool"]["Id"] + + # Given 10 users + user_count = 10 + for i in range(user_count): + conn.admin_create_user(UserPoolId=user_pool_id, + Username=str(uuid.uuid4())) + + max_results = 5 + result = conn.list_users(UserPoolId=user_pool_id, Limit=max_results) + result["Users"].should.have.length_of(max_results) + result.should.have.key("PaginationToken") + + next_token = result["PaginationToken"] + result_2 = conn.list_users(UserPoolId=user_pool_id, + Limit=max_results, PaginationToken=next_token) + result_2["Users"].should.have.length_of(max_results) + result_2.shouldnt.have.key("PaginationToken") + + +@mock_cognitoidp +def test_list_users_when_limit_more_than_total_items(): + conn = boto3.client("cognito-idp", "us-west-2") + user_pool_id = conn.create_user_pool(PoolName=str(uuid.uuid4()))["UserPool"]["Id"] + + # Given 10 users + user_count = 10 + for i in range(user_count): + conn.admin_create_user(UserPoolId=user_pool_id, + Username=str(uuid.uuid4())) + + max_results = user_count + 5 + result = conn.list_users(UserPoolId=user_pool_id, Limit=max_results) + result["Users"].should.have.length_of(user_count) + result.shouldnt.have.key("PaginationToken") + + @mock_cognitoidp def test_admin_disable_user(): conn = boto3.client("cognito-idp", "us-west-2") From 80f860727fbacece3717d258d9079faae8fc5334 Mon Sep 17 00:00:00 2001 From: Tomoya Iwata Date: Thu, 25 Oct 2018 20:34:53 +0900 Subject: [PATCH 07/40] Add support for IoT attach_policy --- IMPLEMENTATION_COVERAGE.md | 4 ++-- moto/iot/models.py | 8 ++++++++ moto/iot/responses.py | 9 +++++++++ tests/test_iot/test_iot.py | 29 +++++++++++++++++++++++++++++ 4 files changed, 48 insertions(+), 2 deletions(-) diff --git a/IMPLEMENTATION_COVERAGE.md b/IMPLEMENTATION_COVERAGE.md index 7c68c0e3..7a1e2e7a 100644 --- a/IMPLEMENTATION_COVERAGE.md +++ b/IMPLEMENTATION_COVERAGE.md @@ -2376,11 +2376,11 @@ - [ ] unsubscribe_from_event - [ ] update_assessment_target -## iot - 30% implemented +## iot - 31% implemented - [ ] accept_certificate_transfer - [X] add_thing_to_thing_group - [ ] associate_targets_with_job -- [ ] attach_policy +- [X] attach_policy - [X] attach_principal_policy - [X] attach_thing_principal - [ ] cancel_certificate_transfer diff --git a/moto/iot/models.py b/moto/iot/models.py index c36bb985..db9ad381 100644 --- a/moto/iot/models.py +++ b/moto/iot/models.py @@ -429,6 +429,14 @@ class IoTBackend(BaseBackend): pass raise ResourceNotFoundException() + def attach_policy(self, policy_name, target): + principal = self._get_principal(target) + policy = self.get_policy(policy_name) + k = (target, policy_name) + if k in self.principal_policies: + return + self.principal_policies[k] = (principal, policy) + def attach_principal_policy(self, policy_name, principal_arn): principal = self._get_principal(principal_arn) policy = self.get_policy(policy_name) diff --git a/moto/iot/responses.py b/moto/iot/responses.py index 006c4c4c..042e5a31 100644 --- a/moto/iot/responses.py +++ b/moto/iot/responses.py @@ -224,6 +224,15 @@ class IoTResponse(BaseResponse): ) return json.dumps(dict()) + def attach_policy(self): + policy_name = self._get_param("policyName") + target = self._get_param('target') + self.iot_backend.attach_policy( + policy_name=policy_name, + target=target, + ) + return json.dumps(dict()) + def attach_principal_policy(self): policy_name = self._get_param("policyName") principal = self.headers.get('x-amzn-iot-principal') diff --git a/tests/test_iot/test_iot.py b/tests/test_iot/test_iot.py index 5c6effd7..9082203d 100644 --- a/tests/test_iot/test_iot.py +++ b/tests/test_iot/test_iot.py @@ -309,6 +309,35 @@ def test_policy(): @mock_iot def test_principal_policy(): + client = boto3.client('iot', region_name='ap-northeast-1') + policy_name = 'my-policy' + doc = '{}' + client.create_policy(policyName=policy_name, policyDocument=doc) + cert = client.create_keys_and_certificate(setAsActive=True) + cert_arn = cert['certificateArn'] + + client.attach_policy(policyName=policy_name, target=cert_arn) + + res = client.list_principal_policies(principal=cert_arn) + res.should.have.key('policies').which.should.have.length_of(1) + for policy in res['policies']: + policy.should.have.key('policyName').which.should_not.be.none + policy.should.have.key('policyArn').which.should_not.be.none + + res = client.list_policy_principals(policyName=policy_name) + res.should.have.key('principals').which.should.have.length_of(1) + for principal in res['principals']: + principal.should_not.be.none + + client.detach_principal_policy(policyName=policy_name, principal=cert_arn) + res = client.list_principal_policies(principal=cert_arn) + res.should.have.key('policies').which.should.have.length_of(0) + res = client.list_policy_principals(policyName=policy_name) + res.should.have.key('principals').which.should.have.length_of(0) + + +@mock_iot +def test_principal_policy_deprecated(): client = boto3.client('iot', region_name='ap-northeast-1') policy_name = 'my-policy' doc = '{}' From b485122ec6e3bae5f80dc39742b323f4eaf5ac06 Mon Sep 17 00:00:00 2001 From: zane Date: Wed, 24 Oct 2018 14:06:23 -0700 Subject: [PATCH 08/40] refactor to store multiple scrects, use uuid --- .gitignore | 2 +- moto/secretsmanager/models.py | 79 ++++++++++--------- moto/secretsmanager/utils.py | 5 +- .../test_secretsmanager.py | 18 +++-- tests/test_secretsmanager/test_server.py | 41 ++++++++-- 5 files changed, 91 insertions(+), 54 deletions(-) diff --git a/.gitignore b/.gitignore index 7f57e98e..cfda5144 100644 --- a/.gitignore +++ b/.gitignore @@ -15,4 +15,4 @@ python_env .ropeproject/ .pytest_cache/ venv/ - +.vscode/ diff --git a/moto/secretsmanager/models.py b/moto/secretsmanager/models.py index 1404a0ec..7f89e2eb 100644 --- a/moto/secretsmanager/models.py +++ b/moto/secretsmanager/models.py @@ -2,6 +2,7 @@ from __future__ import unicode_literals import time import json +import uuid import boto3 @@ -18,10 +19,6 @@ class SecretsManager(BaseModel): def __init__(self, region_name, **kwargs): self.region = region_name - self.secret_id = kwargs.get('secret_id', '') - self.version_id = kwargs.get('version_id', '') - self.version_stage = kwargs.get('version_stage', '') - self.secret_string = '' class SecretsManagerBackend(BaseBackend): @@ -29,14 +26,7 @@ class SecretsManagerBackend(BaseBackend): def __init__(self, region_name=None, **kwargs): super(SecretsManagerBackend, self).__init__() self.region = region_name - self.secret_id = kwargs.get('secret_id', '') - self.name = kwargs.get('name', '') - self.createdate = int(time.time()) - self.secret_string = '' - self.rotation_enabled = False - self.rotation_lambda_arn = '' - self.auto_rotate_after_days = 0 - self.version_id = '' + self.secrets = {} def reset(self): region_name = self.region @@ -44,36 +34,49 @@ class SecretsManagerBackend(BaseBackend): self.__init__(region_name) def _is_valid_identifier(self, identifier): - return identifier in (self.name, self.secret_id) + return identifier in self.secrets def get_secret_value(self, secret_id, version_id, version_stage): if not self._is_valid_identifier(secret_id): raise ResourceNotFoundException() + secret = self.secrets[secret_id] + response = json.dumps({ - "ARN": secret_arn(self.region, self.secret_id), - "Name": self.name, - "VersionId": "A435958A-D821-4193-B719-B7769357AER4", - "SecretString": self.secret_string, + "ARN": secret_arn(self.region, secret['secret_id']), + "Name": secret['name'], + "VersionId": secret['version_id'], + "SecretString": secret['secret_string'], "VersionStages": [ "AWSCURRENT", ], - "CreatedDate": "2018-05-23 13:16:57.198000" + "CreatedDate": secret['createdate'] }) return response def create_secret(self, name, secret_string, **kwargs): - self.secret_string = secret_string - self.secret_id = name - self.name = name + generated_version_id = str(uuid.uuid4()) + + secret = { + 'secret_string': secret_string, + 'secret_id': name, + 'name': name, + 'createdate': int(time.time()), + 'rotation_enabled': False, + 'rotation_lambda_arn': '', + 'auto_rotate_after_days': 0, + 'version_id': generated_version_id + } + + self.secrets[name] = secret response = json.dumps({ "ARN": secret_arn(self.region, name), - "Name": self.name, - "VersionId": "A435958A-D821-4193-B719-B7769357AER4", + "Name": name, + "VersionId": generated_version_id, }) return response @@ -82,15 +85,17 @@ class SecretsManagerBackend(BaseBackend): if not self._is_valid_identifier(secret_id): raise ResourceNotFoundException + secret = self.secrets[secret_id] + response = json.dumps({ - "ARN": secret_arn(self.region, self.secret_id), - "Name": self.name, + "ARN": secret_arn(self.region, secret['secret_id']), + "Name": secret['name'], "Description": "", "KmsKeyId": "", - "RotationEnabled": self.rotation_enabled, - "RotationLambdaARN": self.rotation_lambda_arn, + "RotationEnabled": secret['rotation_enabled'], + "RotationLambdaARN": secret['rotation_lambda_arn'], "RotationRules": { - "AutomaticallyAfterDays": self.auto_rotate_after_days + "AutomaticallyAfterDays": secret['auto_rotate_after_days'] }, "LastRotatedDate": None, "LastChangedDate": None, @@ -141,17 +146,19 @@ class SecretsManagerBackend(BaseBackend): ) raise InvalidParameterException(msg) - self.version_id = client_request_token or '' - self.rotation_lambda_arn = rotation_lambda_arn or '' + secret = self.secrets[secret_id] + + secret['version_id'] = client_request_token or '' + secret['rotation_lambda_arn'] = rotation_lambda_arn or '' if rotation_rules: - self.auto_rotate_after_days = rotation_rules.get(rotation_days, 0) - if self.auto_rotate_after_days > 0: - self.rotation_enabled = True + secret['auto_rotate_after_days'] = rotation_rules.get(rotation_days, 0) + if secret['auto_rotate_after_days'] > 0: + secret['rotation_enabled'] = True response = json.dumps({ - "ARN": secret_arn(self.region, self.secret_id), - "Name": self.name, - "VersionId": self.version_id + "ARN": secret_arn(self.region, secret['secret_id']), + "Name": secret['name'], + "VersionId": secret['version_id'] }) return response diff --git a/moto/secretsmanager/utils.py b/moto/secretsmanager/utils.py index 2cb92020..231fea29 100644 --- a/moto/secretsmanager/utils.py +++ b/moto/secretsmanager/utils.py @@ -52,8 +52,9 @@ def random_password(password_length, exclude_characters, exclude_numbers, def secret_arn(region, secret_id): - return "arn:aws:secretsmanager:{0}:1234567890:secret:{1}-rIjad".format( - region, secret_id) + id_string = ''.join(random.choice(string.ascii_letters) for _ in range(5)) + return "arn:aws:secretsmanager:{0}:1234567890:secret:{1}-{2}".format( + region, secret_id, id_string) def _exclude_characters(password, exclude_characters): diff --git a/tests/test_secretsmanager/test_secretsmanager.py b/tests/test_secretsmanager/test_secretsmanager.py index ec384a66..0e0b98b1 100644 --- a/tests/test_secretsmanager/test_secretsmanager.py +++ b/tests/test_secretsmanager/test_secretsmanager.py @@ -39,8 +39,7 @@ def test_create_secret(): conn = boto3.client('secretsmanager', region_name='us-east-1') result = conn.create_secret(Name='test-secret', SecretString="foosecret") - assert result['ARN'] == ( - 'arn:aws:secretsmanager:us-east-1:1234567890:secret:test-secret-rIjad') + assert result['ARN'] assert result['Name'] == 'test-secret' secret = conn.get_secret_value(SecretId='test-secret') assert secret['SecretString'] == 'foosecret' @@ -159,10 +158,17 @@ def test_describe_secret(): conn.create_secret(Name='test-secret', SecretString='foosecret') + conn.create_secret(Name='test-secret-2', + SecretString='barsecret') + secret_description = conn.describe_secret(SecretId='test-secret') + secret_description_2 = conn.describe_secret(SecretId='test-secret-2') + assert secret_description # Returned dict is not empty - assert secret_description['ARN'] == ( - 'arn:aws:secretsmanager:us-west-2:1234567890:secret:test-secret-rIjad') + assert secret_description['Name'] == ('test-secret') + assert secret_description['ARN'] != '' # Test arn not empty + assert secret_description_2['Name'] == ('test-secret-2') + assert secret_description_2['ARN'] != '' # Test arn not empty @mock_secretsmanager def test_describe_secret_that_does_not_exist(): @@ -190,9 +196,7 @@ def test_rotate_secret(): rotated_secret = conn.rotate_secret(SecretId=secret_name) assert rotated_secret - assert rotated_secret['ARN'] == ( - 'arn:aws:secretsmanager:us-west-2:1234567890:secret:test-secret-rIjad' - ) + assert rotated_secret['ARN'] != '' # Test arn not empty assert rotated_secret['Name'] == secret_name assert rotated_secret['VersionId'] != '' diff --git a/tests/test_secretsmanager/test_server.py b/tests/test_secretsmanager/test_server.py index e573f9b6..d0f495f5 100644 --- a/tests/test_secretsmanager/test_server.py +++ b/tests/test_secretsmanager/test_server.py @@ -82,11 +82,20 @@ def test_create_secret(): headers={ "X-Amz-Target": "secretsmanager.CreateSecret"}, ) + res_2 = test_client.post('/', + data={"Name": "test-secret-2", + "SecretString": "bar-secret"}, + headers={ + "X-Amz-Target": "secretsmanager.CreateSecret"}, + ) json_data = json.loads(res.data.decode("utf-8")) - assert json_data['ARN'] == ( - 'arn:aws:secretsmanager:us-east-1:1234567890:secret:test-secret-rIjad') + assert json_data['ARN'] != '' assert json_data['Name'] == 'test-secret' + + json_data_2 = json.loads(res_2.data.decode("utf-8")) + assert json_data_2['ARN'] != '' + assert json_data_2['Name'] == 'test-secret-2' @mock_secretsmanager def test_describe_secret(): @@ -107,12 +116,30 @@ def test_describe_secret(): "X-Amz-Target": "secretsmanager.DescribeSecret" }, ) + + create_secret_2 = test_client.post('/', + data={"Name": "test-secret-2", + "SecretString": "barsecret"}, + headers={ + "X-Amz-Target": "secretsmanager.CreateSecret" + }, + ) + describe_secret_2 = test_client.post('/', + data={"SecretId": "test-secret-2"}, + headers={ + "X-Amz-Target": "secretsmanager.DescribeSecret" + }, + ) json_data = json.loads(describe_secret.data.decode("utf-8")) assert json_data # Returned dict is not empty - assert json_data['ARN'] == ( - 'arn:aws:secretsmanager:us-east-1:1234567890:secret:test-secret-rIjad' - ) + assert json_data['ARN'] != '' + assert json_data['Name'] == 'test-secret' + + json_data_2 = json.loads(describe_secret_2.data.decode("utf-8")) + assert json_data_2 # Returned dict is not empty + assert json_data_2['ARN'] != '' + assert json_data_2['Name'] == 'test-secret-2' @mock_secretsmanager def test_describe_secret_that_does_not_exist(): @@ -179,9 +206,7 @@ def test_rotate_secret(): json_data = json.loads(rotate_secret.data.decode("utf-8")) assert json_data # Returned dict is not empty - assert json_data['ARN'] == ( - 'arn:aws:secretsmanager:us-east-1:1234567890:secret:test-secret-rIjad' - ) + assert json_data['ARN'] != '' assert json_data['Name'] == 'test-secret' assert json_data['VersionId'] == client_request_token From 5ca68fbf06adecb0c0478f102a9cc7d331aa36c4 Mon Sep 17 00:00:00 2001 From: Illia Volochii Date: Fri, 26 Oct 2018 22:12:26 +0300 Subject: [PATCH 09/40] Update jsondiff to 1.1.2 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 98780dd5..34002b79 100755 --- a/setup.py +++ b/setup.py @@ -21,7 +21,7 @@ install_requires = [ "python-jose<3.0.0", "mock", "docker>=2.5.1", - "jsondiff==1.1.1", + "jsondiff==1.1.2", "aws-xray-sdk<0.96,>=0.93", "responses>=0.9.0", ] From 090cad8c88169e3f6bd049d023d4e4d2fe309025 Mon Sep 17 00:00:00 2001 From: Jamie Starke Date: Fri, 26 Oct 2018 21:54:01 -0700 Subject: [PATCH 10/40] [1009] Converts ECS Instance full arn to instance_id for storage --- moto/ecs/models.py | 2 ++ tests/test_ecs/test_ecs_boto3.py | 59 ++++++++++++++++++++++++++++++++ 2 files changed, 61 insertions(+) diff --git a/moto/ecs/models.py b/moto/ecs/models.py index d0085384..4a6737ce 100644 --- a/moto/ecs/models.py +++ b/moto/ecs/models.py @@ -769,6 +769,8 @@ class EC2ContainerServiceBackend(BaseBackend): Container instances status should be one of [ACTIVE,DRAINING]") failures = [] container_instance_objects = [] + list_container_instance_ids = [x.split('/')[-1] + for x in list_container_instance_ids] for container_instance_id in list_container_instance_ids: container_instance = self.container_instances[cluster_name].get(container_instance_id, None) if container_instance is not None: diff --git a/tests/test_ecs/test_ecs_boto3.py b/tests/test_ecs/test_ecs_boto3.py index 70c1463e..7274e62c 100644 --- a/tests/test_ecs/test_ecs_boto3.py +++ b/tests/test_ecs/test_ecs_boto3.py @@ -910,6 +910,65 @@ def test_update_container_instances_state(): status='test_status').should.throw(Exception) +@mock_ec2 +@mock_ecs +def test_update_container_instances_state_by_arn(): + ecs_client = boto3.client('ecs', region_name='us-east-1') + ec2 = boto3.resource('ec2', region_name='us-east-1') + + test_cluster_name = 'test_ecs_cluster' + _ = ecs_client.create_cluster( + clusterName=test_cluster_name + ) + + instance_to_create = 3 + test_instance_arns = [] + for i in range(0, instance_to_create): + test_instance = ec2.create_instances( + ImageId="ami-1234abcd", + MinCount=1, + MaxCount=1, + )[0] + + instance_id_document = json.dumps( + ec2_utils.generate_instance_identity_document(test_instance) + ) + + response = ecs_client.register_container_instance( + cluster=test_cluster_name, + instanceIdentityDocument=instance_id_document) + + test_instance_arns.append(response['containerInstance']['containerInstanceArn']) + + response = ecs_client.update_container_instances_state(cluster=test_cluster_name, + containerInstances=test_instance_arns, + status='DRAINING') + len(response['failures']).should.equal(0) + len(response['containerInstances']).should.equal(instance_to_create) + response_statuses = [ci['status'] for ci in response['containerInstances']] + for status in response_statuses: + status.should.equal('DRAINING') + response = ecs_client.update_container_instances_state(cluster=test_cluster_name, + containerInstances=test_instance_arns, + status='DRAINING') + len(response['failures']).should.equal(0) + len(response['containerInstances']).should.equal(instance_to_create) + response_statuses = [ci['status'] for ci in response['containerInstances']] + for status in response_statuses: + status.should.equal('DRAINING') + response = ecs_client.update_container_instances_state(cluster=test_cluster_name, + containerInstances=test_instance_arns, + status='ACTIVE') + len(response['failures']).should.equal(0) + len(response['containerInstances']).should.equal(instance_to_create) + response_statuses = [ci['status'] for ci in response['containerInstances']] + for status in response_statuses: + status.should.equal('ACTIVE') + ecs_client.update_container_instances_state.when.called_with(cluster=test_cluster_name, + containerInstances=test_instance_arns, + status='test_status').should.throw(Exception) + + @mock_ec2 @mock_ecs def test_run_task(): From 249dd7059e9c91712e459355126b0d4b57b25583 Mon Sep 17 00:00:00 2001 From: Tomoya Iwata Date: Sun, 28 Oct 2018 17:13:17 +0900 Subject: [PATCH 11/40] add test case for IoT attach_policy do nothing if policy have already attached to certificate --- tests/test_iot/test_iot.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/tests/test_iot/test_iot.py b/tests/test_iot/test_iot.py index 9082203d..47ea9d59 100644 --- a/tests/test_iot/test_iot.py +++ b/tests/test_iot/test_iot.py @@ -318,6 +318,15 @@ def test_principal_policy(): client.attach_policy(policyName=policy_name, target=cert_arn) + res = client.list_principal_policies(principal=cert_arn) + res.should.have.key('policies').which.should.have.length_of(1) + for policy in res['policies']: + policy.should.have.key('policyName').which.should_not.be.none + policy.should.have.key('policyArn').which.should_not.be.none + + # do nothing if policy have already attached to certificate + client.attach_policy(policyName=policy_name, target=cert_arn) + res = client.list_principal_policies(principal=cert_arn) res.should.have.key('policies').which.should.have.length_of(1) for policy in res['policies']: From 9ba28a05b8b5aa709bde7188c6886f81155fd231 Mon Sep 17 00:00:00 2001 From: George Alton Date: Sun, 28 Oct 2018 11:00:21 +0000 Subject: [PATCH 12/40] avoids copying entire result into a new list --- moto/cognitoidp/models.py | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/moto/cognitoidp/models.py b/moto/cognitoidp/models.py index 521701de..edeb7b12 100644 --- a/moto/cognitoidp/models.py +++ b/moto/cognitoidp/models.py @@ -2,6 +2,7 @@ from __future__ import unicode_literals import datetime import functools +import itertools import json import os import time @@ -43,16 +44,12 @@ def paginate(limit, start_arg="next_token", limit_arg="max_results"): def outer_wrapper(func): @functools.wraps(func) def wrapper(*args, **kwargs): - # setup start = int(default_start if kwargs.get(start_arg) is None else kwargs[start_arg]) - stop = int(limit if kwargs.get(limit_arg) is None else kwargs[limit_arg]) - end = start + stop - # call + lim = int(limit if kwargs.get(limit_arg) is None else kwargs[limit_arg]) + stop = start + lim result = func(*args, **kwargs) - # modify - results = list(result) - limited_results = results[start: end] - next_token = end if end < len(results) else None + limited_results = list(itertools.islice(result, start, stop)) + next_token = stop if stop < len(result) else None return limited_results, next_token return wrapper return outer_wrapper From a0708a70fcc1b265a25fccb01ec1ca67e409a5ce Mon Sep 17 00:00:00 2001 From: Mark Challoner Date: Mon, 29 Oct 2018 13:29:53 +0000 Subject: [PATCH 13/40] Fix Tags parameter on CloudFormation create_change_set method. --- moto/cloudformation/responses.py | 3 ++- .../test_cloudformation_stack_crud_boto3.py | 3 +++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/moto/cloudformation/responses.py b/moto/cloudformation/responses.py index a1295a20..fe436939 100644 --- a/moto/cloudformation/responses.py +++ b/moto/cloudformation/responses.py @@ -87,7 +87,8 @@ class CloudFormationResponse(BaseResponse): role_arn = self._get_param('RoleARN') update_or_create = self._get_param('ChangeSetType', 'CREATE') parameters_list = self._get_list_prefix("Parameters.member") - tags = {tag[0]: tag[1] for tag in self._get_list_prefix("Tags.member")} + tags = dict((item['key'], item['value']) + for item in self._get_list_prefix("Tags.member")) parameters = {param['parameter_key']: param['parameter_value'] for param in parameters_list} if template_url: diff --git a/tests/test_cloudformation/test_cloudformation_stack_crud_boto3.py b/tests/test_cloudformation/test_cloudformation_stack_crud_boto3.py index 9bfae617..064e0fb3 100644 --- a/tests/test_cloudformation/test_cloudformation_stack_crud_boto3.py +++ b/tests/test_cloudformation/test_cloudformation_stack_crud_boto3.py @@ -391,6 +391,9 @@ def test_create_change_set_from_s3_url(): TemplateURL=key_url, ChangeSetName='NewChangeSet', ChangeSetType='CREATE', + Tags=[ + {'Key': 'tag-key', 'Value': 'tag-value'} + ], ) assert 'arn:aws:cloudformation:us-west-1:123456789:changeSet/NewChangeSet/' in response['Id'] assert 'arn:aws:cloudformation:us-east-1:123456789:stack/NewStack' in response['StackId'] From 3d71a67794be7d4d3e2542d513f9e376ddb7635c Mon Sep 17 00:00:00 2001 From: Tomoya Iwata Date: Tue, 30 Oct 2018 14:38:59 +0900 Subject: [PATCH 14/40] Add some validations for IoT delete operations fix #1908 --- moto/iot/exceptions.py | 17 +++++++ moto/iot/models.py | 30 ++++++++++++- tests/test_iot/test_iot.py | 92 ++++++++++++++++++++++++++++++++++++++ 3 files changed, 138 insertions(+), 1 deletion(-) diff --git a/moto/iot/exceptions.py b/moto/iot/exceptions.py index 7bbdb706..3af3751d 100644 --- a/moto/iot/exceptions.py +++ b/moto/iot/exceptions.py @@ -31,3 +31,20 @@ class VersionConflictException(IoTClientError): 'VersionConflictException', 'The version for thing %s does not match the expected version.' % name ) + + +class CertificateStateException(IoTClientError): + def __init__(self, msg, cert_id): + self.code = 406 + super(CertificateStateException, self).__init__( + 'CertificateStateException', + '%s Id: %s' % (msg, cert_id) + ) + + +class DeleteConflictException(IoTClientError): + def __init__(self, msg): + self.code = 409 + super(DeleteConflictException, self).__init__( + 'DeleteConflictException', msg + ) diff --git a/moto/iot/models.py b/moto/iot/models.py index c36bb985..4b0b260e 100644 --- a/moto/iot/models.py +++ b/moto/iot/models.py @@ -13,6 +13,8 @@ import boto3 from moto.core import BaseBackend, BaseModel from .exceptions import ( + CertificateStateException, + DeleteConflictException, ResourceNotFoundException, InvalidRequestException, VersionConflictException @@ -378,7 +380,25 @@ class IoTBackend(BaseBackend): return certificate, key_pair def delete_certificate(self, certificate_id): - self.describe_certificate(certificate_id) + cert = self.describe_certificate(certificate_id) + if cert.status == 'ACTIVE': + raise CertificateStateException( + 'Certificate must be deactivated (not ACTIVE) before deletion.', certificate_id) + + certs = [k[0] for k, v in self.principal_things.items() + if self._get_principal(k[0]).certificate_id == certificate_id] + if len(certs) > 0: + raise DeleteConflictException( + 'Things must be detached before deletion (arn: %s)' % certs[0] + ) + + certs = [k[0] for k, v in self.principal_policies.items() + if self._get_principal(k[0]).certificate_id == certificate_id] + if len(certs) > 0: + raise DeleteConflictException( + 'Certificate policies must be detached before deletion (arn: %s)' % certs[0] + ) + del self.certificates[certificate_id] def describe_certificate(self, certificate_id): @@ -411,6 +431,14 @@ class IoTBackend(BaseBackend): return policies[0] def delete_policy(self, policy_name): + + policies = [k[1] for k, v in self.principal_policies.items() if k[1] == policy_name] + if len(policies) > 0: + raise DeleteConflictException( + 'The policy cannot be deleted as the policy is attached to one or more principals (name=%s)' + % policy_name + ) + policy = self.get_policy(policy_name) del self.policies[policy.name] diff --git a/tests/test_iot/test_iot.py b/tests/test_iot/test_iot.py index 5c6effd7..21b670d4 100644 --- a/tests/test_iot/test_iot.py +++ b/tests/test_iot/test_iot.py @@ -5,6 +5,8 @@ import sure # noqa import boto3 from moto import mock_iot +from botocore.exceptions import ClientError +from nose.tools import assert_raises @mock_iot @@ -261,6 +263,96 @@ def test_certs(): res.should.have.key('certificates').which.should.have.length_of(0) +@mock_iot +def test_delete_policy_validation(): + doc = """{ + "Version": "2012-10-17", + "Statement":[ + { + "Effect":"Allow", + "Action":[ + "iot: *" + ], + "Resource":"*" + } + ] + } + """ + client = boto3.client('iot', region_name='ap-northeast-1') + cert = client.create_keys_and_certificate(setAsActive=True) + cert_arn = cert['certificateArn'] + policy_name = 'my-policy' + client.create_policy(policyName=policy_name, policyDocument=doc) + client.attach_principal_policy(policyName=policy_name, principal=cert_arn) + + with assert_raises(ClientError) as e: + client.delete_policy(policyName=policy_name) + e.exception.response['Error']['Message'].should.contain( + 'The policy cannot be deleted as the policy is attached to one or more principals (name=%s)' % policy_name) + res = client.list_policies() + res.should.have.key('policies').which.should.have.length_of(1) + + client.detach_principal_policy(policyName=policy_name, principal=cert_arn) + client.delete_policy(policyName=policy_name) + res = client.list_policies() + res.should.have.key('policies').which.should.have.length_of(0) + + +@mock_iot +def test_delete_certificate_validation(): + doc = """{ + "Version": "2012-10-17", + "Statement":[ + { + "Effect":"Allow", + "Action":[ + "iot: *" + ], + "Resource":"*" + } + ] + } + """ + client = boto3.client('iot', region_name='ap-northeast-1') + cert = client.create_keys_and_certificate(setAsActive=True) + cert_id = cert['certificateId'] + cert_arn = cert['certificateArn'] + policy_name = 'my-policy' + thing_name = 'thing-1' + client.create_policy(policyName=policy_name, policyDocument=doc) + client.attach_principal_policy(policyName=policy_name, principal=cert_arn) + client.create_thing(thingName=thing_name) + client.attach_thing_principal(thingName=thing_name, principal=cert_arn) + + with assert_raises(ClientError) as e: + client.delete_certificate(certificateId=cert_id) + e.exception.response['Error']['Message'].should.contain( + 'Certificate must be deactivated (not ACTIVE) before deletion.') + res = client.list_certificates() + res.should.have.key('certificates').which.should.have.length_of(1) + + client.update_certificate(certificateId=cert_id, newStatus='REVOKED') + with assert_raises(ClientError) as e: + client.delete_certificate(certificateId=cert_id) + e.exception.response['Error']['Message'].should.contain( + 'Things must be detached before deletion (arn: %s)' % cert_arn) + res = client.list_certificates() + res.should.have.key('certificates').which.should.have.length_of(1) + + client.detach_thing_principal(thingName=thing_name, principal=cert_arn) + with assert_raises(ClientError) as e: + client.delete_certificate(certificateId=cert_id) + e.exception.response['Error']['Message'].should.contain( + 'Certificate policies must be detached before deletion (arn: %s)' % cert_arn) + res = client.list_certificates() + res.should.have.key('certificates').which.should.have.length_of(1) + + client.detach_principal_policy(policyName=policy_name, principal=cert_arn) + client.delete_certificate(certificateId=cert_id) + res = client.list_certificates() + res.should.have.key('certificates').which.should.have.length_of(0) + + @mock_iot def test_certs_create_inactive(): client = boto3.client('iot', region_name='ap-northeast-1') From e38eea751f55d2e139f3271a73f69d35ff64129b Mon Sep 17 00:00:00 2001 From: jamesandres Date: Wed, 31 Oct 2018 11:39:49 +0000 Subject: [PATCH 15/40] Go easier on the CPU when moto sqs is idle For our local development setup we have found that moto is using around 25% CPU constantly. Digging in with gdb it turned out that it was burning that CPU in the sleeping loop. Here i'm increasing the sleep by 10x which brings the idle CPU usage down by 10x (to ~2%). I'm not familiar enough with the moto/sqs codebase to know if lengthening this sleep will have an adverse effect; however, in other Python dev I've noticed that (in Python 2.7 anyway..) Python threading won't context switch a thread until a sleep of at least 0.01 seconds is performed (shockingly long!). So based on this guesswork I suspect sleeping for 0.01 seconds won't cause any grief. --- moto/sqs/models.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/moto/sqs/models.py b/moto/sqs/models.py index f3262a98..1404ded7 100644 --- a/moto/sqs/models.py +++ b/moto/sqs/models.py @@ -534,7 +534,7 @@ class SQSBackend(BaseBackend): break import time - time.sleep(0.001) + time.sleep(0.01) continue previous_result_count = len(result) From 67cb2e25bbc365484806daccc97d56cc95d6afe1 Mon Sep 17 00:00:00 2001 From: Pall Valmundsson Date: Thu, 1 Nov 2018 19:51:17 +0000 Subject: [PATCH 16/40] Support IAM Credential Report in boto3 Lowercase XML element names in API responses seem to cause issues for boto3. --- moto/iam/responses.py | 8 ++++---- tests/test_iam/test_iam.py | 21 +++++++++++++++++++++ 2 files changed, 25 insertions(+), 4 deletions(-) diff --git a/moto/iam/responses.py b/moto/iam/responses.py index 22558f3f..91ba0954 100644 --- a/moto/iam/responses.py +++ b/moto/iam/responses.py @@ -1243,8 +1243,8 @@ LIST_ACCESS_KEYS_TEMPLATE = """ CREDENTIAL_REPORT_GENERATING = """ - STARTED - No report exists. Starting a new report generation task + STARTED + No report exists. Starting a new report generation task fa788a82-aa8a-11e4-a278-1786c418872b" @@ -1253,7 +1253,7 @@ CREDENTIAL_REPORT_GENERATING = """ CREDENTIAL_REPORT_GENERATED = """ - COMPLETE + COMPLETE fa788a82-aa8a-11e4-a278-1786c418872b" @@ -1262,7 +1262,7 @@ CREDENTIAL_REPORT_GENERATED = """ CREDENTIAL_REPORT = """ - {{ report }} + {{ report }} 2015-02-02T20:02:02Z text/csv diff --git a/tests/test_iam/test_iam.py b/tests/test_iam/test_iam.py index bc23ff71..34e19888 100644 --- a/tests/test_iam/test_iam.py +++ b/tests/test_iam/test_iam.py @@ -536,6 +536,14 @@ def test_generate_credential_report(): result['generate_credential_report_response'][ 'generate_credential_report_result']['state'].should.equal('COMPLETE') +@mock_iam +def test_generate_credential_report_boto3(): + conn = boto3.client('iam', region_name='us-east-1') + result = conn.generate_credential_report() + result['State'].should.equal('STARTED') + result = conn.generate_credential_report() + result['State'].should.equal('COMPLETE') + @mock_iam_deprecated() def test_get_credential_report(): @@ -551,6 +559,19 @@ def test_get_credential_report(): 'get_credential_report_result']['content'].encode('ascii')).decode('ascii') report.should.match(r'.*my-user.*') +@mock_iam +def test_get_credential_report(): + conn = boto3.client('iam', region_name='us-east-1') + conn.create_user(UserName='my-user') + with assert_raises(ClientError): + conn.get_credential_report() + result = conn.generate_credential_report() + while result['State'] != 'COMPLETE': + result = conn.generate_credential_report() + result = conn.get_credential_report() + report = result['Content'].decode('utf-8') + report.should.match(r'.*my-user.*') + @requires_boto_gte('2.39') @mock_iam_deprecated() From e9f8bec91a46c939fc8c23d3b6cac1de30383585 Mon Sep 17 00:00:00 2001 From: Pall Valmundsson Date: Thu, 1 Nov 2018 19:58:09 +0000 Subject: [PATCH 17/40] Fix boto3 IAM test function names --- tests/test_iam/test_iam.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_iam/test_iam.py b/tests/test_iam/test_iam.py index 34e19888..c33bd7fa 100644 --- a/tests/test_iam/test_iam.py +++ b/tests/test_iam/test_iam.py @@ -537,7 +537,7 @@ def test_generate_credential_report(): 'generate_credential_report_result']['state'].should.equal('COMPLETE') @mock_iam -def test_generate_credential_report_boto3(): +def test_boto3_generate_credential_report(): conn = boto3.client('iam', region_name='us-east-1') result = conn.generate_credential_report() result['State'].should.equal('STARTED') @@ -560,7 +560,7 @@ def test_get_credential_report(): report.should.match(r'.*my-user.*') @mock_iam -def test_get_credential_report(): +def test_boto3_get_credential_report(): conn = boto3.client('iam', region_name='us-east-1') conn.create_user(UserName='my-user') with assert_raises(ClientError): From b4b0ae50773f0cd9192f0c908e54010d924ed8ef Mon Sep 17 00:00:00 2001 From: Mike Grima Date: Fri, 19 Oct 2018 16:40:58 -0700 Subject: [PATCH 18/40] Some IAM fixes. - Fixed InstanceProfiles having `Path` set to `None`. - Added in some dynamic `CreateDate`s. - Fixed missing Instance Profile ID's being sent over --- moto/iam/models.py | 2 + moto/iam/responses.py | 104 ++++++++++++++++++------------------- tests/test_iam/test_iam.py | 67 +++++++++++++----------- 3 files changed, 90 insertions(+), 83 deletions(-) diff --git a/moto/iam/models.py b/moto/iam/models.py index 4a5240a0..accdf033 100644 --- a/moto/iam/models.py +++ b/moto/iam/models.py @@ -117,6 +117,7 @@ class Role(BaseModel): self.path = path self.policies = {} self.managed_policies = {} + self.create_date = datetime.now(pytz.utc) @classmethod def create_from_cloudformation_json(cls, resource_name, cloudformation_json, region_name): @@ -168,6 +169,7 @@ class InstanceProfile(BaseModel): self.name = name self.path = path self.roles = roles if roles else [] + self.create_date = datetime.now(pytz.utc) @classmethod def create_from_cloudformation_json(cls, resource_name, cloudformation_json, region_name): diff --git a/moto/iam/responses.py b/moto/iam/responses.py index 22558f3f..43b438c9 100644 --- a/moto/iam/responses.py +++ b/moto/iam/responses.py @@ -201,7 +201,7 @@ class IamResponse(BaseResponse): def create_instance_profile(self): profile_name = self._get_param('InstanceProfileName') - path = self._get_param('Path') + path = self._get_param('Path', '/') profile = iam_backend.create_instance_profile( profile_name, path, role_ids=[]) @@ -734,7 +734,7 @@ CREATE_INSTANCE_PROFILE_TEMPLATE = """ {% for profile in instance_profiles %} - {{ profile.id }} - - {% for role in profile.roles %} - - {{ role.path }} - {{ role.arn }} - {{ role.name }} - {{ role.assume_policy_document }} - 2012-05-09T15:45:35Z - {{ role.id }} - - {% endfor %} - - {{ profile.name }} - {{ profile.path }} - {{ profile.arn }} - 2012-05-09T16:27:11Z + {{ profile.id }} + + {% for role in profile.roles %} + + {{ role.path }} + {{ role.arn }} + {{ role.name }} + {{ role.assume_policy_document }} + {{ role.create_date }} + {{ role.id }} + + {% endfor %} + + {{ profile.name }} + {{ profile.path }} + {{ profile.arn }} + {{ profile.create_date }} {% endfor %} @@ -1382,7 +1382,7 @@ GET_ACCOUNT_AUTHORIZATION_DETAILS_TEMPLATE = """{{ user.path }} {{ user.name }} {{ user.arn }} - 2012-05-09T15:45:35Z + {{ user.created_iso_8601 }} {% endfor %} @@ -1401,7 +1401,7 @@ GET_ACCOUNT_AUTHORIZATION_DETAILS_TEMPLATE = """{{ group.name }} {{ group.path }} {{ group.arn }} - 2012-05-09T16:27:11Z + {{ group.create_date }} {% endfor %} @@ -1421,23 +1421,23 @@ GET_ACCOUNT_AUTHORIZATION_DETAILS_TEMPLATE = """ {% for profile in instance_profiles %} - {{ profile.id }} - - {% for role in profile.roles %} - - {{ role.path }} - {{ role.arn }} - {{ role.name }} - {{ role.assume_role_policy_document }} - 2012-05-09T15:45:35Z - {{ role.id }} - - {% endfor %} - - {{ profile.name }} - {{ profile.path }} - {{ profile.arn }} - 2012-05-09T16:27:11Z + {{ profile.id }} + + {% for role in profile.roles %} + + {{ role.path }} + {{ role.arn }} + {{ role.name }} + {{ role.assume_role_policy_document }} + {{ role.create_date }} + {{ role.id }} + + {% endfor %} + + {{ profile.name }} + {{ profile.path }} + {{ profile.arn }} + {{ profile.create_date }} {% endfor %} @@ -1445,7 +1445,7 @@ GET_ACCOUNT_AUTHORIZATION_DETAILS_TEMPLATE = """{{ role.arn }} {{ role.name }} {{ role.assume_role_policy_document }} - 2014-07-30T17:09:20Z + {{ role.create_date }} {{ role.id }} {% endfor %} @@ -1474,9 +1474,9 @@ GET_ACCOUNT_AUTHORIZATION_DETAILS_TEMPLATE = """ {{ policy.arn }} 1 - 2012-05-09T16:27:11Z + {{ policy.create_datetime }} true - 2012-05-09T16:27:11Z + {{ policy.update_datetime }} {% endfor %} diff --git a/tests/test_iam/test_iam.py b/tests/test_iam/test_iam.py index bc23ff71..72b9205d 100644 --- a/tests/test_iam/test_iam.py +++ b/tests/test_iam/test_iam.py @@ -108,6 +108,10 @@ def test_create_role_and_instance_profile(): conn.list_roles().roles[0].role_name.should.equal('my-role') + # Test with an empty path: + profile = conn.create_instance_profile('my-other-profile') + profile.path.should.equal('/') + @mock_iam_deprecated() def test_remove_role_from_instance_profile(): @@ -700,10 +704,10 @@ def test_get_account_authorization_details(): import json conn = boto3.client('iam', region_name='us-east-1') conn.create_role(RoleName="my-role", AssumeRolePolicyDocument="some policy", Path="/my-path/") - conn.create_user(Path='/', UserName='testCloudAuxUser') - conn.create_group(Path='/', GroupName='testCloudAuxGroup') + conn.create_user(Path='/', UserName='testUser') + conn.create_group(Path='/', GroupName='testGroup') conn.create_policy( - PolicyName='testCloudAuxPolicy', + PolicyName='testPolicy', Path='/', PolicyDocument=json.dumps({ "Version": "2012-10-17", @@ -715,46 +719,47 @@ def test_get_account_authorization_details(): } ] }), - Description='Test CloudAux Policy' + Description='Test Policy' ) + conn.create_instance_profile(InstanceProfileName='ipn') + conn.add_role_to_instance_profile(InstanceProfileName='ipn', RoleName='my-role') + result = conn.get_account_authorization_details(Filter=['Role']) - len(result['RoleDetailList']) == 1 - len(result['UserDetailList']) == 0 - len(result['GroupDetailList']) == 0 - len(result['Policies']) == 0 + assert len(result['RoleDetailList']) == 1 + assert len(result['UserDetailList']) == 0 + assert len(result['GroupDetailList']) == 0 + assert len(result['Policies']) == 0 + assert len(result['RoleDetailList'][0]['InstanceProfileList']) == 1 result = conn.get_account_authorization_details(Filter=['User']) - len(result['RoleDetailList']) == 0 - len(result['UserDetailList']) == 1 - len(result['GroupDetailList']) == 0 - len(result['Policies']) == 0 + assert len(result['RoleDetailList']) == 0 + assert len(result['UserDetailList']) == 1 + assert len(result['GroupDetailList']) == 0 + assert len(result['Policies']) == 0 result = conn.get_account_authorization_details(Filter=['Group']) - len(result['RoleDetailList']) == 0 - len(result['UserDetailList']) == 0 - len(result['GroupDetailList']) == 1 - len(result['Policies']) == 0 + assert len(result['RoleDetailList']) == 0 + assert len(result['UserDetailList']) == 0 + assert len(result['GroupDetailList']) == 1 + assert len(result['Policies']) == 0 result = conn.get_account_authorization_details(Filter=['LocalManagedPolicy']) - len(result['RoleDetailList']) == 0 - len(result['UserDetailList']) == 0 - len(result['GroupDetailList']) == 0 - len(result['Policies']) == 1 + assert len(result['RoleDetailList']) == 0 + assert len(result['UserDetailList']) == 0 + assert len(result['GroupDetailList']) == 0 + assert len(result['Policies']) == 1 # Check for greater than 1 since this should always be greater than one but might change. # See iam/aws_managed_policies.py result = conn.get_account_authorization_details(Filter=['AWSManagedPolicy']) - len(result['RoleDetailList']) == 0 - len(result['UserDetailList']) == 0 - len(result['GroupDetailList']) == 0 - len(result['Policies']) > 1 + assert len(result['RoleDetailList']) == 0 + assert len(result['UserDetailList']) == 0 + assert len(result['GroupDetailList']) == 0 + assert len(result['Policies']) > 1 result = conn.get_account_authorization_details() - len(result['RoleDetailList']) == 1 - len(result['UserDetailList']) == 1 - len(result['GroupDetailList']) == 1 - len(result['Policies']) > 1 - - - + assert len(result['RoleDetailList']) == 1 + assert len(result['UserDetailList']) == 1 + assert len(result['GroupDetailList']) == 1 + assert len(result['Policies']) > 1 From 6e17ba51c6ee91b1a6109da5887706c07b39679e Mon Sep 17 00:00:00 2001 From: Mike Grima Date: Sun, 21 Oct 2018 17:24:45 -0700 Subject: [PATCH 19/40] Fixed a truncation bug for `list_user_policies`. --- moto/iam/responses.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/moto/iam/responses.py b/moto/iam/responses.py index 43b438c9..3d54b933 100644 --- a/moto/iam/responses.py +++ b/moto/iam/responses.py @@ -1199,8 +1199,8 @@ LIST_USER_POLICIES_TEMPLATE = """ {{ policy }} {% endfor %} + false - false 7a62c49f-347e-4fc4-9331-6e8eEXAMPLE From 94b5438d76271c342b98e81655c1e258a68538ad Mon Sep 17 00:00:00 2001 From: Mike Grima Date: Wed, 24 Oct 2018 18:00:52 -0700 Subject: [PATCH 20/40] Added IAM User signing certificate support --- moto/iam/exceptions.py | 8 ++++ moto/iam/models.py | 59 ++++++++++++++++++++++++++- moto/iam/responses.py | 82 ++++++++++++++++++++++++++++++++++++++ moto/iam/utils.py | 3 +- tests/test_iam/test_iam.py | 76 +++++++++++++++++++++++++++++++++++ 5 files changed, 225 insertions(+), 3 deletions(-) diff --git a/moto/iam/exceptions.py b/moto/iam/exceptions.py index 84f15f51..61922ea1 100644 --- a/moto/iam/exceptions.py +++ b/moto/iam/exceptions.py @@ -24,3 +24,11 @@ class IAMReportNotPresentException(RESTError): def __init__(self, message): super(IAMReportNotPresentException, self).__init__( "ReportNotPresent", message) + + +class MalformedCertificate(RESTError): + code = 400 + + def __init__(self, cert): + super(MalformedCertificate, self).__init__( + 'MalformedCertificate', 'Certificate {cert} is malformed'.format(cert=cert)) diff --git a/moto/iam/models.py b/moto/iam/models.py index accdf033..20804f4d 100644 --- a/moto/iam/models.py +++ b/moto/iam/models.py @@ -1,14 +1,18 @@ from __future__ import unicode_literals import base64 +import sys from datetime import datetime import json +from cryptography import x509 +from cryptography.hazmat.backends import default_backend + import pytz from moto.core import BaseBackend, BaseModel from moto.core.utils import iso_8601_datetime_without_milliseconds from .aws_managed_policies import aws_managed_policies_data -from .exceptions import IAMNotFoundException, IAMConflictException, IAMReportNotPresentException +from .exceptions import IAMNotFoundException, IAMConflictException, IAMReportNotPresentException, MalformedCertificate from .utils import random_access_key, random_alphanumeric, random_resource_id, random_policy_id ACCOUNT_ID = 123456789012 @@ -215,6 +219,16 @@ class Certificate(BaseModel): return "arn:aws:iam::{0}:server-certificate{1}{2}".format(ACCOUNT_ID, self.path, self.cert_name) +class SigningCertificate(BaseModel): + + def __init__(self, id, user_name, body): + self.id = id + self.user_name = user_name + self.body = body + self.upload_date = datetime.strftime(datetime.utcnow(), "%Y-%m-%d-%H-%M-%S") + self.status = 'Active' + + class AccessKey(BaseModel): def __init__(self, user_name): @@ -299,6 +313,7 @@ class User(BaseModel): self.access_keys = [] self.password = None self.password_reset_required = False + self.signing_certificates = {} @property def arn(self): @@ -767,6 +782,48 @@ class IAMBackend(BaseBackend): return users + def upload_signing_certificate(self, user_name, body): + user = self.get_user(user_name) + cert_id = random_resource_id(size=32) + + # Validate the signing cert: + try: + if sys.version_info < (3, 0): + data = bytes(body) + else: + data = bytes(body, 'utf8') + + x509.load_pem_x509_certificate(data, default_backend()) + + except Exception: + raise MalformedCertificate(body) + + user.signing_certificates[cert_id] = SigningCertificate(cert_id, user_name, body) + + return user.signing_certificates[cert_id] + + def delete_signing_certificate(self, user_name, cert_id): + user = self.get_user(user_name) + + try: + del user.signing_certificates[cert_id] + except KeyError: + raise IAMNotFoundException("The Certificate with id {id} cannot be found.".format(id=cert_id)) + + def list_signing_certificates(self, user_name): + user = self.get_user(user_name) + + return list(user.signing_certificates.values()) + + def update_signing_certificate(self, user_name, cert_id, status): + user = self.get_user(user_name) + + try: + user.signing_certificates[cert_id].status = status + + except KeyError: + raise IAMNotFoundException("The Certificate with id {id} cannot be found.".format(id=cert_id)) + def create_login_profile(self, user_name, password): # This does not currently deal with PasswordPolicyViolation. user = self.get_user(user_name) diff --git a/moto/iam/responses.py b/moto/iam/responses.py index 3d54b933..fa33cefe 100644 --- a/moto/iam/responses.py +++ b/moto/iam/responses.py @@ -552,6 +552,38 @@ class IamResponse(BaseResponse): roles=account_details['roles'] ) + def upload_signing_certificate(self): + user_name = self._get_param('UserName') + cert_body = self._get_param('CertificateBody') + + cert = iam_backend.upload_signing_certificate(user_name, cert_body) + template = self.response_template(UPLOAD_SIGNING_CERTIFICATE_TEMPLATE) + return template.render(cert=cert) + + def update_signing_certificate(self): + user_name = self._get_param('UserName') + cert_id = self._get_param('CertificateId') + status = self._get_param('Status') + + iam_backend.update_signing_certificate(user_name, cert_id, status) + template = self.response_template(UPDATE_SIGNING_CERTIFICATE_TEMPLATE) + return template.render() + + def delete_signing_certificate(self): + user_name = self._get_param('UserName') + cert_id = self._get_param('CertificateId') + + iam_backend.delete_signing_certificate(user_name, cert_id) + template = self.response_template(DELETE_SIGNING_CERTIFICATE_TEMPLATE) + return template.render() + + def list_signing_certificates(self): + user_name = self._get_param('UserName') + + certs = iam_backend.list_signing_certificates(user_name) + template = self.response_template(LIST_SIGNING_CERTIFICATES_TEMPLATE) + return template.render(user_name=user_name, certificates=certs) + ATTACH_ROLE_POLICY_TEMPLATE = """ @@ -1485,3 +1517,53 @@ GET_ACCOUNT_AUTHORIZATION_DETAILS_TEMPLATE = """92e79ae7-7399-11e4-8c85-4b53eEXAMPLE """ + + +UPLOAD_SIGNING_CERTIFICATE_TEMPLATE = """ + + + {{ cert.user_name }} + {{ cert.id }} + {{ cert.body }} + {{ cert.status }} + + + + 7a62c49f-347e-4fc4-9331-6e8eEXAMPLE + +""" + + +UPDATE_SIGNING_CERTIFICATE_TEMPLATE = """ + + EXAMPLE8-90ab-cdef-fedc-ba987EXAMPLE + +""" + + +DELETE_SIGNING_CERTIFICATE_TEMPLATE = """ + + 7a62c49f-347e-4fc4-9331-6e8eEXAMPLE + +""" + + +LIST_SIGNING_CERTIFICATES_TEMPLATE = """ + + {{ user_name }} + + {% for cert in certificates %} + + {{ user_name }} + {{ cert.id }} + {{ cert.body }} + {{ cert.status }} + + {% endfor %} + + false + + + 7a62c49f-347e-4fc4-9331-6e8eEXAMPLE + +""" diff --git a/moto/iam/utils.py b/moto/iam/utils.py index 1fae85a6..f59bdfff 100644 --- a/moto/iam/utils.py +++ b/moto/iam/utils.py @@ -12,8 +12,7 @@ def random_alphanumeric(length): ) -def random_resource_id(): - size = 20 +def random_resource_id(size=20): chars = list(range(10)) + list(string.ascii_lowercase) return ''.join(six.text_type(random.choice(chars)) for x in range(size)) diff --git a/tests/test_iam/test_iam.py b/tests/test_iam/test_iam.py index 72b9205d..7bf38f48 100644 --- a/tests/test_iam/test_iam.py +++ b/tests/test_iam/test_iam.py @@ -14,6 +14,19 @@ from nose.tools import raises from tests.helpers import requires_boto_gte +MOCK_CERT = """-----BEGIN CERTIFICATE----- +MIIBpzCCARACCQCY5yOdxCTrGjANBgkqhkiG9w0BAQsFADAXMRUwEwYDVQQKDAxt +b3RvIHRlc3RpbmcwIBcNMTgxMTA1MTkwNTIwWhgPMjI5MjA4MTkxOTA1MjBaMBcx +FTATBgNVBAoMDG1vdG8gdGVzdGluZzCBnzANBgkqhkiG9w0BAQEFAAOBjQAwgYkC +gYEA1Jn3g2h7LD3FLqdpcYNbFXCS4V4eDpuTCje9vKFcC3pi/01147X3zdfPy8Mt +ZhKxcREOwm4NXykh23P9KW7fBovpNwnbYsbPqj8Hf1ZaClrgku1arTVhEnKjx8zO +vaR/bVLCss4uE0E0VM1tJn/QGQsfthFsjuHtwx8uIWz35tUCAwEAATANBgkqhkiG +9w0BAQsFAAOBgQBWdOQ7bDc2nWkUhFjZoNIZrqjyNdjlMUndpwREVD7FQ/DuxJMj +FyDHrtlrS80dPUQWNYHw++oACDpWO01LGLPPrGmuO/7cOdojPEd852q5gd+7W9xt +8vUH+pBa6IBLbvBp+szli51V3TLSWcoyy4ceJNQU2vCkTLoFdS0RLd/7tQ== +-----END CERTIFICATE-----""" + + @mock_iam_deprecated() def test_get_all_server_certs(): conn = boto.connect_iam() @@ -763,3 +776,66 @@ def test_get_account_authorization_details(): assert len(result['UserDetailList']) == 1 assert len(result['GroupDetailList']) == 1 assert len(result['Policies']) > 1 + + +@mock_iam +def test_signing_certs(): + client = boto3.client('iam', region_name='us-east-1') + + # Create the IAM user first: + client.create_user(UserName='testing') + + # Upload the cert: + resp = client.upload_signing_certificate(UserName='testing', CertificateBody=MOCK_CERT)['Certificate'] + cert_id = resp['CertificateId'] + + assert resp['UserName'] == 'testing' + assert resp['Status'] == 'Active' + assert resp['CertificateBody'] == MOCK_CERT + assert resp['CertificateId'] + + # Upload a the cert with an invalid body: + with assert_raises(ClientError) as ce: + client.upload_signing_certificate(UserName='testing', CertificateBody='notacert') + assert ce.exception.response['Error']['Code'] == 'MalformedCertificate' + + # Upload with an invalid user: + with assert_raises(ClientError): + client.upload_signing_certificate(UserName='notauser', CertificateBody=MOCK_CERT) + + # Update: + client.update_signing_certificate(UserName='testing', CertificateId=cert_id, Status='Inactive') + + with assert_raises(ClientError): + client.update_signing_certificate(UserName='notauser', CertificateId=cert_id, Status='Inactive') + + with assert_raises(ClientError) as ce: + client.update_signing_certificate(UserName='testing', CertificateId='x' * 32, Status='Inactive') + + assert ce.exception.response['Error']['Message'] == 'The Certificate with id {id} cannot be found.'.format( + id='x' * 32) + + # List the certs: + resp = client.list_signing_certificates(UserName='testing')['Certificates'] + assert len(resp) == 1 + assert resp[0]['CertificateBody'] == MOCK_CERT + assert resp[0]['Status'] == 'Inactive' # Changed with the update call above. + + with assert_raises(ClientError): + client.list_signing_certificates(UserName='notauser') + + # Delete: + client.delete_signing_certificate(UserName='testing', CertificateId=cert_id) + + with assert_raises(ClientError): + client.delete_signing_certificate(UserName='notauser', CertificateId=cert_id) + + with assert_raises(ClientError) as ce: + client.delete_signing_certificate(UserName='testing', CertificateId=cert_id) + + assert ce.exception.response['Error']['Message'] == 'The Certificate with id {id} cannot be found.'.format( + id=cert_id) + + # Verify that it's not in the list: + resp = client.list_signing_certificates(UserName='testing') + assert not resp['Certificates'] From b66965e6e879a1647ca031448d8536f2e129d2c4 Mon Sep 17 00:00:00 2001 From: adam davis Date: Fri, 2 Nov 2018 16:04:17 -0700 Subject: [PATCH 21/40] Adding cloudformation-validate. Cfn-lint does the heavy lifting. --- moto/batch/responses.py | 2 +- moto/cloudformation/models.py | 4 + moto/cloudformation/responses.py | 27 +++++ moto/cloudformation/utils.py | 33 ++++++ setup.py | 1 + tests/test_cloudformation/test_validate.py | 115 +++++++++++++++++++++ 6 files changed, 181 insertions(+), 1 deletion(-) create mode 100644 tests/test_cloudformation/test_validate.py diff --git a/moto/batch/responses.py b/moto/batch/responses.py index e626b7d4..7fb60618 100644 --- a/moto/batch/responses.py +++ b/moto/batch/responses.py @@ -27,7 +27,7 @@ class BatchResponse(BaseResponse): elif not hasattr(self, '_json'): try: self._json = json.loads(self.body) - except json.JSONDecodeError: + except ValueError: print() return self._json diff --git a/moto/cloudformation/models.py b/moto/cloudformation/models.py index e5ab7255..6ec821b4 100644 --- a/moto/cloudformation/models.py +++ b/moto/cloudformation/models.py @@ -13,6 +13,7 @@ from .utils import ( generate_changeset_id, generate_stack_id, yaml_tag_constructor, + validate_template_cfn_lint, ) from .exceptions import ValidationError @@ -270,6 +271,9 @@ class CloudFormationBackend(BaseBackend): next_token = str(token + 100) if len(all_exports) > token + 100 else None return exports, next_token + def validate_template(self, template): + return validate_template_cfn_lint(template) + def _validate_export_uniqueness(self, stack): new_stack_export_names = [x.name for x in stack.exports] export_names = self.exports.keys() diff --git a/moto/cloudformation/responses.py b/moto/cloudformation/responses.py index a1295a20..0964287a 100644 --- a/moto/cloudformation/responses.py +++ b/moto/cloudformation/responses.py @@ -1,6 +1,7 @@ from __future__ import unicode_literals import json +import yaml from six.moves.urllib.parse import urlparse from moto.core.responses import BaseResponse @@ -294,6 +295,32 @@ class CloudFormationResponse(BaseResponse): template = self.response_template(LIST_EXPORTS_RESPONSE) return template.render(exports=exports, next_token=next_token) + def validate_template(self): + cfn_lint = self.cloudformation_backend.validate_template(self._get_param('TemplateBody')) + if cfn_lint: + raise ValidationError(cfn_lint[0].message) + description = "" + try: + description = json.loads(self._get_param('TemplateBody'))['Description'] + except (ValueError, KeyError): + pass + try: + description = yaml.load(self._get_param('TemplateBody'))['Description'] + except (yaml.ParserError, KeyError): + pass + template = self.response_template(VALIDATE_STACK_RESPONSE_TEMPLATE) + return template.render(description=description) + + +VALIDATE_STACK_RESPONSE_TEMPLATE = """ + + + + +{{ description }} + + +""" CREATE_STACK_RESPONSE_TEMPLATE = """ diff --git a/moto/cloudformation/utils.py b/moto/cloudformation/utils.py index f3b8874e..f963ce7c 100644 --- a/moto/cloudformation/utils.py +++ b/moto/cloudformation/utils.py @@ -3,6 +3,9 @@ import uuid import six import random import yaml +import os + +from cfnlint import decode, core def generate_stack_id(stack_name): @@ -38,3 +41,33 @@ def yaml_tag_constructor(loader, tag, node): key = 'Fn::{}'.format(tag[1:]) return {key: _f(loader, tag, node)} + + +def validate_template_cfn_lint(template): + + # Save the template to a temporary file -- cfn-lint requires a file + filename = "file.tmp" + with open(filename, "w") as file: + file.write(template) + abs_filename = os.path.abspath(filename) + + # decode handles both yaml and json + template, matches = decode.decode(abs_filename, False) + + # Set cfn-lint to info + core.configure_logging(None) + + # Initialize the ruleset to be applied (no overrules, no excludes) + rules = core.get_rules([], [], []) + + # Use us-east-1 region (spec file) for validation + regions = ['us-east-1'] + + # Process all the rules and gather the errors + matches = core.run_checks( + abs_filename, + template, + rules, + regions) + + return matches diff --git a/setup.py b/setup.py index 98780dd5..873ca16b 100755 --- a/setup.py +++ b/setup.py @@ -24,6 +24,7 @@ install_requires = [ "jsondiff==1.1.1", "aws-xray-sdk<0.96,>=0.93", "responses>=0.9.0", + "cfn-lint" ] extras_require = { diff --git a/tests/test_cloudformation/test_validate.py b/tests/test_cloudformation/test_validate.py new file mode 100644 index 00000000..e2c3af05 --- /dev/null +++ b/tests/test_cloudformation/test_validate.py @@ -0,0 +1,115 @@ +from collections import OrderedDict +import json +import yaml +import os +import boto3 +from nose.tools import raises +import botocore + + +from moto.cloudformation.exceptions import ValidationError +from moto.cloudformation.models import FakeStack +from moto.cloudformation.parsing import resource_class_from_type, parse_condition, Export +from moto.sqs.models import Queue +from moto.s3.models import FakeBucket +from moto.cloudformation.utils import yaml_tag_constructor +from boto.cloudformation.stack import Output +from moto import mock_cloudformation, mock_s3, mock_sqs, mock_ec2 + +json_template = { + "AWSTemplateFormatVersion": "2010-09-09", + "Description": "Stack 1", + "Resources": { + "EC2Instance1": { + "Type": "AWS::EC2::Instance", + "Properties": { + "ImageId": "ami-d3adb33f", + "KeyName": "dummy", + "InstanceType": "t2.micro", + "Tags": [ + { + "Key": "Description", + "Value": "Test tag" + }, + { + "Key": "Name", + "Value": "Name tag for tests" + } + ] + } + } + } +} + +# One resource is required +json_bad_template = { + "AWSTemplateFormatVersion": "2010-09-09", + "Description": "Stack 1" +} + +dummy_template_json = json.dumps(json_template) +dummy_bad_template_json = json.dumps(json_bad_template) + + +@mock_cloudformation +def test_boto3_json_validate_successful(): + cf_conn = boto3.client('cloudformation', region_name='us-east-1') + response = cf_conn.validate_template( + TemplateBody=dummy_template_json, + ) + assert response['Description'] == "Stack 1" + assert response['Parameters'] == [] + assert response['ResponseMetadata']['HTTPStatusCode'] == 200 + +@mock_cloudformation +def test_boto3_json_invalid_missing_resource(): + cf_conn = boto3.client('cloudformation', region_name='us-east-1') + try: + cf_conn.validate_template( + TemplateBody=dummy_bad_template_json, + ) + assert False + except botocore.exceptions.ClientError as e: + assert str(e) == 'An error occurred (ValidationError) when calling the ValidateTemplate operation: Stack' \ + ' with id Missing top level item Resources to file module does not exist' + assert True + + +yaml_template = """ + AWSTemplateFormatVersion: '2010-09-09' + Description: Simple CloudFormation Test Template + Resources: + S3Bucket: + Type: AWS::S3::Bucket + Properties: + AccessControl: PublicRead + BucketName: cf-test-bucket-1 +""" + +yaml_bad_template = """ + AWSTemplateFormatVersion: '2010-09-09' + Description: Simple CloudFormation Test Template +""" + +@mock_cloudformation +def test_boto3_yaml_validate_successful(): + cf_conn = boto3.client('cloudformation', region_name='us-east-1') + response = cf_conn.validate_template( + TemplateBody=yaml_template, + ) + assert response['Description'] == "Simple CloudFormation Test Template" + assert response['Parameters'] == [] + assert response['ResponseMetadata']['HTTPStatusCode'] == 200 + +@mock_cloudformation +def test_boto3_yaml_invalid_missing_resource(): + cf_conn = boto3.client('cloudformation', region_name='us-east-1') + try: + cf_conn.validate_template( + TemplateBody=yaml_bad_template, + ) + assert False + except botocore.exceptions.ClientError as e: + assert str(e) == 'An error occurred (ValidationError) when calling the ValidateTemplate operation: Stack' \ + ' with id Missing top level item Resources to file module does not exist' + assert True From 4af9407ef4ac22ba2a13a442d44ced30ef74cef8 Mon Sep 17 00:00:00 2001 From: Justin Dray Date: Tue, 6 Nov 2018 00:03:09 +0000 Subject: [PATCH 22/40] Update to jose 3.x --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 046ecf6e..811be385 100755 --- a/setup.py +++ b/setup.py @@ -18,7 +18,7 @@ install_requires = [ "pyaml", "pytz", "python-dateutil<3.0.0,>=2.1", - "python-jose<3.0.0", + "python-jose<4.0.0", "mock", "docker>=2.5.1", "jsondiff==1.1.1", From e52158f811818b84f86736727de8868b76180d64 Mon Sep 17 00:00:00 2001 From: Tomoya Iwata Date: Tue, 6 Nov 2018 17:12:13 +0900 Subject: [PATCH 23/40] Add support for IoT detach_policy --- IMPLEMENTATION_COVERAGE.md | 4 ++-- moto/iot/models.py | 9 +++++++++ moto/iot/responses.py | 9 +++++++++ tests/test_iot/test_iot.py | 7 ++++++- 4 files changed, 26 insertions(+), 3 deletions(-) diff --git a/IMPLEMENTATION_COVERAGE.md b/IMPLEMENTATION_COVERAGE.md index 7a1e2e7a..0a00dec9 100644 --- a/IMPLEMENTATION_COVERAGE.md +++ b/IMPLEMENTATION_COVERAGE.md @@ -2376,7 +2376,7 @@ - [ ] unsubscribe_from_event - [ ] update_assessment_target -## iot - 31% implemented +## iot - 32% implemented - [ ] accept_certificate_transfer - [X] add_thing_to_thing_group - [ ] associate_targets_with_job @@ -2429,7 +2429,7 @@ - [X] describe_thing_group - [ ] describe_thing_registration_task - [X] describe_thing_type -- [ ] detach_policy +- [X] detach_policy - [X] detach_principal_policy - [X] detach_thing_principal - [ ] disable_topic_rule diff --git a/moto/iot/models.py b/moto/iot/models.py index db9ad381..5c33aecd 100644 --- a/moto/iot/models.py +++ b/moto/iot/models.py @@ -445,6 +445,15 @@ class IoTBackend(BaseBackend): return self.principal_policies[k] = (principal, policy) + def detach_policy(self, policy_name, target): + # this may raises ResourceNotFoundException + self._get_principal(target) + self.get_policy(policy_name) + k = (target, policy_name) + if k not in self.principal_policies: + raise ResourceNotFoundException() + del self.principal_policies[k] + def detach_principal_policy(self, policy_name, principal_arn): # this may raises ResourceNotFoundException self._get_principal(principal_arn) diff --git a/moto/iot/responses.py b/moto/iot/responses.py index 042e5a31..214576f5 100644 --- a/moto/iot/responses.py +++ b/moto/iot/responses.py @@ -242,6 +242,15 @@ class IoTResponse(BaseResponse): ) return json.dumps(dict()) + def detach_policy(self): + policy_name = self._get_param("policyName") + target = self._get_param('target') + self.iot_backend.detach_policy( + policy_name=policy_name, + target=target, + ) + return json.dumps(dict()) + def detach_principal_policy(self): policy_name = self._get_param("policyName") principal = self.headers.get('x-amzn-iot-principal') diff --git a/tests/test_iot/test_iot.py b/tests/test_iot/test_iot.py index 47ea9d59..161c5832 100644 --- a/tests/test_iot/test_iot.py +++ b/tests/test_iot/test_iot.py @@ -5,6 +5,8 @@ import sure # noqa import boto3 from moto import mock_iot +from botocore.exceptions import ClientError +from nose.tools import assert_raises @mock_iot @@ -338,11 +340,14 @@ def test_principal_policy(): for principal in res['principals']: principal.should_not.be.none - client.detach_principal_policy(policyName=policy_name, principal=cert_arn) + client.detach_policy(policyName=policy_name, target=cert_arn) res = client.list_principal_policies(principal=cert_arn) res.should.have.key('policies').which.should.have.length_of(0) res = client.list_policy_principals(policyName=policy_name) res.should.have.key('principals').which.should.have.length_of(0) + with assert_raises(ClientError) as e: + client.detach_policy(policyName=policy_name, target=cert_arn) + e.exception.response['Error']['Code'].should.equal('ResourceNotFoundException') @mock_iot From 0b57ffe26a894ecc7d39a9c1155dadedc0c5f7ba Mon Sep 17 00:00:00 2001 From: Karl Gutwin Date: Wed, 7 Nov 2018 15:03:25 -0500 Subject: [PATCH 24/40] Add StreamSpecification to dynamodb2 package --- moto/dynamodb2/models.py | 22 +++++++++- moto/dynamodb2/responses.py | 9 +++- tests/test_dynamodb2/test_dynamodb.py | 59 +++++++++++++++++++++++++++ 3 files changed, 88 insertions(+), 2 deletions(-) diff --git a/moto/dynamodb2/models.py b/moto/dynamodb2/models.py index a54c4f7d..16e97ea2 100644 --- a/moto/dynamodb2/models.py +++ b/moto/dynamodb2/models.py @@ -294,7 +294,7 @@ class Item(BaseModel): class Table(BaseModel): - def __init__(self, table_name, schema=None, attr=None, throughput=None, indexes=None, global_indexes=None): + def __init__(self, table_name, schema=None, attr=None, throughput=None, indexes=None, global_indexes=None, streams=None): self.name = table_name self.attr = attr self.schema = schema @@ -325,10 +325,18 @@ class Table(BaseModel): 'TimeToLiveStatus': 'DISABLED' # One of 'ENABLING'|'DISABLING'|'ENABLED'|'DISABLED', # 'AttributeName': 'string' # Can contain this } + self.set_stream_specification(streams) def _generate_arn(self, name): return 'arn:aws:dynamodb:us-east-1:123456789011:table/' + name + def set_stream_specification(self, streams): + self.stream_specification = streams + if streams and streams.get('StreamEnabled'): + self.latest_stream_label = datetime.datetime.utcnow().isoformat() + else: + self.latest_stream_label = None + def describe(self, base_key='TableDescription'): results = { base_key: { @@ -345,6 +353,11 @@ class Table(BaseModel): 'LocalSecondaryIndexes': [index for index in self.indexes], } } + if self.stream_specification: + results[base_key]['StreamSpecification'] = self.stream_specification + if self.latest_stream_label: + results[base_key]['LatestStreamLabel'] = self.latest_stream_label + results[base_key]['LatestStreamArn'] = self.table_arn + '/stream/' + self.latest_stream_label return results def __len__(self): @@ -680,6 +693,13 @@ class DynamoDBBackend(BaseBackend): table.throughput = throughput return table + def update_table_streams(self, name, stream_specification): + table = self.tables[name] + if stream_specification['StreamEnabled'] and table.latest_stream_label: + raise ValueError('Table already has stream enabled') + table.set_stream_specification(stream_specification) + return table + def update_table_global_indexes(self, name, global_index_updates): table = self.tables[name] gsis_by_name = dict((i['IndexName'], i) for i in table.global_indexes) diff --git a/moto/dynamodb2/responses.py b/moto/dynamodb2/responses.py index e2f1ef1c..73bd3ae3 100644 --- a/moto/dynamodb2/responses.py +++ b/moto/dynamodb2/responses.py @@ -104,13 +104,16 @@ class DynamoHandler(BaseResponse): # getting the indexes global_indexes = body.get("GlobalSecondaryIndexes", []) local_secondary_indexes = body.get("LocalSecondaryIndexes", []) + # get the stream specification + streams = body.get("StreamSpecification") table = self.dynamodb_backend.create_table(table_name, schema=key_schema, throughput=throughput, attr=attr, global_indexes=global_indexes, - indexes=local_secondary_indexes) + indexes=local_secondary_indexes, + streams=streams) if table is not None: return dynamo_json_dump(table.describe()) else: @@ -163,12 +166,16 @@ class DynamoHandler(BaseResponse): def update_table(self): name = self.body['TableName'] + table = self.dynamodb_backend.get_table(name) if 'GlobalSecondaryIndexUpdates' in self.body: table = self.dynamodb_backend.update_table_global_indexes( name, self.body['GlobalSecondaryIndexUpdates']) if 'ProvisionedThroughput' in self.body: throughput = self.body["ProvisionedThroughput"] table = self.dynamodb_backend.update_table_throughput(name, throughput) + if 'StreamSpecification' in self.body: + table = self.dynamodb_backend.update_table_streams(name, self.body['StreamSpecification']) + return dynamo_json_dump(table.describe()) def describe_table(self): diff --git a/tests/test_dynamodb2/test_dynamodb.py b/tests/test_dynamodb2/test_dynamodb.py index afc919dd..7f30bbcc 100644 --- a/tests/test_dynamodb2/test_dynamodb.py +++ b/tests/test_dynamodb2/test_dynamodb.py @@ -1336,3 +1336,62 @@ def test_query_global_secondary_index_when_created_via_update_table_resource(): assert len(forum_and_subject_items) == 1 assert forum_and_subject_items[0] == {'user_id': Decimal('1'), 'forum_name': 'cats', 'subject': 'my pet is the cutest'} + + +@mock_dynamodb2 +def test_dynamodb_streams_1(): + conn = boto3.client('dynamodb', region_name='us-east-1') + + resp = conn.create_table( + TableName='test-streams', + KeySchema=[{'AttributeName': 'id', 'KeyType': 'HASH'}], + AttributeDefinitions=[{'AttributeName': 'id', 'AttributeType': 'S'}], + ProvisionedThroughput={'ReadCapacityUnits': 1, 'WriteCapacityUnits': 1}, + StreamSpecification={ + 'StreamEnabled': True, + 'StreamViewType': 'NEW_AND_OLD_IMAGES' + } + ) + + assert 'StreamSpecification' in resp['TableDescription'] + assert resp['TableDescription']['StreamSpecification'] == { + 'StreamEnabled': True, + 'StreamViewType': 'NEW_AND_OLD_IMAGES' + } + assert 'LatestStreamLabel' in resp['TableDescription'] + assert 'LatestStreamArn' in resp['TableDescription'] + + resp = conn.delete_table(TableName='test-streams') + + assert 'StreamSpecification' in resp['TableDescription'] + + +@mock_dynamodb2 +def test_dynamodb_streams_2(): + conn = boto3.client('dynamodb', region_name='us-east-1') + + resp = conn.create_table( + TableName='test-stream-update', + KeySchema=[{'AttributeName': 'id', 'KeyType': 'HASH'}], + AttributeDefinitions=[{'AttributeName': 'id', 'AttributeType': 'S'}], + ProvisionedThroughput={'ReadCapacityUnits': 1, 'WriteCapacityUnits': 1}, + ) + + assert 'StreamSpecification' not in resp['TableDescription'] + + resp = conn.update_table( + TableName='test-stream-update', + StreamSpecification={ + 'StreamEnabled': True, + 'StreamViewType': 'NEW_IMAGE' + } + ) + + assert 'StreamSpecification' in resp['TableDescription'] + assert resp['TableDescription']['StreamSpecification'] == { + 'StreamEnabled': True, + 'StreamViewType': 'NEW_IMAGE' + } + assert 'LatestStreamLabel' in resp['TableDescription'] + assert 'LatestStreamArn' in resp['TableDescription'] + From 519899f74f6d50c33ffe82d8d6fee2c86afd3a8e Mon Sep 17 00:00:00 2001 From: Karl Gutwin Date: Wed, 7 Nov 2018 17:10:00 -0500 Subject: [PATCH 25/40] Much of the way towards complete DynamoDB Streams implementation --- moto/__init__.py | 1 + moto/backends.py | 1 + moto/dynamodb2/models.py | 37 ++++++- moto/dynamodbstreams/__init__.py | 6 ++ moto/dynamodbstreams/models.py | 99 +++++++++++++++++++ moto/dynamodbstreams/responses.py | 29 ++++++ moto/dynamodbstreams/urls.py | 10 ++ setup.cfg | 2 +- .../test_dynamodbstreams.py | 96 ++++++++++++++++++ 9 files changed, 278 insertions(+), 3 deletions(-) create mode 100644 moto/dynamodbstreams/__init__.py create mode 100644 moto/dynamodbstreams/models.py create mode 100644 moto/dynamodbstreams/responses.py create mode 100644 moto/dynamodbstreams/urls.py create mode 100644 tests/test_dynamodbstreams/test_dynamodbstreams.py diff --git a/moto/__init__.py b/moto/__init__.py index dd3593d5..e86c499a 100644 --- a/moto/__init__.py +++ b/moto/__init__.py @@ -16,6 +16,7 @@ from .cognitoidp import mock_cognitoidp, mock_cognitoidp_deprecated # flake8: n from .datapipeline import mock_datapipeline, mock_datapipeline_deprecated # flake8: noqa from .dynamodb import mock_dynamodb, mock_dynamodb_deprecated # flake8: noqa from .dynamodb2 import mock_dynamodb2, mock_dynamodb2_deprecated # flake8: noqa +from .dynamodbstreams import mock_dynamodbstreams # flake8: noqa from .ec2 import mock_ec2, mock_ec2_deprecated # flake8: noqa from .ecr import mock_ecr, mock_ecr_deprecated # flake8: noqa from .ecs import mock_ecs, mock_ecs_deprecated # flake8: noqa diff --git a/moto/backends.py b/moto/backends.py index d9542438..7df167a0 100644 --- a/moto/backends.py +++ b/moto/backends.py @@ -12,6 +12,7 @@ from moto.core import moto_api_backends from moto.datapipeline import datapipeline_backends from moto.dynamodb import dynamodb_backends from moto.dynamodb2 import dynamodb_backends2 +from moto.dynamodbstreams import dynamodbstreams_backends from moto.ec2 import ec2_backends from moto.ecr import ecr_backends from moto.ecs import ecs_backends diff --git a/moto/dynamodb2/models.py b/moto/dynamodb2/models.py index 16e97ea2..d58241cf 100644 --- a/moto/dynamodb2/models.py +++ b/moto/dynamodb2/models.py @@ -292,6 +292,34 @@ class Item(BaseModel): 'ADD not supported for %s' % ', '.join(update_action['Value'].keys())) +class StreamShard(BaseModel): + def __init__(self, table): + self.table = table + self.id = 'shardId-00000001541626099285-f35f62ef' + self.starting_sequence_number = 1100000000017454423009 + self.items = [] + self.created_on = datetime.datetime.utcnow() + + def to_json(self): + return { + 'ShardId': self.id, + 'SequenceNumberRange': { + 'StartingSequenceNumber': str(self.starting_sequence_number) + } + } + + def add(self, old, new): + t = self.table.stream_specification['StreamViewType'] + if t == 'KEYS_ONLY': + self.items.append(new.key) + elif t == 'NEW_IMAGE': + self.items.append(new) + elif t == 'OLD_IMAGE': + self.items.append(old) + elif t == 'NEW_AND_OLD_IMAGES': + self.items.append((old, new)) + + class Table(BaseModel): def __init__(self, table_name, schema=None, attr=None, throughput=None, indexes=None, global_indexes=None, streams=None): @@ -334,8 +362,10 @@ class Table(BaseModel): self.stream_specification = streams if streams and streams.get('StreamEnabled'): self.latest_stream_label = datetime.datetime.utcnow().isoformat() + self.stream_shard = StreamShard(self) else: self.latest_stream_label = None + self.stream_shard = None def describe(self, base_key='TableDescription'): results = { @@ -398,6 +428,7 @@ class Table(BaseModel): else: range_value = None + current = self.get_item(hash_value, lookup_range_value) item = Item(hash_value, self.hash_key_type, range_value, self.range_key_type, item_attrs) @@ -413,8 +444,6 @@ class Table(BaseModel): else: lookup_range_value = DynamoType(expected_range_value) - current = self.get_item(hash_value, lookup_range_value) - if current is None: current_attr = {} elif hasattr(current, 'attrs'): @@ -445,6 +474,10 @@ class Table(BaseModel): self.items[hash_value][range_value] = item else: self.items[hash_value] = item + + if self.stream_shard is not None: + self.stream_shard.add(current, item) + return item def __nonzero__(self): diff --git a/moto/dynamodbstreams/__init__.py b/moto/dynamodbstreams/__init__.py new file mode 100644 index 00000000..b35879eb --- /dev/null +++ b/moto/dynamodbstreams/__init__.py @@ -0,0 +1,6 @@ +from __future__ import unicode_literals +from .models import dynamodbstreams_backends +from ..core.models import base_decorator + +dynamodbstreams_backend = dynamodbstreams_backends['us-east-1'] +mock_dynamodbstreams = base_decorator(dynamodbstreams_backends) diff --git a/moto/dynamodbstreams/models.py b/moto/dynamodbstreams/models.py new file mode 100644 index 00000000..35704de6 --- /dev/null +++ b/moto/dynamodbstreams/models.py @@ -0,0 +1,99 @@ +from __future__ import unicode_literals + +import os +import json +import boto3 +import base64 +import datetime + +from moto.core import BaseBackend, BaseModel +from moto.dynamodb2.models import dynamodb_backends + + +class ShardIterator(BaseModel): + def __init__(self, stream_shard, shard_iterator_type, sequence_number=None): + self.id = base64.b64encode(os.urandom(472)).decode('utf-8') + self.stream_shard = stream_shard + self.shard_iterator_type = shard_iterator_type + if shard_iterator_type == 'TRIM_HORIZON': + self.sequence_number = stream_shard.starting_sequence_number + elif shard_iterator_type == 'LATEST': + self.sequence_number = stream_shard.starting_sequence_number + len(stream_shard.items) + elif shard_iterator_type == 'AT_SEQUENCE_NUMBER': + self.sequence_number = sequence_number + elif shard_iterator_type == 'AFTER_SEQUENCE_NUMBER': + self.sequence_number = sequence_number + 1 + + def to_json(self): + return { + 'ShardIterator': '{}/stream/{}|1|{}'.format( + self.stream_shard.table.table_arn, + self.stream_shard.table.latest_stream_label, + self.id) + } + + +class DynamoDBStreamsBackend(BaseBackend): + def __init__(self, region): + self.region = region + + def reset(self): + region = self.region + self.__dict__ = {} + self.__init__(region) + + @property + def dynamodb(self): + return dynamodb_backends[self.region] + + def _get_table_from_arn(self, arn): + table_name = arn.split(':', 6)[5].split('/')[1] + return self.dynamodb.get_table(table_name) + + def describe_stream(self, arn): + table = self._get_table_from_arn(arn) + resp = {'StreamDescription': { + 'StreamArn': arn, + 'StreamLabel': table.latest_stream_label, + 'StreamStatus': ('ENABLED' if table.latest_stream_label + else 'DISABLED'), + 'StreamViewType': table.stream_specification['StreamViewType'], + 'CreationRequestDateTime': table.stream_shard.created_on.isoformat(), + 'TableName': table.name, + 'KeySchema': table.schema, + 'Shards': ([table.stream_shard.to_json()] if table.stream_shard + else []) + }} + + return json.dumps(resp) + + def list_streams(self, table_name=None): + streams = [] + for table in self.dynamodb.tables.values(): + if table_name is not None and table.name != table_name: + continue + if table.latest_stream_label: + d = table.describe(base_key='Table') + streams.append({ + 'StreamArn': d['Table']['LatestStreamArn'], + 'TableName': d['Table']['TableName'], + 'StreamLabel': d['Table']['LatestStreamLabel'] + }) + + return json.dumps({'Streams': streams}) + + def get_shard_iterator(self, arn, shard_id, shard_iterator_type, sequence_number=None): + table = self._get_table_from_arn(arn) + assert table.stream_shard.id == shard_id + + shard_iterator = ShardIterator(table.stream_shard, shard_iterator_type, + sequence_number) + + return json.dumps(shard_iterator.to_json()) + + + +available_regions = boto3.session.Session().get_available_regions( + 'dynamodbstreams') +dynamodbstreams_backends = {region: DynamoDBStreamsBackend(region=region) + for region in available_regions} diff --git a/moto/dynamodbstreams/responses.py b/moto/dynamodbstreams/responses.py new file mode 100644 index 00000000..ef05c171 --- /dev/null +++ b/moto/dynamodbstreams/responses.py @@ -0,0 +1,29 @@ +from __future__ import unicode_literals + +import json + +from moto.core.responses import BaseResponse + +from .models import dynamodbstreams_backends + + +class DynamoDBStreamsHandler(BaseResponse): + + @property + def backend(self): + return dynamodbstreams_backends[self.region] + + def describe_stream(self): + arn = self._get_param('StreamArn') + return self.backend.describe_stream(arn) + + def list_streams(self): + table_name = self._get_param('TableName') + return self.backend.list_streams(table_name) + + def get_shard_iterator(self): + arn = self._get_param('StreamArn') + shard_id = self._get_param('ShardId') + shard_iterator_type = self._get_param('ShardIteratorType') + return self.backend.get_shard_iterator(arn, shard_id, + shard_iterator_type) diff --git a/moto/dynamodbstreams/urls.py b/moto/dynamodbstreams/urls.py new file mode 100644 index 00000000..1d0f94c3 --- /dev/null +++ b/moto/dynamodbstreams/urls.py @@ -0,0 +1,10 @@ +from __future__ import unicode_literals +from .responses import DynamoDBStreamsHandler + +url_bases = [ + "https?://streams.dynamodb.(.+).amazonaws.com" +] + +url_paths = { + "{0}/$": DynamoDBStreamsHandler.dispatch, +} diff --git a/setup.cfg b/setup.cfg index fb04c16a..9dbd988d 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,7 +1,7 @@ [nosetests] verbosity=1 detailed-errors=1 -with-coverage=1 +#with-coverage=1 cover-package=moto [bdist_wheel] diff --git a/tests/test_dynamodbstreams/test_dynamodbstreams.py b/tests/test_dynamodbstreams/test_dynamodbstreams.py new file mode 100644 index 00000000..a10445aa --- /dev/null +++ b/tests/test_dynamodbstreams/test_dynamodbstreams.py @@ -0,0 +1,96 @@ +from __future__ import unicode_literals, print_function + +import boto3 +from moto import mock_dynamodb2, mock_dynamodbstreams + + +class TestClass(): + stream_arn = None + mocks = [] + + def setup(self): + self.mocks = [mock_dynamodb2(), mock_dynamodbstreams()] + for m in self.mocks: + m.start() + + # create a table with a stream + conn = boto3.client('dynamodb', region_name='us-east-1') + + resp = conn.create_table( + TableName='test-streams', + KeySchema=[{'AttributeName': 'id', 'KeyType': 'HASH'}], + AttributeDefinitions=[{'AttributeName': 'id', + 'AttributeType': 'S'}], + ProvisionedThroughput={'ReadCapacityUnits': 1, + 'WriteCapacityUnits': 1}, + StreamSpecification={ + 'StreamEnabled': True, + 'StreamViewType': 'NEW_AND_OLD_IMAGES' + } + ) + self.stream_arn = resp['TableDescription']['LatestStreamArn'] + + def teardown(self): + conn = boto3.client('dynamodb', region_name='us-east-1') + conn.delete_table(TableName='test-streams') + self.stream_arn = None + + for m in self.mocks: + m.stop() + + + def test_verify_stream(self): + conn = boto3.client('dynamodb', region_name='us-east-1') + resp = conn.describe_table(TableName='test-streams') + assert 'LatestStreamArn' in resp['Table'] + + def test_describe_stream(self): + conn = boto3.client('dynamodbstreams', region_name='us-east-1') + + resp = conn.describe_stream(StreamArn=self.stream_arn) + assert 'StreamDescription' in resp + desc = resp['StreamDescription'] + assert desc['StreamArn'] == self.stream_arn + assert desc['TableName'] == 'test-streams' + + def test_list_streams(self): + conn = boto3.client('dynamodbstreams', region_name='us-east-1') + + resp = conn.list_streams() + assert resp['Streams'][0]['StreamArn'] == self.stream_arn + + resp = conn.list_streams(TableName='no-stream') + assert not resp['Streams'] + + def test_get_shard_iterator(self): + conn = boto3.client('dynamodbstreams', region_name='us-east-1') + + resp = conn.describe_stream(StreamArn=self.stream_arn) + shard_id = resp['StreamDescription']['Shards'][0]['ShardId'] + + resp = conn.get_shard_iterator( + StreamArn=self.stream_arn, + ShardId=shard_id, + ShardIteratorType='TRIM_HORIZON' + ) + assert 'ShardIterator' in resp + + def test_get_records(self): + conn = boto3.client('dynamodbstreams', region_name='us-east-1') + + resp = conn.describe_stream(StreamArn=self.stream_arn) + shard_id = resp['StreamDescription']['Shards'][0]['ShardId'] + + resp = conn.get_shard_iterator( + StreamArn=self.stream_arn, + ShardId=shard_id, + ShardIteratorType='TRIM_HORIZON' + ) + iterator_id = resp['ShardIterator'] + + resp = conn.get_records(ShardIterator=iterator_id) + assert 'Records' in resp + + # TODO: Add tests for inserting records into the stream, and + # the various stream types + From 0f6086f708112c024eb7f3079339b1698a507012 Mon Sep 17 00:00:00 2001 From: Karl Gutwin Date: Thu, 8 Nov 2018 10:54:54 -0500 Subject: [PATCH 26/40] Finalize implementation of DynamoDB Streams --- moto/dynamodb2/models.py | 97 ++++++++++++++----- moto/dynamodbstreams/models.py | 44 +++++++-- moto/dynamodbstreams/responses.py | 7 ++ setup.cfg | 2 +- .../test_dynamodbstreams.py | 49 +++++++++- 5 files changed, 166 insertions(+), 33 deletions(-) diff --git a/moto/dynamodb2/models.py b/moto/dynamodb2/models.py index d58241cf..4283c038 100644 --- a/moto/dynamodb2/models.py +++ b/moto/dynamodb2/models.py @@ -1,10 +1,11 @@ from __future__ import unicode_literals -from collections import defaultdict +from collections import defaultdict, namedtuple import copy import datetime import decimal import json import re +import uuid import boto3 from moto.compat import OrderedDict @@ -292,6 +293,44 @@ class Item(BaseModel): 'ADD not supported for %s' % ', '.join(update_action['Value'].keys())) +class StreamRecord(BaseModel): + def __init__(self, table, stream_type, event_name, old, new, seq): + old_a = old.to_json()['Attributes'] if old is not None else {} + new_a = new.to_json()['Attributes'] if new is not None else {} + + rec = old if old is not None else new + keys = {table.hash_key_attr: rec.hash_key.to_json()} + if table.range_key_attr is not None: + keys[table.range_key_attr] = rec.range_key.to_json() + + self.record = { + 'eventID': uuid.uuid4().hex, + 'eventName': event_name, + 'eventSource': 'aws:dynamodb', + 'eventVersion': '1.0', + 'awsRegion': 'us-east-1', + 'dynamodb': { + 'StreamViewType': stream_type, + 'ApproximateCreationDateTime': datetime.datetime.utcnow().isoformat(), + 'SequenceNumber': seq, + 'SizeBytes': 1, + 'Keys': keys + } + } + + if stream_type in ('NEW_IMAGE', 'NEW_AND_OLD_IMAGES'): + self.record['dynamodb']['NewImage'] = new_a + if stream_type in ('OLD_IMAGE', 'NEW_AND_OLD_IMAGES'): + self.record['dynamodb']['OldImage'] = old_a + + # This is a substantial overestimate but it's the easiest to do now + self.record['dynamodb']['SizeBytes'] = len( + json.dumps(self.record['dynamodb'])) + + def to_json(self): + return self.record + + class StreamShard(BaseModel): def __init__(self, table): self.table = table @@ -310,15 +349,22 @@ class StreamShard(BaseModel): def add(self, old, new): t = self.table.stream_specification['StreamViewType'] - if t == 'KEYS_ONLY': - self.items.append(new.key) - elif t == 'NEW_IMAGE': - self.items.append(new) - elif t == 'OLD_IMAGE': - self.items.append(old) - elif t == 'NEW_AND_OLD_IMAGES': - self.items.append((old, new)) - + if old is None: + event_name = 'INSERT' + elif new is None: + event_name = 'DELETE' + else: + event_name = 'MODIFY' + seq = len(self.items) + self.starting_sequence_number + self.items.append( + StreamRecord(self.table, t, event_name, old, new, seq)) + + def get(self, start, quantity): + start -= self.starting_sequence_number + assert start >= 0 + end = start + quantity + return [i.to_json() for i in self.items[start:end]] + class Table(BaseModel): @@ -428,22 +474,22 @@ class Table(BaseModel): else: range_value = None + if expected is None: + expected = {} + lookup_range_value = range_value + else: + expected_range_value = expected.get( + self.range_key_attr, {}).get("Value") + if(expected_range_value is None): + lookup_range_value = range_value + else: + lookup_range_value = DynamoType(expected_range_value) current = self.get_item(hash_value, lookup_range_value) + item = Item(hash_value, self.hash_key_type, range_value, self.range_key_type, item_attrs) if not overwrite: - if expected is None: - expected = {} - lookup_range_value = range_value - else: - expected_range_value = expected.get( - self.range_key_attr, {}).get("Value") - if(expected_range_value is None): - lookup_range_value = range_value - else: - lookup_range_value = DynamoType(expected_range_value) - if current is None: current_attr = {} elif hasattr(current, 'attrs'): @@ -508,9 +554,14 @@ class Table(BaseModel): def delete_item(self, hash_key, range_key): try: if range_key: - return self.items[hash_key].pop(range_key) + item = self.items[hash_key].pop(range_key) else: - return self.items.pop(hash_key) + item = self.items.pop(hash_key) + + if self.stream_shard is not None: + self.stream_shard.add(item, None) + + return item except KeyError: return None diff --git a/moto/dynamodbstreams/models.py b/moto/dynamodbstreams/models.py index 35704de6..7b43eb74 100644 --- a/moto/dynamodbstreams/models.py +++ b/moto/dynamodbstreams/models.py @@ -11,8 +11,9 @@ from moto.dynamodb2.models import dynamodb_backends class ShardIterator(BaseModel): - def __init__(self, stream_shard, shard_iterator_type, sequence_number=None): + def __init__(self, streams_backend, stream_shard, shard_iterator_type, sequence_number=None): self.id = base64.b64encode(os.urandom(472)).decode('utf-8') + self.streams_backend = streams_backend self.stream_shard = stream_shard self.shard_iterator_type = shard_iterator_type if shard_iterator_type == 'TRIM_HORIZON': @@ -24,18 +25,43 @@ class ShardIterator(BaseModel): elif shard_iterator_type == 'AFTER_SEQUENCE_NUMBER': self.sequence_number = sequence_number + 1 + @property + def arn(self): + return '{}/stream/{}|1|{}'.format( + self.stream_shard.table.table_arn, + self.stream_shard.table.latest_stream_label, + self.id) + def to_json(self): return { - 'ShardIterator': '{}/stream/{}|1|{}'.format( - self.stream_shard.table.table_arn, - self.stream_shard.table.latest_stream_label, - self.id) + 'ShardIterator': self.arn + } + + def get(self, limit=1000): + items = self.stream_shard.get(self.sequence_number, limit) + try: + last_sequence_number = max(i['dynamodb']['SequenceNumber'] for i in items) + new_shard_iterator = ShardIterator(self.streams_backend, + self.stream_shard, + 'AFTER_SEQUENCE_NUMBER', + last_sequence_number) + except ValueError: + new_shard_iterator = ShardIterator(self.streams_backend, + self.stream_shard, + 'AT_SEQUENCE_NUMBER', + self.sequence_number) + + self.streams_backend.shard_iterators[new_shard_iterator.arn] = new_shard_iterator + return { + 'NextShardIterator': new_shard_iterator.arn, + 'Records': items } class DynamoDBStreamsBackend(BaseBackend): def __init__(self, region): self.region = region + self.shard_iterators = {} def reset(self): region = self.region @@ -86,11 +112,17 @@ class DynamoDBStreamsBackend(BaseBackend): table = self._get_table_from_arn(arn) assert table.stream_shard.id == shard_id - shard_iterator = ShardIterator(table.stream_shard, shard_iterator_type, + shard_iterator = ShardIterator(self, table.stream_shard, + shard_iterator_type, sequence_number) + self.shard_iterators[shard_iterator.arn] = shard_iterator return json.dumps(shard_iterator.to_json()) + def get_records(self, iterator_arn, limit): + shard_iterator = self.shard_iterators[iterator_arn] + return json.dumps(shard_iterator.get(limit)) + available_regions = boto3.session.Session().get_available_regions( diff --git a/moto/dynamodbstreams/responses.py b/moto/dynamodbstreams/responses.py index ef05c171..c07377d3 100644 --- a/moto/dynamodbstreams/responses.py +++ b/moto/dynamodbstreams/responses.py @@ -27,3 +27,10 @@ class DynamoDBStreamsHandler(BaseResponse): shard_iterator_type = self._get_param('ShardIteratorType') return self.backend.get_shard_iterator(arn, shard_id, shard_iterator_type) + + def get_records(self): + arn = self._get_param('ShardIterator') + limit = self._get_param('Limit') + if limit is None: + limit = 1000 + return self.backend.get_records(arn, limit) diff --git a/setup.cfg b/setup.cfg index 9dbd988d..fb04c16a 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,7 +1,7 @@ [nosetests] verbosity=1 detailed-errors=1 -#with-coverage=1 +with-coverage=1 cover-package=moto [bdist_wheel] diff --git a/tests/test_dynamodbstreams/test_dynamodbstreams.py b/tests/test_dynamodbstreams/test_dynamodbstreams.py index a10445aa..94c0c51b 100644 --- a/tests/test_dynamodbstreams/test_dynamodbstreams.py +++ b/tests/test_dynamodbstreams/test_dynamodbstreams.py @@ -75,7 +75,7 @@ class TestClass(): ) assert 'ShardIterator' in resp - def test_get_records(self): + def test_get_records_empty(self): conn = boto3.client('dynamodbstreams', region_name='us-east-1') resp = conn.describe_stream(StreamArn=self.stream_arn) @@ -90,7 +90,50 @@ class TestClass(): resp = conn.get_records(ShardIterator=iterator_id) assert 'Records' in resp + assert len(resp['Records']) == 0 - # TODO: Add tests for inserting records into the stream, and - # the various stream types + def test_get_records_seq(self): + conn = boto3.client('dynamodb', region_name='us-east-1') + + conn.put_item( + TableName='test-streams', + Item={ + 'id': {'S': 'entry1'}, + 'first_col': {'S': 'foo'} + } + ) + conn.put_item( + TableName='test-streams', + Item={ + 'id': {'S': 'entry1'}, + 'first_col': {'S': 'bar'}, + 'second_col': {'S': 'baz'} + } + ) + conn.delete_item( + TableName='test-streams', + Key={'id': {'S': 'entry1'}} + ) + conn = boto3.client('dynamodbstreams', region_name='us-east-1') + + resp = conn.describe_stream(StreamArn=self.stream_arn) + shard_id = resp['StreamDescription']['Shards'][0]['ShardId'] + + resp = conn.get_shard_iterator( + StreamArn=self.stream_arn, + ShardId=shard_id, + ShardIteratorType='TRIM_HORIZON' + ) + iterator_id = resp['ShardIterator'] + + resp = conn.get_records(ShardIterator=iterator_id) + assert len(resp['Records']) == 3 + assert resp['Records'][0]['eventName'] == 'INSERT' + assert resp['Records'][1]['eventName'] == 'MODIFY' + assert resp['Records'][2]['eventName'] == 'DELETE' + + # now try fetching from the next shard iterator, it should be + # empty + resp = conn.get_records(ShardIterator=resp['NextShardIterator']) + assert len(resp['Records']) == 0 From ff6a57f44391e2e8523c1ca49022ec39ed0facd6 Mon Sep 17 00:00:00 2001 From: Karl Gutwin Date: Thu, 8 Nov 2018 11:08:24 -0500 Subject: [PATCH 27/40] Fix flake8 failures --- moto/backends.py | 1 + moto/dynamodb2/models.py | 16 ++++++++-------- moto/dynamodb2/responses.py | 2 +- moto/dynamodbstreams/models.py | 14 ++++++-------- moto/dynamodbstreams/responses.py | 4 +--- 5 files changed, 17 insertions(+), 20 deletions(-) diff --git a/moto/backends.py b/moto/backends.py index 7df167a0..1a333415 100644 --- a/moto/backends.py +++ b/moto/backends.py @@ -60,6 +60,7 @@ BACKENDS = { 'datapipeline': datapipeline_backends, 'dynamodb': dynamodb_backends, 'dynamodb2': dynamodb_backends2, + 'dynamodbstreams': dynamodbstreams_backends, 'ec2': ec2_backends, 'ecr': ecr_backends, 'ecs': ecs_backends, diff --git a/moto/dynamodb2/models.py b/moto/dynamodb2/models.py index 4283c038..e2882f1e 100644 --- a/moto/dynamodb2/models.py +++ b/moto/dynamodb2/models.py @@ -1,5 +1,5 @@ from __future__ import unicode_literals -from collections import defaultdict, namedtuple +from collections import defaultdict import copy import datetime import decimal @@ -302,7 +302,7 @@ class StreamRecord(BaseModel): keys = {table.hash_key_attr: rec.hash_key.to_json()} if table.range_key_attr is not None: keys[table.range_key_attr] = rec.range_key.to_json() - + self.record = { 'eventID': uuid.uuid4().hex, 'eventName': event_name, @@ -317,7 +317,7 @@ class StreamRecord(BaseModel): 'Keys': keys } } - + if stream_type in ('NEW_IMAGE', 'NEW_AND_OLD_IMAGES'): self.record['dynamodb']['NewImage'] = new_a if stream_type in ('OLD_IMAGE', 'NEW_AND_OLD_IMAGES'): @@ -364,7 +364,7 @@ class StreamShard(BaseModel): assert start >= 0 end = start + quantity return [i.to_json() for i in self.items[start:end]] - + class Table(BaseModel): @@ -485,7 +485,7 @@ class Table(BaseModel): else: lookup_range_value = DynamoType(expected_range_value) current = self.get_item(hash_value, lookup_range_value) - + item = Item(hash_value, self.hash_key_type, range_value, self.range_key_type, item_attrs) @@ -520,10 +520,10 @@ class Table(BaseModel): self.items[hash_value][range_value] = item else: self.items[hash_value] = item - + if self.stream_shard is not None: self.stream_shard.add(current, item) - + return item def __nonzero__(self): @@ -560,7 +560,7 @@ class Table(BaseModel): if self.stream_shard is not None: self.stream_shard.add(item, None) - + return item except KeyError: return None diff --git a/moto/dynamodb2/responses.py b/moto/dynamodb2/responses.py index 73bd3ae3..d2bf99bf 100644 --- a/moto/dynamodb2/responses.py +++ b/moto/dynamodb2/responses.py @@ -175,7 +175,7 @@ class DynamoHandler(BaseResponse): table = self.dynamodb_backend.update_table_throughput(name, throughput) if 'StreamSpecification' in self.body: table = self.dynamodb_backend.update_table_streams(name, self.body['StreamSpecification']) - + return dynamo_json_dump(table.describe()) def describe_table(self): diff --git a/moto/dynamodbstreams/models.py b/moto/dynamodbstreams/models.py index 7b43eb74..41cc6e28 100644 --- a/moto/dynamodbstreams/models.py +++ b/moto/dynamodbstreams/models.py @@ -4,7 +4,6 @@ import os import json import boto3 import base64 -import datetime from moto.core import BaseBackend, BaseModel from moto.dynamodb2.models import dynamodb_backends @@ -31,7 +30,7 @@ class ShardIterator(BaseModel): self.stream_shard.table.table_arn, self.stream_shard.table.latest_stream_label, self.id) - + def to_json(self): return { 'ShardIterator': self.arn @@ -67,7 +66,7 @@ class DynamoDBStreamsBackend(BaseBackend): region = self.region self.__dict__ = {} self.__init__(region) - + @property def dynamodb(self): return dynamodb_backends[self.region] @@ -75,7 +74,7 @@ class DynamoDBStreamsBackend(BaseBackend): def _get_table_from_arn(self, arn): table_name = arn.split(':', 6)[5].split('/')[1] return self.dynamodb.get_table(table_name) - + def describe_stream(self, arn): table = self._get_table_from_arn(arn) resp = {'StreamDescription': { @@ -90,7 +89,7 @@ class DynamoDBStreamsBackend(BaseBackend): 'Shards': ([table.stream_shard.to_json()] if table.stream_shard else []) }} - + return json.dumps(resp) def list_streams(self, table_name=None): @@ -105,7 +104,7 @@ class DynamoDBStreamsBackend(BaseBackend): 'TableName': d['Table']['TableName'], 'StreamLabel': d['Table']['LatestStreamLabel'] }) - + return json.dumps({'Streams': streams}) def get_shard_iterator(self, arn, shard_id, shard_iterator_type, sequence_number=None): @@ -116,14 +115,13 @@ class DynamoDBStreamsBackend(BaseBackend): shard_iterator_type, sequence_number) self.shard_iterators[shard_iterator.arn] = shard_iterator - + return json.dumps(shard_iterator.to_json()) def get_records(self, iterator_arn, limit): shard_iterator = self.shard_iterators[iterator_arn] return json.dumps(shard_iterator.get(limit)) - available_regions = boto3.session.Session().get_available_regions( 'dynamodbstreams') diff --git a/moto/dynamodbstreams/responses.py b/moto/dynamodbstreams/responses.py index c07377d3..c9c11361 100644 --- a/moto/dynamodbstreams/responses.py +++ b/moto/dynamodbstreams/responses.py @@ -1,7 +1,5 @@ from __future__ import unicode_literals -import json - from moto.core.responses import BaseResponse from .models import dynamodbstreams_backends @@ -12,7 +10,7 @@ class DynamoDBStreamsHandler(BaseResponse): @property def backend(self): return dynamodbstreams_backends[self.region] - + def describe_stream(self): arn = self._get_param('StreamArn') return self.backend.describe_stream(arn) From 9d190aa04e4c66831449a1a5cd3673c35e995bf9 Mon Sep 17 00:00:00 2001 From: Karl Gutwin Date: Thu, 8 Nov 2018 13:22:24 -0500 Subject: [PATCH 28/40] Tweak functionality and add tests --- moto/dynamodb2/models.py | 17 ++-- .../test_dynamodbstreams.py | 99 ++++++++++++++++++- 2 files changed, 108 insertions(+), 8 deletions(-) diff --git a/moto/dynamodb2/models.py b/moto/dynamodb2/models.py index e2882f1e..d15b9fce 100644 --- a/moto/dynamodb2/models.py +++ b/moto/dynamodb2/models.py @@ -406,10 +406,13 @@ class Table(BaseModel): def set_stream_specification(self, streams): self.stream_specification = streams - if streams and streams.get('StreamEnabled'): + if streams and (streams.get('StreamEnabled') + or streams.get('StreamViewType')): + self.stream_specification['StreamEnabled'] = True self.latest_stream_label = datetime.datetime.utcnow().isoformat() self.stream_shard = StreamShard(self) else: + self.stream_specification = {'StreamEnabled': False} self.latest_stream_label = None self.stream_shard = None @@ -429,11 +432,11 @@ class Table(BaseModel): 'LocalSecondaryIndexes': [index for index in self.indexes], } } - if self.stream_specification: + if self.stream_specification and self.stream_specification['StreamEnabled']: results[base_key]['StreamSpecification'] = self.stream_specification - if self.latest_stream_label: - results[base_key]['LatestStreamLabel'] = self.latest_stream_label - results[base_key]['LatestStreamArn'] = self.table_arn + '/stream/' + self.latest_stream_label + if self.latest_stream_label: + results[base_key]['LatestStreamLabel'] = self.latest_stream_label + results[base_key]['LatestStreamArn'] = self.table_arn + '/stream/' + self.latest_stream_label return results def __len__(self): @@ -779,7 +782,9 @@ class DynamoDBBackend(BaseBackend): def update_table_streams(self, name, stream_specification): table = self.tables[name] - if stream_specification['StreamEnabled'] and table.latest_stream_label: + if ((stream_specification.get('StreamEnabled') + or stream_specification.get('StreamViewType')) + and table.latest_stream_label): raise ValueError('Table already has stream enabled') table.set_stream_specification(stream_specification) return table diff --git a/tests/test_dynamodbstreams/test_dynamodbstreams.py b/tests/test_dynamodbstreams/test_dynamodbstreams.py index 94c0c51b..7e402562 100644 --- a/tests/test_dynamodbstreams/test_dynamodbstreams.py +++ b/tests/test_dynamodbstreams/test_dynamodbstreams.py @@ -1,10 +1,12 @@ from __future__ import unicode_literals, print_function +from nose.tools import assert_raises + import boto3 from moto import mock_dynamodb2, mock_dynamodbstreams -class TestClass(): +class TestCore(): stream_arn = None mocks = [] @@ -84,7 +86,7 @@ class TestClass(): resp = conn.get_shard_iterator( StreamArn=self.stream_arn, ShardId=shard_id, - ShardIteratorType='TRIM_HORIZON' + ShardIteratorType='LATEST' ) iterator_id = resp['ShardIterator'] @@ -137,3 +139,96 @@ class TestClass(): # empty resp = conn.get_records(ShardIterator=resp['NextShardIterator']) assert len(resp['Records']) == 0 + + +class TestEdges(): + mocks = [] + + def setup(self): + self.mocks = [mock_dynamodb2(), mock_dynamodbstreams()] + for m in self.mocks: + m.start() + + def teardown(self): + for m in self.mocks: + m.stop() + + + def test_enable_stream_on_table(self): + conn = boto3.client('dynamodb', region_name='us-east-1') + resp = conn.create_table( + TableName='test-streams', + KeySchema=[{'AttributeName': 'id', 'KeyType': 'HASH'}], + AttributeDefinitions=[{'AttributeName': 'id', + 'AttributeType': 'S'}], + ProvisionedThroughput={'ReadCapacityUnits': 1, + 'WriteCapacityUnits': 1} + ) + assert 'StreamSpecification' not in resp['TableDescription'] + + resp = conn.update_table( + TableName='test-streams', + StreamSpecification={ + 'StreamViewType': 'KEYS_ONLY' + } + ) + assert 'StreamSpecification' in resp['TableDescription'] + assert resp['TableDescription']['StreamSpecification'] == { + 'StreamEnabled': True, + 'StreamViewType': 'KEYS_ONLY' + } + assert 'LatestStreamLabel' in resp['TableDescription'] + + # now try to enable it again + with assert_raises(ValueError): + resp = conn.update_table( + TableName='test-streams', + StreamSpecification={ + 'StreamViewType': 'OLD_IMAGES' + } + ) + + def test_stream_with_range_key(self): + dyn = boto3.client('dynamodb', region_name='us-east-1') + + resp = dyn.create_table( + TableName='test-streams', + KeySchema=[{'AttributeName': 'id', 'KeyType': 'HASH'}, + {'AttributeName': 'color', 'KeyType': 'RANGE'}], + AttributeDefinitions=[{'AttributeName': 'id', + 'AttributeType': 'S'}, + {'AttributeName': 'color', + 'AttributeType': 'S'}], + ProvisionedThroughput={'ReadCapacityUnits': 1, + 'WriteCapacityUnits': 1}, + StreamSpecification={ + 'StreamViewType': 'NEW_IMAGES' + } + ) + stream_arn = resp['TableDescription']['LatestStreamArn'] + + streams = boto3.client('dynamodbstreams', region_name='us-east-1') + resp = streams.describe_stream(StreamArn=stream_arn) + shard_id = resp['StreamDescription']['Shards'][0]['ShardId'] + + resp = streams.get_shard_iterator( + StreamArn=stream_arn, + ShardId=shard_id, + ShardIteratorType='LATEST' + ) + iterator_id = resp['ShardIterator'] + + dyn.put_item( + TableName='test-streams', + Item={'id': {'S': 'row1'}, 'color': {'S': 'blue'}} + ) + dyn.put_item( + TableName='test-streams', + Item={'id': {'S': 'row2'}, 'color': {'S': 'green'}} + ) + + resp = streams.get_records(ShardIterator=iterator_id) + assert len(resp['Records']) == 2 + assert resp['Records'][0]['eventName'] == 'INSERT' + assert resp['Records'][1]['eventName'] == 'INSERT' + From e7b01292e96172a1d716fc80c7df628e00c2b83e Mon Sep 17 00:00:00 2001 From: Karl Gutwin Date: Thu, 8 Nov 2018 13:57:44 -0500 Subject: [PATCH 29/40] Fix test failures in server mode --- moto/dynamodb2/responses.py | 6 +++++- moto/server.py | 11 +++++++---- tests/test_dynamodbstreams/test_dynamodbstreams.py | 2 +- 3 files changed, 13 insertions(+), 6 deletions(-) diff --git a/moto/dynamodb2/responses.py b/moto/dynamodb2/responses.py index d2bf99bf..79dd749e 100644 --- a/moto/dynamodb2/responses.py +++ b/moto/dynamodb2/responses.py @@ -174,7 +174,11 @@ class DynamoHandler(BaseResponse): throughput = self.body["ProvisionedThroughput"] table = self.dynamodb_backend.update_table_throughput(name, throughput) if 'StreamSpecification' in self.body: - table = self.dynamodb_backend.update_table_streams(name, self.body['StreamSpecification']) + try: + table = self.dynamodb_backend.update_table_streams(name, self.body['StreamSpecification']) + except ValueError: + er = 'com.amazonaws.dynamodb.v20111205#ResourceInUseException' + return self.error(er, 'Cannot enable stream') return dynamo_json_dump(table.describe()) diff --git a/moto/server.py b/moto/server.py index ba247047..5ad02d38 100644 --- a/moto/server.py +++ b/moto/server.py @@ -80,10 +80,13 @@ class DomainDispatcherApplication(object): region = 'us-east-1' if service == 'dynamodb': - dynamo_api_version = environ['HTTP_X_AMZ_TARGET'].split("_")[1].split(".")[0] - # If Newer API version, use dynamodb2 - if dynamo_api_version > "20111205": - host = "dynamodb2" + if environ['HTTP_X_AMZ_TARGET'].startswith('DynamoDBStreams'): + host = 'dynamodbstreams' + else: + dynamo_api_version = environ['HTTP_X_AMZ_TARGET'].split("_")[1].split(".")[0] + # If Newer API version, use dynamodb2 + if dynamo_api_version > "20111205": + host = "dynamodb2" else: host = "{service}.{region}.amazonaws.com".format( service=service, region=region) diff --git a/tests/test_dynamodbstreams/test_dynamodbstreams.py b/tests/test_dynamodbstreams/test_dynamodbstreams.py index 7e402562..b60c2105 100644 --- a/tests/test_dynamodbstreams/test_dynamodbstreams.py +++ b/tests/test_dynamodbstreams/test_dynamodbstreams.py @@ -180,7 +180,7 @@ class TestEdges(): assert 'LatestStreamLabel' in resp['TableDescription'] # now try to enable it again - with assert_raises(ValueError): + with assert_raises(conn.exceptions.ResourceInUseException): resp = conn.update_table( TableName='test-streams', StreamSpecification={ From 770ad1db565e345b1f11edfcba0225307d52be5d Mon Sep 17 00:00:00 2001 From: Karl Gutwin Date: Thu, 8 Nov 2018 16:21:06 -0500 Subject: [PATCH 30/40] Correct behavior of ReturnValues parameter to put_item and update_item --- moto/dynamodb2/responses.py | 38 +++++++++++++++- tests/test_dynamodb2/test_dynamodb.py | 63 +++++++++++++++++++++++++++ 2 files changed, 99 insertions(+), 2 deletions(-) diff --git a/moto/dynamodb2/responses.py b/moto/dynamodb2/responses.py index e2f1ef1c..cb716d56 100644 --- a/moto/dynamodb2/responses.py +++ b/moto/dynamodb2/responses.py @@ -183,6 +183,7 @@ class DynamoHandler(BaseResponse): def put_item(self): name = self.body['TableName'] item = self.body['Item'] + return_values = self.body.get('ReturnValues', 'NONE') if has_empty_keys_or_values(item): return get_empty_str_error() @@ -193,6 +194,13 @@ class DynamoHandler(BaseResponse): else: expected = None + if return_values == 'ALL_OLD': + existing_item = self.dynamodb_backend.get_item(name, item) + if existing_item: + existing_attributes = existing_item.to_json()['Attributes'] + else: + existing_attributes = {} + # Attempt to parse simple ConditionExpressions into an Expected # expression if not expected: @@ -228,6 +236,10 @@ class DynamoHandler(BaseResponse): 'TableName': name, 'CapacityUnits': 1 } + if return_values == 'ALL_OLD': + item_dict['Attributes'] = existing_attributes + else: + item_dict.pop('Attributes', None) return dynamo_json_dump(item_dict) else: er = 'com.amazonaws.dynamodb.v20111205#ResourceNotFoundException' @@ -527,9 +539,9 @@ class DynamoHandler(BaseResponse): return dynamo_json_dump(item_dict) def update_item(self): - name = self.body['TableName'] key = self.body['Key'] + return_values = self.body.get('ReturnValues', 'NONE') update_expression = self.body.get('UpdateExpression') attribute_updates = self.body.get('AttributeUpdates') expression_attribute_names = self.body.get( @@ -537,6 +549,10 @@ class DynamoHandler(BaseResponse): expression_attribute_values = self.body.get( 'ExpressionAttributeValues', {}) existing_item = self.dynamodb_backend.get_item(name, key) + if existing_item: + existing_attributes = existing_item.to_json()['Attributes'] + else: + existing_attributes = {} if has_empty_keys_or_values(expression_attribute_values): return get_empty_str_error() @@ -591,8 +607,26 @@ class DynamoHandler(BaseResponse): 'TableName': name, 'CapacityUnits': 0.5 } - if not existing_item: + unchanged_attributes = { + k for k in existing_attributes.keys() + if existing_attributes[k] == item_dict['Attributes'].get(k) + } + changed_attributes = set(existing_attributes.keys()).union(item_dict['Attributes'].keys()).difference(unchanged_attributes) + + if return_values == 'NONE': item_dict['Attributes'] = {} + elif return_values == 'ALL_OLD': + item_dict['Attributes'] = existing_attributes + elif return_values == 'UPDATED_OLD': + item_dict['Attributes'] = { + k: v for k, v in existing_attributes.items() + if k in changed_attributes + } + elif return_values == 'UPDATED_NEW': + item_dict['Attributes'] = { + k: v for k, v in item_dict['Attributes'].items() + if k in changed_attributes + } return dynamo_json_dump(item_dict) diff --git a/tests/test_dynamodb2/test_dynamodb.py b/tests/test_dynamodb2/test_dynamodb.py index afc919dd..5bfe8f9b 100644 --- a/tests/test_dynamodb2/test_dynamodb.py +++ b/tests/test_dynamodb2/test_dynamodb.py @@ -1246,6 +1246,69 @@ def test_update_if_not_exists(): assert resp['Items'][0]['created_at'] == 123 +# https://github.com/spulec/moto/issues/1937 +@mock_dynamodb2 +def test_update_return_attributes(): + dynamodb = boto3.client('dynamodb', region_name='us-east-1') + + dynamodb.create_table( + TableName='moto-test', + KeySchema=[{'AttributeName': 'id', 'KeyType': 'HASH'}], + AttributeDefinitions=[{'AttributeName': 'id', 'AttributeType': 'S'}], + ProvisionedThroughput={'ReadCapacityUnits': 1, 'WriteCapacityUnits': 1} + ) + + def update(col, to, rv): + return dynamodb.update_item( + TableName='moto-test', + Key={'id': {'S': 'foo'}}, + AttributeUpdates={col: {'Value': {'S': to}, 'Action': 'PUT'}}, + ReturnValues=rv + ) + + r = update('col1', 'val1', 'ALL_NEW') + assert r['Attributes'] == {'id': {'S': 'foo'}, 'col1': {'S': 'val1'}} + + r = update('col1', 'val2', 'ALL_OLD') + assert r['Attributes'] == {'id': {'S': 'foo'}, 'col1': {'S': 'val1'}} + + r = update('col2', 'val3', 'UPDATED_NEW') + assert r['Attributes'] == {'col2': {'S': 'val3'}} + + r = update('col2', 'val4', 'UPDATED_OLD') + assert r['Attributes'] == {'col2': {'S': 'val3'}} + + r = update('col1', 'val5', 'NONE') + assert r['Attributes'] == {} + + +@mock_dynamodb2 +def test_put_return_attributes(): + dynamodb = boto3.client('dynamodb', region_name='us-east-1') + + dynamodb.create_table( + TableName='moto-test', + KeySchema=[{'AttributeName': 'id', 'KeyType': 'HASH'}], + AttributeDefinitions=[{'AttributeName': 'id', 'AttributeType': 'S'}], + ProvisionedThroughput={'ReadCapacityUnits': 1, 'WriteCapacityUnits': 1} + ) + + r = dynamodb.put_item( + TableName='moto-test', + Item={'id': {'S': 'foo'}, 'col1': {'S': 'val1'}}, + ReturnValues='NONE' + ) + assert 'Attributes' not in r + + r = dynamodb.put_item( + TableName='moto-test', + Item={'id': {'S': 'foo'}, 'col1': {'S': 'val2'}}, + ReturnValues='ALL_OLD' + ) + assert r['Attributes'] == {'id': {'S': 'foo'}, 'col1': {'S': 'val1'}} + + + @mock_dynamodb2 def test_query_global_secondary_index_when_created_via_update_table_resource(): dynamodb = boto3.resource('dynamodb', region_name='us-east-1') From 46c0f8915f913b92190986ba74ea2fb266dd28c9 Mon Sep 17 00:00:00 2001 From: Karl Gutwin Date: Fri, 9 Nov 2018 09:32:19 -0500 Subject: [PATCH 31/40] Fix flake8 failures (boo) --- moto/dynamodb2/models.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/moto/dynamodb2/models.py b/moto/dynamodb2/models.py index d15b9fce..8187ceaf 100644 --- a/moto/dynamodb2/models.py +++ b/moto/dynamodb2/models.py @@ -406,8 +406,7 @@ class Table(BaseModel): def set_stream_specification(self, streams): self.stream_specification = streams - if streams and (streams.get('StreamEnabled') - or streams.get('StreamViewType')): + if streams and (streams.get('StreamEnabled') or streams.get('StreamViewType')): self.stream_specification['StreamEnabled'] = True self.latest_stream_label = datetime.datetime.utcnow().isoformat() self.stream_shard = StreamShard(self) @@ -782,9 +781,7 @@ class DynamoDBBackend(BaseBackend): def update_table_streams(self, name, stream_specification): table = self.tables[name] - if ((stream_specification.get('StreamEnabled') - or stream_specification.get('StreamViewType')) - and table.latest_stream_label): + if (stream_specification.get('StreamEnabled') or stream_specification.get('StreamViewType')) and table.latest_stream_label: raise ValueError('Table already has stream enabled') table.set_stream_specification(stream_specification) return table From 2ec32c80f797039da8d7d7dd7096241f960ab33f Mon Sep 17 00:00:00 2001 From: Karl Gutwin Date: Fri, 9 Nov 2018 13:21:38 -0500 Subject: [PATCH 32/40] Merge in functionality from #1899 --- moto/dynamodb2/responses.py | 15 ++++++++++++++- tests/test_dynamodb2/test_dynamodb.py | 19 ++++++++++++++++++- 2 files changed, 32 insertions(+), 2 deletions(-) diff --git a/moto/dynamodb2/responses.py b/moto/dynamodb2/responses.py index cb716d56..03abf73c 100644 --- a/moto/dynamodb2/responses.py +++ b/moto/dynamodb2/responses.py @@ -185,6 +185,10 @@ class DynamoHandler(BaseResponse): item = self.body['Item'] return_values = self.body.get('ReturnValues', 'NONE') + if return_values not in ('ALL_OLD', 'NONE'): + er = 'com.amazonaws.dynamodb.v20111205#ValidationException' + return self.error(er, 'Return values set to invalid value') + if has_empty_keys_or_values(item): return get_empty_str_error() @@ -524,7 +528,11 @@ class DynamoHandler(BaseResponse): def delete_item(self): name = self.body['TableName'] keys = self.body['Key'] - return_values = self.body.get('ReturnValues', '') + return_values = self.body.get('ReturnValues', 'NONE') + if return_values not in ('ALL_OLD', 'NONE'): + er = 'com.amazonaws.dynamodb.v20111205#ValidationException' + return self.error(er, 'Return values set to invalid value') + table = self.dynamodb_backend.get_table(name) if not table: er = 'com.amazonaws.dynamodb.v20120810#ConditionalCheckFailedException' @@ -554,6 +562,11 @@ class DynamoHandler(BaseResponse): else: existing_attributes = {} + if return_values not in ('NONE', 'ALL_OLD', 'ALL_NEW', 'UPDATED_OLD', + 'UPDATED_NEW'): + er = 'com.amazonaws.dynamodb.v20111205#ValidationException' + return self.error(er, 'Return values set to invalid value') + if has_empty_keys_or_values(expression_attribute_values): return get_empty_str_error() diff --git a/tests/test_dynamodb2/test_dynamodb.py b/tests/test_dynamodb2/test_dynamodb.py index 5bfe8f9b..47a714b1 100644 --- a/tests/test_dynamodb2/test_dynamodb.py +++ b/tests/test_dynamodb2/test_dynamodb.py @@ -1000,6 +1000,11 @@ def test_delete_item(): response = table.scan() assert response['Count'] == 2 + # Test ReturnValues validation + with assert_raises(ClientError) as ex: + table.delete_item(Key={'client': 'client1', 'app': 'app1'}, + ReturnValues='ALL_NEW') + # Test deletion and returning old value response = table.delete_item(Key={'client': 'client1', 'app': 'app1'}, ReturnValues='ALL_OLD') response['Attributes'].should.contain('client') @@ -1281,6 +1286,9 @@ def test_update_return_attributes(): r = update('col1', 'val5', 'NONE') assert r['Attributes'] == {} + with assert_raises(ClientError) as ex: + r = update('col1', 'val6', 'WRONG') + @mock_dynamodb2 def test_put_return_attributes(): @@ -1306,7 +1314,16 @@ def test_put_return_attributes(): ReturnValues='ALL_OLD' ) assert r['Attributes'] == {'id': {'S': 'foo'}, 'col1': {'S': 'val1'}} - + + with assert_raises(ClientError) as ex: + dynamodb.put_item( + TableName='moto-test', + Item={'id': {'S': 'foo'}, 'col1': {'S': 'val3'}}, + ReturnValues='ALL_NEW' + ) + ex.exception.response['Error']['Code'].should.equal('ValidationException') + ex.exception.response['ResponseMetadata']['HTTPStatusCode'].should.equal(400) + ex.exception.response['Error']['Message'].should.equal('Return values set to invalid value') @mock_dynamodb2 From 6e6f23a1b0b7f4c9bce4060b5d32433c776c2dba Mon Sep 17 00:00:00 2001 From: Mark Challoner Date: Wed, 14 Nov 2018 12:03:42 +0000 Subject: [PATCH 33/40] Set deleted status on vpc peer deletion. --- moto/ec2/models.py | 9 ++++++--- tests/test_ec2/test_vpc_peering.py | 3 ++- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/moto/ec2/models.py b/moto/ec2/models.py index b94cac47..f7d1eb04 100755 --- a/moto/ec2/models.py +++ b/moto/ec2/models.py @@ -2230,6 +2230,10 @@ class VPCPeeringConnectionStatus(object): self.code = code self.message = message + def deleted(self): + self.code = 'deleted' + self.message = 'Deleted by {deleter ID}' + def initiating(self): self.code = 'initiating-request' self.message = 'Initiating Request to {accepter ID}' @@ -2292,9 +2296,8 @@ class VPCPeeringConnectionBackend(object): return self.vpc_pcxs.get(vpc_pcx_id) def delete_vpc_peering_connection(self, vpc_pcx_id): - deleted = self.vpc_pcxs.pop(vpc_pcx_id, None) - if not deleted: - raise InvalidVPCPeeringConnectionIdError(vpc_pcx_id) + deleted = self.get_vpc_peering_connection(vpc_pcx_id) + deleted._status.deleted() return deleted def accept_vpc_peering_connection(self, vpc_pcx_id): diff --git a/tests/test_ec2/test_vpc_peering.py b/tests/test_ec2/test_vpc_peering.py index 1f98791b..082499a7 100644 --- a/tests/test_ec2/test_vpc_peering.py +++ b/tests/test_ec2/test_vpc_peering.py @@ -89,7 +89,8 @@ def test_vpc_peering_connections_delete(): verdict.should.equal(True) all_vpc_pcxs = conn.get_all_vpc_peering_connections() - all_vpc_pcxs.should.have.length_of(0) + all_vpc_pcxs.should.have.length_of(1) + all_vpc_pcxs[0]._status.code.should.equal('deleted') with assert_raises(EC2ResponseError) as cm: conn.delete_vpc_peering_connection("pcx-1234abcd") From b5eb724773d23f34fc9093bb06b6cdb5813c668b Mon Sep 17 00:00:00 2001 From: Jon Miller Date: Thu, 15 Nov 2018 15:24:45 -0800 Subject: [PATCH 34/40] Add default path to Role & InstanceProfile --- moto/iam/models.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/moto/iam/models.py b/moto/iam/models.py index 4a5240a0..522915a2 100644 --- a/moto/iam/models.py +++ b/moto/iam/models.py @@ -114,7 +114,7 @@ class Role(BaseModel): self.id = role_id self.name = name self.assume_role_policy_document = assume_role_policy_document - self.path = path + self.path = path or '/' self.policies = {} self.managed_policies = {} @@ -166,7 +166,7 @@ class InstanceProfile(BaseModel): def __init__(self, instance_profile_id, name, path, roles): self.id = instance_profile_id self.name = name - self.path = path + self.path = path or '/' self.roles = roles if roles else [] @classmethod From cf5bd7665cff971b8f0e470e96773bcbe1e5ad36 Mon Sep 17 00:00:00 2001 From: Lorenz Hufnagel Date: Fri, 16 Nov 2018 12:23:39 +0100 Subject: [PATCH 35/40] Mock AWS credentials https://github.com/spulec/moto/issues/1924 --- .travis.yml | 2 -- moto/core/models.py | 6 ++++++ 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 3a5de0fa..d386102f 100644 --- a/.travis.yml +++ b/.travis.yml @@ -23,8 +23,6 @@ matrix: sudo: true before_install: - export BOTO_CONFIG=/dev/null - - export AWS_SECRET_ACCESS_KEY=foobar_secret - - export AWS_ACCESS_KEY_ID=foobar_key install: # We build moto first so the docker container doesn't try to compile it as well, also note we don't use # -d for docker run so the logs show up in travis diff --git a/moto/core/models.py b/moto/core/models.py index 19267ca0..9fe1e96b 100644 --- a/moto/core/models.py +++ b/moto/core/models.py @@ -4,6 +4,7 @@ from __future__ import absolute_import import functools import inspect +import os import re import six from io import BytesIO @@ -21,6 +22,11 @@ from .utils import ( ) +# "Mock" the AWS credentials as they can't be mocked in Botocore currently +os.environ.setdefault("AWS_ACCESS_KEY_ID", "foobar_key") +os.environ.setdefault("AWS_SECRET_ACCESS_KEY", "foobar_secret") + + class BaseMockAWS(object): nested_count = 0 From 009b02bcd521626996d38d4983d02a4cf3d890de Mon Sep 17 00:00:00 2001 From: Jon Michaelchuck Date: Mon, 26 Nov 2018 15:56:46 -0800 Subject: [PATCH 36/40] Raise a client error if PutBucketTags request contains duplicate keys A PutBucketTags request with duplicate keys will raise a ClientError with code InvalidTag and message 'Cannot provide multiple Tags with the same key'. --- moto/s3/exceptions.py | 9 +++++++++ moto/s3/models.py | 5 ++++- tests/test_s3/test_s3.py | 18 ++++++++++++++++++ 3 files changed, 31 insertions(+), 1 deletion(-) diff --git a/moto/s3/exceptions.py b/moto/s3/exceptions.py index 26515dfd..f78e2494 100644 --- a/moto/s3/exceptions.py +++ b/moto/s3/exceptions.py @@ -178,3 +178,12 @@ class InvalidStorageClass(S3ClientError): "InvalidStorageClass", "The storage class you specified is not valid", *args, **kwargs) + +class DuplicateTagKeys(S3ClientError): + code = 400 + + def __init__(self, *args, **kwargs): + super(DuplicateTagKeys, self).__init__( + "InvalidTag", + "Cannot provide multiple Tags with the same key", + *args, **kwargs) diff --git a/moto/s3/models.py b/moto/s3/models.py index bb4d7848..fd53417f 100644 --- a/moto/s3/models.py +++ b/moto/s3/models.py @@ -15,7 +15,7 @@ from bisect import insort from moto.core import BaseBackend, BaseModel from moto.core.utils import iso_8601_datetime_with_milliseconds, rfc_1123_datetime from .exceptions import BucketAlreadyExists, MissingBucket, InvalidPart, EntityTooSmall, MissingKey, \ - InvalidNotificationDestination, MalformedXML, InvalidStorageClass + InvalidNotificationDestination, MalformedXML, InvalidStorageClass, DuplicateTagKeys from .utils import clean_key_name, _VersionedKeyStore UPLOAD_ID_BYTES = 43 @@ -773,6 +773,9 @@ class S3Backend(BaseBackend): return key def put_bucket_tagging(self, bucket_name, tagging): + tag_keys = [tag.key for tag in tagging.tag_set.tags] + if len(tag_keys) != len(set(tag_keys)): + raise DuplicateTagKeys() bucket = self.get_bucket(bucket_name) bucket.set_tags(tagging) diff --git a/tests/test_s3/test_s3.py b/tests/test_s3/test_s3.py index 6e339abb..ffafc0df 100644 --- a/tests/test_s3/test_s3.py +++ b/tests/test_s3/test_s3.py @@ -1553,6 +1553,24 @@ def test_boto3_put_bucket_tagging(): }) resp['ResponseMetadata']['HTTPStatusCode'].should.equal(200) + # With duplicate tag keys: + with assert_raises(ClientError) as err: + resp = s3.put_bucket_tagging(Bucket=bucket_name, + Tagging={ + "TagSet": [ + { + "Key": "TagOne", + "Value": "ValueOne" + }, + { + "Key": "TagOne", + "Value": "ValueOneAgain" + } + ] + }) + e = err.exception + e.response["Error"]["Code"].should.equal("InvalidTag") + e.response["Error"]["Message"].should.equal("Cannot provide multiple Tags with the same key") @mock_s3 def test_boto3_get_bucket_tagging(): From d29869bf9b678c5573f0ff343dd204c2459142a3 Mon Sep 17 00:00:00 2001 From: Jon Michaelchuck Date: Tue, 27 Nov 2018 08:32:30 -0800 Subject: [PATCH 37/40] flake8 fix --- moto/s3/exceptions.py | 1 + 1 file changed, 1 insertion(+) diff --git a/moto/s3/exceptions.py b/moto/s3/exceptions.py index f78e2494..c7d82ddf 100644 --- a/moto/s3/exceptions.py +++ b/moto/s3/exceptions.py @@ -179,6 +179,7 @@ class InvalidStorageClass(S3ClientError): "The storage class you specified is not valid", *args, **kwargs) + class DuplicateTagKeys(S3ClientError): code = 400 From 5db35ef168ff0ae9a1ebd397885cdaa326221f5c Mon Sep 17 00:00:00 2001 From: amitchakote7 Date: Thu, 29 Nov 2018 15:39:39 +1100 Subject: [PATCH 38/40] Added TargetGroupARNs to DESCRIBE_AUTOSCALING_GROUPS_TEMPLATE --- moto/autoscaling/responses.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/moto/autoscaling/responses.py b/moto/autoscaling/responses.py index 5586c51d..845db013 100644 --- a/moto/autoscaling/responses.py +++ b/moto/autoscaling/responses.py @@ -508,6 +508,15 @@ DESCRIBE_AUTOSCALING_GROUPS_TEMPLATE = """ {% endif %} + {% if group.target_group_arns %} + + {% for target_group_arn in group.target_group_arns %} + {{ target_group_arn }} + {% endfor %} + + {% else %} + + {% endif %} {{ group.min_size }} {% if group.vpc_zone_identifier %} {{ group.vpc_zone_identifier }} From 08d17a7a13f6ea9d655a3b336f1cc595ee9c4ff4 Mon Sep 17 00:00:00 2001 From: zane Date: Fri, 21 Dec 2018 14:04:52 -0800 Subject: [PATCH 39/40] adding Tags support --- moto/secretsmanager/models.py | 12 ++++-------- moto/secretsmanager/responses.py | 4 +++- .../test_secretsmanager/test_secretsmanager.py | 17 +++++++++++++++++ 3 files changed, 24 insertions(+), 9 deletions(-) diff --git a/moto/secretsmanager/models.py b/moto/secretsmanager/models.py index 7f89e2eb..1350ab46 100644 --- a/moto/secretsmanager/models.py +++ b/moto/secretsmanager/models.py @@ -56,7 +56,7 @@ class SecretsManagerBackend(BaseBackend): return response - def create_secret(self, name, secret_string, **kwargs): + def create_secret(self, name, secret_string, tags, **kwargs): generated_version_id = str(uuid.uuid4()) @@ -68,7 +68,8 @@ class SecretsManagerBackend(BaseBackend): 'rotation_enabled': False, 'rotation_lambda_arn': '', 'auto_rotate_after_days': 0, - 'version_id': generated_version_id + 'version_id': generated_version_id, + 'tags': tags } self.secrets[name] = secret @@ -101,12 +102,7 @@ class SecretsManagerBackend(BaseBackend): "LastChangedDate": None, "LastAccessedDate": None, "DeletedDate": None, - "Tags": [ - { - "Key": "", - "Value": "" - }, - ] + "Tags": secret['tags'] }) return response diff --git a/moto/secretsmanager/responses.py b/moto/secretsmanager/responses.py index b8b6872a..932e7bfd 100644 --- a/moto/secretsmanager/responses.py +++ b/moto/secretsmanager/responses.py @@ -19,9 +19,11 @@ class SecretsManagerResponse(BaseResponse): def create_secret(self): name = self._get_param('Name') secret_string = self._get_param('SecretString') + tags = self._get_param('Tags', if_none=[]) return secretsmanager_backends[self.region].create_secret( name=name, - secret_string=secret_string + secret_string=secret_string, + tags=tags ) def get_random_password(self): diff --git a/tests/test_secretsmanager/test_secretsmanager.py b/tests/test_secretsmanager/test_secretsmanager.py index 0e0b98b1..16928242 100644 --- a/tests/test_secretsmanager/test_secretsmanager.py +++ b/tests/test_secretsmanager/test_secretsmanager.py @@ -44,6 +44,23 @@ def test_create_secret(): secret = conn.get_secret_value(SecretId='test-secret') assert secret['SecretString'] == 'foosecret' +@mock_secretsmanager +def test_create_secret_with_tags(): + conn = boto3.client('secretsmanager', region_name='us-east-1') + secret_name = 'test-secret-with-tags' + + result = conn.create_secret( + Name=secret_name, + SecretString="foosecret", + Tags=[{"Key": "Foo", "Value": "Bar"}, {"Key": "Mykey", "Value": "Myvalue"}] + ) + assert result['ARN'] + assert result['Name'] == secret_name + secret_value = conn.get_secret_value(SecretId=secret_name) + assert secret_value['SecretString'] == 'foosecret' + secret_details = conn.describe_secret(SecretId=secret_name) + assert secret_details['Tags'] == [{"Key": "Foo", "Value": "Bar"}, {"Key": "Mykey", "Value": "Myvalue"}] + @mock_secretsmanager def test_get_random_password_default_length(): conn = boto3.client('secretsmanager', region_name='us-west-2') From f4767c805ed7c81513dea2bb8be3432b739675a0 Mon Sep 17 00:00:00 2001 From: Steve Pulec Date: Fri, 28 Dec 2018 19:38:09 -0500 Subject: [PATCH 40/40] Uncomment EMR FailureDetails. Closes #1891. --- moto/emr/responses.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/moto/emr/responses.py b/moto/emr/responses.py index 49e37ab9..933e0177 100644 --- a/moto/emr/responses.py +++ b/moto/emr/responses.py @@ -613,13 +613,11 @@ DESCRIBE_STEP_TEMPLATE = """