From 1267e201c53039146955a3df0085d520f5f1d063 Mon Sep 17 00:00:00 2001 From: Robert Rose Date: Thu, 28 Jun 2018 12:31:08 -0700 Subject: [PATCH 01/72] Updated index.rst to fix overflow The EC2 endpoint status was overflowing and creating a scroll bar on my screen. It was bugging me so I fixed it via the GitHub web interface. Will test to ensure it builds correctly when I get home from work. --- docs/index.rst | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/index.rst b/docs/index.rst index 32134240..66e12e4b 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -34,11 +34,11 @@ Currently implemented Services: | - DynamoDB2 | - @mock_dynamodb2 | - core endpoints + partial indexes| +-----------------------+---------------------+-----------------------------------+ | EC2 | @mock_ec2 | core endpoints done | -| - AMI | | core endpoints done | -| - EBS | | core endpoints done | -| - Instances | | all endpoints done | -| - Security Groups | | core endpoints done | -| - Tags | | all endpoints done | +| - AMI | | - core endpoints done | +| - EBS | | - core endpoints done | +| - Instances | | - all endpoints done | +| - Security Groups | | - core endpoints done | +| - Tags | | - all endpoints done | +-----------------------+---------------------+-----------------------------------+ | ECS | @mock_ecs | basic endpoints done | +-----------------------+---------------------+-----------------------------------+ From f23288d9b92d8519239a2beeeb0dc27b97676f17 Mon Sep 17 00:00:00 2001 From: Ferruvich Date: Tue, 7 Aug 2018 13:55:13 +0200 Subject: [PATCH 02/72] Changed the 'create_access_token' function in order to add the extra data into 'create_jwt' function --- moto/cognitoidp/models.py | 24 +++++++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) diff --git a/moto/cognitoidp/models.py b/moto/cognitoidp/models.py index 52a73f89..1119edcb 100644 --- a/moto/cognitoidp/models.py +++ b/moto/cognitoidp/models.py @@ -84,7 +84,11 @@ class CognitoIdpUserPool(BaseModel): return refresh_token def create_access_token(self, client_id, username): - access_token, expires_in = self.create_jwt(client_id, username) + extra_data = self.get_user_extra_data_by_client_id( + client_id, username + ) + access_token, expires_in = self.create_jwt(client_id, username, + extra_data=extra_data) self.access_tokens[access_token] = (client_id, username) return access_token, expires_in @@ -97,6 +101,21 @@ class CognitoIdpUserPool(BaseModel): id_token, _ = self.create_id_token(client_id, username) return access_token, id_token, expires_in + def get_user_extra_data_by_client_id(self, client_id, username): + extra_data = {} + current_client = self.clients.get(client_id, None) + if current_client: + for readable_field in current_client.get_readable_fields(): + attribute = list(filter( + lambda f: f['Name'] == readable_field, + self.users.get(username).attributes + )) + if len(attribute) > 0: + extra_data.update({ + attribute[0]['Name']: attribute[0]['Value'] + }) + return extra_data + class CognitoIdpUserPoolDomain(BaseModel): @@ -138,6 +157,9 @@ class CognitoIdpUserPoolClient(BaseModel): return user_pool_client_json + def get_readable_fields(self): + return self.extended_config.get('ReadAttributes', []) + class CognitoIdpIdentityProvider(BaseModel): From 4776228b6824e31a43060082ef4ef15b490de3af Mon Sep 17 00:00:00 2001 From: Ferruvich Date: Tue, 7 Aug 2018 13:55:32 +0200 Subject: [PATCH 03/72] Testing new feature --- tests/test_cognitoidp/test_cognitoidp.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/tests/test_cognitoidp/test_cognitoidp.py b/tests/test_cognitoidp/test_cognitoidp.py index b2bd469c..fda24146 100644 --- a/tests/test_cognitoidp/test_cognitoidp.py +++ b/tests/test_cognitoidp/test_cognitoidp.py @@ -325,6 +325,7 @@ def test_delete_identity_providers(): def test_admin_create_user(): conn = boto3.client("cognito-idp", "us-west-2") + username = str(uuid.uuid4()) username = str(uuid.uuid4()) value = str(uuid.uuid4()) user_pool_id = conn.create_user_pool(PoolName=str(uuid.uuid4()))["UserPool"]["Id"] @@ -399,15 +400,22 @@ def authentication_flow(conn): username = str(uuid.uuid4()) temporary_password = str(uuid.uuid4()) user_pool_id = conn.create_user_pool(PoolName=str(uuid.uuid4()))["UserPool"]["Id"] + user_attribute_name = str(uuid.uuid4()) + user_attribute_value = str(uuid.uuid4()) client_id = conn.create_user_pool_client( UserPoolId=user_pool_id, ClientName=str(uuid.uuid4()), + ReadAttributes=[user_attribute_name] )["UserPoolClient"]["ClientId"] conn.admin_create_user( UserPoolId=user_pool_id, Username=username, TemporaryPassword=temporary_password, + UserAttributes=[{ + 'Name': user_attribute_name, + 'Value': user_attribute_value + }] ) result = conn.admin_initiate_auth( @@ -446,6 +454,9 @@ def authentication_flow(conn): "access_token": result["AuthenticationResult"]["AccessToken"], "username": username, "password": new_password, + "additional_fields": { + user_attribute_name: user_attribute_value + } } @@ -475,6 +486,8 @@ def test_token_legitimacy(): access_claims = json.loads(jws.verify(access_token, json_web_key, "RS256")) access_claims["iss"].should.equal(issuer) access_claims["aud"].should.equal(client_id) + for k, v in outputs["additional_fields"].items(): + access_claims[k].should.equal(v) @mock_cognitoidp From df28ec03d2cc8c44387fa0ac41f57a3431dd6a45 Mon Sep 17 00:00:00 2001 From: Ferruvich Date: Fri, 10 Aug 2018 16:07:27 +0200 Subject: [PATCH 04/72] Added extra test --- tests/test_cognitoidp/test_cognitoidp.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tests/test_cognitoidp/test_cognitoidp.py b/tests/test_cognitoidp/test_cognitoidp.py index fda24146..d1d57f30 100644 --- a/tests/test_cognitoidp/test_cognitoidp.py +++ b/tests/test_cognitoidp/test_cognitoidp.py @@ -4,8 +4,10 @@ import boto3 import json import os import uuid +import jwt from jose import jws + from moto import mock_cognitoidp import sure # noqa @@ -446,6 +448,9 @@ def authentication_flow(conn): result["AuthenticationResult"]["IdToken"].should_not.be.none result["AuthenticationResult"]["AccessToken"].should_not.be.none + jwt.decode( + result["AuthenticationResult"]["AccessToken"], verify=False + ).get(user_attribute_name, '').should.be.equal(user_attribute_value) return { "user_pool_id": user_pool_id, From aae3ab6b900b879841e75474a9fcc716f238009f Mon Sep 17 00:00:00 2001 From: Ferruvich Date: Fri, 10 Aug 2018 16:26:09 +0200 Subject: [PATCH 05/72] Added package dependency --- requirements-dev.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/requirements-dev.txt b/requirements-dev.txt index 655be061..11f59e06 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -15,3 +15,4 @@ click==6.7 inflection==0.3.1 lxml==4.0.0 beautifulsoup4==4.6.0 +PyJWT==1.6.4 From f9762a5ecb98599109172d0af5295b6f13d1c57f Mon Sep 17 00:00:00 2001 From: Ferruvich Date: Fri, 10 Aug 2018 16:52:48 +0200 Subject: [PATCH 06/72] Removing failing test in order to check coverage and Travis test --- requirements-dev.txt | 1 - tests/test_cognitoidp/test_cognitoidp.py | 4 ---- 2 files changed, 5 deletions(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index 11f59e06..655be061 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -15,4 +15,3 @@ click==6.7 inflection==0.3.1 lxml==4.0.0 beautifulsoup4==4.6.0 -PyJWT==1.6.4 diff --git a/tests/test_cognitoidp/test_cognitoidp.py b/tests/test_cognitoidp/test_cognitoidp.py index d1d57f30..98153fee 100644 --- a/tests/test_cognitoidp/test_cognitoidp.py +++ b/tests/test_cognitoidp/test_cognitoidp.py @@ -4,7 +4,6 @@ import boto3 import json import os import uuid -import jwt from jose import jws @@ -448,9 +447,6 @@ def authentication_flow(conn): result["AuthenticationResult"]["IdToken"].should_not.be.none result["AuthenticationResult"]["AccessToken"].should_not.be.none - jwt.decode( - result["AuthenticationResult"]["AccessToken"], verify=False - ).get(user_attribute_name, '').should.be.equal(user_attribute_value) return { "user_pool_id": user_pool_id, From 2253bbf36123cf542f95f33b0037984686d199d6 Mon Sep 17 00:00:00 2001 From: Ferruvich Date: Tue, 7 Aug 2018 13:55:13 +0200 Subject: [PATCH 07/72] Changed the 'create_access_token' function in order to add the extra data into 'create_jwt' function --- moto/cognitoidp/models.py | 24 +++++++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) diff --git a/moto/cognitoidp/models.py b/moto/cognitoidp/models.py index 52a73f89..1119edcb 100644 --- a/moto/cognitoidp/models.py +++ b/moto/cognitoidp/models.py @@ -84,7 +84,11 @@ class CognitoIdpUserPool(BaseModel): return refresh_token def create_access_token(self, client_id, username): - access_token, expires_in = self.create_jwt(client_id, username) + extra_data = self.get_user_extra_data_by_client_id( + client_id, username + ) + access_token, expires_in = self.create_jwt(client_id, username, + extra_data=extra_data) self.access_tokens[access_token] = (client_id, username) return access_token, expires_in @@ -97,6 +101,21 @@ class CognitoIdpUserPool(BaseModel): id_token, _ = self.create_id_token(client_id, username) return access_token, id_token, expires_in + def get_user_extra_data_by_client_id(self, client_id, username): + extra_data = {} + current_client = self.clients.get(client_id, None) + if current_client: + for readable_field in current_client.get_readable_fields(): + attribute = list(filter( + lambda f: f['Name'] == readable_field, + self.users.get(username).attributes + )) + if len(attribute) > 0: + extra_data.update({ + attribute[0]['Name']: attribute[0]['Value'] + }) + return extra_data + class CognitoIdpUserPoolDomain(BaseModel): @@ -138,6 +157,9 @@ class CognitoIdpUserPoolClient(BaseModel): return user_pool_client_json + def get_readable_fields(self): + return self.extended_config.get('ReadAttributes', []) + class CognitoIdpIdentityProvider(BaseModel): From ab8f4dd159fd66ba8f9dedb9e8445a3ca33fb308 Mon Sep 17 00:00:00 2001 From: Ferruvich Date: Tue, 7 Aug 2018 13:55:32 +0200 Subject: [PATCH 08/72] Testing new feature --- tests/test_cognitoidp/test_cognitoidp.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/tests/test_cognitoidp/test_cognitoidp.py b/tests/test_cognitoidp/test_cognitoidp.py index b2bd469c..fda24146 100644 --- a/tests/test_cognitoidp/test_cognitoidp.py +++ b/tests/test_cognitoidp/test_cognitoidp.py @@ -325,6 +325,7 @@ def test_delete_identity_providers(): def test_admin_create_user(): conn = boto3.client("cognito-idp", "us-west-2") + username = str(uuid.uuid4()) username = str(uuid.uuid4()) value = str(uuid.uuid4()) user_pool_id = conn.create_user_pool(PoolName=str(uuid.uuid4()))["UserPool"]["Id"] @@ -399,15 +400,22 @@ def authentication_flow(conn): username = str(uuid.uuid4()) temporary_password = str(uuid.uuid4()) user_pool_id = conn.create_user_pool(PoolName=str(uuid.uuid4()))["UserPool"]["Id"] + user_attribute_name = str(uuid.uuid4()) + user_attribute_value = str(uuid.uuid4()) client_id = conn.create_user_pool_client( UserPoolId=user_pool_id, ClientName=str(uuid.uuid4()), + ReadAttributes=[user_attribute_name] )["UserPoolClient"]["ClientId"] conn.admin_create_user( UserPoolId=user_pool_id, Username=username, TemporaryPassword=temporary_password, + UserAttributes=[{ + 'Name': user_attribute_name, + 'Value': user_attribute_value + }] ) result = conn.admin_initiate_auth( @@ -446,6 +454,9 @@ def authentication_flow(conn): "access_token": result["AuthenticationResult"]["AccessToken"], "username": username, "password": new_password, + "additional_fields": { + user_attribute_name: user_attribute_value + } } @@ -475,6 +486,8 @@ def test_token_legitimacy(): access_claims = json.loads(jws.verify(access_token, json_web_key, "RS256")) access_claims["iss"].should.equal(issuer) access_claims["aud"].should.equal(client_id) + for k, v in outputs["additional_fields"].items(): + access_claims[k].should.equal(v) @mock_cognitoidp From 9adb7c818df7bbaf842c49b82cba8579fb7bd8a9 Mon Sep 17 00:00:00 2001 From: Ferruvich Date: Fri, 10 Aug 2018 16:07:27 +0200 Subject: [PATCH 09/72] Added extra test --- tests/test_cognitoidp/test_cognitoidp.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tests/test_cognitoidp/test_cognitoidp.py b/tests/test_cognitoidp/test_cognitoidp.py index fda24146..d1d57f30 100644 --- a/tests/test_cognitoidp/test_cognitoidp.py +++ b/tests/test_cognitoidp/test_cognitoidp.py @@ -4,8 +4,10 @@ import boto3 import json import os import uuid +import jwt from jose import jws + from moto import mock_cognitoidp import sure # noqa @@ -446,6 +448,9 @@ def authentication_flow(conn): result["AuthenticationResult"]["IdToken"].should_not.be.none result["AuthenticationResult"]["AccessToken"].should_not.be.none + jwt.decode( + result["AuthenticationResult"]["AccessToken"], verify=False + ).get(user_attribute_name, '').should.be.equal(user_attribute_value) return { "user_pool_id": user_pool_id, From c2b8a7bbb0d1b53b20e7b019a1c8b65b9359abab Mon Sep 17 00:00:00 2001 From: Ferruvich Date: Fri, 10 Aug 2018 16:26:09 +0200 Subject: [PATCH 10/72] Added package dependency --- requirements-dev.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/requirements-dev.txt b/requirements-dev.txt index 111cd5f3..54802ad8 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -15,3 +15,4 @@ click==6.7 inflection==0.3.1 lxml==4.2.3 beautifulsoup4==4.6.0 +PyJWT==1.6.4 From e69d2834e8905f42eb02ae0a451acfe891e64a50 Mon Sep 17 00:00:00 2001 From: Ferruvich Date: Fri, 10 Aug 2018 16:52:48 +0200 Subject: [PATCH 11/72] Removing failing test in order to check coverage and Travis test --- requirements-dev.txt | 1 - tests/test_cognitoidp/test_cognitoidp.py | 4 ---- 2 files changed, 5 deletions(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index 54802ad8..111cd5f3 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -15,4 +15,3 @@ click==6.7 inflection==0.3.1 lxml==4.2.3 beautifulsoup4==4.6.0 -PyJWT==1.6.4 diff --git a/tests/test_cognitoidp/test_cognitoidp.py b/tests/test_cognitoidp/test_cognitoidp.py index d1d57f30..98153fee 100644 --- a/tests/test_cognitoidp/test_cognitoidp.py +++ b/tests/test_cognitoidp/test_cognitoidp.py @@ -4,7 +4,6 @@ import boto3 import json import os import uuid -import jwt from jose import jws @@ -448,9 +447,6 @@ def authentication_flow(conn): result["AuthenticationResult"]["IdToken"].should_not.be.none result["AuthenticationResult"]["AccessToken"].should_not.be.none - jwt.decode( - result["AuthenticationResult"]["AccessToken"], verify=False - ).get(user_attribute_name, '').should.be.equal(user_attribute_value) return { "user_pool_id": user_pool_id, From da76ea633bfc6fef1ac1619dd311c379b253b833 Mon Sep 17 00:00:00 2001 From: Ferruvich Date: Fri, 28 Sep 2018 16:31:57 +0200 Subject: [PATCH 12/72] Removed redundand line --- tests/test_cognitoidp/test_cognitoidp.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/test_cognitoidp/test_cognitoidp.py b/tests/test_cognitoidp/test_cognitoidp.py index 98153fee..b879cc42 100644 --- a/tests/test_cognitoidp/test_cognitoidp.py +++ b/tests/test_cognitoidp/test_cognitoidp.py @@ -326,7 +326,6 @@ def test_delete_identity_providers(): def test_admin_create_user(): conn = boto3.client("cognito-idp", "us-west-2") - username = str(uuid.uuid4()) username = str(uuid.uuid4()) value = str(uuid.uuid4()) user_pool_id = conn.create_user_pool(PoolName=str(uuid.uuid4()))["UserPool"]["Id"] From edbc57e00d74bdfc39a604e692c5607ca57ac705 Mon Sep 17 00:00:00 2001 From: Ashley Gould Date: Sat, 14 Jul 2018 11:35:37 -0700 Subject: [PATCH 13/72] add support for AWS Organizations endpoints covers so far: - create_organization - describe_organization - create_account - describe_account - list_accounts all tests passing. could use some advise from maintaners. --- moto/__init__.py | 1 + moto/backends.py | 2 + moto/organizations/__init__.py | 6 + moto/organizations/models.py | 131 ++++++++++++ moto/organizations/responses.py | 48 +++++ moto/organizations/urls.py | 10 + moto/organizations/utils.py | 34 +++ tests/test_organizations/__init__.py | 0 tests/test_organizations/object_syntax.py | 24 +++ .../test_organizations_boto3.py | 193 ++++++++++++++++++ .../test_organizations_utils.py | 26 +++ 11 files changed, 475 insertions(+) create mode 100644 moto/organizations/__init__.py create mode 100644 moto/organizations/models.py create mode 100644 moto/organizations/responses.py create mode 100644 moto/organizations/urls.py create mode 100644 moto/organizations/utils.py create mode 100644 tests/test_organizations/__init__.py create mode 100644 tests/test_organizations/object_syntax.py create mode 100644 tests/test_organizations/test_organizations_boto3.py create mode 100644 tests/test_organizations/test_organizations_utils.py diff --git a/moto/__init__.py b/moto/__init__.py index 0ce5e54d..2301b7ca 100644 --- a/moto/__init__.py +++ b/moto/__init__.py @@ -27,6 +27,7 @@ from .glacier import mock_glacier, mock_glacier_deprecated # flake8: noqa from .iam import mock_iam, mock_iam_deprecated # flake8: noqa from .kinesis import mock_kinesis, mock_kinesis_deprecated # flake8: noqa from .kms import mock_kms, mock_kms_deprecated # flake8: noqa +from .organizations import mock_organizations # flake8: noqa from .opsworks import mock_opsworks, mock_opsworks_deprecated # flake8: noqa from .polly import mock_polly # flake8: noqa from .rds import mock_rds, mock_rds_deprecated # flake8: noqa diff --git a/moto/backends.py b/moto/backends.py index cd8fe174..25fcec09 100644 --- a/moto/backends.py +++ b/moto/backends.py @@ -26,6 +26,7 @@ from moto.kinesis import kinesis_backends from moto.kms import kms_backends from moto.logs import logs_backends from moto.opsworks import opsworks_backends +from moto.organizations import organizations_backends from moto.polly import polly_backends from moto.rds2 import rds2_backends from moto.redshift import redshift_backends @@ -72,6 +73,7 @@ BACKENDS = { 'kinesis': kinesis_backends, 'kms': kms_backends, 'opsworks': opsworks_backends, + 'organizations': organizations_backends, 'polly': polly_backends, 'redshift': redshift_backends, 'rds': rds2_backends, diff --git a/moto/organizations/__init__.py b/moto/organizations/__init__.py new file mode 100644 index 00000000..372782dd --- /dev/null +++ b/moto/organizations/__init__.py @@ -0,0 +1,6 @@ +from __future__ import unicode_literals +from .models import organizations_backend +from ..core.models import base_decorator + +organizations_backends = {"global": organizations_backend} +mock_organizations = base_decorator(organizations_backends) diff --git a/moto/organizations/models.py b/moto/organizations/models.py new file mode 100644 index 00000000..50212af2 --- /dev/null +++ b/moto/organizations/models.py @@ -0,0 +1,131 @@ +from __future__ import unicode_literals + +import datetime +import time + +from moto.core import BaseBackend, BaseModel +from moto.core.utils import unix_time +from moto.organizations import utils + +MASTER_ACCOUNT_ID = '123456789012' +MASTER_ACCOUNT_EMAIL = 'fakeorg@moto-example.com' +ORGANIZATION_ARN_FORMAT = 'arn:aws:organizations::{0}:organization/{1}' +MASTER_ACCOUNT_ARN_FORMAT = 'arn:aws:organizations::{0}:account/{1}/{0}' +ACCOUNT_ARN_FORMAT = 'arn:aws:organizations::{0}:account/{1}/{2}' + + +class FakeOrganization(BaseModel): + + def __init__(self, feature_set): + self.id = utils.make_random_org_id() + self.feature_set = feature_set + self.master_account_id = MASTER_ACCOUNT_ID + self.master_account_email = MASTER_ACCOUNT_EMAIL + self.available_policy_types = [{ + 'Type': 'SERVICE_CONTROL_POLICY', + 'Status': 'ENABLED' + }] + + @property + def arn(self): + return ORGANIZATION_ARN_FORMAT.format(self.master_account_id, self.id) + + @property + def master_account_arn(self): + return MASTER_ACCOUNT_ARN_FORMAT.format(self.master_account_id, self.id) + + def _describe(self): + return { + 'Organization': { + 'Id': self.id, + 'Arn': self.arn, + 'FeatureSet': self.feature_set, + 'MasterAccountArn': self.master_account_arn, + 'MasterAccountId': self.master_account_id, + 'MasterAccountEmail': self.master_account_email, + 'AvailablePolicyTypes': self.available_policy_types, + } + } + + +class FakeAccount(BaseModel): + + def __init__(self, organization, **kwargs): + self.organization_id = organization.id + self.master_account_id = organization.master_account_id + self.create_account_status_id = utils.make_random_create_account_status_id() + self.account_id = utils.make_random_account_id() + self.account_name = kwargs['AccountName'] + self.email = kwargs['Email'] + self.create_time = datetime.datetime.utcnow() + self.status = 'ACTIVE' + self.joined_method = 'CREATED' + + @property + def arn(self): + return ACCOUNT_ARN_FORMAT.format( + self.master_account_id, + self.organization_id, + self.account_id + ) + + @property + def create_account_status(self): + return { + 'CreateAccountStatus': { + 'Id': self.create_account_status_id, + 'AccountName': self.account_name, + 'State': 'SUCCEEDED', + 'RequestedTimestamp': unix_time(self.create_time), + 'CompletedTimestamp': unix_time(self.create_time), + 'AccountId': self.account_id, + } + } + + def describe(self): + return { + 'Account': { + 'Id': self.account_id, + 'Arn': self.arn, + 'Email': self.email, + 'Name': self.account_name, + 'Status': self.status, + 'JoinedMethod': self.joined_method, + 'JoinedTimestamp': unix_time(self.create_time), + } + } + + +class OrganizationsBackend(BaseBackend): + + def __init__(self): + self.org = None + self.accounts = [] + + def create_organization(self, **kwargs): + self.org = FakeOrganization(kwargs['FeatureSet']) + return self.org._describe() + + def describe_organization(self): + return self.org._describe() + + def create_account(self, **kwargs): + new_account = FakeAccount(self.org, **kwargs) + self.accounts.append(new_account) + return new_account.create_account_status + + def describe_account(self, **kwargs): + account = [account for account in self.accounts + if account.account_id == kwargs['AccountId']][0] + return account.describe() + + def list_accounts(self): + return dict( + Accounts=[account.describe()['Account'] for account in self.accounts] + ) + + +organizations_backend = OrganizationsBackend() + + + diff --git a/moto/organizations/responses.py b/moto/organizations/responses.py new file mode 100644 index 00000000..10288ded --- /dev/null +++ b/moto/organizations/responses.py @@ -0,0 +1,48 @@ +from __future__ import unicode_literals +import json + +from moto.core.responses import BaseResponse +from .models import organizations_backend + + +class OrganizationsResponse(BaseResponse): + + @property + def organizations_backend(self): + return organizations_backend + + @property + def request_params(self): + try: + return json.loads(self.body) + except ValueError: + return {} + + def _get_param(self, param, default=None): + return self.request_params.get(param, default) + + def create_organization(self): + return json.dumps( + self.organizations_backend.create_organization(**self.request_params) + ) + + def describe_organization(self): + return json.dumps( + self.organizations_backend.describe_organization() + ) + + def create_account(self): + return json.dumps( + self.organizations_backend.create_account(**self.request_params) + ) + + def describe_account(self): + return json.dumps( + self.organizations_backend.describe_account(**self.request_params) + ) + + def list_accounts(self): + return json.dumps( + self.organizations_backend.list_accounts() + ) + diff --git a/moto/organizations/urls.py b/moto/organizations/urls.py new file mode 100644 index 00000000..7911f5b5 --- /dev/null +++ b/moto/organizations/urls.py @@ -0,0 +1,10 @@ +from __future__ import unicode_literals +from .responses import OrganizationsResponse + +url_bases = [ + "https?://organizations.(.+).amazonaws.com", +] + +url_paths = { + '{0}/$': OrganizationsResponse.dispatch, +} diff --git a/moto/organizations/utils.py b/moto/organizations/utils.py new file mode 100644 index 00000000..5916dc5e --- /dev/null +++ b/moto/organizations/utils.py @@ -0,0 +1,34 @@ +from __future__ import unicode_literals + +import random +import string + +CHARSET=string.ascii_lowercase + string.digits +ORG_ID_SIZE = 10 +ROOT_ID_SIZE = 4 +ACCOUNT_ID_SIZE = 12 +CREATE_ACCOUNT_STATUS_ID_SIZE = 8 + + +def make_random_org_id(): + # The regex pattern for an organization ID string requires "o-" + # followed by from 10 to 32 lower-case letters or digits. + # e.g. 'o-vipjnq5z86' + return 'o-' + ''.join(random.choice(CHARSET) for x in range(ORG_ID_SIZE)) + +def make_random_root_id(): + # The regex pattern for a root ID string requires "r-" followed by + # from 4 to 32 lower-case letters or digits. + # e.g. 'r-3zwx' + return 'r-' + ''.join(random.choice(CHARSET) for x in range(ROOT_ID_SIZE)) + +def make_random_account_id(): + # The regex pattern for an account ID string requires exactly 12 digits. + # e.g. '488633172133' + return ''.join([random.choice(string.digits) for n in range(ACCOUNT_ID_SIZE)]) + +def make_random_create_account_status_id(): + # The regex pattern for an create account request ID string requires + # "car-" followed by from 8 to 32 lower-case letters or digits. + # e.g. 'car-35gxzwrp' + return 'car-' + ''.join(random.choice(CHARSET) for x in range(CREATE_ACCOUNT_STATUS_ID_SIZE)) diff --git a/tests/test_organizations/__init__.py b/tests/test_organizations/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/test_organizations/object_syntax.py b/tests/test_organizations/object_syntax.py new file mode 100644 index 00000000..35437be6 --- /dev/null +++ b/tests/test_organizations/object_syntax.py @@ -0,0 +1,24 @@ +""" +Temporary functions for checking object structures while specing out +models. This module will go away. +""" + +import yaml +import moto +from moto import organizations as orgs + + +# utils +print(orgs.utils.make_random_org_id()) +print(orgs.utils.make_random_root_id()) +print(orgs.utils.make_random_account_id()) +print(orgs.utils.make_random_create_account_id()) + +# models +my_org = orgs.models.FakeOrganization(feature_set = 'ALL') +print(yaml.dump(my_org._describe())) +#assert False + +my_account = orgs.models.FakeAccount(my_org, AccountName='blee01', Email='blee01@moto-example.org') +print(yaml.dump(my_account)) +#assert False diff --git a/tests/test_organizations/test_organizations_boto3.py b/tests/test_organizations/test_organizations_boto3.py new file mode 100644 index 00000000..fa70f330 --- /dev/null +++ b/tests/test_organizations/test_organizations_boto3.py @@ -0,0 +1,193 @@ +from __future__ import unicode_literals + +import boto3 +import botocore.exceptions +import sure # noqa +import yaml +import re +import datetime + +import moto +from moto import mock_organizations +from moto.organizations.models import ( + MASTER_ACCOUNT_ID, + MASTER_ACCOUNT_EMAIL, + ORGANIZATION_ARN_FORMAT, + MASTER_ACCOUNT_ARN_FORMAT, + ACCOUNT_ARN_FORMAT, +) +from .test_organizations_utils import ( + ORG_ID_REGEX, + ROOT_ID_REGEX, + ACCOUNT_ID_REGEX, + CREATE_ACCOUNT_STATUS_ID_REGEX, +) + +EMAIL_REGEX = "^.+@[a-zA-Z0-9-.]+.[a-zA-Z]{2,3}|[0-9]{1,3}$" + + +def validate_organization(response): + org = response['Organization'] + sorted(org.keys()).should.equal([ + 'Arn', + 'AvailablePolicyTypes', + 'FeatureSet', + 'Id', + 'MasterAccountArn', + 'MasterAccountEmail', + 'MasterAccountId', + ]) + org['Id'].should.match(ORG_ID_REGEX) + org['MasterAccountId'].should.equal(MASTER_ACCOUNT_ID) + org['MasterAccountArn'].should.equal(MASTER_ACCOUNT_ARN_FORMAT.format( + org['MasterAccountId'], + org['Id'], + )) + org['Arn'].should.equal(ORGANIZATION_ARN_FORMAT.format( + org['MasterAccountId'], + org['Id'], + )) + org['MasterAccountEmail'].should.equal(MASTER_ACCOUNT_EMAIL) + org['FeatureSet'].should.be.within(['ALL', 'CONSOLIDATED_BILLING']) + org['AvailablePolicyTypes'].should.equal([{ + 'Type': 'SERVICE_CONTROL_POLICY', + 'Status': 'ENABLED' + }]) + # + #'Organization': { + # 'Id': 'string', + # 'Arn': 'string', + # 'FeatureSet': 'ALL'|'CONSOLIDATED_BILLING', + # 'MasterAccountArn': 'string', + # 'MasterAccountId': 'string', + # 'MasterAccountEmail': 'string', + # 'AvailablePolicyTypes': [ + # { + # 'Type': 'SERVICE_CONTROL_POLICY', + # 'Status': 'ENABLED'|'PENDING_ENABLE'|'PENDING_DISABLE' + # }, + # ] + #} + +def validate_account(org, account): + sorted(account.keys()).should.equal([ + 'Arn', + 'Email', + 'Id', + 'JoinedMethod', + 'JoinedTimestamp', + 'Name', + 'Status', + ]) + account['Id'].should.match(ACCOUNT_ID_REGEX) + account['Arn'].should.equal(ACCOUNT_ARN_FORMAT.format( + org['MasterAccountId'], + org['Id'], + account['Id'], + )) + account['Email'].should.match(EMAIL_REGEX) + account['JoinedMethod'].should.be.within(['INVITED', 'CREATED']) + account['Status'].should.be.within(['ACTIVE', 'SUSPENDED']) + account['Name'].should.be.a(str) + account['JoinedTimestamp'].should.be.a(datetime.datetime) + #'Account': { + # 'Id': 'string', + # 'Arn': 'string', + # 'Email': 'string', + # 'Name': 'string', + # 'Status': 'ACTIVE'|'SUSPENDED', + # 'JoinedMethod': 'INVITED'|'CREATED', + # 'JoinedTimestamp': datetime(2015, 1, 1) + #} + +def validate_create_account_status(create_status): + sorted(create_status.keys()).should.equal([ + 'AccountId', + 'AccountName', + 'CompletedTimestamp', + 'Id', + 'RequestedTimestamp', + 'State', + ]) + create_status['Id'].should.match(CREATE_ACCOUNT_STATUS_ID_REGEX) + create_status['AccountId'].should.match(ACCOUNT_ID_REGEX) + create_status['AccountName'].should.be.a(str) + create_status['State'].should.equal('SUCCEEDED') + create_status['RequestedTimestamp'].should.be.a(datetime.datetime) + create_status['CompletedTimestamp'].should.be.a(datetime.datetime) + #'CreateAccountStatus': { + # 'Id': 'string', + # 'AccountName': 'string', + # 'State': 'IN_PROGRESS'|'SUCCEEDED'|'FAILED', + # 'RequestedTimestamp': datetime(2015, 1, 1), + # 'CompletedTimestamp': datetime(2015, 1, 1), + # 'AccountId': 'string', + # 'FailureReason': 'ACCOUNT_LIMIT_EXCEEDED'|'EMAIL_ALREADY_EXISTS'|'INVALID_ADDRESS'|'INVALID_EMAIL'|'CONCURRENT_ACCOUNT_MODIFICATION'|'INTERNAL_FAILURE' + #} + +@mock_organizations +def test_create_organization(): + client = boto3.client('organizations', region_name='us-east-1') + response = client.create_organization(FeatureSet='ALL') + #print(yaml.dump(response)) + validate_organization(response) + response['Organization']['FeatureSet'].should.equal('ALL') + #assert False + +@mock_organizations +def test_describe_organization(): + client = boto3.client('organizations', region_name='us-east-1') + client.create_organization(FeatureSet='ALL') + response = client.describe_organization() + #print(yaml.dump(response)) + validate_organization(response) + #assert False + + +mockname = 'mock-account' +mockdomain = 'moto-example.org' +mockemail = '@'.join([mockname, mockdomain]) + +@mock_organizations +def test_create_account(): + client = boto3.client('organizations', region_name='us-east-1') + client.create_organization(FeatureSet='ALL') + create_status = client.create_account( + AccountName=mockname, Email=mockemail)['CreateAccountStatus'] + #print(yaml.dump(create_status, default_flow_style=False)) + validate_create_account_status(create_status) + create_status['AccountName'].should.equal(mockname) + #assert False + +@mock_organizations +def test_describe_account(): + client = boto3.client('organizations', region_name='us-east-1') + org = client.create_organization(FeatureSet='ALL')['Organization'] + account_id = client.create_account( + AccountName=mockname, Email=mockemail)['CreateAccountStatus']['AccountId'] + response = client.describe_account(AccountId=account_id) + #print(yaml.dump(response, default_flow_style=False)) + validate_account(org, response['Account']) + response['Account']['Name'].should.equal(mockname) + response['Account']['Email'].should.equal(mockemail) + #assert False + +@mock_organizations +def test_list_accounts(): + client = boto3.client('organizations', region_name='us-east-1') + org = client.create_organization(FeatureSet='ALL')['Organization'] + for i in range(5): + name = mockname + str(i) + email = name + '@' + mockdomain + client.create_account(AccountName=name, Email=email) + response = client.list_accounts() + #print(yaml.dump(response, default_flow_style=False)) + response.should.have.key('Accounts') + accounts = response['Accounts'] + len(accounts).should.equal(5) + for account in accounts: + validate_account(org, account) + accounts[3]['Name'].should.equal(mockname + '3') + accounts[2]['Email'].should.equal(mockname + '2' + '@' + mockdomain) + #assert False + diff --git a/tests/test_organizations/test_organizations_utils.py b/tests/test_organizations/test_organizations_utils.py new file mode 100644 index 00000000..8144c327 --- /dev/null +++ b/tests/test_organizations/test_organizations_utils.py @@ -0,0 +1,26 @@ +from __future__ import unicode_literals + +import sure # noqa +import moto +from moto.organizations import utils + +ORG_ID_REGEX = r'o-[a-z0-9]{%s}' % utils.ORG_ID_SIZE +ROOT_ID_REGEX = r'r-[a-z0-9]{%s}' % utils.ROOT_ID_SIZE +ACCOUNT_ID_REGEX = r'[0-9]{%s}' % utils.ACCOUNT_ID_SIZE +CREATE_ACCOUNT_STATUS_ID_REGEX = r'car-[a-z0-9]{%s}' % utils.CREATE_ACCOUNT_STATUS_ID_SIZE + +def test_make_random_org_id(): + org_id = utils.make_random_org_id() + org_id.should.match(ORG_ID_REGEX) + +def test_make_random_root_id(): + org_id = utils.make_random_root_id() + org_id.should.match(ROOT_ID_REGEX) + +def test_make_random_account_id(): + account_id = utils.make_random_account_id() + account_id.should.match(ACCOUNT_ID_REGEX) + +def test_make_random_create_account_status_id(): + create_account_status_id = utils.make_random_create_account_status_id() + create_account_status_id.should.match(CREATE_ACCOUNT_STATUS_ID_REGEX) From f20898da0ef6c9a8c46866fb596e713054afc349 Mon Sep 17 00:00:00 2001 From: Ashley Gould Date: Sat, 14 Jul 2018 12:28:19 -0700 Subject: [PATCH 14/72] add info on organizations support to docs --- IMPLEMENTATION_COVERAGE.md | 12 ++++++------ README.md | 2 ++ 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/IMPLEMENTATION_COVERAGE.md b/IMPLEMENTATION_COVERAGE.md index 411f55a8..837850ac 100644 --- a/IMPLEMENTATION_COVERAGE.md +++ b/IMPLEMENTATION_COVERAGE.md @@ -3147,22 +3147,22 @@ - [ ] update_server - [ ] update_server_engine_attributes -## organizations - 0% implemented +## organizations - 8% implemented - [ ] accept_handshake - [ ] attach_policy - [ ] cancel_handshake -- [ ] create_account -- [ ] create_organization +- [X] create_account +- [X] create_organization - [ ] create_organizational_unit - [ ] create_policy - [ ] decline_handshake - [ ] delete_organization - [ ] delete_organizational_unit - [ ] delete_policy -- [ ] describe_account +- [X] describe_account - [ ] describe_create_account_status - [ ] describe_handshake -- [ ] describe_organization +- [X] describe_organization - [ ] describe_organizational_unit - [ ] describe_policy - [ ] detach_policy @@ -3173,7 +3173,7 @@ - [ ] enable_policy_type - [ ] invite_account_to_organization - [ ] leave_organization -- [ ] list_accounts +- [X] list_accounts - [ ] list_accounts_for_parent - [ ] list_aws_service_access_for_organization - [ ] list_children diff --git a/README.md b/README.md index a6926a58..e4fcb650 100644 --- a/README.md +++ b/README.md @@ -112,6 +112,8 @@ It gets even better! Moto isn't just for Python code and it isn't just for S3. L |------------------------------------------------------------------------------| | KMS | @mock_kms | basic endpoints done | |------------------------------------------------------------------------------| +| Organizations | @mock_organizations | some endpoints done | +|------------------------------------------------------------------------------| | Polly | @mock_polly | all endpoints done | |------------------------------------------------------------------------------| | RDS | @mock_rds | core endpoints done | From c40d2be646884e333098bdb5fb1941f8f327cba4 Mon Sep 17 00:00:00 2001 From: Ashley Gould Date: Sat, 14 Jul 2018 13:23:15 -0700 Subject: [PATCH 15/72] organizations: clean up for flake8 --- moto/organizations/models.py | 8 +-- moto/organizations/responses.py | 1 - moto/organizations/utils.py | 11 ++-- tests/test_organizations/object_syntax.py | 3 +- .../test_organizations_boto3.py | 53 +++++-------------- .../test_organizations_utils.py | 5 +- 6 files changed, 26 insertions(+), 55 deletions(-) diff --git a/moto/organizations/models.py b/moto/organizations/models.py index 50212af2..e1d59c1d 100644 --- a/moto/organizations/models.py +++ b/moto/organizations/models.py @@ -1,7 +1,6 @@ from __future__ import unicode_literals import datetime -import time from moto.core import BaseBackend, BaseModel from moto.core.utils import unix_time @@ -25,7 +24,7 @@ class FakeOrganization(BaseModel): 'Type': 'SERVICE_CONTROL_POLICY', 'Status': 'ENABLED' }] - + @property def arn(self): return ORGANIZATION_ARN_FORMAT.format(self.master_account_id, self.id) @@ -115,7 +114,7 @@ class OrganizationsBackend(BaseBackend): return new_account.create_account_status def describe_account(self, **kwargs): - account = [account for account in self.accounts + account = [account for account in self.accounts if account.account_id == kwargs['AccountId']][0] return account.describe() @@ -126,6 +125,3 @@ class OrganizationsBackend(BaseBackend): organizations_backend = OrganizationsBackend() - - - diff --git a/moto/organizations/responses.py b/moto/organizations/responses.py index 10288ded..7c8d4501 100644 --- a/moto/organizations/responses.py +++ b/moto/organizations/responses.py @@ -45,4 +45,3 @@ class OrganizationsResponse(BaseResponse): return json.dumps( self.organizations_backend.list_accounts() ) - diff --git a/moto/organizations/utils.py b/moto/organizations/utils.py index 5916dc5e..1b111c1a 100644 --- a/moto/organizations/utils.py +++ b/moto/organizations/utils.py @@ -3,7 +3,7 @@ from __future__ import unicode_literals import random import string -CHARSET=string.ascii_lowercase + string.digits +CHARSET = string.ascii_lowercase + string.digits ORG_ID_SIZE = 10 ROOT_ID_SIZE = 4 ACCOUNT_ID_SIZE = 12 @@ -11,24 +11,27 @@ CREATE_ACCOUNT_STATUS_ID_SIZE = 8 def make_random_org_id(): - # The regex pattern for an organization ID string requires "o-" + # The regex pattern for an organization ID string requires "o-" # followed by from 10 to 32 lower-case letters or digits. # e.g. 'o-vipjnq5z86' return 'o-' + ''.join(random.choice(CHARSET) for x in range(ORG_ID_SIZE)) + def make_random_root_id(): - # The regex pattern for a root ID string requires "r-" followed by + # The regex pattern for a root ID string requires "r-" followed by # from 4 to 32 lower-case letters or digits. # e.g. 'r-3zwx' return 'r-' + ''.join(random.choice(CHARSET) for x in range(ROOT_ID_SIZE)) + def make_random_account_id(): # The regex pattern for an account ID string requires exactly 12 digits. # e.g. '488633172133' return ''.join([random.choice(string.digits) for n in range(ACCOUNT_ID_SIZE)]) + def make_random_create_account_status_id(): - # The regex pattern for an create account request ID string requires + # The regex pattern for an create account request ID string requires # "car-" followed by from 8 to 32 lower-case letters or digits. # e.g. 'car-35gxzwrp' return 'car-' + ''.join(random.choice(CHARSET) for x in range(CREATE_ACCOUNT_STATUS_ID_SIZE)) diff --git a/tests/test_organizations/object_syntax.py b/tests/test_organizations/object_syntax.py index 35437be6..2779d1d0 100644 --- a/tests/test_organizations/object_syntax.py +++ b/tests/test_organizations/object_syntax.py @@ -4,7 +4,6 @@ models. This module will go away. """ import yaml -import moto from moto import organizations as orgs @@ -15,7 +14,7 @@ print(orgs.utils.make_random_account_id()) print(orgs.utils.make_random_create_account_id()) # models -my_org = orgs.models.FakeOrganization(feature_set = 'ALL') +my_org = orgs.models.FakeOrganization(feature_set='ALL') print(yaml.dump(my_org._describe())) #assert False diff --git a/tests/test_organizations/test_organizations_boto3.py b/tests/test_organizations/test_organizations_boto3.py index fa70f330..7ae8d957 100644 --- a/tests/test_organizations/test_organizations_boto3.py +++ b/tests/test_organizations/test_organizations_boto3.py @@ -1,13 +1,9 @@ from __future__ import unicode_literals import boto3 -import botocore.exceptions import sure # noqa -import yaml -import re import datetime -import moto from moto import mock_organizations from moto.organizations.models import ( MASTER_ACCOUNT_ID, @@ -53,21 +49,7 @@ def validate_organization(response): 'Type': 'SERVICE_CONTROL_POLICY', 'Status': 'ENABLED' }]) - # - #'Organization': { - # 'Id': 'string', - # 'Arn': 'string', - # 'FeatureSet': 'ALL'|'CONSOLIDATED_BILLING', - # 'MasterAccountArn': 'string', - # 'MasterAccountId': 'string', - # 'MasterAccountEmail': 'string', - # 'AvailablePolicyTypes': [ - # { - # 'Type': 'SERVICE_CONTROL_POLICY', - # 'Status': 'ENABLED'|'PENDING_ENABLE'|'PENDING_DISABLE' - # }, - # ] - #} + def validate_account(org, account): sorted(account.keys()).should.equal([ @@ -84,21 +66,13 @@ def validate_account(org, account): org['MasterAccountId'], org['Id'], account['Id'], - )) + )) account['Email'].should.match(EMAIL_REGEX) account['JoinedMethod'].should.be.within(['INVITED', 'CREATED']) account['Status'].should.be.within(['ACTIVE', 'SUSPENDED']) account['Name'].should.be.a(str) account['JoinedTimestamp'].should.be.a(datetime.datetime) - #'Account': { - # 'Id': 'string', - # 'Arn': 'string', - # 'Email': 'string', - # 'Name': 'string', - # 'Status': 'ACTIVE'|'SUSPENDED', - # 'JoinedMethod': 'INVITED'|'CREATED', - # 'JoinedTimestamp': datetime(2015, 1, 1) - #} + def validate_create_account_status(create_status): sorted(create_status.keys()).should.equal([ @@ -115,15 +89,7 @@ def validate_create_account_status(create_status): create_status['State'].should.equal('SUCCEEDED') create_status['RequestedTimestamp'].should.be.a(datetime.datetime) create_status['CompletedTimestamp'].should.be.a(datetime.datetime) - #'CreateAccountStatus': { - # 'Id': 'string', - # 'AccountName': 'string', - # 'State': 'IN_PROGRESS'|'SUCCEEDED'|'FAILED', - # 'RequestedTimestamp': datetime(2015, 1, 1), - # 'CompletedTimestamp': datetime(2015, 1, 1), - # 'AccountId': 'string', - # 'FailureReason': 'ACCOUNT_LIMIT_EXCEEDED'|'EMAIL_ALREADY_EXISTS'|'INVALID_ADDRESS'|'INVALID_EMAIL'|'CONCURRENT_ACCOUNT_MODIFICATION'|'INTERNAL_FAILURE' - #} + @mock_organizations def test_create_organization(): @@ -134,6 +100,7 @@ def test_create_organization(): response['Organization']['FeatureSet'].should.equal('ALL') #assert False + @mock_organizations def test_describe_organization(): client = boto3.client('organizations', region_name='us-east-1') @@ -148,23 +115,27 @@ mockname = 'mock-account' mockdomain = 'moto-example.org' mockemail = '@'.join([mockname, mockdomain]) + @mock_organizations def test_create_account(): client = boto3.client('organizations', region_name='us-east-1') client.create_organization(FeatureSet='ALL') create_status = client.create_account( - AccountName=mockname, Email=mockemail)['CreateAccountStatus'] + AccountName=mockname, Email=mockemail + )['CreateAccountStatus'] #print(yaml.dump(create_status, default_flow_style=False)) validate_create_account_status(create_status) create_status['AccountName'].should.equal(mockname) #assert False + @mock_organizations def test_describe_account(): client = boto3.client('organizations', region_name='us-east-1') org = client.create_organization(FeatureSet='ALL')['Organization'] account_id = client.create_account( - AccountName=mockname, Email=mockemail)['CreateAccountStatus']['AccountId'] + AccountName=mockname, Email=mockemail + )['CreateAccountStatus']['AccountId'] response = client.describe_account(AccountId=account_id) #print(yaml.dump(response, default_flow_style=False)) validate_account(org, response['Account']) @@ -172,6 +143,7 @@ def test_describe_account(): response['Account']['Email'].should.equal(mockemail) #assert False + @mock_organizations def test_list_accounts(): client = boto3.client('organizations', region_name='us-east-1') @@ -190,4 +162,3 @@ def test_list_accounts(): accounts[3]['Name'].should.equal(mockname + '3') accounts[2]['Email'].should.equal(mockname + '2' + '@' + mockdomain) #assert False - diff --git a/tests/test_organizations/test_organizations_utils.py b/tests/test_organizations/test_organizations_utils.py index 8144c327..3e29d5cb 100644 --- a/tests/test_organizations/test_organizations_utils.py +++ b/tests/test_organizations/test_organizations_utils.py @@ -1,7 +1,6 @@ from __future__ import unicode_literals import sure # noqa -import moto from moto.organizations import utils ORG_ID_REGEX = r'o-[a-z0-9]{%s}' % utils.ORG_ID_SIZE @@ -9,18 +8,22 @@ ROOT_ID_REGEX = r'r-[a-z0-9]{%s}' % utils.ROOT_ID_SIZE ACCOUNT_ID_REGEX = r'[0-9]{%s}' % utils.ACCOUNT_ID_SIZE CREATE_ACCOUNT_STATUS_ID_REGEX = r'car-[a-z0-9]{%s}' % utils.CREATE_ACCOUNT_STATUS_ID_SIZE + def test_make_random_org_id(): org_id = utils.make_random_org_id() org_id.should.match(ORG_ID_REGEX) + def test_make_random_root_id(): org_id = utils.make_random_root_id() org_id.should.match(ROOT_ID_REGEX) + def test_make_random_account_id(): account_id = utils.make_random_account_id() account_id.should.match(ACCOUNT_ID_REGEX) + def test_make_random_create_account_status_id(): create_account_status_id = utils.make_random_create_account_status_id() create_account_status_id.should.match(CREATE_ACCOUNT_STATUS_ID_REGEX) From 6c0c6148f15b7a9133a0c9cce190e6870f787d6e Mon Sep 17 00:00:00 2001 From: Ashley Gould Date: Sun, 15 Jul 2018 10:31:16 -0700 Subject: [PATCH 16/72] organizations: add endpoint list_roots --- moto/organizations/models.py | 70 +++++++++++++++++-- moto/organizations/responses.py | 5 ++ moto/organizations/utils.py | 14 ++++ tests/test_organizations/object_syntax.py | 6 +- .../test_organizations_boto3.py | 24 +++++++ .../test_organizations_utils.py | 11 ++- 6 files changed, 122 insertions(+), 8 deletions(-) diff --git a/moto/organizations/models.py b/moto/organizations/models.py index e1d59c1d..26565403 100644 --- a/moto/organizations/models.py +++ b/moto/organizations/models.py @@ -11,7 +11,7 @@ MASTER_ACCOUNT_EMAIL = 'fakeorg@moto-example.com' ORGANIZATION_ARN_FORMAT = 'arn:aws:organizations::{0}:organization/{1}' MASTER_ACCOUNT_ARN_FORMAT = 'arn:aws:organizations::{0}:account/{1}/{0}' ACCOUNT_ARN_FORMAT = 'arn:aws:organizations::{0}:account/{1}/{2}' - +ROOT_ARN_FORMAT = 'arn:aws:organizations::{0}:root/{1}/{2}' class FakeOrganization(BaseModel): @@ -33,7 +33,7 @@ class FakeOrganization(BaseModel): def master_account_arn(self): return MASTER_ACCOUNT_ARN_FORMAT.format(self.master_account_id, self.id) - def _describe(self): + def describe(self): return { 'Organization': { 'Id': self.id, @@ -95,18 +95,80 @@ class FakeAccount(BaseModel): } +class FakeOrganizationalUnit(BaseModel): + + def __init__(self, organization, **kwargs): + self.organization_id = organization.id + self.master_account_id = organization.master_account_id + self.id = utils.make_random_ou_id() + self.name = kwargs['Name'] + + @property + def arn(self): + return OU_ARN_FORMAT.format( + self.master_account_id, + self.organization_id, + self.id + ) + + def describe(self): + return { + 'OrganizationalUnit': { + 'Id': self.id, + 'Arn': self.arn, + 'Name': self.name, + } + } + + +class FakeRoot(BaseModel): + + def __init__(self, organization, **kwargs): + self.organization_id = organization.id + self.master_account_id = organization.master_account_id + self.id = utils.make_random_root_id() + self.name = 'Root' + self.policy_types = [{ + 'Type': 'SERVICE_CONTROL_POLICY', + 'Status': 'ENABLED' + }] + + @property + def arn(self): + return ROOT_ARN_FORMAT.format( + self.master_account_id, + self.organization_id, + self.id + ) + + def describe(self): + return { + 'Id': self.id, + 'Arn': self.arn, + 'Name': self.name, + 'PolicyTypes': self.policy_types + } + + class OrganizationsBackend(BaseBackend): def __init__(self): self.org = None self.accounts = [] + self.roots = [] def create_organization(self, **kwargs): self.org = FakeOrganization(kwargs['FeatureSet']) - return self.org._describe() + self.roots.append(FakeRoot(self.org)) + return self.org.describe() def describe_organization(self): - return self.org._describe() + return self.org.describe() + + def list_roots(self): + return dict( + Roots=[root.describe() for root in self.roots] + ) def create_account(self, **kwargs): new_account = FakeAccount(self.org, **kwargs) diff --git a/moto/organizations/responses.py b/moto/organizations/responses.py index 7c8d4501..1804a3fc 100644 --- a/moto/organizations/responses.py +++ b/moto/organizations/responses.py @@ -31,6 +31,11 @@ class OrganizationsResponse(BaseResponse): self.organizations_backend.describe_organization() ) + def list_roots(self): + return json.dumps( + self.organizations_backend.list_roots() + ) + def create_account(self): return json.dumps( self.organizations_backend.create_account(**self.request_params) diff --git a/moto/organizations/utils.py b/moto/organizations/utils.py index 1b111c1a..c7e5c71c 100644 --- a/moto/organizations/utils.py +++ b/moto/organizations/utils.py @@ -7,6 +7,7 @@ CHARSET = string.ascii_lowercase + string.digits ORG_ID_SIZE = 10 ROOT_ID_SIZE = 4 ACCOUNT_ID_SIZE = 12 +OU_ID_SUFFIX_SIZE = 8 CREATE_ACCOUNT_STATUS_ID_SIZE = 8 @@ -24,6 +25,19 @@ def make_random_root_id(): return 'r-' + ''.join(random.choice(CHARSET) for x in range(ROOT_ID_SIZE)) +def make_random_ou_id(root_id): + # The regex pattern for an organizational unit ID string requires "ou-" + # followed by from 4 to 32 lower-case letters or digits (the ID of the root + # that contains the OU) followed by a second "-" dash and from 8 to 32 + # additional lower-case letters or digits. + # e.g. ou-g8sd-5oe3bjaw + return '-'.join([ + 'ou', + root_id.partition('-')[2], + ''.join(random.choice(CHARSET) for x in range(OU_ID_SUFFIX_SIZE)), + ]) + + def make_random_account_id(): # The regex pattern for an account ID string requires exactly 12 digits. # e.g. '488633172133' diff --git a/tests/test_organizations/object_syntax.py b/tests/test_organizations/object_syntax.py index 2779d1d0..3fb86b9d 100644 --- a/tests/test_organizations/object_syntax.py +++ b/tests/test_organizations/object_syntax.py @@ -9,9 +9,11 @@ from moto import organizations as orgs # utils print(orgs.utils.make_random_org_id()) -print(orgs.utils.make_random_root_id()) +root_id = orgs.utils.make_random_root_id() +print(root_id) +print(orgs.utils.make_random_ou_id(root_id)) print(orgs.utils.make_random_account_id()) -print(orgs.utils.make_random_create_account_id()) +print(orgs.utils.make_random_create_account_status_id()) # models my_org = orgs.models.FakeOrganization(feature_set='ALL') diff --git a/tests/test_organizations/test_organizations_boto3.py b/tests/test_organizations/test_organizations_boto3.py index 7ae8d957..f4dc3b44 100644 --- a/tests/test_organizations/test_organizations_boto3.py +++ b/tests/test_organizations/test_organizations_boto3.py @@ -3,6 +3,7 @@ from __future__ import unicode_literals import boto3 import sure # noqa import datetime +import yaml from moto import mock_organizations from moto.organizations.models import ( @@ -11,6 +12,7 @@ from moto.organizations.models import ( ORGANIZATION_ARN_FORMAT, MASTER_ACCOUNT_ARN_FORMAT, ACCOUNT_ARN_FORMAT, + ROOT_ARN_FORMAT, ) from .test_organizations_utils import ( ORG_ID_REGEX, @@ -111,6 +113,28 @@ def test_describe_organization(): #assert False +@mock_organizations +def test_list_roots(): + client = boto3.client('organizations', region_name='us-east-1') + org = client.create_organization(FeatureSet='ALL')['Organization'] + response = client.list_roots() + #print(yaml.dump(response, default_flow_style=False)) + response.should.have.key('Roots').should.be.a(list) + response['Roots'].should_not.be.empty + root = response['Roots'][0] + root.should.have.key('Id').should.match(ROOT_ID_REGEX) + root.should.have.key('Arn').should.equal(ROOT_ARN_FORMAT.format( + org['MasterAccountId'], + org['Id'], + root['Id'], + )) + root.should.have.key('Name').should.be.a(str) + root.should.have.key('PolicyTypes').should.be.a(list) + root['PolicyTypes'][0].should.have.key('Type').should.equal('SERVICE_CONTROL_POLICY') + root['PolicyTypes'][0].should.have.key('Status').should.equal('ENABLED') + #assert False + + mockname = 'mock-account' mockdomain = 'moto-example.org' mockemail = '@'.join([mockname, mockdomain]) diff --git a/tests/test_organizations/test_organizations_utils.py b/tests/test_organizations/test_organizations_utils.py index 3e29d5cb..d2720144 100644 --- a/tests/test_organizations/test_organizations_utils.py +++ b/tests/test_organizations/test_organizations_utils.py @@ -5,6 +5,7 @@ from moto.organizations import utils ORG_ID_REGEX = r'o-[a-z0-9]{%s}' % utils.ORG_ID_SIZE ROOT_ID_REGEX = r'r-[a-z0-9]{%s}' % utils.ROOT_ID_SIZE +OU_ID_REGEX = r'ou-[a-z0-9]{%s}-[a-z0-9]{%s}' % (utils.ROOT_ID_SIZE, utils.OU_ID_SUFFIX_SIZE) ACCOUNT_ID_REGEX = r'[0-9]{%s}' % utils.ACCOUNT_ID_SIZE CREATE_ACCOUNT_STATUS_ID_REGEX = r'car-[a-z0-9]{%s}' % utils.CREATE_ACCOUNT_STATUS_ID_SIZE @@ -15,8 +16,14 @@ def test_make_random_org_id(): def test_make_random_root_id(): - org_id = utils.make_random_root_id() - org_id.should.match(ROOT_ID_REGEX) + root_id = utils.make_random_root_id() + root_id.should.match(ROOT_ID_REGEX) + + +def test_make_random_ou_id(): + root_id = utils.make_random_root_id() + ou_id = utils.make_random_ou_id(root_id) + ou_id.should.match(OU_ID_REGEX) def test_make_random_account_id(): From beebb9abc871f826597e12ba29bcdaa2cd78f4af Mon Sep 17 00:00:00 2001 From: Ashley Gould Date: Sun, 15 Jul 2018 11:49:26 -0700 Subject: [PATCH 17/72] organizations: add 2 more endpoints create_organizational_unit describe_organizational_unit --- IMPLEMENTATION_COVERAGE.md | 8 +-- moto/organizations/models.py | 72 +++++++++++-------- moto/organizations/responses.py | 10 +++ .../test_organizations_boto3.py | 49 +++++++++++++ 4 files changed, 107 insertions(+), 32 deletions(-) diff --git a/IMPLEMENTATION_COVERAGE.md b/IMPLEMENTATION_COVERAGE.md index 837850ac..7b4bc5e4 100644 --- a/IMPLEMENTATION_COVERAGE.md +++ b/IMPLEMENTATION_COVERAGE.md @@ -3147,13 +3147,13 @@ - [ ] update_server - [ ] update_server_engine_attributes -## organizations - 8% implemented +## organizations - 19% implemented - [ ] accept_handshake - [ ] attach_policy - [ ] cancel_handshake - [X] create_account - [X] create_organization -- [ ] create_organizational_unit +- [X] create_organizational_unit - [ ] create_policy - [ ] decline_handshake - [ ] delete_organization @@ -3163,7 +3163,7 @@ - [ ] describe_create_account_status - [ ] describe_handshake - [X] describe_organization -- [ ] describe_organizational_unit +- [X] describe_organizational_unit - [ ] describe_policy - [ ] detach_policy - [ ] disable_aws_service_access @@ -3184,7 +3184,7 @@ - [ ] list_parents - [ ] list_policies - [ ] list_policies_for_target -- [ ] list_roots +- [X] list_roots - [ ] list_targets_for_policy - [ ] move_account - [ ] remove_account_from_organization diff --git a/moto/organizations/models.py b/moto/organizations/models.py index 26565403..461072ce 100644 --- a/moto/organizations/models.py +++ b/moto/organizations/models.py @@ -12,6 +12,7 @@ ORGANIZATION_ARN_FORMAT = 'arn:aws:organizations::{0}:organization/{1}' MASTER_ACCOUNT_ARN_FORMAT = 'arn:aws:organizations::{0}:account/{1}/{0}' ACCOUNT_ARN_FORMAT = 'arn:aws:organizations::{0}:account/{1}/{2}' ROOT_ARN_FORMAT = 'arn:aws:organizations::{0}:root/{1}/{2}' +OU_ARN_FORMAT = 'arn:aws:organizations::{0}:ou/{1}/{2}' class FakeOrganization(BaseModel): @@ -95,32 +96,6 @@ class FakeAccount(BaseModel): } -class FakeOrganizationalUnit(BaseModel): - - def __init__(self, organization, **kwargs): - self.organization_id = organization.id - self.master_account_id = organization.master_account_id - self.id = utils.make_random_ou_id() - self.name = kwargs['Name'] - - @property - def arn(self): - return OU_ARN_FORMAT.format( - self.master_account_id, - self.organization_id, - self.id - ) - - def describe(self): - return { - 'OrganizationalUnit': { - 'Id': self.id, - 'Arn': self.arn, - 'Name': self.name, - } - } - - class FakeRoot(BaseModel): def __init__(self, organization, **kwargs): @@ -150,12 +125,40 @@ class FakeRoot(BaseModel): } + +class FakeOrganizationalUnit(BaseModel): + + def __init__(self, organization, root_id, **kwargs): + self.organization_id = organization.id + self.master_account_id = organization.master_account_id + self.id = utils.make_random_ou_id(root_id) + self.name = kwargs['Name'] + self.parent_id = kwargs['ParentId'] + + @property + def arn(self): + return OU_ARN_FORMAT.format( + self.master_account_id, + self.organization_id, + self.id + ) + + def describe(self): + return { + 'OrganizationalUnit': { + 'Id': self.id, + 'Arn': self.arn, + 'Name': self.name, + } + } + class OrganizationsBackend(BaseBackend): def __init__(self): self.org = None self.accounts = [] self.roots = [] + self.ou = [] def create_organization(self, **kwargs): self.org = FakeOrganization(kwargs['FeatureSet']) @@ -170,14 +173,27 @@ class OrganizationsBackend(BaseBackend): Roots=[root.describe() for root in self.roots] ) + def create_organizational_unit(self, **kwargs): + new_ou = FakeOrganizationalUnit(self.org, self.roots[0].id, **kwargs) + self.ou.append(new_ou) + return new_ou.describe() + + def describe_organizational_unit(self, **kwargs): + ou = [ + ou for ou in self.ou if ou.id == kwargs['OrganizationalUnitId'] + ].pop(0) + return ou.describe() + def create_account(self, **kwargs): new_account = FakeAccount(self.org, **kwargs) self.accounts.append(new_account) return new_account.create_account_status def describe_account(self, **kwargs): - account = [account for account in self.accounts - if account.account_id == kwargs['AccountId']][0] + account = [ + account for account in self.accounts + if account.account_id == kwargs['AccountId'] + ].pop(0) return account.describe() def list_accounts(self): diff --git a/moto/organizations/responses.py b/moto/organizations/responses.py index 1804a3fc..4f0643cf 100644 --- a/moto/organizations/responses.py +++ b/moto/organizations/responses.py @@ -36,6 +36,16 @@ class OrganizationsResponse(BaseResponse): self.organizations_backend.list_roots() ) + def create_organizational_unit(self): + return json.dumps( + self.organizations_backend.create_organizational_unit(**self.request_params) + ) + + def describe_organizational_unit(self): + return json.dumps( + self.organizations_backend.describe_organizational_unit(**self.request_params) + ) + def create_account(self): return json.dumps( self.organizations_backend.create_account(**self.request_params) diff --git a/tests/test_organizations/test_organizations_boto3.py b/tests/test_organizations/test_organizations_boto3.py index f4dc3b44..0e398bd4 100644 --- a/tests/test_organizations/test_organizations_boto3.py +++ b/tests/test_organizations/test_organizations_boto3.py @@ -13,10 +13,12 @@ from moto.organizations.models import ( MASTER_ACCOUNT_ARN_FORMAT, ACCOUNT_ARN_FORMAT, ROOT_ARN_FORMAT, + OU_ARN_FORMAT, ) from .test_organizations_utils import ( ORG_ID_REGEX, ROOT_ID_REGEX, + OU_ID_REGEX, ACCOUNT_ID_REGEX, CREATE_ACCOUNT_STATUS_ID_REGEX, ) @@ -53,6 +55,18 @@ def validate_organization(response): }]) +def validate_organizationa_unit(org, response): + response.should.have.key('OrganizationalUnit').should.be.a(dict) + ou = response['OrganizationalUnit'] + ou.should.have.key('Id').should.match(OU_ID_REGEX) + ou.should.have.key('Arn').should.equal(OU_ARN_FORMAT.format( + org['MasterAccountId'], + org['Id'], + ou['Id'], + )) + ou.should.have.key('Name').should.equal(ou_name) + + def validate_account(org, account): sorted(account.keys()).should.equal([ 'Arn', @@ -113,6 +127,9 @@ def test_describe_organization(): #assert False +# Organizational Units +ou_name = 'ou01' + @mock_organizations def test_list_roots(): client = boto3.client('organizations', region_name='us-east-1') @@ -135,6 +152,38 @@ def test_list_roots(): #assert False +@mock_organizations +def test_create_organizational_unit(): + client = boto3.client('organizations', region_name='us-east-1') + org = client.create_organization(FeatureSet='ALL')['Organization'] + root_id = client.list_roots()['Roots'][0]['Id'] + response = client.create_organizational_unit( + ParentId=root_id, + Name=ou_name, + ) + #print(yaml.dump(response, default_flow_style=False)) + validate_organizationa_unit(org, response) + #assert False + + +@mock_organizations +def test_describe_organizational_unit(): + client = boto3.client('organizations', region_name='us-east-1') + org = client.create_organization(FeatureSet='ALL')['Organization'] + root_id = client.list_roots()['Roots'][0]['Id'] + ou_id = client.create_organizational_unit( + ParentId=root_id, + Name=ou_name, + )['OrganizationalUnit']['Id'] + response = client.describe_organizational_unit( + OrganizationalUnitId=ou_id, + ) + print(yaml.dump(response, default_flow_style=False)) + validate_organizationa_unit(org, response) + #assert False + + +# Accounts mockname = 'mock-account' mockdomain = 'moto-example.org' mockemail = '@'.join([mockname, mockdomain]) From fc2447c6a49139ce9fd06bbc48aa23f3fd1bcae7 Mon Sep 17 00:00:00 2001 From: Ashley Gould Date: Sun, 15 Jul 2018 13:58:27 -0700 Subject: [PATCH 18/72] organiziaions: 2 new endpoints: list_organizational_units_for_parents list_parents --- IMPLEMENTATION_COVERAGE.md | 6 +- moto/organizations/models.py | 32 ++++++++++- moto/organizations/responses.py | 10 ++++ .../test_organizations_boto3.py | 57 ++++++++++++++++--- 4 files changed, 92 insertions(+), 13 deletions(-) diff --git a/IMPLEMENTATION_COVERAGE.md b/IMPLEMENTATION_COVERAGE.md index 7b4bc5e4..84c574bc 100644 --- a/IMPLEMENTATION_COVERAGE.md +++ b/IMPLEMENTATION_COVERAGE.md @@ -3147,7 +3147,7 @@ - [ ] update_server - [ ] update_server_engine_attributes -## organizations - 19% implemented +## organizations - 20% implemented - [ ] accept_handshake - [ ] attach_policy - [ ] cancel_handshake @@ -3180,8 +3180,8 @@ - [ ] list_create_account_status - [ ] list_handshakes_for_account - [ ] list_handshakes_for_organization -- [ ] list_organizational_units_for_parent -- [ ] list_parents +- [X] list_organizational_units_for_parent +- [X] list_parents - [ ] list_policies - [ ] list_policies_for_target - [X] list_roots diff --git a/moto/organizations/models.py b/moto/organizations/models.py index 461072ce..b46c6279 100644 --- a/moto/organizations/models.py +++ b/moto/organizations/models.py @@ -14,6 +14,7 @@ ACCOUNT_ARN_FORMAT = 'arn:aws:organizations::{0}:account/{1}/{2}' ROOT_ARN_FORMAT = 'arn:aws:organizations::{0}:root/{1}/{2}' OU_ARN_FORMAT = 'arn:aws:organizations::{0}:ou/{1}/{2}' + class FakeOrganization(BaseModel): def __init__(self, feature_set): @@ -125,7 +126,6 @@ class FakeRoot(BaseModel): } - class FakeOrganizationalUnit(BaseModel): def __init__(self, organization, root_id, **kwargs): @@ -152,6 +152,7 @@ class FakeOrganizationalUnit(BaseModel): } } + class OrganizationsBackend(BaseBackend): def __init__(self): @@ -184,6 +185,35 @@ class OrganizationsBackend(BaseBackend): ].pop(0) return ou.describe() + def list_organizational_units_for_parent(self, **kwargs): + return dict( + OrganizationalUnits=[ + { + 'Id': ou.id, + 'Arn': ou.arn, + 'Name': ou.name, + } + for ou in self.ou + if ou.parent_id == kwargs['ParentId'] + ] + ) + + def list_parents(self, **kwargs): + parent_id = [ + ou.parent_id for ou in self.ou if ou.id == kwargs['ChildId'] + ].pop(0) + root_parents = [ + dict(Id=root.id, Type='ROOT') + for root in self.roots + if root.id == parent_id + ] + ou_parents = [ + dict(Id=ou.id, Type='ORGANIZATIONAL_UNIT') + for ou in self.ou + if ou.id == parent_id + ] + return dict(Parents=root_parents + ou_parents) + def create_account(self, **kwargs): new_account = FakeAccount(self.org, **kwargs) self.accounts.append(new_account) diff --git a/moto/organizations/responses.py b/moto/organizations/responses.py index 4f0643cf..9fdb7331 100644 --- a/moto/organizations/responses.py +++ b/moto/organizations/responses.py @@ -46,6 +46,16 @@ class OrganizationsResponse(BaseResponse): self.organizations_backend.describe_organizational_unit(**self.request_params) ) + def list_organizational_units_for_parent(self): + return json.dumps( + self.organizations_backend.list_organizational_units_for_parent(**self.request_params) + ) + + def list_parents(self): + return json.dumps( + self.organizations_backend.list_parents(**self.request_params) + ) + def create_account(self): return json.dumps( self.organizations_backend.create_account(**self.request_params) diff --git a/tests/test_organizations/test_organizations_boto3.py b/tests/test_organizations/test_organizations_boto3.py index 0e398bd4..8375ffbe 100644 --- a/tests/test_organizations/test_organizations_boto3.py +++ b/tests/test_organizations/test_organizations_boto3.py @@ -55,7 +55,7 @@ def validate_organization(response): }]) -def validate_organizationa_unit(org, response): +def validate_organizational_unit(org, response): response.should.have.key('OrganizationalUnit').should.be.a(dict) ou = response['OrganizationalUnit'] ou.should.have.key('Id').should.match(OU_ID_REGEX) @@ -64,7 +64,7 @@ def validate_organizationa_unit(org, response): org['Id'], ou['Id'], )) - ou.should.have.key('Name').should.equal(ou_name) + ou.should.have.key('Name').should.be.a(str) def validate_account(org, account): @@ -128,7 +128,6 @@ def test_describe_organization(): # Organizational Units -ou_name = 'ou01' @mock_organizations def test_list_roots(): @@ -157,12 +156,14 @@ def test_create_organizational_unit(): client = boto3.client('organizations', region_name='us-east-1') org = client.create_organization(FeatureSet='ALL')['Organization'] root_id = client.list_roots()['Roots'][0]['Id'] + ou_name = 'ou01' response = client.create_organizational_unit( ParentId=root_id, Name=ou_name, ) #print(yaml.dump(response, default_flow_style=False)) - validate_organizationa_unit(org, response) + validate_organizational_unit(org, response) + response['OrganizationalUnit']['Name'].should.equal(ou_name) #assert False @@ -173,13 +174,51 @@ def test_describe_organizational_unit(): root_id = client.list_roots()['Roots'][0]['Id'] ou_id = client.create_organizational_unit( ParentId=root_id, - Name=ou_name, + Name='ou01', )['OrganizationalUnit']['Id'] - response = client.describe_organizational_unit( - OrganizationalUnitId=ou_id, - ) + response = client.describe_organizational_unit(OrganizationalUnitId=ou_id) print(yaml.dump(response, default_flow_style=False)) - validate_organizationa_unit(org, response) + validate_organizational_unit(org, response) + #assert False + + +@mock_organizations +def test_list_organizational_units_for_parent(): + client = boto3.client('organizations', region_name='us-east-1') + org = client.create_organization(FeatureSet='ALL')['Organization'] + root_id = client.list_roots()['Roots'][0]['Id'] + client.create_organizational_unit(ParentId=root_id, Name='ou01') + client.create_organizational_unit(ParentId=root_id, Name='ou02') + client.create_organizational_unit(ParentId=root_id, Name='ou03') + response = client.list_organizational_units_for_parent(ParentId=root_id) + print(yaml.dump(response, default_flow_style=False)) + response.should.have.key('OrganizationalUnits').should.be.a(list) + for ou in response['OrganizationalUnits']: + validate_organizational_unit(org, dict(OrganizationalUnit=ou)) + #assert False + + +@mock_organizations +def test_list_parents(): + client = boto3.client('organizations', region_name='us-east-1') + org = client.create_organization(FeatureSet='ALL')['Organization'] + root_id = client.list_roots()['Roots'][0]['Id'] + + ou01 = client.create_organizational_unit(ParentId=root_id, Name='ou01') + ou01_id = ou01['OrganizationalUnit']['Id'] + response01 = client.list_parents(ChildId=ou01_id) + #print(yaml.dump(response01, default_flow_style=False)) + response01.should.have.key('Parents').should.be.a(list) + response01['Parents'][0].should.have.key('Id').should.equal(root_id) + response01['Parents'][0].should.have.key('Type').should.equal('ROOT') + + ou02 = client.create_organizational_unit(ParentId=ou01_id, Name='ou02') + ou02_id = ou02['OrganizationalUnit']['Id'] + response02 = client.list_parents(ChildId=ou02_id) + #print(yaml.dump(response02, default_flow_style=False)) + response02.should.have.key('Parents').should.be.a(list) + response02['Parents'][0].should.have.key('Id').should.equal(ou01_id) + response02['Parents'][0].should.have.key('Type').should.equal('ORGANIZATIONAL_UNIT') #assert False From 009dcdb21a10399f7b4fb6f9465267de0e7324bb Mon Sep 17 00:00:00 2001 From: Ashley Gould Date: Sun, 15 Jul 2018 15:25:34 -0700 Subject: [PATCH 19/72] organizations: and another 2 endpoints: list_accounts_for_parent move_account --- IMPLEMENTATION_COVERAGE.md | 6 ++-- moto/organizations/models.py | 25 +++++++++++-- moto/organizations/responses.py | 10 ++++++ .../test_organizations_boto3.py | 36 +++++++++++++++++++ 4 files changed, 72 insertions(+), 5 deletions(-) diff --git a/IMPLEMENTATION_COVERAGE.md b/IMPLEMENTATION_COVERAGE.md index 84c574bc..e1943aa9 100644 --- a/IMPLEMENTATION_COVERAGE.md +++ b/IMPLEMENTATION_COVERAGE.md @@ -3147,7 +3147,7 @@ - [ ] update_server - [ ] update_server_engine_attributes -## organizations - 20% implemented +## organizations - 28% implemented - [ ] accept_handshake - [ ] attach_policy - [ ] cancel_handshake @@ -3174,7 +3174,7 @@ - [ ] invite_account_to_organization - [ ] leave_organization - [X] list_accounts -- [ ] list_accounts_for_parent +- [X] list_accounts_for_parent - [ ] list_aws_service_access_for_organization - [ ] list_children - [ ] list_create_account_status @@ -3186,7 +3186,7 @@ - [ ] list_policies_for_target - [X] list_roots - [ ] list_targets_for_policy -- [ ] move_account +- [X] move_account - [ ] remove_account_from_organization - [ ] update_organizational_unit - [ ] update_policy diff --git a/moto/organizations/models.py b/moto/organizations/models.py index b46c6279..bbcc1479 100644 --- a/moto/organizations/models.py +++ b/moto/organizations/models.py @@ -51,7 +51,7 @@ class FakeOrganization(BaseModel): class FakeAccount(BaseModel): - def __init__(self, organization, **kwargs): + def __init__(self, organization, root_id, **kwargs): self.organization_id = organization.id self.master_account_id = organization.master_account_id self.create_account_status_id = utils.make_random_create_account_status_id() @@ -61,6 +61,7 @@ class FakeAccount(BaseModel): self.create_time = datetime.datetime.utcnow() self.status = 'ACTIVE' self.joined_method = 'CREATED' + self.parent_id = root_id @property def arn(self): @@ -215,7 +216,7 @@ class OrganizationsBackend(BaseBackend): return dict(Parents=root_parents + ou_parents) def create_account(self, **kwargs): - new_account = FakeAccount(self.org, **kwargs) + new_account = FakeAccount(self.org, self.roots[0].id, **kwargs) self.accounts.append(new_account) return new_account.create_account_status @@ -231,5 +232,25 @@ class OrganizationsBackend(BaseBackend): Accounts=[account.describe()['Account'] for account in self.accounts] ) + def list_accounts_for_parent(self, **kwargs): + return dict( + Accounts=[ + account.describe()['Account'] + for account in self.accounts + if account.parent_id == kwargs['ParentId'] + ] + ) + + def move_account(self, **kwargs): + new_parent_id = kwargs['DestinationParentId'] + all_parent_id = [parent.id for parent in self.roots + self.ou] + account = [ + account for account in self.accounts if account.account_id == kwargs['AccountId'] + ].pop(0) + assert new_parent_id in all_parent_id + assert account.parent_id == kwargs['SourceParentId'] + index = self.accounts.index(account) + self.accounts[index].parent_id = new_parent_id + organizations_backend = OrganizationsBackend() diff --git a/moto/organizations/responses.py b/moto/organizations/responses.py index 9fdb7331..6684ae68 100644 --- a/moto/organizations/responses.py +++ b/moto/organizations/responses.py @@ -70,3 +70,13 @@ class OrganizationsResponse(BaseResponse): return json.dumps( self.organizations_backend.list_accounts() ) + + def list_accounts_for_parent(self): + return json.dumps( + self.organizations_backend.list_accounts_for_parent(**self.request_params) + ) + + def move_account(self): + return json.dumps( + self.organizations_backend.move_account(**self.request_params) + ) diff --git a/tests/test_organizations/test_organizations_boto3.py b/tests/test_organizations/test_organizations_boto3.py index 8375ffbe..5355e271 100644 --- a/tests/test_organizations/test_organizations_boto3.py +++ b/tests/test_organizations/test_organizations_boto3.py @@ -274,3 +274,39 @@ def test_list_accounts(): accounts[3]['Name'].should.equal(mockname + '3') accounts[2]['Email'].should.equal(mockname + '2' + '@' + mockdomain) #assert False + + +@mock_organizations +def test_list_accounts_for_parent(): + client = boto3.client('organizations', region_name='us-east-1') + org = client.create_organization(FeatureSet='ALL')['Organization'] + root_id = client.list_roots()['Roots'][0]['Id'] + account_id = client.create_account( + AccountName=mockname, + Email=mockemail, + )['CreateAccountStatus']['AccountId'] + response = client.list_accounts_for_parent(ParentId=root_id) + #print(yaml.dump(response, default_flow_style=False)) + account_id.should.be.within([account['Id'] for account in response['Accounts']]) + #assert False + + +@mock_organizations +def test_move_account(): + client = boto3.client('organizations', region_name='us-east-1') + org = client.create_organization(FeatureSet='ALL')['Organization'] + root_id = client.list_roots()['Roots'][0]['Id'] + account_id = client.create_account( + AccountName=mockname, Email=mockemail + )['CreateAccountStatus']['AccountId'] + ou01 = client.create_organizational_unit(ParentId=root_id, Name='ou01') + ou01_id = ou01['OrganizationalUnit']['Id'] + client.move_account( + AccountId=account_id, + SourceParentId=root_id, + DestinationParentId=ou01_id, + ) + response = client.list_accounts_for_parent(ParentId=ou01_id) + #print(yaml.dump(response, default_flow_style=False)) + account_id.should.be.within([account['Id'] for account in response['Accounts']]) + #assert False From 9b5c6c4f0f04392ab40df7404020fd70340e7347 Mon Sep 17 00:00:00 2001 From: Ashley Gould Date: Sun, 15 Jul 2018 18:48:56 -0700 Subject: [PATCH 20/72] organizations.model.FakeAccount: rename attributes: account_id -> id account_name -> name --- moto/organizations/models.py | 71 ++++++++++++++++++------------------ 1 file changed, 36 insertions(+), 35 deletions(-) diff --git a/moto/organizations/models.py b/moto/organizations/models.py index bbcc1479..8a00918d 100644 --- a/moto/organizations/models.py +++ b/moto/organizations/models.py @@ -55,8 +55,8 @@ class FakeAccount(BaseModel): self.organization_id = organization.id self.master_account_id = organization.master_account_id self.create_account_status_id = utils.make_random_create_account_status_id() - self.account_id = utils.make_random_account_id() - self.account_name = kwargs['AccountName'] + self.id = utils.make_random_account_id() + self.name = kwargs['AccountName'] self.email = kwargs['Email'] self.create_time = datetime.datetime.utcnow() self.status = 'ACTIVE' @@ -68,7 +68,7 @@ class FakeAccount(BaseModel): return ACCOUNT_ARN_FORMAT.format( self.master_account_id, self.organization_id, - self.account_id + self.id ) @property @@ -76,21 +76,21 @@ class FakeAccount(BaseModel): return { 'CreateAccountStatus': { 'Id': self.create_account_status_id, - 'AccountName': self.account_name, + 'AccountName': self.name, 'State': 'SUCCEEDED', 'RequestedTimestamp': unix_time(self.create_time), 'CompletedTimestamp': unix_time(self.create_time), - 'AccountId': self.account_id, + 'AccountId': self.id, } } def describe(self): return { 'Account': { - 'Id': self.account_id, + 'Id': self.id, 'Arn': self.arn, 'Email': self.email, - 'Name': self.account_name, + 'Name': self.name, 'Status': self.status, 'JoinedMethod': self.joined_method, 'JoinedTimestamp': unix_time(self.create_time), @@ -98,6 +98,33 @@ class FakeAccount(BaseModel): } +class FakeOrganizationalUnit(BaseModel): + + def __init__(self, organization, root_id, **kwargs): + self.organization_id = organization.id + self.master_account_id = organization.master_account_id + self.id = utils.make_random_ou_id(root_id) + self.name = kwargs['Name'] + self.parent_id = kwargs['ParentId'] + + @property + def arn(self): + return OU_ARN_FORMAT.format( + self.master_account_id, + self.organization_id, + self.id + ) + + def describe(self): + return { + 'OrganizationalUnit': { + 'Id': self.id, + 'Arn': self.arn, + 'Name': self.name, + } + } + + class FakeRoot(BaseModel): def __init__(self, organization, **kwargs): @@ -127,32 +154,6 @@ class FakeRoot(BaseModel): } -class FakeOrganizationalUnit(BaseModel): - - def __init__(self, organization, root_id, **kwargs): - self.organization_id = organization.id - self.master_account_id = organization.master_account_id - self.id = utils.make_random_ou_id(root_id) - self.name = kwargs['Name'] - self.parent_id = kwargs['ParentId'] - - @property - def arn(self): - return OU_ARN_FORMAT.format( - self.master_account_id, - self.organization_id, - self.id - ) - - def describe(self): - return { - 'OrganizationalUnit': { - 'Id': self.id, - 'Arn': self.arn, - 'Name': self.name, - } - } - class OrganizationsBackend(BaseBackend): @@ -223,7 +224,7 @@ class OrganizationsBackend(BaseBackend): def describe_account(self, **kwargs): account = [ account for account in self.accounts - if account.account_id == kwargs['AccountId'] + if account.id == kwargs['AccountId'] ].pop(0) return account.describe() @@ -245,7 +246,7 @@ class OrganizationsBackend(BaseBackend): new_parent_id = kwargs['DestinationParentId'] all_parent_id = [parent.id for parent in self.roots + self.ou] account = [ - account for account in self.accounts if account.account_id == kwargs['AccountId'] + account for account in self.accounts if account.id == kwargs['AccountId'] ].pop(0) assert new_parent_id in all_parent_id assert account.parent_id == kwargs['SourceParentId'] From 30a9aa33e55e35070eb2468c0d377eb79e7a14e3 Mon Sep 17 00:00:00 2001 From: Ashley Gould Date: Sun, 15 Jul 2018 20:39:13 -0700 Subject: [PATCH 21/72] organizations: endpoint list_parents now support account_id param refactered classes: FakeRoot inherits from FakeOrganizationsUnit add root_id attribute to class FakeOrganization dropped 'roots' attribute from class OrganizationaBackend --- moto/organizations/models.py | 84 ++++++++++--------- .../test_organizations_boto3.py | 79 +++++++++++------ 2 files changed, 98 insertions(+), 65 deletions(-) diff --git a/moto/organizations/models.py b/moto/organizations/models.py index 8a00918d..609ce383 100644 --- a/moto/organizations/models.py +++ b/moto/organizations/models.py @@ -1,6 +1,7 @@ from __future__ import unicode_literals import datetime +import re from moto.core import BaseBackend, BaseModel from moto.core.utils import unix_time @@ -19,6 +20,7 @@ class FakeOrganization(BaseModel): def __init__(self, feature_set): self.id = utils.make_random_org_id() + self.root_id = utils.make_random_root_id() self.feature_set = feature_set self.master_account_id = MASTER_ACCOUNT_ID self.master_account_email = MASTER_ACCOUNT_EMAIL @@ -51,7 +53,7 @@ class FakeOrganization(BaseModel): class FakeAccount(BaseModel): - def __init__(self, organization, root_id, **kwargs): + def __init__(self, organization, **kwargs): self.organization_id = organization.id self.master_account_id = organization.master_account_id self.create_account_status_id = utils.make_random_create_account_status_id() @@ -61,7 +63,7 @@ class FakeAccount(BaseModel): self.create_time = datetime.datetime.utcnow() self.status = 'ACTIVE' self.joined_method = 'CREATED' - self.parent_id = root_id + self.parent_id = organization.root_id @property def arn(self): @@ -100,16 +102,18 @@ class FakeAccount(BaseModel): class FakeOrganizationalUnit(BaseModel): - def __init__(self, organization, root_id, **kwargs): + def __init__(self, organization, **kwargs): + self.type = 'ORGANIZATIONAL_UNIT' self.organization_id = organization.id self.master_account_id = organization.master_account_id - self.id = utils.make_random_ou_id(root_id) - self.name = kwargs['Name'] - self.parent_id = kwargs['ParentId'] + self.id = utils.make_random_ou_id(organization.root_id) + self.name = kwargs.get('Name') + self.parent_id = kwargs.get('ParentId') + self._arn_format = OU_ARN_FORMAT @property def arn(self): - return OU_ARN_FORMAT.format( + return self._arn_format.format( self.master_account_id, self.organization_id, self.id @@ -125,25 +129,18 @@ class FakeOrganizationalUnit(BaseModel): } -class FakeRoot(BaseModel): +class FakeRoot(FakeOrganizationalUnit): def __init__(self, organization, **kwargs): - self.organization_id = organization.id - self.master_account_id = organization.master_account_id - self.id = utils.make_random_root_id() + super().__init__(organization, **kwargs) + self.type = 'ROOT' + self.id = organization.root_id self.name = 'Root' self.policy_types = [{ 'Type': 'SERVICE_CONTROL_POLICY', 'Status': 'ENABLED' }] - - @property - def arn(self): - return ROOT_ARN_FORMAT.format( - self.master_account_id, - self.organization_id, - self.id - ) + self._arn_format = ROOT_ARN_FORMAT def describe(self): return { @@ -154,18 +151,16 @@ class FakeRoot(BaseModel): } - class OrganizationsBackend(BaseBackend): def __init__(self): self.org = None self.accounts = [] - self.roots = [] self.ou = [] def create_organization(self, **kwargs): self.org = FakeOrganization(kwargs['FeatureSet']) - self.roots.append(FakeRoot(self.org)) + self.ou.append(FakeRoot(self.org)) return self.org.describe() def describe_organization(self): @@ -173,11 +168,11 @@ class OrganizationsBackend(BaseBackend): def list_roots(self): return dict( - Roots=[root.describe() for root in self.roots] + Roots=[ou.describe() for ou in self.ou if isinstance(ou, FakeRoot)] ) def create_organizational_unit(self, **kwargs): - new_ou = FakeOrganizationalUnit(self.org, self.roots[0].id, **kwargs) + new_ou = FakeOrganizationalUnit(self.org, **kwargs) self.ou.append(new_ou) return new_ou.describe() @@ -201,23 +196,29 @@ class OrganizationsBackend(BaseBackend): ) def list_parents(self, **kwargs): - parent_id = [ - ou.parent_id for ou in self.ou if ou.id == kwargs['ChildId'] - ].pop(0) - root_parents = [ - dict(Id=root.id, Type='ROOT') - for root in self.roots - if root.id == parent_id - ] - ou_parents = [ - dict(Id=ou.id, Type='ORGANIZATIONAL_UNIT') - for ou in self.ou - if ou.id == parent_id - ] - return dict(Parents=root_parents + ou_parents) + if re.compile(r'[0-9]{12}').match(kwargs['ChildId']): + parent_id = [ + account.parent_id for account in self.accounts + if account.id == kwargs['ChildId'] + ].pop(0) + else: + parent_id = [ + ou.parent_id for ou in self.ou + if ou.id == kwargs['ChildId'] + ].pop(0) + return dict( + Parents=[ + { + 'Id': ou.id, + 'Type': ou.type, + } + for ou in self.ou + if ou.id == parent_id + ] + ) def create_account(self, **kwargs): - new_account = FakeAccount(self.org, self.roots[0].id, **kwargs) + new_account = FakeAccount(self.org, **kwargs) self.accounts.append(new_account) return new_account.create_account_status @@ -244,9 +245,10 @@ class OrganizationsBackend(BaseBackend): def move_account(self, **kwargs): new_parent_id = kwargs['DestinationParentId'] - all_parent_id = [parent.id for parent in self.roots + self.ou] + all_parent_id = [parent.id for parent in self.ou] account = [ - account for account in self.accounts if account.id == kwargs['AccountId'] + account for account in self.accounts + if account.id == kwargs['AccountId'] ].pop(0) assert new_parent_id in all_parent_id assert account.parent_id == kwargs['SourceParentId'] diff --git a/tests/test_organizations/test_organizations_boto3.py b/tests/test_organizations/test_organizations_boto3.py index 5355e271..2b692489 100644 --- a/tests/test_organizations/test_organizations_boto3.py +++ b/tests/test_organizations/test_organizations_boto3.py @@ -198,30 +198,6 @@ def test_list_organizational_units_for_parent(): #assert False -@mock_organizations -def test_list_parents(): - client = boto3.client('organizations', region_name='us-east-1') - org = client.create_organization(FeatureSet='ALL')['Organization'] - root_id = client.list_roots()['Roots'][0]['Id'] - - ou01 = client.create_organizational_unit(ParentId=root_id, Name='ou01') - ou01_id = ou01['OrganizationalUnit']['Id'] - response01 = client.list_parents(ChildId=ou01_id) - #print(yaml.dump(response01, default_flow_style=False)) - response01.should.have.key('Parents').should.be.a(list) - response01['Parents'][0].should.have.key('Id').should.equal(root_id) - response01['Parents'][0].should.have.key('Type').should.equal('ROOT') - - ou02 = client.create_organizational_unit(ParentId=ou01_id, Name='ou02') - ou02_id = ou02['OrganizationalUnit']['Id'] - response02 = client.list_parents(ChildId=ou02_id) - #print(yaml.dump(response02, default_flow_style=False)) - response02.should.have.key('Parents').should.be.a(list) - response02['Parents'][0].should.have.key('Id').should.equal(ou01_id) - response02['Parents'][0].should.have.key('Type').should.equal('ORGANIZATIONAL_UNIT') - #assert False - - # Accounts mockname = 'mock-account' mockdomain = 'moto-example.org' @@ -310,3 +286,58 @@ def test_move_account(): #print(yaml.dump(response, default_flow_style=False)) account_id.should.be.within([account['Id'] for account in response['Accounts']]) #assert False + + +@mock_organizations +def test_list_parents_for_ou(): + client = boto3.client('organizations', region_name='us-east-1') + org = client.create_organization(FeatureSet='ALL')['Organization'] + root_id = client.list_roots()['Roots'][0]['Id'] + ou01 = client.create_organizational_unit(ParentId=root_id, Name='ou01') + ou01_id = ou01['OrganizationalUnit']['Id'] + response01 = client.list_parents(ChildId=ou01_id) + #print(yaml.dump(response01, default_flow_style=False)) + response01.should.have.key('Parents').should.be.a(list) + response01['Parents'][0].should.have.key('Id').should.equal(root_id) + response01['Parents'][0].should.have.key('Type').should.equal('ROOT') + ou02 = client.create_organizational_unit(ParentId=ou01_id, Name='ou02') + ou02_id = ou02['OrganizationalUnit']['Id'] + response02 = client.list_parents(ChildId=ou02_id) + #print(yaml.dump(response02, default_flow_style=False)) + response02.should.have.key('Parents').should.be.a(list) + response02['Parents'][0].should.have.key('Id').should.equal(ou01_id) + response02['Parents'][0].should.have.key('Type').should.equal('ORGANIZATIONAL_UNIT') + #assert False + + +@mock_organizations +def test_list_parents_for_accounts(): + client = boto3.client('organizations', region_name='us-east-1') + org = client.create_organization(FeatureSet='ALL')['Organization'] + root_id = client.list_roots()['Roots'][0]['Id'] + ou01 = client.create_organizational_unit(ParentId=root_id, Name='ou01') + ou01_id = ou01['OrganizationalUnit']['Id'] + account01_id = client.create_account( + AccountName='account01', + Email='account01@moto-example.org' + )['CreateAccountStatus']['AccountId'] + account02_id = client.create_account( + AccountName='account02', + Email='account02@moto-example.org' + )['CreateAccountStatus']['AccountId'] + client.move_account( + AccountId=account02_id, + SourceParentId=root_id, + DestinationParentId=ou01_id, + ) + response01 = client.list_parents(ChildId=account01_id) + #print(yaml.dump(response01, default_flow_style=False)) + response01.should.have.key('Parents').should.be.a(list) + response01['Parents'][0].should.have.key('Id').should.equal(root_id) + response01['Parents'][0].should.have.key('Type').should.equal('ROOT') + response02 = client.list_parents(ChildId=account02_id) + #print(yaml.dump(response02, default_flow_style=False)) + response02.should.have.key('Parents').should.be.a(list) + response02['Parents'][0].should.have.key('Id').should.equal(ou01_id) + response02['Parents'][0].should.have.key('Type').should.equal('ORGANIZATIONAL_UNIT') + #assert False From 8f400b7110b7804ab7445c5aab3e909376833e79 Mon Sep 17 00:00:00 2001 From: Ashley Gould Date: Sun, 15 Jul 2018 22:19:42 -0700 Subject: [PATCH 22/72] organizations: add endpoint list_chilren --- moto/organizations/models.py | 60 ++++++++++++------- moto/organizations/responses.py | 5 ++ .../test_organizations_boto3.py | 41 +++++++++++++ 3 files changed, 84 insertions(+), 22 deletions(-) diff --git a/moto/organizations/models.py b/moto/organizations/models.py index 609ce383..2e0d6b40 100644 --- a/moto/organizations/models.py +++ b/moto/organizations/models.py @@ -195,28 +195,6 @@ class OrganizationsBackend(BaseBackend): ] ) - def list_parents(self, **kwargs): - if re.compile(r'[0-9]{12}').match(kwargs['ChildId']): - parent_id = [ - account.parent_id for account in self.accounts - if account.id == kwargs['ChildId'] - ].pop(0) - else: - parent_id = [ - ou.parent_id for ou in self.ou - if ou.id == kwargs['ChildId'] - ].pop(0) - return dict( - Parents=[ - { - 'Id': ou.id, - 'Type': ou.type, - } - for ou in self.ou - if ou.id == parent_id - ] - ) - def create_account(self, **kwargs): new_account = FakeAccount(self.org, **kwargs) self.accounts.append(new_account) @@ -255,5 +233,43 @@ class OrganizationsBackend(BaseBackend): index = self.accounts.index(account) self.accounts[index].parent_id = new_parent_id + def list_parents(self, **kwargs): + if re.compile(r'[0-9]{12}').match(kwargs['ChildId']): + obj_list = self.accounts + else: + obj_list = self.ou + parent_id = [ + obj.parent_id for obj in obj_list + if obj.id == kwargs['ChildId'] + ].pop(0) + return dict( + Parents=[ + { + 'Id': ou.id, + 'Type': ou.type, + } + for ou in self.ou + if ou.id == parent_id + ] + ) + + def list_children(self, **kwargs): + if kwargs['ChildType'] == 'ACCOUNT': + obj_list = self.accounts + elif kwargs['ChildType'] == 'ORGANIZATIONAL_UNIT': + obj_list = self.ou + else: + raise ValueError + return dict( + Children=[ + { + 'Id': obj.id, + 'Type': kwargs['ChildType'], + } + for obj in obj_list + if obj.parent_id == kwargs['ParentId'] + ] + ) + organizations_backend = OrganizationsBackend() diff --git a/moto/organizations/responses.py b/moto/organizations/responses.py index 6684ae68..966c3fbf 100644 --- a/moto/organizations/responses.py +++ b/moto/organizations/responses.py @@ -80,3 +80,8 @@ class OrganizationsResponse(BaseResponse): return json.dumps( self.organizations_backend.move_account(**self.request_params) ) + + def list_children(self): + return json.dumps( + self.organizations_backend.list_children(**self.request_params) + ) diff --git a/tests/test_organizations/test_organizations_boto3.py b/tests/test_organizations/test_organizations_boto3.py index 2b692489..496bcd74 100644 --- a/tests/test_organizations/test_organizations_boto3.py +++ b/tests/test_organizations/test_organizations_boto3.py @@ -341,3 +341,44 @@ def test_list_parents_for_accounts(): response02['Parents'][0].should.have.key('Id').should.equal(ou01_id) response02['Parents'][0].should.have.key('Type').should.equal('ORGANIZATIONAL_UNIT') #assert False + + +@mock_organizations +def test_list_chidlren(): + client = boto3.client('organizations', region_name='us-east-1') + org = client.create_organization(FeatureSet='ALL')['Organization'] + root_id = client.list_roots()['Roots'][0]['Id'] + ou01 = client.create_organizational_unit(ParentId=root_id, Name='ou01') + ou01_id = ou01['OrganizationalUnit']['Id'] + ou02 = client.create_organizational_unit(ParentId=ou01_id, Name='ou02') + ou02_id = ou02['OrganizationalUnit']['Id'] + account01_id = client.create_account( + AccountName='account01', + Email='account01@moto-example.org' + )['CreateAccountStatus']['AccountId'] + account02_id = client.create_account( + AccountName='account02', + Email='account02@moto-example.org' + )['CreateAccountStatus']['AccountId'] + client.move_account( + AccountId=account02_id, + SourceParentId=root_id, + DestinationParentId=ou01_id, + ) + response01 = client.list_children(ParentId=root_id, ChildType='ACCOUNT') + response02 = client.list_children(ParentId=root_id, ChildType='ORGANIZATIONAL_UNIT') + response03 = client.list_children(ParentId=ou01_id, ChildType='ACCOUNT') + response04 = client.list_children(ParentId=ou01_id, ChildType='ORGANIZATIONAL_UNIT') + #print(yaml.dump(response01, default_flow_style=False)) + #print(yaml.dump(response02, default_flow_style=False)) + #print(yaml.dump(response03, default_flow_style=False)) + #print(yaml.dump(response04, default_flow_style=False)) + response01['Children'][0]['Id'].should.equal(account01_id) + response01['Children'][0]['Type'].should.equal('ACCOUNT') + response02['Children'][0]['Id'].should.equal(ou01_id) + response02['Children'][0]['Type'].should.equal('ORGANIZATIONAL_UNIT') + response03['Children'][0]['Id'].should.equal(account02_id) + response03['Children'][0]['Type'].should.equal('ACCOUNT') + response04['Children'][0]['Id'].should.equal(ou02_id) + response04['Children'][0]['Type'].should.equal('ORGANIZATIONAL_UNIT') + #assert False From 01912bdca7e2603035c4905c0274f835045d2a2a Mon Sep 17 00:00:00 2001 From: Ashley Gould Date: Mon, 16 Jul 2018 07:23:06 -0700 Subject: [PATCH 23/72] organizations: fix python 2.7 test errors --- moto/organizations/models.py | 2 +- tests/test_organizations/test_organizations_boto3.py | 9 +++++---- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/moto/organizations/models.py b/moto/organizations/models.py index 2e0d6b40..a165395f 100644 --- a/moto/organizations/models.py +++ b/moto/organizations/models.py @@ -132,7 +132,7 @@ class FakeOrganizationalUnit(BaseModel): class FakeRoot(FakeOrganizationalUnit): def __init__(self, organization, **kwargs): - super().__init__(organization, **kwargs) + super(FakeRoot, self).__init__(organization, **kwargs) self.type = 'ROOT' self.id = organization.root_id self.name = 'Root' diff --git a/tests/test_organizations/test_organizations_boto3.py b/tests/test_organizations/test_organizations_boto3.py index 496bcd74..0ac2b61b 100644 --- a/tests/test_organizations/test_organizations_boto3.py +++ b/tests/test_organizations/test_organizations_boto3.py @@ -4,6 +4,7 @@ import boto3 import sure # noqa import datetime import yaml +import six from moto import mock_organizations from moto.organizations.models import ( @@ -64,7 +65,7 @@ def validate_organizational_unit(org, response): org['Id'], ou['Id'], )) - ou.should.have.key('Name').should.be.a(str) + ou.should.have.key('Name').should.be.a(six.string_types) def validate_account(org, account): @@ -86,7 +87,7 @@ def validate_account(org, account): account['Email'].should.match(EMAIL_REGEX) account['JoinedMethod'].should.be.within(['INVITED', 'CREATED']) account['Status'].should.be.within(['ACTIVE', 'SUSPENDED']) - account['Name'].should.be.a(str) + account['Name'].should.be.a(six.string_types) account['JoinedTimestamp'].should.be.a(datetime.datetime) @@ -101,7 +102,7 @@ def validate_create_account_status(create_status): ]) create_status['Id'].should.match(CREATE_ACCOUNT_STATUS_ID_REGEX) create_status['AccountId'].should.match(ACCOUNT_ID_REGEX) - create_status['AccountName'].should.be.a(str) + create_status['AccountName'].should.be.a(six.string_types) create_status['State'].should.equal('SUCCEEDED') create_status['RequestedTimestamp'].should.be.a(datetime.datetime) create_status['CompletedTimestamp'].should.be.a(datetime.datetime) @@ -144,7 +145,7 @@ def test_list_roots(): org['Id'], root['Id'], )) - root.should.have.key('Name').should.be.a(str) + root.should.have.key('Name').should.be.a(six.string_types) root.should.have.key('PolicyTypes').should.be.a(list) root['PolicyTypes'][0].should.have.key('Type').should.equal('SERVICE_CONTROL_POLICY') root['PolicyTypes'][0].should.have.key('Status').should.equal('ENABLED') From 40e422b74d56e545f4a1a903213a56bebff5c53f Mon Sep 17 00:00:00 2001 From: Ashley Gould Date: Thu, 19 Jul 2018 11:50:24 -0700 Subject: [PATCH 24/72] [issue #1720] Add support for AWS Organizations ready for pull request did a little cleanup refactoring local tests pass --- moto/organizations/models.py | 22 +-- moto/organizations/utils.py | 8 + tests/test_organizations/object_syntax.py | 25 --- .../organizations_test_utils.py | 136 ++++++++++++++++ .../test_organizations_boto3.py | 153 +----------------- .../test_organizations_utils.py | 36 ----- 6 files changed, 158 insertions(+), 222 deletions(-) delete mode 100644 tests/test_organizations/object_syntax.py create mode 100644 tests/test_organizations/organizations_test_utils.py delete mode 100644 tests/test_organizations/test_organizations_utils.py diff --git a/moto/organizations/models.py b/moto/organizations/models.py index a165395f..0f3c6740 100644 --- a/moto/organizations/models.py +++ b/moto/organizations/models.py @@ -7,14 +7,6 @@ from moto.core import BaseBackend, BaseModel from moto.core.utils import unix_time from moto.organizations import utils -MASTER_ACCOUNT_ID = '123456789012' -MASTER_ACCOUNT_EMAIL = 'fakeorg@moto-example.com' -ORGANIZATION_ARN_FORMAT = 'arn:aws:organizations::{0}:organization/{1}' -MASTER_ACCOUNT_ARN_FORMAT = 'arn:aws:organizations::{0}:account/{1}/{0}' -ACCOUNT_ARN_FORMAT = 'arn:aws:organizations::{0}:account/{1}/{2}' -ROOT_ARN_FORMAT = 'arn:aws:organizations::{0}:root/{1}/{2}' -OU_ARN_FORMAT = 'arn:aws:organizations::{0}:ou/{1}/{2}' - class FakeOrganization(BaseModel): @@ -22,8 +14,8 @@ class FakeOrganization(BaseModel): self.id = utils.make_random_org_id() self.root_id = utils.make_random_root_id() self.feature_set = feature_set - self.master_account_id = MASTER_ACCOUNT_ID - self.master_account_email = MASTER_ACCOUNT_EMAIL + self.master_account_id = utils.MASTER_ACCOUNT_ID + self.master_account_email = utils.MASTER_ACCOUNT_EMAIL self.available_policy_types = [{ 'Type': 'SERVICE_CONTROL_POLICY', 'Status': 'ENABLED' @@ -31,11 +23,11 @@ class FakeOrganization(BaseModel): @property def arn(self): - return ORGANIZATION_ARN_FORMAT.format(self.master_account_id, self.id) + return utils.ORGANIZATION_ARN_FORMAT.format(self.master_account_id, self.id) @property def master_account_arn(self): - return MASTER_ACCOUNT_ARN_FORMAT.format(self.master_account_id, self.id) + return utils.MASTER_ACCOUNT_ARN_FORMAT.format(self.master_account_id, self.id) def describe(self): return { @@ -67,7 +59,7 @@ class FakeAccount(BaseModel): @property def arn(self): - return ACCOUNT_ARN_FORMAT.format( + return utils.ACCOUNT_ARN_FORMAT.format( self.master_account_id, self.organization_id, self.id @@ -109,7 +101,7 @@ class FakeOrganizationalUnit(BaseModel): self.id = utils.make_random_ou_id(organization.root_id) self.name = kwargs.get('Name') self.parent_id = kwargs.get('ParentId') - self._arn_format = OU_ARN_FORMAT + self._arn_format = utils.OU_ARN_FORMAT @property def arn(self): @@ -140,7 +132,7 @@ class FakeRoot(FakeOrganizationalUnit): 'Type': 'SERVICE_CONTROL_POLICY', 'Status': 'ENABLED' }] - self._arn_format = ROOT_ARN_FORMAT + self._arn_format = utils.ROOT_ARN_FORMAT def describe(self): return { diff --git a/moto/organizations/utils.py b/moto/organizations/utils.py index c7e5c71c..007afa6e 100644 --- a/moto/organizations/utils.py +++ b/moto/organizations/utils.py @@ -3,6 +3,14 @@ from __future__ import unicode_literals import random import string +MASTER_ACCOUNT_ID = '123456789012' +MASTER_ACCOUNT_EMAIL = 'fakeorg@moto-example.com' +ORGANIZATION_ARN_FORMAT = 'arn:aws:organizations::{0}:organization/{1}' +MASTER_ACCOUNT_ARN_FORMAT = 'arn:aws:organizations::{0}:account/{1}/{0}' +ACCOUNT_ARN_FORMAT = 'arn:aws:organizations::{0}:account/{1}/{2}' +ROOT_ARN_FORMAT = 'arn:aws:organizations::{0}:root/{1}/{2}' +OU_ARN_FORMAT = 'arn:aws:organizations::{0}:ou/{1}/{2}' + CHARSET = string.ascii_lowercase + string.digits ORG_ID_SIZE = 10 ROOT_ID_SIZE = 4 diff --git a/tests/test_organizations/object_syntax.py b/tests/test_organizations/object_syntax.py deleted file mode 100644 index 3fb86b9d..00000000 --- a/tests/test_organizations/object_syntax.py +++ /dev/null @@ -1,25 +0,0 @@ -""" -Temporary functions for checking object structures while specing out -models. This module will go away. -""" - -import yaml -from moto import organizations as orgs - - -# utils -print(orgs.utils.make_random_org_id()) -root_id = orgs.utils.make_random_root_id() -print(root_id) -print(orgs.utils.make_random_ou_id(root_id)) -print(orgs.utils.make_random_account_id()) -print(orgs.utils.make_random_create_account_status_id()) - -# models -my_org = orgs.models.FakeOrganization(feature_set='ALL') -print(yaml.dump(my_org._describe())) -#assert False - -my_account = orgs.models.FakeAccount(my_org, AccountName='blee01', Email='blee01@moto-example.org') -print(yaml.dump(my_account)) -#assert False diff --git a/tests/test_organizations/organizations_test_utils.py b/tests/test_organizations/organizations_test_utils.py new file mode 100644 index 00000000..6548b183 --- /dev/null +++ b/tests/test_organizations/organizations_test_utils.py @@ -0,0 +1,136 @@ +from __future__ import unicode_literals + +import six +import sure # noqa +import datetime +from moto.organizations import utils + +EMAIL_REGEX = "^.+@[a-zA-Z0-9-.]+.[a-zA-Z]{2,3}|[0-9]{1,3}$" +ORG_ID_REGEX = r'o-[a-z0-9]{%s}' % utils.ORG_ID_SIZE +ROOT_ID_REGEX = r'r-[a-z0-9]{%s}' % utils.ROOT_ID_SIZE +OU_ID_REGEX = r'ou-[a-z0-9]{%s}-[a-z0-9]{%s}' % (utils.ROOT_ID_SIZE, utils.OU_ID_SUFFIX_SIZE) +ACCOUNT_ID_REGEX = r'[0-9]{%s}' % utils.ACCOUNT_ID_SIZE +CREATE_ACCOUNT_STATUS_ID_REGEX = r'car-[a-z0-9]{%s}' % utils.CREATE_ACCOUNT_STATUS_ID_SIZE + + +def test_make_random_org_id(): + org_id = utils.make_random_org_id() + org_id.should.match(ORG_ID_REGEX) + + +def test_make_random_root_id(): + root_id = utils.make_random_root_id() + root_id.should.match(ROOT_ID_REGEX) + + +def test_make_random_ou_id(): + root_id = utils.make_random_root_id() + ou_id = utils.make_random_ou_id(root_id) + ou_id.should.match(OU_ID_REGEX) + + +def test_make_random_account_id(): + account_id = utils.make_random_account_id() + account_id.should.match(ACCOUNT_ID_REGEX) + + +def test_make_random_create_account_status_id(): + create_account_status_id = utils.make_random_create_account_status_id() + create_account_status_id.should.match(CREATE_ACCOUNT_STATUS_ID_REGEX) + + +def validate_organization(response): + org = response['Organization'] + sorted(org.keys()).should.equal([ + 'Arn', + 'AvailablePolicyTypes', + 'FeatureSet', + 'Id', + 'MasterAccountArn', + 'MasterAccountEmail', + 'MasterAccountId', + ]) + org['Id'].should.match(ORG_ID_REGEX) + org['MasterAccountId'].should.equal(utils.MASTER_ACCOUNT_ID) + org['MasterAccountArn'].should.equal(utils.MASTER_ACCOUNT_ARN_FORMAT.format( + org['MasterAccountId'], + org['Id'], + )) + org['Arn'].should.equal(utils.ORGANIZATION_ARN_FORMAT.format( + org['MasterAccountId'], + org['Id'], + )) + org['MasterAccountEmail'].should.equal(utils.MASTER_ACCOUNT_EMAIL) + org['FeatureSet'].should.be.within(['ALL', 'CONSOLIDATED_BILLING']) + org['AvailablePolicyTypes'].should.equal([{ + 'Type': 'SERVICE_CONTROL_POLICY', + 'Status': 'ENABLED' + }]) + + +def validate_roots(org, response): + response.should.have.key('Roots').should.be.a(list) + response['Roots'].should_not.be.empty + root = response['Roots'][0] + root.should.have.key('Id').should.match(ROOT_ID_REGEX) + root.should.have.key('Arn').should.equal(utils.ROOT_ARN_FORMAT.format( + org['MasterAccountId'], + org['Id'], + root['Id'], + )) + root.should.have.key('Name').should.be.a(six.string_types) + root.should.have.key('PolicyTypes').should.be.a(list) + root['PolicyTypes'][0].should.have.key('Type').should.equal('SERVICE_CONTROL_POLICY') + root['PolicyTypes'][0].should.have.key('Status').should.equal('ENABLED') + + +def validate_organizational_unit(org, response): + response.should.have.key('OrganizationalUnit').should.be.a(dict) + ou = response['OrganizationalUnit'] + ou.should.have.key('Id').should.match(OU_ID_REGEX) + ou.should.have.key('Arn').should.equal(utils.OU_ARN_FORMAT.format( + org['MasterAccountId'], + org['Id'], + ou['Id'], + )) + ou.should.have.key('Name').should.be.a(six.string_types) + + +def validate_account(org, account): + sorted(account.keys()).should.equal([ + 'Arn', + 'Email', + 'Id', + 'JoinedMethod', + 'JoinedTimestamp', + 'Name', + 'Status', + ]) + account['Id'].should.match(ACCOUNT_ID_REGEX) + account['Arn'].should.equal(utils.ACCOUNT_ARN_FORMAT.format( + org['MasterAccountId'], + org['Id'], + account['Id'], + )) + account['Email'].should.match(EMAIL_REGEX) + account['JoinedMethod'].should.be.within(['INVITED', 'CREATED']) + account['Status'].should.be.within(['ACTIVE', 'SUSPENDED']) + account['Name'].should.be.a(six.string_types) + account['JoinedTimestamp'].should.be.a(datetime.datetime) + + +def validate_create_account_status(create_status): + sorted(create_status.keys()).should.equal([ + 'AccountId', + 'AccountName', + 'CompletedTimestamp', + 'Id', + 'RequestedTimestamp', + 'State', + ]) + create_status['Id'].should.match(CREATE_ACCOUNT_STATUS_ID_REGEX) + create_status['AccountId'].should.match(ACCOUNT_ID_REGEX) + create_status['AccountName'].should.be.a(six.string_types) + create_status['State'].should.equal('SUCCEEDED') + create_status['RequestedTimestamp'].should.be.a(datetime.datetime) + create_status['CompletedTimestamp'].should.be.a(datetime.datetime) diff --git a/tests/test_organizations/test_organizations_boto3.py b/tests/test_organizations/test_organizations_boto3.py index 0ac2b61b..c2f06774 100644 --- a/tests/test_organizations/test_organizations_boto3.py +++ b/tests/test_organizations/test_organizations_boto3.py @@ -2,120 +2,24 @@ from __future__ import unicode_literals import boto3 import sure # noqa -import datetime import yaml -import six from moto import mock_organizations -from moto.organizations.models import ( - MASTER_ACCOUNT_ID, - MASTER_ACCOUNT_EMAIL, - ORGANIZATION_ARN_FORMAT, - MASTER_ACCOUNT_ARN_FORMAT, - ACCOUNT_ARN_FORMAT, - ROOT_ARN_FORMAT, - OU_ARN_FORMAT, +from .organizations_test_utils import ( + validate_organization, + validate_roots, + validate_organizational_unit, + validate_account, + validate_create_account_status, ) -from .test_organizations_utils import ( - ORG_ID_REGEX, - ROOT_ID_REGEX, - OU_ID_REGEX, - ACCOUNT_ID_REGEX, - CREATE_ACCOUNT_STATUS_ID_REGEX, -) - -EMAIL_REGEX = "^.+@[a-zA-Z0-9-.]+.[a-zA-Z]{2,3}|[0-9]{1,3}$" - - -def validate_organization(response): - org = response['Organization'] - sorted(org.keys()).should.equal([ - 'Arn', - 'AvailablePolicyTypes', - 'FeatureSet', - 'Id', - 'MasterAccountArn', - 'MasterAccountEmail', - 'MasterAccountId', - ]) - org['Id'].should.match(ORG_ID_REGEX) - org['MasterAccountId'].should.equal(MASTER_ACCOUNT_ID) - org['MasterAccountArn'].should.equal(MASTER_ACCOUNT_ARN_FORMAT.format( - org['MasterAccountId'], - org['Id'], - )) - org['Arn'].should.equal(ORGANIZATION_ARN_FORMAT.format( - org['MasterAccountId'], - org['Id'], - )) - org['MasterAccountEmail'].should.equal(MASTER_ACCOUNT_EMAIL) - org['FeatureSet'].should.be.within(['ALL', 'CONSOLIDATED_BILLING']) - org['AvailablePolicyTypes'].should.equal([{ - 'Type': 'SERVICE_CONTROL_POLICY', - 'Status': 'ENABLED' - }]) - - -def validate_organizational_unit(org, response): - response.should.have.key('OrganizationalUnit').should.be.a(dict) - ou = response['OrganizationalUnit'] - ou.should.have.key('Id').should.match(OU_ID_REGEX) - ou.should.have.key('Arn').should.equal(OU_ARN_FORMAT.format( - org['MasterAccountId'], - org['Id'], - ou['Id'], - )) - ou.should.have.key('Name').should.be.a(six.string_types) - - -def validate_account(org, account): - sorted(account.keys()).should.equal([ - 'Arn', - 'Email', - 'Id', - 'JoinedMethod', - 'JoinedTimestamp', - 'Name', - 'Status', - ]) - account['Id'].should.match(ACCOUNT_ID_REGEX) - account['Arn'].should.equal(ACCOUNT_ARN_FORMAT.format( - org['MasterAccountId'], - org['Id'], - account['Id'], - )) - account['Email'].should.match(EMAIL_REGEX) - account['JoinedMethod'].should.be.within(['INVITED', 'CREATED']) - account['Status'].should.be.within(['ACTIVE', 'SUSPENDED']) - account['Name'].should.be.a(six.string_types) - account['JoinedTimestamp'].should.be.a(datetime.datetime) - - -def validate_create_account_status(create_status): - sorted(create_status.keys()).should.equal([ - 'AccountId', - 'AccountName', - 'CompletedTimestamp', - 'Id', - 'RequestedTimestamp', - 'State', - ]) - create_status['Id'].should.match(CREATE_ACCOUNT_STATUS_ID_REGEX) - create_status['AccountId'].should.match(ACCOUNT_ID_REGEX) - create_status['AccountName'].should.be.a(six.string_types) - create_status['State'].should.equal('SUCCEEDED') - create_status['RequestedTimestamp'].should.be.a(datetime.datetime) - create_status['CompletedTimestamp'].should.be.a(datetime.datetime) @mock_organizations def test_create_organization(): client = boto3.client('organizations', region_name='us-east-1') response = client.create_organization(FeatureSet='ALL') - #print(yaml.dump(response)) validate_organization(response) response['Organization']['FeatureSet'].should.equal('ALL') - #assert False @mock_organizations @@ -123,9 +27,7 @@ def test_describe_organization(): client = boto3.client('organizations', region_name='us-east-1') client.create_organization(FeatureSet='ALL') response = client.describe_organization() - #print(yaml.dump(response)) validate_organization(response) - #assert False # Organizational Units @@ -135,21 +37,7 @@ def test_list_roots(): client = boto3.client('organizations', region_name='us-east-1') org = client.create_organization(FeatureSet='ALL')['Organization'] response = client.list_roots() - #print(yaml.dump(response, default_flow_style=False)) - response.should.have.key('Roots').should.be.a(list) - response['Roots'].should_not.be.empty - root = response['Roots'][0] - root.should.have.key('Id').should.match(ROOT_ID_REGEX) - root.should.have.key('Arn').should.equal(ROOT_ARN_FORMAT.format( - org['MasterAccountId'], - org['Id'], - root['Id'], - )) - root.should.have.key('Name').should.be.a(six.string_types) - root.should.have.key('PolicyTypes').should.be.a(list) - root['PolicyTypes'][0].should.have.key('Type').should.equal('SERVICE_CONTROL_POLICY') - root['PolicyTypes'][0].should.have.key('Status').should.equal('ENABLED') - #assert False + validate_roots(org, response) @mock_organizations @@ -162,10 +50,8 @@ def test_create_organizational_unit(): ParentId=root_id, Name=ou_name, ) - #print(yaml.dump(response, default_flow_style=False)) validate_organizational_unit(org, response) response['OrganizationalUnit']['Name'].should.equal(ou_name) - #assert False @mock_organizations @@ -178,9 +64,7 @@ def test_describe_organizational_unit(): Name='ou01', )['OrganizationalUnit']['Id'] response = client.describe_organizational_unit(OrganizationalUnitId=ou_id) - print(yaml.dump(response, default_flow_style=False)) validate_organizational_unit(org, response) - #assert False @mock_organizations @@ -192,11 +76,9 @@ def test_list_organizational_units_for_parent(): client.create_organizational_unit(ParentId=root_id, Name='ou02') client.create_organizational_unit(ParentId=root_id, Name='ou03') response = client.list_organizational_units_for_parent(ParentId=root_id) - print(yaml.dump(response, default_flow_style=False)) response.should.have.key('OrganizationalUnits').should.be.a(list) for ou in response['OrganizationalUnits']: validate_organizational_unit(org, dict(OrganizationalUnit=ou)) - #assert False # Accounts @@ -212,10 +94,8 @@ def test_create_account(): create_status = client.create_account( AccountName=mockname, Email=mockemail )['CreateAccountStatus'] - #print(yaml.dump(create_status, default_flow_style=False)) validate_create_account_status(create_status) create_status['AccountName'].should.equal(mockname) - #assert False @mock_organizations @@ -226,11 +106,9 @@ def test_describe_account(): AccountName=mockname, Email=mockemail )['CreateAccountStatus']['AccountId'] response = client.describe_account(AccountId=account_id) - #print(yaml.dump(response, default_flow_style=False)) validate_account(org, response['Account']) response['Account']['Name'].should.equal(mockname) response['Account']['Email'].should.equal(mockemail) - #assert False @mock_organizations @@ -242,7 +120,6 @@ def test_list_accounts(): email = name + '@' + mockdomain client.create_account(AccountName=name, Email=email) response = client.list_accounts() - #print(yaml.dump(response, default_flow_style=False)) response.should.have.key('Accounts') accounts = response['Accounts'] len(accounts).should.equal(5) @@ -250,7 +127,6 @@ def test_list_accounts(): validate_account(org, account) accounts[3]['Name'].should.equal(mockname + '3') accounts[2]['Email'].should.equal(mockname + '2' + '@' + mockdomain) - #assert False @mock_organizations @@ -263,9 +139,7 @@ def test_list_accounts_for_parent(): Email=mockemail, )['CreateAccountStatus']['AccountId'] response = client.list_accounts_for_parent(ParentId=root_id) - #print(yaml.dump(response, default_flow_style=False)) account_id.should.be.within([account['Id'] for account in response['Accounts']]) - #assert False @mock_organizations @@ -284,9 +158,7 @@ def test_move_account(): DestinationParentId=ou01_id, ) response = client.list_accounts_for_parent(ParentId=ou01_id) - #print(yaml.dump(response, default_flow_style=False)) account_id.should.be.within([account['Id'] for account in response['Accounts']]) - #assert False @mock_organizations @@ -297,18 +169,15 @@ def test_list_parents_for_ou(): ou01 = client.create_organizational_unit(ParentId=root_id, Name='ou01') ou01_id = ou01['OrganizationalUnit']['Id'] response01 = client.list_parents(ChildId=ou01_id) - #print(yaml.dump(response01, default_flow_style=False)) response01.should.have.key('Parents').should.be.a(list) response01['Parents'][0].should.have.key('Id').should.equal(root_id) response01['Parents'][0].should.have.key('Type').should.equal('ROOT') ou02 = client.create_organizational_unit(ParentId=ou01_id, Name='ou02') ou02_id = ou02['OrganizationalUnit']['Id'] response02 = client.list_parents(ChildId=ou02_id) - #print(yaml.dump(response02, default_flow_style=False)) response02.should.have.key('Parents').should.be.a(list) response02['Parents'][0].should.have.key('Id').should.equal(ou01_id) response02['Parents'][0].should.have.key('Type').should.equal('ORGANIZATIONAL_UNIT') - #assert False @mock_organizations @@ -332,16 +201,13 @@ def test_list_parents_for_accounts(): DestinationParentId=ou01_id, ) response01 = client.list_parents(ChildId=account01_id) - #print(yaml.dump(response01, default_flow_style=False)) response01.should.have.key('Parents').should.be.a(list) response01['Parents'][0].should.have.key('Id').should.equal(root_id) response01['Parents'][0].should.have.key('Type').should.equal('ROOT') response02 = client.list_parents(ChildId=account02_id) - #print(yaml.dump(response02, default_flow_style=False)) response02.should.have.key('Parents').should.be.a(list) response02['Parents'][0].should.have.key('Id').should.equal(ou01_id) response02['Parents'][0].should.have.key('Type').should.equal('ORGANIZATIONAL_UNIT') - #assert False @mock_organizations @@ -370,10 +236,6 @@ def test_list_chidlren(): response02 = client.list_children(ParentId=root_id, ChildType='ORGANIZATIONAL_UNIT') response03 = client.list_children(ParentId=ou01_id, ChildType='ACCOUNT') response04 = client.list_children(ParentId=ou01_id, ChildType='ORGANIZATIONAL_UNIT') - #print(yaml.dump(response01, default_flow_style=False)) - #print(yaml.dump(response02, default_flow_style=False)) - #print(yaml.dump(response03, default_flow_style=False)) - #print(yaml.dump(response04, default_flow_style=False)) response01['Children'][0]['Id'].should.equal(account01_id) response01['Children'][0]['Type'].should.equal('ACCOUNT') response02['Children'][0]['Id'].should.equal(ou01_id) @@ -382,4 +244,3 @@ def test_list_chidlren(): response03['Children'][0]['Type'].should.equal('ACCOUNT') response04['Children'][0]['Id'].should.equal(ou02_id) response04['Children'][0]['Type'].should.equal('ORGANIZATIONAL_UNIT') - #assert False diff --git a/tests/test_organizations/test_organizations_utils.py b/tests/test_organizations/test_organizations_utils.py deleted file mode 100644 index d2720144..00000000 --- a/tests/test_organizations/test_organizations_utils.py +++ /dev/null @@ -1,36 +0,0 @@ -from __future__ import unicode_literals - -import sure # noqa -from moto.organizations import utils - -ORG_ID_REGEX = r'o-[a-z0-9]{%s}' % utils.ORG_ID_SIZE -ROOT_ID_REGEX = r'r-[a-z0-9]{%s}' % utils.ROOT_ID_SIZE -OU_ID_REGEX = r'ou-[a-z0-9]{%s}-[a-z0-9]{%s}' % (utils.ROOT_ID_SIZE, utils.OU_ID_SUFFIX_SIZE) -ACCOUNT_ID_REGEX = r'[0-9]{%s}' % utils.ACCOUNT_ID_SIZE -CREATE_ACCOUNT_STATUS_ID_REGEX = r'car-[a-z0-9]{%s}' % utils.CREATE_ACCOUNT_STATUS_ID_SIZE - - -def test_make_random_org_id(): - org_id = utils.make_random_org_id() - org_id.should.match(ORG_ID_REGEX) - - -def test_make_random_root_id(): - root_id = utils.make_random_root_id() - root_id.should.match(ROOT_ID_REGEX) - - -def test_make_random_ou_id(): - root_id = utils.make_random_root_id() - ou_id = utils.make_random_ou_id(root_id) - ou_id.should.match(OU_ID_REGEX) - - -def test_make_random_account_id(): - account_id = utils.make_random_account_id() - account_id.should.match(ACCOUNT_ID_REGEX) - - -def test_make_random_create_account_status_id(): - create_account_status_id = utils.make_random_create_account_status_id() - create_account_status_id.should.match(CREATE_ACCOUNT_STATUS_ID_REGEX) From 05928b1497831e38a642baadc438b8238d65b4bf Mon Sep 17 00:00:00 2001 From: Ashley Gould Date: Fri, 20 Jul 2018 13:54:53 -0700 Subject: [PATCH 25/72] [issue #1720] Add support for AWS Organizations added exception handling in class OrganizationsBackend --- IMPLEMENTATION_COVERAGE.md | 4 +- README.md | 2 +- moto/organizations/models.py | 76 ++++++++++++------- .../test_organizations_boto3.py | 69 ++++++++++++++++- 4 files changed, 120 insertions(+), 31 deletions(-) diff --git a/IMPLEMENTATION_COVERAGE.md b/IMPLEMENTATION_COVERAGE.md index e1943aa9..d19d2473 100644 --- a/IMPLEMENTATION_COVERAGE.md +++ b/IMPLEMENTATION_COVERAGE.md @@ -3147,7 +3147,7 @@ - [ ] update_server - [ ] update_server_engine_attributes -## organizations - 28% implemented +## organizations - 30% implemented - [ ] accept_handshake - [ ] attach_policy - [ ] cancel_handshake @@ -3176,7 +3176,7 @@ - [X] list_accounts - [X] list_accounts_for_parent - [ ] list_aws_service_access_for_organization -- [ ] list_children +- [X] list_children - [ ] list_create_account_status - [ ] list_handshakes_for_account - [ ] list_handshakes_for_organization diff --git a/README.md b/README.md index e4fcb650..189bf2c4 100644 --- a/README.md +++ b/README.md @@ -112,7 +112,7 @@ It gets even better! Moto isn't just for Python code and it isn't just for S3. L |------------------------------------------------------------------------------| | KMS | @mock_kms | basic endpoints done | |------------------------------------------------------------------------------| -| Organizations | @mock_organizations | some endpoints done | +| Organizations | @mock_organizations | some core endpoints done | |------------------------------------------------------------------------------| | Polly | @mock_polly | all endpoints done | |------------------------------------------------------------------------------| diff --git a/moto/organizations/models.py b/moto/organizations/models.py index 0f3c6740..91896f53 100644 --- a/moto/organizations/models.py +++ b/moto/organizations/models.py @@ -4,6 +4,7 @@ import datetime import re from moto.core import BaseBackend, BaseModel +from moto.core.exceptions import RESTError from moto.core.utils import unix_time from moto.organizations import utils @@ -168,13 +169,31 @@ class OrganizationsBackend(BaseBackend): self.ou.append(new_ou) return new_ou.describe() + def get_organizational_unit_by_id(self, ou_id): + ou = next((ou for ou in self.ou if ou.id == ou_id), None) + if ou is None: + raise RESTError( + 'OrganizationalUnitNotFoundException', + "You specified an organizational unit that doesn't exist." + ) + return ou + + def validate_parent_id(self, parent_id): + try: + self.get_organizational_unit_by_id(parent_id) + except RESTError as e: + raise RESTError( + 'ParentNotFoundException', + "You specified parent that doesn't exist." + ) + return parent_id + def describe_organizational_unit(self, **kwargs): - ou = [ - ou for ou in self.ou if ou.id == kwargs['OrganizationalUnitId'] - ].pop(0) + ou = self.get_organizational_unit_by_id(kwargs['OrganizationalUnitId']) return ou.describe() def list_organizational_units_for_parent(self, **kwargs): + parent_id = self.validate_parent_id(kwargs['ParentId']) return dict( OrganizationalUnits=[ { @@ -183,7 +202,7 @@ class OrganizationsBackend(BaseBackend): 'Name': ou.name, } for ou in self.ou - if ou.parent_id == kwargs['ParentId'] + if ou.parent_id == parent_id ] ) @@ -192,11 +211,20 @@ class OrganizationsBackend(BaseBackend): self.accounts.append(new_account) return new_account.create_account_status - def describe_account(self, **kwargs): - account = [ + def get_account_by_id(self, account_id): + account = next(( account for account in self.accounts - if account.id == kwargs['AccountId'] - ].pop(0) + if account.id == account_id + ), None) + if account is None: + raise RESTError( + 'AccountNotFoundException', + "You specified an account that doesn't exist." + ) + return account + + def describe_account(self, **kwargs): + account = self.get_account_by_id(kwargs['AccountId']) return account.describe() def list_accounts(self): @@ -205,35 +233,27 @@ class OrganizationsBackend(BaseBackend): ) def list_accounts_for_parent(self, **kwargs): + parent_id = self.validate_parent_id(kwargs['ParentId']) return dict( Accounts=[ account.describe()['Account'] for account in self.accounts - if account.parent_id == kwargs['ParentId'] + if account.parent_id == parent_id ] ) def move_account(self, **kwargs): - new_parent_id = kwargs['DestinationParentId'] - all_parent_id = [parent.id for parent in self.ou] - account = [ - account for account in self.accounts - if account.id == kwargs['AccountId'] - ].pop(0) - assert new_parent_id in all_parent_id - assert account.parent_id == kwargs['SourceParentId'] + new_parent_id = self.validate_parent_id(kwargs['DestinationParentId']) + self.validate_parent_id(kwargs['SourceParentId']) + account = self.get_account_by_id(kwargs['AccountId']) index = self.accounts.index(account) self.accounts[index].parent_id = new_parent_id def list_parents(self, **kwargs): if re.compile(r'[0-9]{12}').match(kwargs['ChildId']): - obj_list = self.accounts + child_object = self.get_account_by_id(kwargs['ChildId']) else: - obj_list = self.ou - parent_id = [ - obj.parent_id for obj in obj_list - if obj.id == kwargs['ChildId'] - ].pop(0) + child_object = self.get_organizational_unit_by_id(kwargs['ChildId']) return dict( Parents=[ { @@ -241,17 +261,21 @@ class OrganizationsBackend(BaseBackend): 'Type': ou.type, } for ou in self.ou - if ou.id == parent_id + if ou.id == child_object.parent_id ] ) def list_children(self, **kwargs): + parent_id = self.validate_parent_id(kwargs['ParentId']) if kwargs['ChildType'] == 'ACCOUNT': obj_list = self.accounts elif kwargs['ChildType'] == 'ORGANIZATIONAL_UNIT': obj_list = self.ou else: - raise ValueError + raise RESTError( + 'InvalidInputException', + 'You specified an invalid value.' + ) return dict( Children=[ { @@ -259,7 +283,7 @@ class OrganizationsBackend(BaseBackend): 'Type': kwargs['ChildType'], } for obj in obj_list - if obj.parent_id == kwargs['ParentId'] + if obj.parent_id == parent_id ] ) diff --git a/tests/test_organizations/test_organizations_boto3.py b/tests/test_organizations/test_organizations_boto3.py index c2f06774..ae9bacd8 100644 --- a/tests/test_organizations/test_organizations_boto3.py +++ b/tests/test_organizations/test_organizations_boto3.py @@ -2,9 +2,11 @@ from __future__ import unicode_literals import boto3 import sure # noqa -import yaml +from botocore.exceptions import ClientError +from nose.tools import assert_raises from moto import mock_organizations +from moto.organizations import utils from .organizations_test_utils import ( validate_organization, validate_roots, @@ -67,6 +69,20 @@ def test_describe_organizational_unit(): validate_organizational_unit(org, response) +@mock_organizations +def test_describe_organizational_unit_exception(): + client = boto3.client('organizations', region_name='us-east-1') + org = client.create_organization(FeatureSet='ALL')['Organization'] + with assert_raises(ClientError) as e: + response = client.describe_organizational_unit( + OrganizationalUnitId=utils.make_random_root_id() + ) + ex = e.exception + ex.operation_name.should.equal('DescribeOrganizationalUnit') + ex.response['Error']['Code'].should.equal('400') + ex.response['Error']['Message'].should.contain('OrganizationalUnitNotFoundException') + + @mock_organizations def test_list_organizational_units_for_parent(): client = boto3.client('organizations', region_name='us-east-1') @@ -81,6 +97,19 @@ def test_list_organizational_units_for_parent(): validate_organizational_unit(org, dict(OrganizationalUnit=ou)) +@mock_organizations +def test_list_organizational_units_for_parent_exception(): + client = boto3.client('organizations', region_name='us-east-1') + with assert_raises(ClientError) as e: + response = client.list_organizational_units_for_parent( + ParentId=utils.make_random_root_id() + ) + ex = e.exception + ex.operation_name.should.equal('ListOrganizationalUnitsForParent') + ex.response['Error']['Code'].should.equal('400') + ex.response['Error']['Message'].should.contain('ParentNotFoundException') + + # Accounts mockname = 'mock-account' mockdomain = 'moto-example.org' @@ -111,6 +140,17 @@ def test_describe_account(): response['Account']['Email'].should.equal(mockemail) +@mock_organizations +def test_describe_account_exception(): + client = boto3.client('organizations', region_name='us-east-1') + with assert_raises(ClientError) as e: + response = client.describe_account(AccountId=utils.make_random_account_id()) + ex = e.exception + ex.operation_name.should.equal('DescribeAccount') + ex.response['Error']['Code'].should.equal('400') + ex.response['Error']['Message'].should.contain('AccountNotFoundException') + + @mock_organizations def test_list_accounts(): client = boto3.client('organizations', region_name='us-east-1') @@ -211,7 +251,7 @@ def test_list_parents_for_accounts(): @mock_organizations -def test_list_chidlren(): +def test_list_children(): client = boto3.client('organizations', region_name='us-east-1') org = client.create_organization(FeatureSet='ALL')['Organization'] root_id = client.list_roots()['Roots'][0]['Id'] @@ -244,3 +284,28 @@ def test_list_chidlren(): response03['Children'][0]['Type'].should.equal('ACCOUNT') response04['Children'][0]['Id'].should.equal(ou02_id) response04['Children'][0]['Type'].should.equal('ORGANIZATIONAL_UNIT') + + +@mock_organizations +def test_list_children_exception(): + client = boto3.client('organizations', region_name='us-east-1') + org = client.create_organization(FeatureSet='ALL')['Organization'] + root_id = client.list_roots()['Roots'][0]['Id'] + with assert_raises(ClientError) as e: + response = client.list_children( + ParentId=utils.make_random_root_id(), + ChildType='ACCOUNT' + ) + ex = e.exception + ex.operation_name.should.equal('ListChildren') + ex.response['Error']['Code'].should.equal('400') + ex.response['Error']['Message'].should.contain('ParentNotFoundException') + with assert_raises(ClientError) as e: + response = client.list_children( + ParentId=root_id, + ChildType='BLEE' + ) + ex = e.exception + ex.operation_name.should.equal('ListChildren') + ex.response['Error']['Code'].should.equal('400') + ex.response['Error']['Message'].should.contain('InvalidInputException') From 4356e951e10795fcb9b73ac6b67540fbfeda2001 Mon Sep 17 00:00:00 2001 From: Ashley Gould Date: Fri, 20 Jul 2018 14:07:04 -0700 Subject: [PATCH 26/72] [issue #1720] Add support for AWS Organizations fix travis build error --- moto/organizations/models.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/moto/organizations/models.py b/moto/organizations/models.py index 91896f53..c02b1a49 100644 --- a/moto/organizations/models.py +++ b/moto/organizations/models.py @@ -181,7 +181,7 @@ class OrganizationsBackend(BaseBackend): def validate_parent_id(self, parent_id): try: self.get_organizational_unit_by_id(parent_id) - except RESTError as e: + except RESTError: raise RESTError( 'ParentNotFoundException', "You specified parent that doesn't exist." From b8be517be08c5d7b42315ac1af7da23d978d5f9f Mon Sep 17 00:00:00 2001 From: Ashley Gould Date: Thu, 26 Jul 2018 12:20:43 -0700 Subject: [PATCH 27/72] organizations support: add exception handling for describe_organizations --- moto/organizations/models.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/moto/organizations/models.py b/moto/organizations/models.py index c02b1a49..9d5fe388 100644 --- a/moto/organizations/models.py +++ b/moto/organizations/models.py @@ -157,6 +157,11 @@ class OrganizationsBackend(BaseBackend): return self.org.describe() def describe_organization(self): + if not self.org: + raise RESTError( + 'AWSOrganizationsNotInUseException', + "Your account is not a member of an organization." + ) return self.org.describe() def list_roots(self): From 3afb2862c0cdb8e9d229b7c7e4141e1abfa57d00 Mon Sep 17 00:00:00 2001 From: William Richard Date: Mon, 1 Oct 2018 16:30:23 -0400 Subject: [PATCH 28/72] Filter event log ids should be strings Based on the boto docs, eventId should be returned as a string. https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/logs.html#CloudWatchLogs.Client.filter_log_events --- moto/logs/models.py | 2 +- tests/test_logs/test_logs.py | 6 +++++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/moto/logs/models.py b/moto/logs/models.py index a4ff9db4..ca1fdc4a 100644 --- a/moto/logs/models.py +++ b/moto/logs/models.py @@ -19,7 +19,7 @@ class LogEvent: def to_filter_dict(self): return { - "eventId": self.eventId, + "eventId": str(self.eventId), "ingestionTime": self.ingestionTime, # "logStreamName": "message": self.message, diff --git a/tests/test_logs/test_logs.py b/tests/test_logs/test_logs.py index 05bd3c82..e3d46fd8 100644 --- a/tests/test_logs/test_logs.py +++ b/tests/test_logs/test_logs.py @@ -121,4 +121,8 @@ def test_filter_logs_interleaved(): interleaved=True, ) events = res['events'] - events.should.have.length_of(2) + for original_message, resulting_event in zip(messages, events): + resulting_event['eventId'].should.equal(str(resulting_event['eventId'])) + resulting_event['timestamp'].should.equal(original_message['timestamp']) + resulting_event['message'].should.equal(original_message['message']) + From ea4fcaa82a969ffdc2482fbd18c41cf117e22cc9 Mon Sep 17 00:00:00 2001 From: Jon Beilke Date: Wed, 3 Oct 2018 00:40:28 -0500 Subject: [PATCH 29/72] add support for NoncurrentVersionTransition, NoncurrentVersionExpiration, and AbortIncompleteMultipartUpload actions to s3 lifecycle rules --- moto/s3/models.py | 24 ++++-- moto/s3/responses.py | 16 ++++ tests/test_s3/test_s3_lifecycle.py | 129 +++++++++++++++++++++++++++++ 3 files changed, 163 insertions(+), 6 deletions(-) diff --git a/moto/s3/models.py b/moto/s3/models.py index f3994b5d..4b1a343d 100644 --- a/moto/s3/models.py +++ b/moto/s3/models.py @@ -341,8 +341,9 @@ class LifecycleAndFilter(BaseModel): class LifecycleRule(BaseModel): def __init__(self, id=None, prefix=None, lc_filter=None, status=None, expiration_days=None, - expiration_date=None, transition_days=None, expired_object_delete_marker=None, - transition_date=None, storage_class=None): + expiration_date=None, transition_days=None, transition_date=None, storage_class=None, + expired_object_delete_marker=None, nve_noncurrent_days=None, nvt_noncurrent_days=None, + nvt_storage_class=None, aimu_days=None): self.id = id self.prefix = prefix self.filter = lc_filter @@ -351,8 +352,12 @@ class LifecycleRule(BaseModel): self.expiration_date = expiration_date self.transition_days = transition_days self.transition_date = transition_date - self.expired_object_delete_marker = expired_object_delete_marker self.storage_class = storage_class + self.expired_object_delete_marker = expired_object_delete_marker + self.nve_noncurrent_days = nve_noncurrent_days + self.nvt_noncurrent_days = nvt_noncurrent_days + self.nvt_storage_class = nvt_storage_class + self.aimu_days = aimu_days class CorsRule(BaseModel): @@ -414,8 +419,12 @@ class FakeBucket(BaseModel): def set_lifecycle(self, rules): self.rules = [] for rule in rules: + # Extract actions from Lifecycle rule expiration = rule.get('Expiration') transition = rule.get('Transition') + nve = rule.get('NoncurrentVersionExpiration') + nvt = rule.get('NoncurrentVersionTransition') + aimu = rule.get('AbortIncompleteMultipartUpload') eodm = None if expiration and expiration.get("ExpiredObjectDeleteMarker") is not None: @@ -459,11 +468,14 @@ class FakeBucket(BaseModel): status=rule['Status'], expiration_days=expiration.get('Days') if expiration else None, expiration_date=expiration.get('Date') if expiration else None, - expired_object_delete_marker=eodm, transition_days=transition.get('Days') if transition else None, transition_date=transition.get('Date') if transition else None, - storage_class=transition[ - 'StorageClass'] if transition else None, + storage_class=transition.get('StorageClass') if transition else None, + expired_object_delete_marker=eodm, + nve_noncurrent_days = nve.get('NoncurrentDays') if nve else None, + nvt_noncurrent_days = nvt.get('NoncurrentDays') if nvt else None, + nvt_storage_class = nvt.get('StorageClass') if nvt else None, + aimu_days=aimu.get('DaysAfterInitiation') if aimu else None, )) def delete_lifecycle(self): diff --git a/moto/s3/responses.py b/moto/s3/responses.py index f8dc7e42..de101a19 100755 --- a/moto/s3/responses.py +++ b/moto/s3/responses.py @@ -1228,6 +1228,22 @@ S3_BUCKET_LIFECYCLE_CONFIGURATION = """ {% endif %} {% endif %} + {% if rule.nvt_noncurrent_days and rule.nvt_storage_class %} + + {{ rule.nvt_noncurrent_days }} + {{ rule.nvt_storage_class }} + + {% endif %} + {% if rule.nve_noncurrent_days %} + + {{ rule.nve_noncurrent_days }} + + {% endif %} + {% if rule.aimu_days %} + + {{ rule.aimu_days }} + + {% endif %} {% endfor %} diff --git a/tests/test_s3/test_s3_lifecycle.py b/tests/test_s3/test_s3_lifecycle.py index d176e95c..9c4bd49b 100644 --- a/tests/test_s3/test_s3_lifecycle.py +++ b/tests/test_s3/test_s3_lifecycle.py @@ -191,6 +191,135 @@ def test_lifecycle_with_eodm(): assert err.exception.response["Error"]["Code"] == "MalformedXML" +@mock_s3 +def test_lifecycle_with_nve(): + client = boto3.client("s3") + client.create_bucket(Bucket="bucket") + + lfc = { + "Rules": [ + { + "NoncurrentVersionExpiration": { + "NoncurrentDays": 30 + }, + "ID": "wholebucket", + "Filter": { + "Prefix": "" + }, + "Status": "Enabled" + } + ] + } + client.put_bucket_lifecycle_configuration(Bucket="bucket", LifecycleConfiguration=lfc) + result = client.get_bucket_lifecycle_configuration(Bucket="bucket") + assert len(result["Rules"]) == 1 + assert result["Rules"][0]["NoncurrentVersionExpiration"]["NoncurrentDays"] == 30 + + # Change NoncurrentDays: + lfc["Rules"][0]["NoncurrentVersionExpiration"]["NoncurrentDays"] = 10 + client.put_bucket_lifecycle_configuration(Bucket="bucket", LifecycleConfiguration=lfc) + result = client.get_bucket_lifecycle_configuration(Bucket="bucket") + assert len(result["Rules"]) == 1 + assert result["Rules"][0]["NoncurrentVersionExpiration"]["NoncurrentDays"] == 10 + + # With failures for missing children: + del lfc["Rules"][0]["NoncurrentVersionExpiration"]["NoncurrentDays"] + with assert_raises(ClientError) as err: + client.put_bucket_lifecycle_configuration(Bucket="bucket", LifecycleConfiguration=lfc) + assert err.exception.response["Error"]["Code"] == "MalformedXML" + + +@mock_s3 +def test_lifecycle_with_nvt(): + client = boto3.client("s3") + client.create_bucket(Bucket="bucket") + + lfc = { + "Rules": [ + { + "NoncurrentVersionTransition": { + "NoncurrentDays": 30, + "StorageClass": "ONEZONE_IA" + }, + "ID": "wholebucket", + "Filter": { + "Prefix": "" + }, + "Status": "Enabled" + } + ] + } + client.put_bucket_lifecycle_configuration(Bucket="bucket", LifecycleConfiguration=lfc) + result = client.get_bucket_lifecycle_configuration(Bucket="bucket") + assert len(result["Rules"]) == 1 + assert result["Rules"][0]["NoncurrentVersionTransition"]["NoncurrentDays"] == 30 + assert result["Rules"][0]["NoncurrentVersionTransition"]["StorageClass"] == "ONEZONE_IA" + + # Change NoncurrentDays: + lfc["Rules"][0]["NoncurrentVersionTransition"]["NoncurrentDays"] = 10 + client.put_bucket_lifecycle_configuration(Bucket="bucket", LifecycleConfiguration=lfc) + result = client.get_bucket_lifecycle_configuration(Bucket="bucket") + assert len(result["Rules"]) == 1 + assert result["Rules"][0]["NoncurrentVersionTransition"]["NoncurrentDays"] == 10 + + # Change StorageClass: + lfc["Rules"][0]["NoncurrentVersionTransition"]["StorageClass"] = "GLACIER" + client.put_bucket_lifecycle_configuration(Bucket="bucket", LifecycleConfiguration=lfc) + result = client.get_bucket_lifecycle_configuration(Bucket="bucket") + assert len(result["Rules"]) == 1 + assert result["Rules"][0]["NoncurrentVersionTransition"]["StorageClass"] == "GLACIER" + + # With failures for missing children: + del lfc["Rules"][0]["NoncurrentVersionTransition"]["NoncurrentDays"] + with assert_raises(ClientError) as err: + client.put_bucket_lifecycle_configuration(Bucket="bucket", LifecycleConfiguration=lfc) + assert err.exception.response["Error"]["Code"] == "MalformedXML" + lfc["Rules"][0]["NoncurrentVersionTransition"]["NoncurrentDays"] = 30 + + del lfc["Rules"][0]["NoncurrentVersionTransition"]["StorageClass"] + with assert_raises(ClientError) as err: + client.put_bucket_lifecycle_configuration(Bucket="bucket", LifecycleConfiguration=lfc) + assert err.exception.response["Error"]["Code"] == "MalformedXML" + + +@mock_s3 +def test_lifecycle_with_aimu(): + client = boto3.client("s3") + client.create_bucket(Bucket="bucket") + + lfc = { + "Rules": [ + { + "AbortIncompleteMultipartUpload": { + "DaysAfterInitiation": 7 + }, + "ID": "wholebucket", + "Filter": { + "Prefix": "" + }, + "Status": "Enabled" + } + ] + } + client.put_bucket_lifecycle_configuration(Bucket="bucket", LifecycleConfiguration=lfc) + result = client.get_bucket_lifecycle_configuration(Bucket="bucket") + assert len(result["Rules"]) == 1 + assert result["Rules"][0]["AbortIncompleteMultipartUpload"]["DaysAfterInitiation"] == 7 + + # Change DaysAfterInitiation: + lfc["Rules"][0]["AbortIncompleteMultipartUpload"]["DaysAfterInitiation"] = 30 + client.put_bucket_lifecycle_configuration(Bucket="bucket", LifecycleConfiguration=lfc) + result = client.get_bucket_lifecycle_configuration(Bucket="bucket") + assert len(result["Rules"]) == 1 + assert result["Rules"][0]["AbortIncompleteMultipartUpload"]["DaysAfterInitiation"] == 30 + + # With failures for missing children: + del lfc["Rules"][0]["AbortIncompleteMultipartUpload"]["DaysAfterInitiation"] + with assert_raises(ClientError) as err: + client.put_bucket_lifecycle_configuration(Bucket="bucket", LifecycleConfiguration=lfc) + assert err.exception.response["Error"]["Code"] == "MalformedXML" + + @mock_s3_deprecated def test_lifecycle_with_glacier_transition(): conn = boto.s3.connect_to_region("us-west-1") From 691a8722a898edaa824d941a79a11a7e6f6627dd Mon Sep 17 00:00:00 2001 From: Jon Beilke Date: Wed, 3 Oct 2018 00:45:47 -0500 Subject: [PATCH 30/72] formatting fix for flake8 due to extra spaces --- moto/s3/models.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/moto/s3/models.py b/moto/s3/models.py index 4b1a343d..d889b8ca 100644 --- a/moto/s3/models.py +++ b/moto/s3/models.py @@ -472,9 +472,9 @@ class FakeBucket(BaseModel): transition_date=transition.get('Date') if transition else None, storage_class=transition.get('StorageClass') if transition else None, expired_object_delete_marker=eodm, - nve_noncurrent_days = nve.get('NoncurrentDays') if nve else None, - nvt_noncurrent_days = nvt.get('NoncurrentDays') if nvt else None, - nvt_storage_class = nvt.get('StorageClass') if nvt else None, + nve_noncurrent_days=nve.get('NoncurrentDays') if nve else None, + nvt_noncurrent_days=nvt.get('NoncurrentDays') if nvt else None, + nvt_storage_class=nvt.get('StorageClass') if nvt else None, aimu_days=aimu.get('DaysAfterInitiation') if aimu else None, )) From 9b5f983cb5b4f02a9d7d0842f4f73e01bcab671b Mon Sep 17 00:00:00 2001 From: Jon Beilke Date: Wed, 3 Oct 2018 01:11:11 -0500 Subject: [PATCH 31/72] add action validation to set_lifecycle() --- moto/s3/models.py | 35 +++++++++++++++++++++++------- tests/test_s3/test_s3_lifecycle.py | 22 +++++++++---------- 2 files changed, 38 insertions(+), 19 deletions(-) diff --git a/moto/s3/models.py b/moto/s3/models.py index d889b8ca..fc977bb9 100644 --- a/moto/s3/models.py +++ b/moto/s3/models.py @@ -419,12 +419,31 @@ class FakeBucket(BaseModel): def set_lifecycle(self, rules): self.rules = [] for rule in rules: - # Extract actions from Lifecycle rule + # Extract and validate actions from Lifecycle rule expiration = rule.get('Expiration') transition = rule.get('Transition') - nve = rule.get('NoncurrentVersionExpiration') - nvt = rule.get('NoncurrentVersionTransition') - aimu = rule.get('AbortIncompleteMultipartUpload') + + nve_noncurrent_days = None + if rule.get('NoncurrentVersionExpiration'): + if not rule["NoncurrentVersionExpiration"].get('NoncurrentDays'): + raise MalformedXML() + nve_noncurrent_days = rule["NoncurrentVersionExpiration"]["NoncurrentDays"] + + nvt_noncurrent_days = None + nvt_storage_class = None + if rule.get('NoncurrentVersionTransition'): + if not rule["NoncurrentVersionTransition"].get('NoncurrentDays'): + raise MalformedXML() + if not rule["NoncurrentVersionTransition"].get('StorageClass'): + raise MalformedXML() + nvt_noncurrent_days = rule["NoncurrentVersionTransition"]["NoncurrentDays"] + nvt_storage_class = rule["NoncurrentVersionTransition"]["StorageClass"] + + aimu_days = None + if rule.get('AbortIncompleteMultipartUpload'): + if not rule["AbortIncompleteMultipartUpload"].get('DaysAfterInitiation'): + raise MalformedXML() + aimu_days = rule["AbortIncompleteMultipartUpload"]["DaysAfterInitiation"] eodm = None if expiration and expiration.get("ExpiredObjectDeleteMarker") is not None: @@ -472,10 +491,10 @@ class FakeBucket(BaseModel): transition_date=transition.get('Date') if transition else None, storage_class=transition.get('StorageClass') if transition else None, expired_object_delete_marker=eodm, - nve_noncurrent_days=nve.get('NoncurrentDays') if nve else None, - nvt_noncurrent_days=nvt.get('NoncurrentDays') if nvt else None, - nvt_storage_class=nvt.get('StorageClass') if nvt else None, - aimu_days=aimu.get('DaysAfterInitiation') if aimu else None, + nve_noncurrent_days=nve_noncurrent_days, + nvt_noncurrent_days=nvt_noncurrent_days, + nvt_storage_class=nvt_storage_class, + aimu_days=aimu_days, )) def delete_lifecycle(self): diff --git a/tests/test_s3/test_s3_lifecycle.py b/tests/test_s3/test_s3_lifecycle.py index 9c4bd49b..ff89dc11 100644 --- a/tests/test_s3/test_s3_lifecycle.py +++ b/tests/test_s3/test_s3_lifecycle.py @@ -237,10 +237,10 @@ def test_lifecycle_with_nvt(): lfc = { "Rules": [ { - "NoncurrentVersionTransition": { + "NoncurrentVersionTransitions": [{ "NoncurrentDays": 30, "StorageClass": "ONEZONE_IA" - }, + }], "ID": "wholebucket", "Filter": { "Prefix": "" @@ -252,31 +252,31 @@ def test_lifecycle_with_nvt(): client.put_bucket_lifecycle_configuration(Bucket="bucket", LifecycleConfiguration=lfc) result = client.get_bucket_lifecycle_configuration(Bucket="bucket") assert len(result["Rules"]) == 1 - assert result["Rules"][0]["NoncurrentVersionTransition"]["NoncurrentDays"] == 30 - assert result["Rules"][0]["NoncurrentVersionTransition"]["StorageClass"] == "ONEZONE_IA" + assert result["Rules"][0]["NoncurrentVersionTransitions"][0]["NoncurrentDays"] == 30 + assert result["Rules"][0]["NoncurrentVersionTransitions"][0]["StorageClass"] == "ONEZONE_IA" # Change NoncurrentDays: - lfc["Rules"][0]["NoncurrentVersionTransition"]["NoncurrentDays"] = 10 + lfc["Rules"][0]["NoncurrentVersionTransitions"][0]["NoncurrentDays"] = 10 client.put_bucket_lifecycle_configuration(Bucket="bucket", LifecycleConfiguration=lfc) result = client.get_bucket_lifecycle_configuration(Bucket="bucket") assert len(result["Rules"]) == 1 - assert result["Rules"][0]["NoncurrentVersionTransition"]["NoncurrentDays"] == 10 + assert result["Rules"][0]["NoncurrentVersionTransitions"][0]["NoncurrentDays"] == 10 # Change StorageClass: - lfc["Rules"][0]["NoncurrentVersionTransition"]["StorageClass"] = "GLACIER" + lfc["Rules"][0]["NoncurrentVersionTransitions"][0]["StorageClass"] = "GLACIER" client.put_bucket_lifecycle_configuration(Bucket="bucket", LifecycleConfiguration=lfc) result = client.get_bucket_lifecycle_configuration(Bucket="bucket") assert len(result["Rules"]) == 1 - assert result["Rules"][0]["NoncurrentVersionTransition"]["StorageClass"] == "GLACIER" + assert result["Rules"][0]["NoncurrentVersionTransitions"][0]["StorageClass"] == "GLACIER" # With failures for missing children: - del lfc["Rules"][0]["NoncurrentVersionTransition"]["NoncurrentDays"] + del lfc["Rules"][0]["NoncurrentVersionTransitions"][0]["NoncurrentDays"] with assert_raises(ClientError) as err: client.put_bucket_lifecycle_configuration(Bucket="bucket", LifecycleConfiguration=lfc) assert err.exception.response["Error"]["Code"] == "MalformedXML" - lfc["Rules"][0]["NoncurrentVersionTransition"]["NoncurrentDays"] = 30 + lfc["Rules"][0]["NoncurrentVersionTransitions"][0]["NoncurrentDays"] = 30 - del lfc["Rules"][0]["NoncurrentVersionTransition"]["StorageClass"] + del lfc["Rules"][0]["NoncurrentVersionTransitions"][0]["StorageClass"] with assert_raises(ClientError) as err: client.put_bucket_lifecycle_configuration(Bucket="bucket", LifecycleConfiguration=lfc) assert err.exception.response["Error"]["Code"] == "MalformedXML" From a1a8ac7286ab30c2888b879bc16ecd2bd91dd1d7 Mon Sep 17 00:00:00 2001 From: Jon Beilke Date: Wed, 3 Oct 2018 01:26:09 -0500 Subject: [PATCH 32/72] check for None in lifecycle actions --- moto/s3/models.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/moto/s3/models.py b/moto/s3/models.py index fc977bb9..39f36982 100644 --- a/moto/s3/models.py +++ b/moto/s3/models.py @@ -425,23 +425,23 @@ class FakeBucket(BaseModel): nve_noncurrent_days = None if rule.get('NoncurrentVersionExpiration'): - if not rule["NoncurrentVersionExpiration"].get('NoncurrentDays'): + if rule["NoncurrentVersionExpiration"].get('NoncurrentDays') is None: raise MalformedXML() nve_noncurrent_days = rule["NoncurrentVersionExpiration"]["NoncurrentDays"] nvt_noncurrent_days = None nvt_storage_class = None if rule.get('NoncurrentVersionTransition'): - if not rule["NoncurrentVersionTransition"].get('NoncurrentDays'): + if rule["NoncurrentVersionTransition"].get('NoncurrentDays') is None: raise MalformedXML() - if not rule["NoncurrentVersionTransition"].get('StorageClass'): + if rule["NoncurrentVersionTransition"].get('StorageClass') is None: raise MalformedXML() nvt_noncurrent_days = rule["NoncurrentVersionTransition"]["NoncurrentDays"] nvt_storage_class = rule["NoncurrentVersionTransition"]["StorageClass"] aimu_days = None if rule.get('AbortIncompleteMultipartUpload'): - if not rule["AbortIncompleteMultipartUpload"].get('DaysAfterInitiation'): + if rule["AbortIncompleteMultipartUpload"].get('DaysAfterInitiation') is None: raise MalformedXML() aimu_days = rule["AbortIncompleteMultipartUpload"]["DaysAfterInitiation"] From 5783d662064e248084675c9358bc7e50c4666c51 Mon Sep 17 00:00:00 2001 From: Ash Berlin-Taylor Date: Tue, 2 Oct 2018 17:25:14 +0100 Subject: [PATCH 33/72] Mock more of the Glue Data Catalog APIs This adds some of the missing Get/Update/Create APIs relating to the Glue data catalog -- but not yet all of them, and none of the Batch* API calls. --- moto/glue/exceptions.py | 59 +++- moto/glue/models.py | 104 ++++++- moto/glue/responses.py | 103 +++++-- tests/test_glue/fixtures/datacatalog.py | 25 ++ tests/test_glue/helpers.py | 81 +++++- tests/test_glue/test_datacatalog.py | 362 ++++++++++++++++++++++-- 6 files changed, 673 insertions(+), 61 deletions(-) diff --git a/moto/glue/exceptions.py b/moto/glue/exceptions.py index 62ea1525..8972adb3 100644 --- a/moto/glue/exceptions.py +++ b/moto/glue/exceptions.py @@ -6,19 +6,56 @@ class GlueClientError(JsonRESTError): code = 400 -class DatabaseAlreadyExistsException(GlueClientError): - def __init__(self): - self.code = 400 - super(DatabaseAlreadyExistsException, self).__init__( - 'DatabaseAlreadyExistsException', - 'Database already exists.' +class AlreadyExistsException(GlueClientError): + def __init__(self, typ): + super(GlueClientError, self).__init__( + 'AlreadyExistsException', + '%s already exists.' % (typ), ) -class TableAlreadyExistsException(GlueClientError): +class DatabaseAlreadyExistsException(AlreadyExistsException): def __init__(self): - self.code = 400 - super(TableAlreadyExistsException, self).__init__( - 'TableAlreadyExistsException', - 'Table already exists.' + super(DatabaseAlreadyExistsException, self).__init__('Database') + + +class TableAlreadyExistsException(AlreadyExistsException): + def __init__(self): + super(TableAlreadyExistsException, self).__init__('Table') + + +class PartitionAlreadyExistsException(AlreadyExistsException): + def __init__(self): + super(PartitionAlreadyExistsException, self).__init__('Partition') + + +class EntityNotFoundException(GlueClientError): + def __init__(self, msg): + super(GlueClientError, self).__init__( + 'EntityNotFoundException', + msg, ) + + +class DatabaseNotFoundException(EntityNotFoundException): + def __init__(self, db): + super(DatabaseNotFoundException, self).__init__( + 'Database %s not found.' % db, + ) + + +class TableNotFoundException(EntityNotFoundException): + def __init__(self, tbl): + super(TableNotFoundException, self).__init__( + 'Table %s not found.' % tbl, + ) + + +class PartitionNotFoundException(EntityNotFoundException): + def __init__(self): + super(PartitionNotFoundException, self).__init__("Cannot find partition.") + + +class VersionNotFoundException(EntityNotFoundException): + def __init__(self): + super(VersionNotFoundException, self).__init__("Version not found.") diff --git a/moto/glue/models.py b/moto/glue/models.py index 09b7d60e..bcf2ec4b 100644 --- a/moto/glue/models.py +++ b/moto/glue/models.py @@ -1,8 +1,19 @@ from __future__ import unicode_literals +import time + from moto.core import BaseBackend, BaseModel from moto.compat import OrderedDict -from.exceptions import DatabaseAlreadyExistsException, TableAlreadyExistsException +from.exceptions import ( + JsonRESTError, + DatabaseAlreadyExistsException, + DatabaseNotFoundException, + TableAlreadyExistsException, + TableNotFoundException, + PartitionAlreadyExistsException, + PartitionNotFoundException, + VersionNotFoundException, +) class GlueBackend(BaseBackend): @@ -19,7 +30,10 @@ class GlueBackend(BaseBackend): return database def get_database(self, database_name): - return self.databases[database_name] + try: + return self.databases[database_name] + except KeyError: + raise DatabaseNotFoundException(database_name) def create_table(self, database_name, table_name, table_input): database = self.get_database(database_name) @@ -33,7 +47,10 @@ class GlueBackend(BaseBackend): def get_table(self, database_name, table_name): database = self.get_database(database_name) - return database.tables[table_name] + try: + return database.tables[table_name] + except KeyError: + raise TableNotFoundException(table_name) def get_tables(self, database_name): database = self.get_database(database_name) @@ -52,9 +69,84 @@ class FakeTable(BaseModel): def __init__(self, database_name, table_name, table_input): self.database_name = database_name self.name = table_name - self.table_input = table_input - self.storage_descriptor = self.table_input.get('StorageDescriptor', {}) - self.partition_keys = self.table_input.get('PartitionKeys', []) + self.partitions = OrderedDict() + self.versions = [] + self.update(table_input) + + def update(self, table_input): + self.versions.append(table_input) + + def get_version(self, ver): + try: + if not isinstance(ver, int): + # "1" goes to [0] + ver = int(ver) - 1 + except ValueError as e: + raise JsonRESTError("InvalidInputException", str(e)) + + try: + return self.versions[ver] + except IndexError: + raise VersionNotFoundException() + + def as_dict(self, version=-1): + obj = { + 'DatabaseName': self.database_name, + 'Name': self.name, + } + obj.update(self.get_version(version)) + return obj + + def create_partition(self, partiton_input): + partition = FakePartition(self.database_name, self.name, partiton_input) + key = str(partition.values) + if key in self.partitions: + raise PartitionAlreadyExistsException() + self.partitions[str(partition.values)] = partition + + def get_partitions(self): + return [p for str_part_values, p in self.partitions.items()] + + def get_partition(self, values): + try: + return self.partitions[str(values)] + except KeyError: + raise PartitionNotFoundException() + + def update_partition(self, old_values, partiton_input): + partition = FakePartition(self.database_name, self.name, partiton_input) + key = str(partition.values) + if old_values == partiton_input['Values']: + # Altering a partition in place. Don't remove it so the order of + # returned partitions doesn't change + if key not in self.partitions: + raise PartitionNotFoundException() + else: + removed = self.partitions.pop(str(old_values), None) + if removed is None: + raise PartitionNotFoundException() + if key in self.partitions: + # Trying to update to overwrite a partition that exists + raise PartitionAlreadyExistsException() + self.partitions[key] = partition + + +class FakePartition(BaseModel): + def __init__(self, database_name, table_name, partiton_input): + self.creation_time = time.time() + self.database_name = database_name + self.table_name = table_name + self.partition_input = partiton_input + self.values = self.partition_input.get('Values', []) + + def as_dict(self): + obj = { + 'DatabaseName': self.database_name, + 'TableName': self.table_name, + 'CreationTime': self.creation_time, + } + obj.update(self.partition_input) + return obj glue_backend = GlueBackend() diff --git a/moto/glue/responses.py b/moto/glue/responses.py index bb64c40d..84cc6f90 100644 --- a/moto/glue/responses.py +++ b/moto/glue/responses.py @@ -37,27 +37,94 @@ class GlueResponse(BaseResponse): database_name = self.parameters.get('DatabaseName') table_name = self.parameters.get('Name') table = self.glue_backend.get_table(database_name, table_name) + + return json.dumps({'Table': table.as_dict()}) + + def update_table(self): + database_name = self.parameters.get('DatabaseName') + table_input = self.parameters.get('TableInput') + table_name = table_input.get('Name') + table = self.glue_backend.get_table(database_name, table_name) + table.update(table_input) + return "" + + def get_table_versions(self): + database_name = self.parameters.get('DatabaseName') + table_name = self.parameters.get('TableName') + table = self.glue_backend.get_table(database_name, table_name) + return json.dumps({ - 'Table': { - 'DatabaseName': table.database_name, - 'Name': table.name, - 'PartitionKeys': table.partition_keys, - 'StorageDescriptor': table.storage_descriptor - } + "TableVersions": [ + { + "Table": table.as_dict(version=n), + "VersionId": str(n + 1), + } for n in range(len(table.versions)) + ], + }) + + def get_table_version(self): + database_name = self.parameters.get('DatabaseName') + table_name = self.parameters.get('TableName') + table = self.glue_backend.get_table(database_name, table_name) + ver_id = self.parameters.get('VersionId') + + return json.dumps({ + "TableVersion": { + "Table": table.as_dict(version=ver_id), + "VersionId": ver_id, + }, }) def get_tables(self): database_name = self.parameters.get('DatabaseName') tables = self.glue_backend.get_tables(database_name) - return json.dumps( - { - 'TableList': [ - { - 'DatabaseName': table.database_name, - 'Name': table.name, - 'PartitionKeys': table.partition_keys, - 'StorageDescriptor': table.storage_descriptor - } for table in tables - ] - } - ) + return json.dumps({ + 'TableList': [ + table.as_dict() for table in tables + ] + }) + + def get_partitions(self): + database_name = self.parameters.get('DatabaseName') + table_name = self.parameters.get('TableName') + if 'Expression' in self.parameters: + raise NotImplementedError("Expression filtering in get_partitions is not implemented in moto") + table = self.glue_backend.get_table(database_name, table_name) + + return json.dumps({ + 'Partitions': [ + p.as_dict() for p in table.get_partitions() + ] + }) + + def get_partition(self): + database_name = self.parameters.get('DatabaseName') + table_name = self.parameters.get('TableName') + values = self.parameters.get('PartitionValues') + + table = self.glue_backend.get_table(database_name, table_name) + + p = table.get_partition(values) + + return json.dumps({'Partition': p.as_dict()}) + + def create_partition(self): + database_name = self.parameters.get('DatabaseName') + table_name = self.parameters.get('TableName') + part_input = self.parameters.get('PartitionInput') + + table = self.glue_backend.get_table(database_name, table_name) + table.create_partition(part_input) + + return "" + + def update_partition(self): + database_name = self.parameters.get('DatabaseName') + table_name = self.parameters.get('TableName') + part_input = self.parameters.get('PartitionInput') + part_to_update = self.parameters.get('PartitionValueList') + + table = self.glue_backend.get_table(database_name, table_name) + table.update_partition(part_to_update, part_input) + + return "" diff --git a/tests/test_glue/fixtures/datacatalog.py b/tests/test_glue/fixtures/datacatalog.py index b2efe415..edad2f0f 100644 --- a/tests/test_glue/fixtures/datacatalog.py +++ b/tests/test_glue/fixtures/datacatalog.py @@ -29,3 +29,28 @@ TABLE_INPUT = { }, 'TableType': 'EXTERNAL_TABLE', } + + +PARTITION_INPUT = { + # 'DatabaseName': 'dbname', + 'StorageDescriptor': { + 'BucketColumns': [], + 'Columns': [], + 'Compressed': False, + 'InputFormat': 'org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat', + 'Location': 's3://.../partition=value', + 'NumberOfBuckets': -1, + 'OutputFormat': 'org.apache.hadoop.hive.ql.io.parquet.MapredParquetOutputFormat', + 'Parameters': {}, + 'SerdeInfo': { + 'Parameters': {'path': 's3://...', 'serialization.format': '1'}, + 'SerializationLibrary': 'org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe'}, + 'SkewedInfo': {'SkewedColumnNames': [], + 'SkewedColumnValueLocationMaps': {}, + 'SkewedColumnValues': []}, + 'SortColumns': [], + 'StoredAsSubDirectories': False, + }, + # 'TableName': 'source_table', + # 'Values': ['2018-06-26'], +} diff --git a/tests/test_glue/helpers.py b/tests/test_glue/helpers.py index 4a51f911..331b9986 100644 --- a/tests/test_glue/helpers.py +++ b/tests/test_glue/helpers.py @@ -2,7 +2,7 @@ from __future__ import unicode_literals import copy -from .fixtures.datacatalog import TABLE_INPUT +from .fixtures.datacatalog import TABLE_INPUT, PARTITION_INPUT def create_database(client, database_name): @@ -17,22 +17,38 @@ def get_database(client, database_name): return client.get_database(Name=database_name) -def create_table_input(table_name, s3_location, columns=[], partition_keys=[]): +def create_table_input(database_name, table_name, columns=[], partition_keys=[]): table_input = copy.deepcopy(TABLE_INPUT) table_input['Name'] = table_name table_input['PartitionKeys'] = partition_keys table_input['StorageDescriptor']['Columns'] = columns - table_input['StorageDescriptor']['Location'] = s3_location + table_input['StorageDescriptor']['Location'] = 's3://my-bucket/{database_name}/{table_name}'.format( + database_name=database_name, + table_name=table_name + ) return table_input -def create_table(client, database_name, table_name, table_input): +def create_table(client, database_name, table_name, table_input=None, **kwargs): + if table_input is None: + table_input = create_table_input(database_name, table_name, **kwargs) + return client.create_table( DatabaseName=database_name, TableInput=table_input ) +def update_table(client, database_name, table_name, table_input=None, **kwargs): + if table_input is None: + table_input = create_table_input(database_name, table_name, **kwargs) + + return client.update_table( + DatabaseName=database_name, + TableInput=table_input, + ) + + def get_table(client, database_name, table_name): return client.get_table( DatabaseName=database_name, @@ -44,3 +60,60 @@ def get_tables(client, database_name): return client.get_tables( DatabaseName=database_name ) + + +def get_table_versions(client, database_name, table_name): + return client.get_table_versions( + DatabaseName=database_name, + TableName=table_name + ) + + +def get_table_version(client, database_name, table_name, version_id): + return client.get_table_version( + DatabaseName=database_name, + TableName=table_name, + VersionId=version_id, + ) + + +def create_partition_input(database_name, table_name, values=[], columns=[]): + root_path = 's3://my-bucket/{database_name}/{table_name}'.format( + database_name=database_name, + table_name=table_name + ) + + part_input = copy.deepcopy(PARTITION_INPUT) + part_input['Values'] = values + part_input['StorageDescriptor']['Columns'] = columns + part_input['StorageDescriptor']['SerdeInfo']['Parameters']['path'] = root_path + return part_input + + +def create_partition(client, database_name, table_name, partiton_input=None, **kwargs): + if partiton_input is None: + partiton_input = create_partition_input(database_name, table_name, **kwargs) + return client.create_partition( + DatabaseName=database_name, + TableName=table_name, + PartitionInput=partiton_input + ) + + +def update_partition(client, database_name, table_name, old_values=[], partiton_input=None, **kwargs): + if partiton_input is None: + partiton_input = create_partition_input(database_name, table_name, **kwargs) + return client.update_partition( + DatabaseName=database_name, + TableName=table_name, + PartitionInput=partiton_input, + PartitionValueList=old_values, + ) + + +def get_partition(client, database_name, table_name, values): + return client.get_partition( + DatabaseName=database_name, + TableName=table_name, + PartitionValues=values, + ) diff --git a/tests/test_glue/test_datacatalog.py b/tests/test_glue/test_datacatalog.py index 7dabeb1f..a457d512 100644 --- a/tests/test_glue/test_datacatalog.py +++ b/tests/test_glue/test_datacatalog.py @@ -1,10 +1,15 @@ from __future__ import unicode_literals import sure # noqa +import re from nose.tools import assert_raises import boto3 from botocore.client import ClientError + +from datetime import datetime +import pytz + from moto import mock_glue from . import helpers @@ -30,7 +35,19 @@ def test_create_database_already_exists(): with assert_raises(ClientError) as exc: helpers.create_database(client, database_name) - exc.exception.response['Error']['Code'].should.equal('DatabaseAlreadyExistsException') + exc.exception.response['Error']['Code'].should.equal('AlreadyExistsException') + + +@mock_glue +def test_get_database_not_exits(): + client = boto3.client('glue', region_name='us-east-1') + database_name = 'nosuchdatabase' + + with assert_raises(ClientError) as exc: + helpers.get_database(client, database_name) + + exc.exception.response['Error']['Code'].should.equal('EntityNotFoundException') + exc.exception.response['Error']['Message'].should.match('Database nosuchdatabase not found') @mock_glue @@ -40,12 +57,7 @@ def test_create_table(): helpers.create_database(client, database_name) table_name = 'myspecialtable' - s3_location = 's3://my-bucket/{database_name}/{table_name}'.format( - database_name=database_name, - table_name=table_name - ) - - table_input = helpers.create_table_input(table_name, s3_location) + table_input = helpers.create_table_input(database_name, table_name) helpers.create_table(client, database_name, table_name, table_input) response = helpers.get_table(client, database_name, table_name) @@ -63,18 +75,12 @@ def test_create_table_already_exists(): helpers.create_database(client, database_name) table_name = 'cantcreatethistabletwice' - s3_location = 's3://my-bucket/{database_name}/{table_name}'.format( - database_name=database_name, - table_name=table_name - ) - - table_input = helpers.create_table_input(table_name, s3_location) - helpers.create_table(client, database_name, table_name, table_input) + helpers.create_table(client, database_name, table_name) with assert_raises(ClientError) as exc: - helpers.create_table(client, database_name, table_name, table_input) + helpers.create_table(client, database_name, table_name) - exc.exception.response['Error']['Code'].should.equal('TableAlreadyExistsException') + exc.exception.response['Error']['Code'].should.equal('AlreadyExistsException') @mock_glue @@ -87,11 +93,7 @@ def test_get_tables(): table_inputs = {} for table_name in table_names: - s3_location = 's3://my-bucket/{database_name}/{table_name}'.format( - database_name=database_name, - table_name=table_name - ) - table_input = helpers.create_table_input(table_name, s3_location) + table_input = helpers.create_table_input(database_name, table_name) table_inputs[table_name] = table_input helpers.create_table(client, database_name, table_name, table_input) @@ -99,10 +101,326 @@ def test_get_tables(): tables = response['TableList'] - assert len(tables) == 3 + tables.should.have.length_of(3) for table in tables: table_name = table['Name'] table_name.should.equal(table_inputs[table_name]['Name']) table['StorageDescriptor'].should.equal(table_inputs[table_name]['StorageDescriptor']) table['PartitionKeys'].should.equal(table_inputs[table_name]['PartitionKeys']) + + +@mock_glue +def test_get_table_versions(): + client = boto3.client('glue', region_name='us-east-1') + database_name = 'myspecialdatabase' + helpers.create_database(client, database_name) + + table_name = 'myfirsttable' + version_inputs = {} + + table_input = helpers.create_table_input(database_name, table_name) + helpers.create_table(client, database_name, table_name, table_input) + version_inputs["1"] = table_input + + columns = [{'Name': 'country', 'Type': 'string'}] + table_input = helpers.create_table_input(database_name, table_name, columns=columns) + helpers.update_table(client, database_name, table_name, table_input) + version_inputs["2"] = table_input + + # Updateing with an indentical input should still create a new version + helpers.update_table(client, database_name, table_name, table_input) + version_inputs["3"] = table_input + + response = helpers.get_table_versions(client, database_name, table_name) + + vers = response['TableVersions'] + + vers.should.have.length_of(3) + vers[0]['Table']['StorageDescriptor']['Columns'].should.equal([]) + vers[-1]['Table']['StorageDescriptor']['Columns'].should.equal(columns) + + for n, ver in enumerate(vers): + n = str(n + 1) + ver['VersionId'].should.equal(n) + ver['Table']['Name'].should.equal(table_name) + ver['Table']['StorageDescriptor'].should.equal(version_inputs[n]['StorageDescriptor']) + ver['Table']['PartitionKeys'].should.equal(version_inputs[n]['PartitionKeys']) + + response = helpers.get_table_version(client, database_name, table_name, "3") + ver = response['TableVersion'] + + ver['VersionId'].should.equal("3") + ver['Table']['Name'].should.equal(table_name) + ver['Table']['StorageDescriptor']['Columns'].should.equal(columns) + + +@mock_glue +def test_get_table_version_not_found(): + client = boto3.client('glue', region_name='us-east-1') + database_name = 'myspecialdatabase' + table_name = 'myfirsttable' + helpers.create_database(client, database_name) + helpers.create_table(client, database_name, table_name) + + with assert_raises(ClientError) as exc: + helpers.get_table_version(client, database_name, 'myfirsttable', "20") + + exc.exception.response['Error']['Code'].should.equal('EntityNotFoundException') + exc.exception.response['Error']['Message'].should.match('version', re.I) + + +@mock_glue +def test_get_table_version_invalid_input(): + client = boto3.client('glue', region_name='us-east-1') + database_name = 'myspecialdatabase' + table_name = 'myfirsttable' + helpers.create_database(client, database_name) + helpers.create_table(client, database_name, table_name) + + with assert_raises(ClientError) as exc: + helpers.get_table_version(client, database_name, 'myfirsttable', "10not-an-int") + + exc.exception.response['Error']['Code'].should.equal('InvalidInputException') + + +@mock_glue +def test_get_table_not_exits(): + client = boto3.client('glue', region_name='us-east-1') + database_name = 'myspecialdatabase' + helpers.create_database(client, database_name) + + with assert_raises(ClientError) as exc: + helpers.get_table(client, database_name, 'myfirsttable') + + exc.exception.response['Error']['Code'].should.equal('EntityNotFoundException') + exc.exception.response['Error']['Message'].should.match('Table myfirsttable not found') + + +@mock_glue +def test_get_table_when_database_not_exits(): + client = boto3.client('glue', region_name='us-east-1') + database_name = 'nosuchdatabase' + + with assert_raises(ClientError) as exc: + helpers.get_table(client, database_name, 'myfirsttable') + + exc.exception.response['Error']['Code'].should.equal('EntityNotFoundException') + exc.exception.response['Error']['Message'].should.match('Database nosuchdatabase not found') + + +@mock_glue +def test_get_partitions_empty(): + client = boto3.client('glue', region_name='us-east-1') + database_name = 'myspecialdatabase' + table_name = 'myfirsttable' + helpers.create_database(client, database_name) + + helpers.create_table(client, database_name, table_name) + + response = client.get_partitions(DatabaseName=database_name, TableName=table_name) + + response['Partitions'].should.have.length_of(0) + + +@mock_glue +def test_create_partition(): + client = boto3.client('glue', region_name='us-east-1') + database_name = 'myspecialdatabase' + table_name = 'myfirsttable' + values = ['2018-10-01'] + helpers.create_database(client, database_name) + + helpers.create_table(client, database_name, table_name) + + before = datetime.now(pytz.utc) + + part_input = helpers.create_partition_input(database_name, table_name, values=values) + helpers.create_partition(client, database_name, table_name, part_input) + + after = datetime.now(pytz.utc) + + response = client.get_partitions(DatabaseName=database_name, TableName=table_name) + + partitions = response['Partitions'] + + partitions.should.have.length_of(1) + + partition = partitions[0] + + partition['TableName'].should.equal(table_name) + partition['StorageDescriptor'].should.equal(part_input['StorageDescriptor']) + partition['Values'].should.equal(values) + partition['CreationTime'].should.be.greater_than(before) + partition['CreationTime'].should.be.lower_than(after) + + +@mock_glue +def test_create_partition_already_exist(): + client = boto3.client('glue', region_name='us-east-1') + database_name = 'myspecialdatabase' + table_name = 'myfirsttable' + values = ['2018-10-01'] + helpers.create_database(client, database_name) + + helpers.create_table(client, database_name, table_name) + + helpers.create_partition(client, database_name, table_name, values=values) + + with assert_raises(ClientError) as exc: + helpers.create_partition(client, database_name, table_name, values=values) + + exc.exception.response['Error']['Code'].should.equal('AlreadyExistsException') + + +@mock_glue +def test_get_partition_not_found(): + client = boto3.client('glue', region_name='us-east-1') + database_name = 'myspecialdatabase' + table_name = 'myfirsttable' + values = ['2018-10-01'] + helpers.create_database(client, database_name) + + helpers.create_table(client, database_name, table_name) + + with assert_raises(ClientError) as exc: + helpers.get_partition(client, database_name, table_name, values) + + exc.exception.response['Error']['Code'].should.equal('EntityNotFoundException') + exc.exception.response['Error']['Message'].should.match('partition') + + +@mock_glue +def test_get_partition(): + client = boto3.client('glue', region_name='us-east-1') + database_name = 'myspecialdatabase' + table_name = 'myfirsttable' + helpers.create_database(client, database_name) + + helpers.create_table(client, database_name, table_name) + + values = [['2018-10-01'], ['2018-09-01']] + + helpers.create_partition(client, database_name, table_name, values=values[0]) + helpers.create_partition(client, database_name, table_name, values=values[1]) + + response = client.get_partition(DatabaseName=database_name, TableName=table_name, PartitionValues=values[1]) + + partition = response['Partition'] + + partition['TableName'].should.equal(table_name) + partition['Values'].should.equal(values[1]) + + +@mock_glue +def test_update_partition_not_found_moving(): + client = boto3.client('glue', region_name='us-east-1') + database_name = 'myspecialdatabase' + table_name = 'myfirsttable' + + helpers.create_database(client, database_name) + helpers.create_table(client, database_name, table_name) + + with assert_raises(ClientError) as exc: + helpers.update_partition(client, database_name, table_name, old_values=['0000-00-00'], values=['2018-10-02']) + + exc.exception.response['Error']['Code'].should.equal('EntityNotFoundException') + exc.exception.response['Error']['Message'].should.match('partition') + + +@mock_glue +def test_update_partition_not_found_change_in_place(): + client = boto3.client('glue', region_name='us-east-1') + database_name = 'myspecialdatabase' + table_name = 'myfirsttable' + values = ['2018-10-01'] + + helpers.create_database(client, database_name) + helpers.create_table(client, database_name, table_name) + + with assert_raises(ClientError) as exc: + helpers.update_partition(client, database_name, table_name, old_values=values, values=values) + + exc.exception.response['Error']['Code'].should.equal('EntityNotFoundException') + exc.exception.response['Error']['Message'].should.match('partition') + + +@mock_glue +def test_update_partition_cannot_overwrite(): + client = boto3.client('glue', region_name='us-east-1') + database_name = 'myspecialdatabase' + table_name = 'myfirsttable' + helpers.create_database(client, database_name) + + helpers.create_table(client, database_name, table_name) + + values = [['2018-10-01'], ['2018-09-01']] + + helpers.create_partition(client, database_name, table_name, values=values[0]) + helpers.create_partition(client, database_name, table_name, values=values[1]) + + with assert_raises(ClientError) as exc: + helpers.update_partition(client, database_name, table_name, old_values=values[0], values=values[1]) + + exc.exception.response['Error']['Code'].should.equal('AlreadyExistsException') + + +@mock_glue +def test_update_partition(): + client = boto3.client('glue', region_name='us-east-1') + database_name = 'myspecialdatabase' + table_name = 'myfirsttable' + values = ['2018-10-01'] + + helpers.create_database(client, database_name) + helpers.create_table(client, database_name, table_name) + helpers.create_partition(client, database_name, table_name, values=values) + + response = helpers.update_partition( + client, + database_name, + table_name, + old_values=values, + values=values, + columns=[{'Name': 'country', 'Type': 'string'}], + ) + + response = client.get_partition(DatabaseName=database_name, TableName=table_name, PartitionValues=values) + partition = response['Partition'] + + partition['TableName'].should.equal(table_name) + partition['StorageDescriptor']['Columns'].should.equal([{'Name': 'country', 'Type': 'string'}]) + + +@mock_glue +def test_update_partition_move(): + client = boto3.client('glue', region_name='us-east-1') + database_name = 'myspecialdatabase' + table_name = 'myfirsttable' + values = ['2018-10-01'] + new_values = ['2018-09-01'] + + helpers.create_database(client, database_name) + helpers.create_table(client, database_name, table_name) + helpers.create_partition(client, database_name, table_name, values=values) + + response = helpers.update_partition( + client, + database_name, + table_name, + old_values=values, + values=new_values, + columns=[{'Name': 'country', 'Type': 'string'}], + ) + + with assert_raises(ClientError) as exc: + helpers.get_partition(client, database_name, table_name, values) + + # Old partition shouldn't exist anymore + exc.exception.response['Error']['Code'].should.equal('EntityNotFoundException') + + response = client.get_partition(DatabaseName=database_name, TableName=table_name, PartitionValues=new_values) + partition = response['Partition'] + + partition['TableName'].should.equal(table_name) + partition['StorageDescriptor']['Columns'].should.equal([{'Name': 'country', 'Type': 'string'}]) From 5b3b52752de5b4edca796c33d9dfd46b99c24a82 Mon Sep 17 00:00:00 2001 From: Jon Beilke Date: Thu, 4 Oct 2018 10:25:16 -0500 Subject: [PATCH 34/72] explicitly check that lifecycle actions are not None when setting lifecycle --- moto/s3/models.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/moto/s3/models.py b/moto/s3/models.py index 39f36982..bb4d7848 100644 --- a/moto/s3/models.py +++ b/moto/s3/models.py @@ -424,14 +424,14 @@ class FakeBucket(BaseModel): transition = rule.get('Transition') nve_noncurrent_days = None - if rule.get('NoncurrentVersionExpiration'): + if rule.get('NoncurrentVersionExpiration') is not None: if rule["NoncurrentVersionExpiration"].get('NoncurrentDays') is None: raise MalformedXML() nve_noncurrent_days = rule["NoncurrentVersionExpiration"]["NoncurrentDays"] nvt_noncurrent_days = None nvt_storage_class = None - if rule.get('NoncurrentVersionTransition'): + if rule.get('NoncurrentVersionTransition') is not None: if rule["NoncurrentVersionTransition"].get('NoncurrentDays') is None: raise MalformedXML() if rule["NoncurrentVersionTransition"].get('StorageClass') is None: @@ -440,7 +440,7 @@ class FakeBucket(BaseModel): nvt_storage_class = rule["NoncurrentVersionTransition"]["StorageClass"] aimu_days = None - if rule.get('AbortIncompleteMultipartUpload'): + if rule.get('AbortIncompleteMultipartUpload') is not None: if rule["AbortIncompleteMultipartUpload"].get('DaysAfterInitiation') is None: raise MalformedXML() aimu_days = rule["AbortIncompleteMultipartUpload"]["DaysAfterInitiation"] From 12e0a38b56794888ce9eb572ab9c3dadaef96dba Mon Sep 17 00:00:00 2001 From: Jon Beilke Date: Fri, 5 Oct 2018 11:04:55 -0500 Subject: [PATCH 35/72] add TODO for testing exceptions with aimu and nve --- tests/test_s3/test_s3_lifecycle.py | 12 ++---------- 1 file changed, 2 insertions(+), 10 deletions(-) diff --git a/tests/test_s3/test_s3_lifecycle.py b/tests/test_s3/test_s3_lifecycle.py index ff89dc11..3d533a64 100644 --- a/tests/test_s3/test_s3_lifecycle.py +++ b/tests/test_s3/test_s3_lifecycle.py @@ -222,11 +222,7 @@ def test_lifecycle_with_nve(): assert len(result["Rules"]) == 1 assert result["Rules"][0]["NoncurrentVersionExpiration"]["NoncurrentDays"] == 10 - # With failures for missing children: - del lfc["Rules"][0]["NoncurrentVersionExpiration"]["NoncurrentDays"] - with assert_raises(ClientError) as err: - client.put_bucket_lifecycle_configuration(Bucket="bucket", LifecycleConfiguration=lfc) - assert err.exception.response["Error"]["Code"] == "MalformedXML" + # TODO: Add test for failures due to missing children @mock_s3 @@ -313,11 +309,7 @@ def test_lifecycle_with_aimu(): assert len(result["Rules"]) == 1 assert result["Rules"][0]["AbortIncompleteMultipartUpload"]["DaysAfterInitiation"] == 30 - # With failures for missing children: - del lfc["Rules"][0]["AbortIncompleteMultipartUpload"]["DaysAfterInitiation"] - with assert_raises(ClientError) as err: - client.put_bucket_lifecycle_configuration(Bucket="bucket", LifecycleConfiguration=lfc) - assert err.exception.response["Error"]["Code"] == "MalformedXML" + # TODO: Add test for failures due to missing children @mock_s3_deprecated From 60ec840eef44c395e83342469ab81c0ca5a46daf Mon Sep 17 00:00:00 2001 From: Jon Beilke Date: Fri, 5 Oct 2018 15:55:47 -0500 Subject: [PATCH 36/72] add disable_key, enable_key, cancel_key_deletion, and schedule_key_deletion actions to KMS endpoint --- moto/kms/models.py | 30 +++++++++++++- moto/kms/responses.py | 46 ++++++++++++++++++++++ tests/test_kms/test_kms.py | 80 ++++++++++++++++++++++++++++++++++++++ 3 files changed, 155 insertions(+), 1 deletion(-) diff --git a/moto/kms/models.py b/moto/kms/models.py index 89ebf008..30a01d36 100644 --- a/moto/kms/models.py +++ b/moto/kms/models.py @@ -12,11 +12,13 @@ class Key(BaseModel): self.id = generate_key_id() self.policy = policy self.key_usage = key_usage + self.key_state = "Enabled" self.description = description self.enabled = True self.region = region self.account_id = "0123456789012" self.key_rotation_status = False + self.deletion_date = None @property def physical_resource_id(self): @@ -27,7 +29,7 @@ class Key(BaseModel): return "arn:aws:kms:{0}:{1}:key/{2}".format(self.region, self.account_id, self.id) def to_dict(self): - return { + key_dict = { "KeyMetadata": { "AWSAccountId": self.account_id, "Arn": self.arn, @@ -36,8 +38,11 @@ class Key(BaseModel): "Enabled": self.enabled, "KeyId": self.id, "KeyUsage": self.key_usage, + "KeyState": self.key_state, } } + key_dict['KeyMetadata']['DeletionDate'] = self.deletion_date if self.key_state == 'PendingDeletion' + return key_dict def delete(self, region_name): kms_backends[region_name].delete_key(self.id) @@ -138,6 +143,29 @@ class KmsBackend(BaseBackend): def get_key_policy(self, key_id): return self.keys[self.get_key_id(key_id)].policy + def disable_key(self, key_id): + if key_id in self.keys: + self.keys[key_id].enabled = False + self.keys[key_id].key_state = 'Disabled' + + def enable_key(self, key_id): + if key_id in self.keys: + self.keys[key_id].enabled = True + self.keys[key_id].key_state = 'Enabled' + + def cancel_key_deletion(self, key_id): + if key_id in self.keys: + self.keys[key_id].key_state = 'Disabled' + self.keys[key_id].deletion_date = None + + def schedule_key_deletion(self, key_id, pending_window_in_days=30): + if key_id in self.keys: + if 7 <= pending_window_in_days <= 30: + self.keys[key_id].enabled = False + self.keys[key_id].key_state = 'PendingDeletion' + self.keys[key_id].deletion_date = datetime.now() + timedelta(days=pending_window_in_days) + return self.keys[key_id].deletion_date + kms_backends = {} for region in boto.kms.regions(): diff --git a/moto/kms/responses.py b/moto/kms/responses.py index 0f544e95..e782d862 100644 --- a/moto/kms/responses.py +++ b/moto/kms/responses.py @@ -233,6 +233,52 @@ class KmsResponse(BaseResponse): value = self.parameters.get("CiphertextBlob") return json.dumps({"Plaintext": base64.b64decode(value).decode("utf-8")}) + def disable_key(self): + key_id = self.parameters.get('KeyId') + _assert_valid_key_id(self.kms_backend.get_key_id(key_id)) + try: + self.kms_backend.disable_key(key_id) + except KeyError: + raise JSONResponseError(404, 'Not Found', body={ + 'message': "Key 'arn:aws:kms:{region}:012345678912:key/{key_id}' does not exist".format(region=self.region, key_id=key_id), + '__type': 'NotFoundException'}) + return json.dumps(None) + + def enable_key(self): + key_id = self.parameters.get('KeyId') + _assert_valid_key_id(self.kms_backend.get_key_id(key_id)) + try: + self.kms_backend.enable_key(key_id) + except KeyError: + raise JSONResponseError(404, 'Not Found', body={ + 'message': "Key 'arn:aws:kms:{region}:012345678912:key/{key_id}' does not exist".format(region=self.region, key_id=key_id), + '__type': 'NotFoundException'}) + return json.dumps(None) + + def cancel_key_deletion(self): + key_id = self.parameters.get('KeyId') + _assert_valid_key_id(self.kms_backend.get_key_id(key_id)) + try: + self.kms_backend.cancel_key_deletion(key_id) + except KeyError: + raise JSONResponseError(404, 'Not Found', body={ + 'message': "Key 'arn:aws:kms:{region}:012345678912:key/{key_id}' does not exist".format(region=self.region, key_id=key_id), + '__type': 'NotFoundException'}) + return json.dumps({'KeyId': key_id}) + + def schedule_key_deletion(self): + key_id = self.parameters.get('KeyId') + _assert_valid_key_id(self.kms_backend.get_key_id(key_id)) + try: + return json.dumps({ + 'KeyId': key_id, + 'DeletionDate': self.kms_backend.schedule_key_deletion(key_id) + }) + except KeyError: + raise JSONResponseError(404, 'Not Found', body={ + 'message': "Key 'arn:aws:kms:{region}:012345678912:key/{key_id}' does not exist".format(region=self.region, key_id=key_id), + '__type': 'NotFoundException'}) + def _assert_valid_key_id(key_id): if not re.match(r'^[A-F0-9]{8}-[A-F0-9]{4}-[A-F0-9]{4}-[A-F0-9]{4}-[A-F0-9]{12}$', key_id, re.IGNORECASE): diff --git a/tests/test_kms/test_kms.py b/tests/test_kms/test_kms.py index 96715de7..9779f02a 100644 --- a/tests/test_kms/test_kms.py +++ b/tests/test_kms/test_kms.py @@ -617,3 +617,83 @@ def test_kms_encrypt_boto3(): response = client.decrypt(CiphertextBlob=response['CiphertextBlob']) response['Plaintext'].should.equal(b'bar') + + +@mock_kms +def test_disable_key(): + client = boto3.client('kms', region_name='us-east-1') + key = client.create_key(description='disable-key') + client.disable_key( + KeyId=key['KeyMetadata']['KeyId'] + ) + + result = client.describe_key(KeyId='disable-key') + assert result["KeyMetadata"]["Enabled"] == False + assert result["KeyMetadata"]["KeyState"] == 'Disabled' + + +@mock_kms +def test_enable_key(): + client = boto3.client('kms', region_name='us-east-1') + key = client.create_key(description='enable-key') + client.disable_key( + KeyId=key['KeyMetadata']['KeyId'] + ) + client.enable_key( + KeyId=key['KeyMetadata']['KeyId'] + ) + + result = client.describe_key(KeyId='enable-key') + assert result["KeyMetadata"]["Enabled"] == True + assert result["KeyMetadata"]["KeyState"] == 'Enabled' + + +@mock_kms +def test_schedule_key_deletion(): + client = boto3.client('kms', region_name='us-east-1') + key = client.create_key(description='schedule-key-deletion') + response = client.schedule_key_deletion( + KeyId=key['KeyMetadata']['KeyId'] + ) + assert response['KeyId'] == 'schedule-key-deletion' + assert response['DeletionDate'] == datetime.now() + timedelta(days=30) + + result = client.describe_key(KeyId='schedule-key-deletion') + assert result["KeyMetadata"]["Enabled"] == False + assert result["KeyMetadata"]["KeyState"] == 'PendingDeletion' + assert 'DeletionDate' in result["KeyMetadata"] + + +@mock_kms +def test_schedule_key_deletion_custom(): + client = boto3.client('kms', region_name='us-east-1') + key = client.create_key(description='schedule-key-deletion') + response = client.schedule_key_deletion( + KeyId=key['KeyMetadata']['KeyId'], + PendingWindowInDays=7 + ) + assert response['KeyId'] == 'schedule-key-deletion' + assert response['DeletionDate'] == datetime.now() + timedelta(days=7) + + result = client.describe_key(KeyId='schedule-key-deletion') + assert result["KeyMetadata"]["Enabled"] == False + assert result["KeyMetadata"]["KeyState"] == 'PendingDeletion' + assert 'DeletionDate' in result["KeyMetadata"] + + +@mock_kms +def test_cancel_key_deletion(): + client = boto3.client('kms', region_name='us-east-1') + key = client.create_key(description='cancel-key-deletion') + client.schedule_key_deletion( + KeyId=key['KeyMetadata']['KeyId'] + ) + response = client.cancel_key_deletion( + KeyId=key['KeyMetadata']['KeyId'] + ) + assert response['KeyId'] == 'cancel-key-deletion' + + result = client.describe_key(KeyId='cancel-key-deletion') + assert result["KeyMetadata"]["Enabled"] == False + assert result["KeyMetadata"]["KeyState"] == 'Disabled' + assert 'DeletionDate' not in result["KeyMetadata"] From 15c24e49f0633b7cbe09378dfe22dc94e9190e1f Mon Sep 17 00:00:00 2001 From: Jon Beilke Date: Fri, 5 Oct 2018 16:00:20 -0500 Subject: [PATCH 37/72] fix formatting for including DeletionDate in response --- moto/kms/models.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/moto/kms/models.py b/moto/kms/models.py index 30a01d36..4ac6ee84 100644 --- a/moto/kms/models.py +++ b/moto/kms/models.py @@ -41,7 +41,8 @@ class Key(BaseModel): "KeyState": self.key_state, } } - key_dict['KeyMetadata']['DeletionDate'] = self.deletion_date if self.key_state == 'PendingDeletion' + if self.key_state == 'PendingDeletion': + key_dict['KeyMetadata']['DeletionDate'] = self.deletion_date return key_dict def delete(self, region_name): From 7e96203020ea4cd873e0a585884b156896f49265 Mon Sep 17 00:00:00 2001 From: Jon Beilke Date: Fri, 5 Oct 2018 16:21:16 -0500 Subject: [PATCH 38/72] add freezegun and test DeletionDate for chedule_key_deletion --- moto/kms/models.py | 1 + tests/test_kms/test_kms.py | 26 +++++++++++++++----------- 2 files changed, 16 insertions(+), 11 deletions(-) diff --git a/moto/kms/models.py b/moto/kms/models.py index 4ac6ee84..113bd173 100644 --- a/moto/kms/models.py +++ b/moto/kms/models.py @@ -4,6 +4,7 @@ import boto.kms from moto.core import BaseBackend, BaseModel from .utils import generate_key_id from collections import defaultdict +from datetime import datetime, timedelta class Key(BaseModel): diff --git a/tests/test_kms/test_kms.py b/tests/test_kms/test_kms.py index 9779f02a..287ef915 100644 --- a/tests/test_kms/test_kms.py +++ b/tests/test_kms/test_kms.py @@ -8,6 +8,8 @@ from boto.kms.exceptions import AlreadyExistsException, NotFoundException import sure # noqa from moto import mock_kms, mock_kms_deprecated from nose.tools import assert_raises +from freezegun import freeze_time +from datetime import datetime, timedelta @mock_kms_deprecated @@ -652,11 +654,12 @@ def test_enable_key(): def test_schedule_key_deletion(): client = boto3.client('kms', region_name='us-east-1') key = client.create_key(description='schedule-key-deletion') - response = client.schedule_key_deletion( - KeyId=key['KeyMetadata']['KeyId'] - ) - assert response['KeyId'] == 'schedule-key-deletion' - assert response['DeletionDate'] == datetime.now() + timedelta(days=30) + with freeze_time("2015-01-01 12:00:00"): + response = client.schedule_key_deletion( + KeyId=key['KeyMetadata']['KeyId'] + ) + assert response['KeyId'] == 'schedule-key-deletion' + assert response['DeletionDate'] == datetime.now() + timedelta(days=30) result = client.describe_key(KeyId='schedule-key-deletion') assert result["KeyMetadata"]["Enabled"] == False @@ -668,12 +671,13 @@ def test_schedule_key_deletion(): def test_schedule_key_deletion_custom(): client = boto3.client('kms', region_name='us-east-1') key = client.create_key(description='schedule-key-deletion') - response = client.schedule_key_deletion( - KeyId=key['KeyMetadata']['KeyId'], - PendingWindowInDays=7 - ) - assert response['KeyId'] == 'schedule-key-deletion' - assert response['DeletionDate'] == datetime.now() + timedelta(days=7) + with freeze_time("2015-01-01 12:00:00"): + response = client.schedule_key_deletion( + KeyId=key['KeyMetadata']['KeyId'], + PendingWindowInDays=7 + ) + assert response['KeyId'] == 'schedule-key-deletion' + assert response['DeletionDate'] == datetime.now() + timedelta(days=7) result = client.describe_key(KeyId='schedule-key-deletion') assert result["KeyMetadata"]["Enabled"] == False From 695b4349ba62865cd3e9987af65c39a761f2c35b Mon Sep 17 00:00:00 2001 From: Jon Beilke Date: Fri, 5 Oct 2018 21:43:12 -0500 Subject: [PATCH 39/72] indentation fix --- moto/kms/responses.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/moto/kms/responses.py b/moto/kms/responses.py index e782d862..dc37c8b5 100644 --- a/moto/kms/responses.py +++ b/moto/kms/responses.py @@ -271,8 +271,8 @@ class KmsResponse(BaseResponse): _assert_valid_key_id(self.kms_backend.get_key_id(key_id)) try: return json.dumps({ - 'KeyId': key_id, - 'DeletionDate': self.kms_backend.schedule_key_deletion(key_id) + 'KeyId': key_id, + 'DeletionDate': self.kms_backend.schedule_key_deletion(key_id) }) except KeyError: raise JSONResponseError(404, 'Not Found', body={ From a29daf411baf056f3fc924abae7e7bcbe224f5cc Mon Sep 17 00:00:00 2001 From: Jon Beilke Date: Fri, 5 Oct 2018 21:56:32 -0500 Subject: [PATCH 40/72] fix invalid variables used in kms testing --- tests/test_kms/test_kms.py | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/tests/test_kms/test_kms.py b/tests/test_kms/test_kms.py index 287ef915..ab7513b4 100644 --- a/tests/test_kms/test_kms.py +++ b/tests/test_kms/test_kms.py @@ -623,8 +623,8 @@ def test_kms_encrypt_boto3(): @mock_kms def test_disable_key(): - client = boto3.client('kms', region_name='us-east-1') - key = client.create_key(description='disable-key') + client = boto3.client('kms') + key = client.create_key(Description='disable-key') client.disable_key( KeyId=key['KeyMetadata']['KeyId'] ) @@ -636,8 +636,8 @@ def test_disable_key(): @mock_kms def test_enable_key(): - client = boto3.client('kms', region_name='us-east-1') - key = client.create_key(description='enable-key') + client = boto3.client('kms') + key = client.create_key(Description='enable-key') client.disable_key( KeyId=key['KeyMetadata']['KeyId'] ) @@ -652,8 +652,8 @@ def test_enable_key(): @mock_kms def test_schedule_key_deletion(): - client = boto3.client('kms', region_name='us-east-1') - key = client.create_key(description='schedule-key-deletion') + client = boto3.client('kms') + key = client.create_key(Description='schedule-key-deletion') with freeze_time("2015-01-01 12:00:00"): response = client.schedule_key_deletion( KeyId=key['KeyMetadata']['KeyId'] @@ -669,8 +669,8 @@ def test_schedule_key_deletion(): @mock_kms def test_schedule_key_deletion_custom(): - client = boto3.client('kms', region_name='us-east-1') - key = client.create_key(description='schedule-key-deletion') + client = boto3.client('kms') + key = client.create_key(Description='schedule-key-deletion') with freeze_time("2015-01-01 12:00:00"): response = client.schedule_key_deletion( KeyId=key['KeyMetadata']['KeyId'], @@ -687,8 +687,8 @@ def test_schedule_key_deletion_custom(): @mock_kms def test_cancel_key_deletion(): - client = boto3.client('kms', region_name='us-east-1') - key = client.create_key(description='cancel-key-deletion') + client = boto3.client('kms') + key = client.create_key(Description='cancel-key-deletion') client.schedule_key_deletion( KeyId=key['KeyMetadata']['KeyId'] ) From 786b9ca519f830424bdddbcb69e0322f40c0d9b0 Mon Sep 17 00:00:00 2001 From: Jon Beilke Date: Fri, 5 Oct 2018 22:17:48 -0500 Subject: [PATCH 41/72] need region for kms client --- tests/test_kms/test_kms.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tests/test_kms/test_kms.py b/tests/test_kms/test_kms.py index ab7513b4..0dc93b25 100644 --- a/tests/test_kms/test_kms.py +++ b/tests/test_kms/test_kms.py @@ -623,7 +623,7 @@ def test_kms_encrypt_boto3(): @mock_kms def test_disable_key(): - client = boto3.client('kms') + client = boto3.client('kms', region_name='us-east-1') key = client.create_key(Description='disable-key') client.disable_key( KeyId=key['KeyMetadata']['KeyId'] @@ -636,7 +636,7 @@ def test_disable_key(): @mock_kms def test_enable_key(): - client = boto3.client('kms') + client = boto3.client('kms', region_name='us-east-1') key = client.create_key(Description='enable-key') client.disable_key( KeyId=key['KeyMetadata']['KeyId'] @@ -652,7 +652,7 @@ def test_enable_key(): @mock_kms def test_schedule_key_deletion(): - client = boto3.client('kms') + client = boto3.client('kms', region_name='us-east-1') key = client.create_key(Description='schedule-key-deletion') with freeze_time("2015-01-01 12:00:00"): response = client.schedule_key_deletion( @@ -669,7 +669,7 @@ def test_schedule_key_deletion(): @mock_kms def test_schedule_key_deletion_custom(): - client = boto3.client('kms') + client = boto3.client('kms', region_name='us-east-1') key = client.create_key(Description='schedule-key-deletion') with freeze_time("2015-01-01 12:00:00"): response = client.schedule_key_deletion( @@ -687,7 +687,7 @@ def test_schedule_key_deletion_custom(): @mock_kms def test_cancel_key_deletion(): - client = boto3.client('kms') + client = boto3.client('kms', region_name='us-east-1') key = client.create_key(Description='cancel-key-deletion') client.schedule_key_deletion( KeyId=key['KeyMetadata']['KeyId'] From 372f749831344f979d349238e15585f68dabc5ca Mon Sep 17 00:00:00 2001 From: Jon Beilke Date: Fri, 5 Oct 2018 22:46:19 -0500 Subject: [PATCH 42/72] format DeletionDate properly for JSON serialization --- moto/kms/responses.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/moto/kms/responses.py b/moto/kms/responses.py index dc37c8b5..fe7d2852 100644 --- a/moto/kms/responses.py +++ b/moto/kms/responses.py @@ -9,6 +9,7 @@ from boto.exception import JSONResponseError from boto.kms.exceptions import AlreadyExistsException, NotFoundException from moto.core.responses import BaseResponse +from moto.core.utils import iso_8601_datetime_without_milliseconds from .models import kms_backends reserved_aliases = [ @@ -272,7 +273,7 @@ class KmsResponse(BaseResponse): try: return json.dumps({ 'KeyId': key_id, - 'DeletionDate': self.kms_backend.schedule_key_deletion(key_id) + 'DeletionDate': iso_8601_datetime_without_milliseconds(self.kms_backend.schedule_key_deletion(key_id)) }) except KeyError: raise JSONResponseError(404, 'Not Found', body={ From f596069dabd1852f67f3a8a63e0e01029089cb43 Mon Sep 17 00:00:00 2001 From: Jon Beilke Date: Fri, 5 Oct 2018 23:35:34 -0500 Subject: [PATCH 43/72] use initial KeyMetadata for identifying keys in KMS tests --- tests/test_kms/test_kms.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/tests/test_kms/test_kms.py b/tests/test_kms/test_kms.py index 0dc93b25..69cd508b 100644 --- a/tests/test_kms/test_kms.py +++ b/tests/test_kms/test_kms.py @@ -658,10 +658,10 @@ def test_schedule_key_deletion(): response = client.schedule_key_deletion( KeyId=key['KeyMetadata']['KeyId'] ) - assert response['KeyId'] == 'schedule-key-deletion' + assert response['KeyId'] == key['KeyMetadata']['KeyId'] assert response['DeletionDate'] == datetime.now() + timedelta(days=30) - result = client.describe_key(KeyId='schedule-key-deletion') + result = client.describe_key(KeyId=key['KeyMetadata']['KeyId']) assert result["KeyMetadata"]["Enabled"] == False assert result["KeyMetadata"]["KeyState"] == 'PendingDeletion' assert 'DeletionDate' in result["KeyMetadata"] @@ -676,10 +676,10 @@ def test_schedule_key_deletion_custom(): KeyId=key['KeyMetadata']['KeyId'], PendingWindowInDays=7 ) - assert response['KeyId'] == 'schedule-key-deletion' + assert response['KeyId'] == key['KeyMetadata']['KeyId'] assert response['DeletionDate'] == datetime.now() + timedelta(days=7) - result = client.describe_key(KeyId='schedule-key-deletion') + result = client.describe_key(KeyId=key['KeyMetadata']['KeyId']) assert result["KeyMetadata"]["Enabled"] == False assert result["KeyMetadata"]["KeyState"] == 'PendingDeletion' assert 'DeletionDate' in result["KeyMetadata"] @@ -695,9 +695,9 @@ def test_cancel_key_deletion(): response = client.cancel_key_deletion( KeyId=key['KeyMetadata']['KeyId'] ) - assert response['KeyId'] == 'cancel-key-deletion' + assert response['KeyId'] == key['KeyMetadata']['KeyId'] - result = client.describe_key(KeyId='cancel-key-deletion') + result = client.describe_key(KeyId=key['KeyMetadata']['KeyId']) assert result["KeyMetadata"]["Enabled"] == False assert result["KeyMetadata"]["KeyState"] == 'Disabled' assert 'DeletionDate' not in result["KeyMetadata"] From 6277983e3f01daa19247e0f217941fd2d3ad4b1d Mon Sep 17 00:00:00 2001 From: Jon Beilke Date: Fri, 5 Oct 2018 23:48:19 -0500 Subject: [PATCH 44/72] missed some KeyMetadata and need to transform datetime for testing --- tests/test_kms/test_kms.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/tests/test_kms/test_kms.py b/tests/test_kms/test_kms.py index 69cd508b..a06a0732 100644 --- a/tests/test_kms/test_kms.py +++ b/tests/test_kms/test_kms.py @@ -7,6 +7,7 @@ from boto.exception import JSONResponseError from boto.kms.exceptions import AlreadyExistsException, NotFoundException import sure # noqa from moto import mock_kms, mock_kms_deprecated +from moto.core.utils import iso_8601_datetime_without_milliseconds from nose.tools import assert_raises from freezegun import freeze_time from datetime import datetime, timedelta @@ -629,7 +630,7 @@ def test_disable_key(): KeyId=key['KeyMetadata']['KeyId'] ) - result = client.describe_key(KeyId='disable-key') + result = client.describe_key(KeyId=key['KeyMetadata']['KeyId']) assert result["KeyMetadata"]["Enabled"] == False assert result["KeyMetadata"]["KeyState"] == 'Disabled' @@ -645,7 +646,7 @@ def test_enable_key(): KeyId=key['KeyMetadata']['KeyId'] ) - result = client.describe_key(KeyId='enable-key') + result = client.describe_key(KeyId=key['KeyMetadata']['KeyId']) assert result["KeyMetadata"]["Enabled"] == True assert result["KeyMetadata"]["KeyState"] == 'Enabled' @@ -659,7 +660,7 @@ def test_schedule_key_deletion(): KeyId=key['KeyMetadata']['KeyId'] ) assert response['KeyId'] == key['KeyMetadata']['KeyId'] - assert response['DeletionDate'] == datetime.now() + timedelta(days=30) + assert response['DeletionDate'] == iso_8601_datetime_without_milliseconds(datetime.now() + timedelta(days=30)) result = client.describe_key(KeyId=key['KeyMetadata']['KeyId']) assert result["KeyMetadata"]["Enabled"] == False @@ -677,7 +678,7 @@ def test_schedule_key_deletion_custom(): PendingWindowInDays=7 ) assert response['KeyId'] == key['KeyMetadata']['KeyId'] - assert response['DeletionDate'] == datetime.now() + timedelta(days=7) + assert response['DeletionDate'] == iso_8601_datetime_without_milliseconds(datetime.now() + timedelta(days=7)) result = client.describe_key(KeyId=key['KeyMetadata']['KeyId']) assert result["KeyMetadata"]["Enabled"] == False From 21c8914efe65465ef90351dfa13441bf74e67427 Mon Sep 17 00:00:00 2001 From: Jon Beilke Date: Sat, 6 Oct 2018 00:13:47 -0500 Subject: [PATCH 45/72] include pending days input for schedule key deletion and update tests since boto client returns DeletionDate as datetime --- moto/kms/responses.py | 3 ++- tests/test_kms/test_kms.py | 5 ++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/moto/kms/responses.py b/moto/kms/responses.py index fe7d2852..831045e3 100644 --- a/moto/kms/responses.py +++ b/moto/kms/responses.py @@ -269,11 +269,12 @@ class KmsResponse(BaseResponse): def schedule_key_deletion(self): key_id = self.parameters.get('KeyId') + pending_window_in_days = self.parameters.get('PendingWindowInDays') _assert_valid_key_id(self.kms_backend.get_key_id(key_id)) try: return json.dumps({ 'KeyId': key_id, - 'DeletionDate': iso_8601_datetime_without_milliseconds(self.kms_backend.schedule_key_deletion(key_id)) + 'DeletionDate': iso_8601_datetime_without_milliseconds(self.kms_backend.schedule_key_deletion(key_id, pending_window_in_days)) }) except KeyError: raise JSONResponseError(404, 'Not Found', body={ diff --git a/tests/test_kms/test_kms.py b/tests/test_kms/test_kms.py index a06a0732..218df1d8 100644 --- a/tests/test_kms/test_kms.py +++ b/tests/test_kms/test_kms.py @@ -7,7 +7,6 @@ from boto.exception import JSONResponseError from boto.kms.exceptions import AlreadyExistsException, NotFoundException import sure # noqa from moto import mock_kms, mock_kms_deprecated -from moto.core.utils import iso_8601_datetime_without_milliseconds from nose.tools import assert_raises from freezegun import freeze_time from datetime import datetime, timedelta @@ -660,7 +659,7 @@ def test_schedule_key_deletion(): KeyId=key['KeyMetadata']['KeyId'] ) assert response['KeyId'] == key['KeyMetadata']['KeyId'] - assert response['DeletionDate'] == iso_8601_datetime_without_milliseconds(datetime.now() + timedelta(days=30)) + assert response['DeletionDate'] == datetime.now() + timedelta(days=30) result = client.describe_key(KeyId=key['KeyMetadata']['KeyId']) assert result["KeyMetadata"]["Enabled"] == False @@ -678,7 +677,7 @@ def test_schedule_key_deletion_custom(): PendingWindowInDays=7 ) assert response['KeyId'] == key['KeyMetadata']['KeyId'] - assert response['DeletionDate'] == iso_8601_datetime_without_milliseconds(datetime.now() + timedelta(days=7)) + assert response['DeletionDate'] == datetime.now() + timedelta(days=7) result = client.describe_key(KeyId=key['KeyMetadata']['KeyId']) assert result["KeyMetadata"]["Enabled"] == False From 59c233f43158d453fd9f2392a8db4d451e34a46d Mon Sep 17 00:00:00 2001 From: Jon Beilke Date: Sat, 6 Oct 2018 00:33:23 -0500 Subject: [PATCH 46/72] avoid needing to import datetime and dealing with timezone vs naive datetimes in tests --- tests/test_kms/test_kms.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/tests/test_kms/test_kms.py b/tests/test_kms/test_kms.py index 218df1d8..b09459e1 100644 --- a/tests/test_kms/test_kms.py +++ b/tests/test_kms/test_kms.py @@ -9,7 +9,6 @@ import sure # noqa from moto import mock_kms, mock_kms_deprecated from nose.tools import assert_raises from freezegun import freeze_time -from datetime import datetime, timedelta @mock_kms_deprecated @@ -659,7 +658,7 @@ def test_schedule_key_deletion(): KeyId=key['KeyMetadata']['KeyId'] ) assert response['KeyId'] == key['KeyMetadata']['KeyId'] - assert response['DeletionDate'] == datetime.now() + timedelta(days=30) + assert response['DeletionDate'] == '2015-01-31T12:00:00.000Z' result = client.describe_key(KeyId=key['KeyMetadata']['KeyId']) assert result["KeyMetadata"]["Enabled"] == False @@ -677,7 +676,7 @@ def test_schedule_key_deletion_custom(): PendingWindowInDays=7 ) assert response['KeyId'] == key['KeyMetadata']['KeyId'] - assert response['DeletionDate'] == datetime.now() + timedelta(days=7) + assert response['DeletionDate'] == '2015-01-08T12:00:00.000Z' result = client.describe_key(KeyId=key['KeyMetadata']['KeyId']) assert result["KeyMetadata"]["Enabled"] == False From 9b25d56a3585fcc5a90a8e606b63cacf170d4351 Mon Sep 17 00:00:00 2001 From: Jon Beilke Date: Sat, 6 Oct 2018 01:18:26 -0500 Subject: [PATCH 47/72] need datetime for tests since thats what boto3 returns and add default for PendingWindowInDays --- moto/kms/models.py | 2 +- moto/kms/responses.py | 5 ++++- tests/test_kms/test_kms.py | 5 +++-- 3 files changed, 8 insertions(+), 4 deletions(-) diff --git a/moto/kms/models.py b/moto/kms/models.py index 113bd173..01b3d971 100644 --- a/moto/kms/models.py +++ b/moto/kms/models.py @@ -160,7 +160,7 @@ class KmsBackend(BaseBackend): self.keys[key_id].key_state = 'Disabled' self.keys[key_id].deletion_date = None - def schedule_key_deletion(self, key_id, pending_window_in_days=30): + def schedule_key_deletion(self, key_id, pending_window_in_days): if key_id in self.keys: if 7 <= pending_window_in_days <= 30: self.keys[key_id].enabled = False diff --git a/moto/kms/responses.py b/moto/kms/responses.py index 831045e3..7caeafca 100644 --- a/moto/kms/responses.py +++ b/moto/kms/responses.py @@ -269,7 +269,10 @@ class KmsResponse(BaseResponse): def schedule_key_deletion(self): key_id = self.parameters.get('KeyId') - pending_window_in_days = self.parameters.get('PendingWindowInDays') + if self.parameters.get('PendingWindowInDays') is None: + pending_window_in_days = 30 + else: + pending_window_in_days = self.parameters.get('PendingWindowInDays') _assert_valid_key_id(self.kms_backend.get_key_id(key_id)) try: return json.dumps({ diff --git a/tests/test_kms/test_kms.py b/tests/test_kms/test_kms.py index b09459e1..e16bd2ce 100644 --- a/tests/test_kms/test_kms.py +++ b/tests/test_kms/test_kms.py @@ -9,6 +9,7 @@ import sure # noqa from moto import mock_kms, mock_kms_deprecated from nose.tools import assert_raises from freezegun import freeze_time +from datetime import datetime, timedelta @mock_kms_deprecated @@ -658,7 +659,7 @@ def test_schedule_key_deletion(): KeyId=key['KeyMetadata']['KeyId'] ) assert response['KeyId'] == key['KeyMetadata']['KeyId'] - assert response['DeletionDate'] == '2015-01-31T12:00:00.000Z' + assert response['DeletionDate'] == datetime(2015, 1, 31, 12, 0, tzinfo=tzlocal()) result = client.describe_key(KeyId=key['KeyMetadata']['KeyId']) assert result["KeyMetadata"]["Enabled"] == False @@ -676,7 +677,7 @@ def test_schedule_key_deletion_custom(): PendingWindowInDays=7 ) assert response['KeyId'] == key['KeyMetadata']['KeyId'] - assert response['DeletionDate'] == '2015-01-08T12:00:00.000Z' + assert response['DeletionDate'] == datetime(2015, 1, 8, 12, 0, tzinfo=tzlocal()) result = client.describe_key(KeyId=key['KeyMetadata']['KeyId']) assert result["KeyMetadata"]["Enabled"] == False From 76baab74ad6a1c4ebe8d3037bf202ca473187875 Mon Sep 17 00:00:00 2001 From: Jon Beilke Date: Sat, 6 Oct 2018 01:33:02 -0500 Subject: [PATCH 48/72] missing tzlocal --- tests/test_kms/test_kms.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/test_kms/test_kms.py b/tests/test_kms/test_kms.py index e16bd2ce..159f45af 100644 --- a/tests/test_kms/test_kms.py +++ b/tests/test_kms/test_kms.py @@ -10,6 +10,7 @@ from moto import mock_kms, mock_kms_deprecated from nose.tools import assert_raises from freezegun import freeze_time from datetime import datetime, timedelta +from dateutil.tz import tzlocal @mock_kms_deprecated From 398dcd8230d8a2e235527bb72a0766b02d3cd6fb Mon Sep 17 00:00:00 2001 From: Jon Beilke Date: Sat, 6 Oct 2018 01:47:22 -0500 Subject: [PATCH 49/72] transform DeletionDate in model instead to accomodate Key.to_dict --- moto/kms/models.py | 5 +++-- moto/kms/responses.py | 3 +-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/moto/kms/models.py b/moto/kms/models.py index 01b3d971..bb39d1b2 100644 --- a/moto/kms/models.py +++ b/moto/kms/models.py @@ -2,6 +2,7 @@ from __future__ import unicode_literals import boto.kms from moto.core import BaseBackend, BaseModel +from moto.core.utils import iso_8601_datetime_without_milliseconds from .utils import generate_key_id from collections import defaultdict from datetime import datetime, timedelta @@ -43,7 +44,7 @@ class Key(BaseModel): } } if self.key_state == 'PendingDeletion': - key_dict['KeyMetadata']['DeletionDate'] = self.deletion_date + key_dict['KeyMetadata']['DeletionDate'] = iso_8601_datetime_without_milliseconds(self.deletion_date) return key_dict def delete(self, region_name): @@ -166,7 +167,7 @@ class KmsBackend(BaseBackend): self.keys[key_id].enabled = False self.keys[key_id].key_state = 'PendingDeletion' self.keys[key_id].deletion_date = datetime.now() + timedelta(days=pending_window_in_days) - return self.keys[key_id].deletion_date + return iso_8601_datetime_without_milliseconds(self.keys[key_id].deletion_date) kms_backends = {} diff --git a/moto/kms/responses.py b/moto/kms/responses.py index 7caeafca..5883f51e 100644 --- a/moto/kms/responses.py +++ b/moto/kms/responses.py @@ -9,7 +9,6 @@ from boto.exception import JSONResponseError from boto.kms.exceptions import AlreadyExistsException, NotFoundException from moto.core.responses import BaseResponse -from moto.core.utils import iso_8601_datetime_without_milliseconds from .models import kms_backends reserved_aliases = [ @@ -277,7 +276,7 @@ class KmsResponse(BaseResponse): try: return json.dumps({ 'KeyId': key_id, - 'DeletionDate': iso_8601_datetime_without_milliseconds(self.kms_backend.schedule_key_deletion(key_id, pending_window_in_days)) + 'DeletionDate': self.kms_backend.schedule_key_deletion(key_id, pending_window_in_days) }) except KeyError: raise JSONResponseError(404, 'Not Found', body={ From c2595b2eef9c147ccc03d2ce7ce2b59b6cec4d35 Mon Sep 17 00:00:00 2001 From: Jon Beilke Date: Mon, 8 Oct 2018 08:29:21 -0500 Subject: [PATCH 50/72] cant manipulate time in server mode tests --- tests/test_kms/test_kms.py | 23 +++++++++++++++++++---- 1 file changed, 19 insertions(+), 4 deletions(-) diff --git a/tests/test_kms/test_kms.py b/tests/test_kms/test_kms.py index 159f45af..4f8503ce 100644 --- a/tests/test_kms/test_kms.py +++ b/tests/test_kms/test_kms.py @@ -655,12 +655,19 @@ def test_enable_key(): def test_schedule_key_deletion(): client = boto3.client('kms', region_name='us-east-1') key = client.create_key(Description='schedule-key-deletion') - with freeze_time("2015-01-01 12:00:00"): + if os.environ.get('TEST_SERVER_MODE', 'false').lower() == 'false': + with freeze_time("2015-01-01 12:00:00"): + response = client.schedule_key_deletion( + KeyId=key['KeyMetadata']['KeyId'] + ) + assert response['KeyId'] == key['KeyMetadata']['KeyId'] + assert response['DeletionDate'] == datetime(2015, 1, 31, 12, 0, tzinfo=tzlocal()) + else: + # Can't manipulate time in server mode response = client.schedule_key_deletion( KeyId=key['KeyMetadata']['KeyId'] ) assert response['KeyId'] == key['KeyMetadata']['KeyId'] - assert response['DeletionDate'] == datetime(2015, 1, 31, 12, 0, tzinfo=tzlocal()) result = client.describe_key(KeyId=key['KeyMetadata']['KeyId']) assert result["KeyMetadata"]["Enabled"] == False @@ -672,13 +679,21 @@ def test_schedule_key_deletion(): def test_schedule_key_deletion_custom(): client = boto3.client('kms', region_name='us-east-1') key = client.create_key(Description='schedule-key-deletion') - with freeze_time("2015-01-01 12:00:00"): + if os.environ.get('TEST_SERVER_MODE', 'false').lower() == 'false': + with freeze_time("2015-01-01 12:00:00"): + response = client.schedule_key_deletion( + KeyId=key['KeyMetadata']['KeyId'], + PendingWindowInDays=7 + ) + assert response['KeyId'] == key['KeyMetadata']['KeyId'] + assert response['DeletionDate'] == datetime(2015, 1, 8, 12, 0, tzinfo=tzlocal()) + else: + # Can't manipulate time in server mode response = client.schedule_key_deletion( KeyId=key['KeyMetadata']['KeyId'], PendingWindowInDays=7 ) assert response['KeyId'] == key['KeyMetadata']['KeyId'] - assert response['DeletionDate'] == datetime(2015, 1, 8, 12, 0, tzinfo=tzlocal()) result = client.describe_key(KeyId=key['KeyMetadata']['KeyId']) assert result["KeyMetadata"]["Enabled"] == False From 181e9690b84db61f180d31f30ea3e92776cb9a49 Mon Sep 17 00:00:00 2001 From: Jon Beilke Date: Mon, 8 Oct 2018 08:38:49 -0500 Subject: [PATCH 51/72] need os for checking server mode env variable --- tests/test_kms/test_kms.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_kms/test_kms.py b/tests/test_kms/test_kms.py index 4f8503ce..8bccae27 100644 --- a/tests/test_kms/test_kms.py +++ b/tests/test_kms/test_kms.py @@ -1,5 +1,5 @@ from __future__ import unicode_literals -import re +import os, re import boto3 import boto.kms From c1ebec1b352cf29b9ba666019954b6667757d738 Mon Sep 17 00:00:00 2001 From: Jon Beilke Date: Mon, 8 Oct 2018 10:17:51 -0500 Subject: [PATCH 52/72] remove start_time from attrib comparison in test_copy_snapshot() --- tests/test_ec2/test_elastic_block_store.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_ec2/test_elastic_block_store.py b/tests/test_ec2/test_elastic_block_store.py index 8930838c..442e41dd 100644 --- a/tests/test_ec2/test_elastic_block_store.py +++ b/tests/test_ec2/test_elastic_block_store.py @@ -615,8 +615,8 @@ def test_copy_snapshot(): dest = dest_ec2.Snapshot(copy_snapshot_response['SnapshotId']) attribs = ['data_encryption_key_id', 'encrypted', - 'kms_key_id', 'owner_alias', 'owner_id', 'progress', - 'start_time', 'state', 'state_message', + 'kms_key_id', 'owner_alias', 'owner_id', + 'progress', 'state', 'state_message', 'tags', 'volume_id', 'volume_size'] for attrib in attribs: From d9577f9d3d8c7e4aa32bdb346a7d8a5c4a55d2f4 Mon Sep 17 00:00:00 2001 From: George Alton Date: Mon, 8 Oct 2018 19:04:47 +0100 Subject: [PATCH 53/72] Ensures a UserPool Id starts like {region}_ --- moto/cognitoidp/models.py | 2 +- tests/test_cognitoidp/test_cognitoidp.py | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/moto/cognitoidp/models.py b/moto/cognitoidp/models.py index 52a73f89..db54ded7 100644 --- a/moto/cognitoidp/models.py +++ b/moto/cognitoidp/models.py @@ -24,7 +24,7 @@ class CognitoIdpUserPool(BaseModel): def __init__(self, region, name, extended_config): self.region = region - self.id = str(uuid.uuid4()) + self.id = "{}_{}".format(self.region, str(uuid.uuid4().hex)) self.name = name self.status = None self.extended_config = extended_config or {} diff --git a/tests/test_cognitoidp/test_cognitoidp.py b/tests/test_cognitoidp/test_cognitoidp.py index b2bd469c..76e86a69 100644 --- a/tests/test_cognitoidp/test_cognitoidp.py +++ b/tests/test_cognitoidp/test_cognitoidp.py @@ -24,6 +24,7 @@ def test_create_user_pool(): ) result["UserPool"]["Id"].should_not.be.none + result["UserPool"]["Id"].should.match(r'[\w-]+_[0-9a-zA-Z]+') result["UserPool"]["Name"].should.equal(name) result["UserPool"]["LambdaConfig"]["PreSignUp"].should.equal(value) From 7a88e634eb0e7780a07645faad5545ce2689531d Mon Sep 17 00:00:00 2001 From: Ashley Gould Date: Mon, 8 Oct 2018 15:27:19 -0700 Subject: [PATCH 54/72] organizations: add exception test for describe_organization endpoint --- tests/test_organizations/test_organizations_boto3.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/tests/test_organizations/test_organizations_boto3.py b/tests/test_organizations/test_organizations_boto3.py index ae9bacd8..dfac5fee 100644 --- a/tests/test_organizations/test_organizations_boto3.py +++ b/tests/test_organizations/test_organizations_boto3.py @@ -32,6 +32,17 @@ def test_describe_organization(): validate_organization(response) +@mock_organizations +def test_describe_organization_exception(): + client = boto3.client('organizations', region_name='us-east-1') + with assert_raises(ClientError) as e: + response = client.describe_organization() + ex = e.exception + ex.operation_name.should.equal('DescribeOrganization') + ex.response['Error']['Code'].should.equal('400') + ex.response['Error']['Message'].should.contain('AWSOrganizationsNotInUseException') + + # Organizational Units @mock_organizations From 9081a160d3df9d36c27af5efc87d81457bd74e4f Mon Sep 17 00:00:00 2001 From: Danny Cosson Date: Tue, 9 Oct 2018 10:28:15 -0700 Subject: [PATCH 55/72] fixes for cognito identity library --- moto/cognitoidentity/responses.py | 5 ++++- moto/cognitoidentity/utils.py | 2 +- tests/test_cognitoidentity/test_cognitoidentity.py | 14 ++++++++++++++ 3 files changed, 19 insertions(+), 2 deletions(-) diff --git a/moto/cognitoidentity/responses.py b/moto/cognitoidentity/responses.py index ea54b2cf..e7b42832 100644 --- a/moto/cognitoidentity/responses.py +++ b/moto/cognitoidentity/responses.py @@ -3,6 +3,7 @@ from __future__ import unicode_literals from moto.core.responses import BaseResponse from .models import cognitoidentity_backends +from .utils import get_random_identity_id class CognitoIdentityResponse(BaseResponse): @@ -31,4 +32,6 @@ class CognitoIdentityResponse(BaseResponse): return cognitoidentity_backends[self.region].get_credentials_for_identity(self._get_param('IdentityId')) def get_open_id_token_for_developer_identity(self): - return cognitoidentity_backends[self.region].get_open_id_token_for_developer_identity(self._get_param('IdentityId')) + return cognitoidentity_backends[self.region].get_open_id_token_for_developer_identity( + self._get_param('IdentityId') or get_random_identity_id(self.region) + ) diff --git a/moto/cognitoidentity/utils.py b/moto/cognitoidentity/utils.py index 35963176..6143d512 100644 --- a/moto/cognitoidentity/utils.py +++ b/moto/cognitoidentity/utils.py @@ -2,4 +2,4 @@ from moto.core.utils import get_random_hex def get_random_identity_id(region): - return "{0}:{0}".format(region, get_random_hex(length=19)) + return "{0}:{1}".format(region, get_random_hex(length=19)) diff --git a/tests/test_cognitoidentity/test_cognitoidentity.py b/tests/test_cognitoidentity/test_cognitoidentity.py index a38107b9..ac79fa22 100644 --- a/tests/test_cognitoidentity/test_cognitoidentity.py +++ b/tests/test_cognitoidentity/test_cognitoidentity.py @@ -31,6 +31,7 @@ def test_create_identity_pool(): # testing a helper function def test_get_random_identity_id(): assert len(get_random_identity_id('us-west-2')) > 0 + assert len(get_random_identity_id('us-west-2').split(':')[1]) == 19 @mock_cognitoidentity @@ -69,3 +70,16 @@ def test_get_open_id_token_for_developer_identity(): ) assert len(result['Token']) assert result['IdentityId'] == '12345' + +@mock_cognitoidentity +def test_get_open_id_token_for_developer_identity_when_no_explicit_identity_id(): + conn = boto3.client('cognito-identity', 'us-west-2') + result = conn.get_open_id_token_for_developer_identity( + IdentityPoolId='us-west-2:12345', + Logins={ + 'someurl': '12345' + }, + TokenDuration=123 + ) + assert len(result['Token']) > 0 + assert len(result['IdentityId']) > 0 From 2c15d71c2c872d2d452e16b61d5faf613073338b Mon Sep 17 00:00:00 2001 From: Kosei Kitahara Date: Thu, 11 Oct 2018 18:21:53 +0900 Subject: [PATCH 56/72] Allow spaces to if_not_exists --- moto/dynamodb2/models.py | 2 +- tests/test_dynamodb2/test_dynamodb.py | 6 ++++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/moto/dynamodb2/models.py b/moto/dynamodb2/models.py index b327c7a4..0fa96f3b 100644 --- a/moto/dynamodb2/models.py +++ b/moto/dynamodb2/models.py @@ -154,7 +154,7 @@ class Item(BaseModel): # If not exists, changes value to a default if needed, else its the same as it was if value.startswith('if_not_exists'): # Function signature - match = re.match(r'.*if_not_exists\((?P.+),\s*(?P.+)\).*', value) + match = re.match(r'.*if_not_exists\s*\((?P.+),\s*(?P.+)\).*', value) if not match: raise TypeError diff --git a/tests/test_dynamodb2/test_dynamodb.py b/tests/test_dynamodb2/test_dynamodb.py index 243de270..70feaf7f 100644 --- a/tests/test_dynamodb2/test_dynamodb.py +++ b/tests/test_dynamodb2/test_dynamodb.py @@ -1220,7 +1220,8 @@ def test_update_if_not_exists(): 'forum_name': 'the-key', 'subject': '123' }, - UpdateExpression='SET created_at = if_not_exists(created_at, :created_at)', + # if_not_exists without space + UpdateExpression='SET created_at=if_not_exists(created_at,:created_at)', ExpressionAttributeValues={ ':created_at': 123 } @@ -1233,7 +1234,8 @@ def test_update_if_not_exists(): 'forum_name': 'the-key', 'subject': '123' }, - UpdateExpression='SET created_at = if_not_exists(created_at, :created_at)', + # if_not_exists with space + UpdateExpression='SET created_at = if_not_exists (created_at, :created_at)', ExpressionAttributeValues={ ':created_at': 456 } From cf157287e707117447ccb637d8ffbf642eaad240 Mon Sep 17 00:00:00 2001 From: Kosei Kitahara Date: Fri, 12 Oct 2018 16:08:05 +0900 Subject: [PATCH 57/72] Fix wrong type if exists --- moto/dynamodb2/models.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/moto/dynamodb2/models.py b/moto/dynamodb2/models.py index b327c7a4..02b2933f 100644 --- a/moto/dynamodb2/models.py +++ b/moto/dynamodb2/models.py @@ -162,12 +162,13 @@ class Item(BaseModel): # If it already exists, get its value so we dont overwrite it if path in self.attrs: - value = self.attrs[path].cast_value + value = self.attrs[path] - if value in expression_attribute_values: - value = DynamoType(expression_attribute_values[value]) - else: - value = DynamoType({"S": value}) + if type(value) != DynamoType: + if value in expression_attribute_values: + value = DynamoType(expression_attribute_values[value]) + else: + value = DynamoType({"S": value}) if '.' not in key: self.attrs[key] = value From 13c2e699328eebca5ae8d92797a9fc031b461752 Mon Sep 17 00:00:00 2001 From: Kosei Kitahara Date: Fri, 12 Oct 2018 16:59:52 +0900 Subject: [PATCH 58/72] Allow extra spaces to attribute_exists and attribute_not_exists too --- moto/dynamodb2/responses.py | 8 ++++---- tests/test_dynamodb2/test_dynamodb.py | 4 ++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/moto/dynamodb2/responses.py b/moto/dynamodb2/responses.py index 493e1783..e2f1ef1c 100644 --- a/moto/dynamodb2/responses.py +++ b/moto/dynamodb2/responses.py @@ -204,9 +204,9 @@ class DynamoHandler(BaseResponse): if cond_items: expected = {} overwrite = False - exists_re = re.compile('^attribute_exists\((.*)\)$') + exists_re = re.compile('^attribute_exists\s*\((.*)\)$') not_exists_re = re.compile( - '^attribute_not_exists\((.*)\)$') + '^attribute_not_exists\s*\((.*)\)$') for cond in cond_items: exists_m = exists_re.match(cond) @@ -556,9 +556,9 @@ class DynamoHandler(BaseResponse): if cond_items: expected = {} - exists_re = re.compile('^attribute_exists\((.*)\)$') + exists_re = re.compile('^attribute_exists\s*\((.*)\)$') not_exists_re = re.compile( - '^attribute_not_exists\((.*)\)$') + '^attribute_not_exists\s*\((.*)\)$') for cond in cond_items: exists_m = exists_re.match(cond) diff --git a/tests/test_dynamodb2/test_dynamodb.py b/tests/test_dynamodb2/test_dynamodb.py index 70feaf7f..afc919dd 100644 --- a/tests/test_dynamodb2/test_dynamodb.py +++ b/tests/test_dynamodb2/test_dynamodb.py @@ -700,8 +700,8 @@ def test_filter_expression(): filter_expr = moto.dynamodb2.comparisons.get_filter_expression('Id IN :v0', {}, {':v0': {'NS': [7, 8, 9]}}) filter_expr.expr(row1).should.be(True) - # attribute function tests - filter_expr = moto.dynamodb2.comparisons.get_filter_expression('attribute_exists(Id) AND attribute_not_exists(User)', {}, {}) + # attribute function tests (with extra spaces) + filter_expr = moto.dynamodb2.comparisons.get_filter_expression('attribute_exists(Id) AND attribute_not_exists (User)', {}, {}) filter_expr.expr(row1).should.be(True) filter_expr = moto.dynamodb2.comparisons.get_filter_expression('attribute_type(Id, N)', {}, {}) From 27ca96519b5517fa51b954972ee95c46f77d8782 Mon Sep 17 00:00:00 2001 From: Steve Pulec Date: Mon, 15 Oct 2018 01:37:25 -0400 Subject: [PATCH 59/72] Fix extra whitespace in s3. Closes #1844. --- moto/s3/responses.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/moto/s3/responses.py b/moto/s3/responses.py index de101a19..962025cb 100755 --- a/moto/s3/responses.py +++ b/moto/s3/responses.py @@ -1449,7 +1449,7 @@ S3_MULTIPART_LIST_RESPONSE = """ STANDARD 1 - {{ count }} + {{ count }} {{ count }} false {% for part in parts %} From 81f96c4ceb1934f317901bea98012bb8926839c6 Mon Sep 17 00:00:00 2001 From: Gary Donovan Date: Wed, 17 Oct 2018 11:08:44 +1100 Subject: [PATCH 60/72] Don't compare a dict_keys object to a list, since it is always False --- moto/dynamodb2/models.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/moto/dynamodb2/models.py b/moto/dynamodb2/models.py index 63ad20df..a54c4f7d 100644 --- a/moto/dynamodb2/models.py +++ b/moto/dynamodb2/models.py @@ -265,9 +265,9 @@ class Item(BaseModel): self.attrs[attribute_name] = DynamoType({"SS": new_value}) elif isinstance(new_value, dict): self.attrs[attribute_name] = DynamoType({"M": new_value}) - elif update_action['Value'].keys() == ['N']: + elif set(update_action['Value'].keys()) == set(['N']): self.attrs[attribute_name] = DynamoType({"N": new_value}) - elif update_action['Value'].keys() == ['NULL']: + elif set(update_action['Value'].keys()) == set(['NULL']): if attribute_name in self.attrs: del self.attrs[attribute_name] else: From 8ae1a2b357ff9a3d4b8c14b22229e5d6138ec9e5 Mon Sep 17 00:00:00 2001 From: Mike Grima Date: Tue, 16 Oct 2018 17:14:23 -0700 Subject: [PATCH 61/72] Fixes for IAM Groups --- moto/iam/models.py | 10 +++++++++- moto/iam/responses.py | 4 +++- tests/test_iam/test_iam_groups.py | 22 ++++++++++++++++++++++ 3 files changed, 34 insertions(+), 2 deletions(-) diff --git a/moto/iam/models.py b/moto/iam/models.py index 4d884fa2..4a5240a0 100644 --- a/moto/iam/models.py +++ b/moto/iam/models.py @@ -255,7 +255,15 @@ class Group(BaseModel): @property def arn(self): - return "arn:aws:iam::{0}:group/{1}".format(ACCOUNT_ID, self.path) + if self.path == '/': + return "arn:aws:iam::{0}:group/{1}".format(ACCOUNT_ID, self.name) + + else: + return "arn:aws:iam::{0}:group/{1}/{2}".format(ACCOUNT_ID, self.path, self.name) + + @property + def create_date(self): + return self.created def get_policy(self, policy_name): try: diff --git a/moto/iam/responses.py b/moto/iam/responses.py index 9e8d2139..f7b373db 100644 --- a/moto/iam/responses.py +++ b/moto/iam/responses.py @@ -285,7 +285,7 @@ class IamResponse(BaseResponse): def create_group(self): group_name = self._get_param('GroupName') - path = self._get_param('Path') + path = self._get_param('Path', '/') group = iam_backend.create_group(group_name, path) template = self.response_template(CREATE_GROUP_TEMPLATE) @@ -1007,6 +1007,7 @@ CREATE_GROUP_TEMPLATE = """ {{ group.name }} {{ group.id }} {{ group.arn }} + {{ group.create_date }} @@ -1021,6 +1022,7 @@ GET_GROUP_TEMPLATE = """ {{ group.name }} {{ group.id }} {{ group.arn }} + {{ group.create_date }} {% for user in group.users %} diff --git a/tests/test_iam/test_iam_groups.py b/tests/test_iam/test_iam_groups.py index 49c7987f..0d4756f7 100644 --- a/tests/test_iam/test_iam_groups.py +++ b/tests/test_iam/test_iam_groups.py @@ -1,4 +1,7 @@ from __future__ import unicode_literals + +from datetime import datetime + import boto import boto3 import sure # noqa @@ -25,6 +28,25 @@ def test_get_group(): conn.get_group('not-group') +@mock_iam() +def test_get_group_current(): + conn = boto3.client('iam', region_name='us-east-1') + conn.create_group(GroupName='my-group') + result = conn.get_group(GroupName='my-group') + + assert result['Group']['Path'] == '/' + assert result['Group']['GroupName'] == 'my-group' + assert isinstance(result['Group']['CreateDate'], datetime) + assert result['Group']['GroupId'] + assert result['Group']['Arn'] == 'arn:aws:iam::123456789012:group/my-group' + assert not result['Users'] + + # Make a group with a different path: + other_group = conn.create_group(GroupName='my-other-group', Path='some/location') + assert other_group['Group']['Path'] == 'some/location' + assert other_group['Group']['Arn'] == 'arn:aws:iam::123456789012:group/some/location/my-other-group' + + @mock_iam_deprecated() def test_get_all_groups(): conn = boto.connect_iam() From 1b42c7bf7a53cd2cfd755216e6a9a3a4e591167d Mon Sep 17 00:00:00 2001 From: Gary Donovan Date: Fri, 27 Jul 2018 16:10:26 +1000 Subject: [PATCH 62/72] Be able to change `enabled` status for cognito-idp users --- IMPLEMENTATION_COVERAGE.md | 4 +-- moto/cognitoidp/models.py | 8 ++++++ moto/cognitoidp/responses.py | 12 +++++++++ tests/test_cognitoidp/test_cognitoidp.py | 32 ++++++++++++++++++++++++ 4 files changed, 54 insertions(+), 2 deletions(-) diff --git a/IMPLEMENTATION_COVERAGE.md b/IMPLEMENTATION_COVERAGE.md index 17b864dc..7c68c0e3 100644 --- a/IMPLEMENTATION_COVERAGE.md +++ b/IMPLEMENTATION_COVERAGE.md @@ -835,8 +835,8 @@ - [ ] admin_delete_user - [ ] admin_delete_user_attributes - [ ] admin_disable_provider_for_user -- [ ] admin_disable_user -- [ ] admin_enable_user +- [X] admin_disable_user +- [X] admin_enable_user - [ ] admin_forget_device - [ ] admin_get_device - [ ] admin_get_user diff --git a/moto/cognitoidp/models.py b/moto/cognitoidp/models.py index 10da0c6f..4c0132d3 100644 --- a/moto/cognitoidp/models.py +++ b/moto/cognitoidp/models.py @@ -394,6 +394,14 @@ class CognitoIdpBackend(BaseBackend): return user_pool.users.values() + def admin_disable_user(self, user_pool_id, username): + user = self.admin_get_user(user_pool_id, username) + user.enabled = False + + def admin_enable_user(self, user_pool_id, username): + user = self.admin_get_user(user_pool_id, username) + user.enabled = True + def admin_delete_user(self, user_pool_id, username): user_pool = self.user_pools.get(user_pool_id) if not user_pool: diff --git a/moto/cognitoidp/responses.py b/moto/cognitoidp/responses.py index e6f20367..50939786 100644 --- a/moto/cognitoidp/responses.py +++ b/moto/cognitoidp/responses.py @@ -160,6 +160,18 @@ class CognitoIdpResponse(BaseResponse): "Users": [user.to_json(extended=True) for user in users] }) + def admin_disable_user(self): + user_pool_id = self._get_param("UserPoolId") + username = self._get_param("Username") + cognitoidp_backends[self.region].admin_disable_user(user_pool_id, username) + return "" + + def admin_enable_user(self): + user_pool_id = self._get_param("UserPoolId") + username = self._get_param("Username") + cognitoidp_backends[self.region].admin_enable_user(user_pool_id, username) + return "" + def admin_delete_user(self): user_pool_id = self._get_param("UserPoolId") username = self._get_param("Username") diff --git a/tests/test_cognitoidp/test_cognitoidp.py b/tests/test_cognitoidp/test_cognitoidp.py index 56d7c08a..b001d8f0 100644 --- a/tests/test_cognitoidp/test_cognitoidp.py +++ b/tests/test_cognitoidp/test_cognitoidp.py @@ -343,6 +343,7 @@ def test_admin_create_user(): result["User"]["Attributes"].should.have.length_of(1) result["User"]["Attributes"][0]["Name"].should.equal("thing") result["User"]["Attributes"][0]["Value"].should.equal(value) + result["User"]["Enabled"].should.equal(True) @mock_cognitoidp @@ -379,6 +380,37 @@ def test_list_users(): result["Users"][0]["Username"].should.equal(username) +@mock_cognitoidp +def test_admin_disable_user(): + conn = boto3.client("cognito-idp", "us-west-2") + + username = str(uuid.uuid4()) + user_pool_id = conn.create_user_pool(PoolName=str(uuid.uuid4()))["UserPool"]["Id"] + conn.admin_create_user(UserPoolId=user_pool_id, Username=username) + + result = conn.admin_disable_user(UserPoolId=user_pool_id, Username=username) + list(result.keys()).should.equal(["ResponseMetadata"]) # No response expected + + conn.admin_get_user(UserPoolId=user_pool_id, Username=username) \ + ["Enabled"].should.equal(False) + + +@mock_cognitoidp +def test_admin_enable_user(): + conn = boto3.client("cognito-idp", "us-west-2") + + username = str(uuid.uuid4()) + user_pool_id = conn.create_user_pool(PoolName=str(uuid.uuid4()))["UserPool"]["Id"] + conn.admin_create_user(UserPoolId=user_pool_id, Username=username) + conn.admin_disable_user(UserPoolId=user_pool_id, Username=username) + + result = conn.admin_enable_user(UserPoolId=user_pool_id, Username=username) + list(result.keys()).should.equal(["ResponseMetadata"]) # No response expected + + conn.admin_get_user(UserPoolId=user_pool_id, Username=username) \ + ["Enabled"].should.equal(True) + + @mock_cognitoidp def test_admin_delete_user(): conn = boto3.client("cognito-idp", "us-west-2") From d9190245108576103db7eb391d38d44dfe39b6dc Mon Sep 17 00:00:00 2001 From: George Alton Date: Wed, 17 Oct 2018 13:44:00 +0100 Subject: [PATCH 63/72] Adds keyId support to apigateway get_usage_plans apigateway is able to filter the result set, returning only usage plans with the given keyId. This commit supports filtering the usage plans returned to the user by filtering the list of usage plans by checking for usage plan keys --- moto/apigateway/models.py | 11 ++++++-- moto/apigateway/responses.py | 3 ++- tests/test_apigateway/test_apigateway.py | 33 ++++++++++++++++++++++++ 3 files changed, 44 insertions(+), 3 deletions(-) diff --git a/moto/apigateway/models.py b/moto/apigateway/models.py index 4094c7a6..db4746a0 100644 --- a/moto/apigateway/models.py +++ b/moto/apigateway/models.py @@ -606,8 +606,15 @@ class APIGatewayBackend(BaseBackend): self.usage_plans[plan['id']] = plan return plan - def get_usage_plans(self): - return list(self.usage_plans.values()) + def get_usage_plans(self, api_key_id=None): + plans = list(self.usage_plans.values()) + if api_key_id is not None: + plans = [ + plan + for plan in plans + if self.usage_plan_keys.get(plan['id'], {}).get(api_key_id, False) + ] + return plans def get_usage_plan(self, usage_plan_id): return self.usage_plans[usage_plan_id] diff --git a/moto/apigateway/responses.py b/moto/apigateway/responses.py index 7364ae2c..bc4d262c 100644 --- a/moto/apigateway/responses.py +++ b/moto/apigateway/responses.py @@ -255,7 +255,8 @@ class APIGatewayResponse(BaseResponse): if self.method == 'POST': usage_plan_response = self.backend.create_usage_plan(json.loads(self.body)) elif self.method == 'GET': - usage_plans_response = self.backend.get_usage_plans() + api_key_id = self.querystring.get("keyId", [None])[0] + usage_plans_response = self.backend.get_usage_plans(api_key_id=api_key_id) return 200, {}, json.dumps({"item": usage_plans_response}) return 200, {}, json.dumps(usage_plan_response) diff --git a/tests/test_apigateway/test_apigateway.py b/tests/test_apigateway/test_apigateway.py index 8a2c4370..5954de8c 100644 --- a/tests/test_apigateway/test_apigateway.py +++ b/tests/test_apigateway/test_apigateway.py @@ -1084,3 +1084,36 @@ def test_create_usage_plan_key_non_existent_api_key(): # Attempt to create a usage plan key for a API key that doesn't exists payload = {'usagePlanId': usage_plan_id, 'keyId': 'non-existent', 'keyType': 'API_KEY' } client.create_usage_plan_key.when.called_with(**payload).should.throw(ClientError) + + +@mock_apigateway +def test_get_usage_plans_using_key_id(): + region_name = 'us-west-2' + client = boto3.client('apigateway', region_name=region_name) + + # Create 2 Usage Plans + # one will be attached to an API Key, the other will remain unattached + attached_plan = client.create_usage_plan(name='Attached') + unattached_plan = client.create_usage_plan(name='Unattached') + + # Create an API key + # to attach to the usage plan + key_name = 'test-api-key' + response = client.create_api_key(name=key_name) + key_id = response["id"] + + # Create a Usage Plan Key + # Attached the Usage Plan and API Key + key_type = 'API_KEY' + payload = {'usagePlanId': attached_plan['id'], 'keyId': key_id, 'keyType': key_type} + response = client.create_usage_plan_key(**payload) + + # All usage plans should be returned when keyId is not included + all_plans = client.get_usage_plans() + len(all_plans['items']).should.equal(2) + + # Only the usage plan attached to the given api key are included + only_plans_with_key = client.get_usage_plans(keyId=key_id) + len(only_plans_with_key['items']).should.equal(1) + only_plans_with_key['items'][0]['name'].should.equal(attached_plan['name']) + only_plans_with_key['items'][0]['id'].should.equal(attached_plan['id']) From 2d2708cfd7c5a9f1457c94e482d82d7ee5757ffc Mon Sep 17 00:00:00 2001 From: George Alton Date: Wed, 17 Oct 2018 18:39:52 +0100 Subject: [PATCH 64/72] Missing users now raise a UserNotFoundException A missing user in a cognito user pool has raises a UserNotFoundException, not a ResourceNotFoundException. This commit corrects the behaviour so that the correct exception is raised --- moto/cognitoidp/models.py | 4 ++-- tests/test_cognitoidp/test_cognitoidp.py | 18 +++++++++++++++++- 2 files changed, 19 insertions(+), 3 deletions(-) diff --git a/moto/cognitoidp/models.py b/moto/cognitoidp/models.py index 4c0132d3..476d470b 100644 --- a/moto/cognitoidp/models.py +++ b/moto/cognitoidp/models.py @@ -383,7 +383,7 @@ class CognitoIdpBackend(BaseBackend): raise ResourceNotFoundError(user_pool_id) if username not in user_pool.users: - raise ResourceNotFoundError(username) + raise UserNotFoundError(username) return user_pool.users[username] @@ -408,7 +408,7 @@ class CognitoIdpBackend(BaseBackend): raise ResourceNotFoundError(user_pool_id) if username not in user_pool.users: - raise ResourceNotFoundError(username) + raise UserNotFoundError(username) del user_pool.users[username] diff --git a/tests/test_cognitoidp/test_cognitoidp.py b/tests/test_cognitoidp/test_cognitoidp.py index b001d8f0..f72a4476 100644 --- a/tests/test_cognitoidp/test_cognitoidp.py +++ b/tests/test_cognitoidp/test_cognitoidp.py @@ -368,6 +368,22 @@ def test_admin_get_user(): result["UserAttributes"][0]["Value"].should.equal(value) +@mock_cognitoidp +def test_admin_get_missing_user(): + conn = boto3.client("cognito-idp", "us-west-2") + + username = str(uuid.uuid4()) + user_pool_id = conn.create_user_pool(PoolName=str(uuid.uuid4()))["UserPool"]["Id"] + + caught = False + try: + conn.admin_get_user(UserPoolId=user_pool_id, Username=username) + except conn.exceptions.UserNotFoundException: + caught = True + + caught.should.be.true + + @mock_cognitoidp def test_list_users(): conn = boto3.client("cognito-idp", "us-west-2") @@ -423,7 +439,7 @@ def test_admin_delete_user(): caught = False try: conn.admin_get_user(UserPoolId=user_pool_id, Username=username) - except conn.exceptions.ResourceNotFoundException: + except conn.exceptions.UserNotFoundException: caught = True caught.should.be.true From 4a7ed0d43e6dff07c26e434b3df2af55521a7632 Mon Sep 17 00:00:00 2001 From: Will Bengtson Date: Wed, 17 Oct 2018 15:48:13 -0700 Subject: [PATCH 65/72] remove the marker since this is truncated --- moto/iam/responses.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/moto/iam/responses.py b/moto/iam/responses.py index f7b373db..22558f3f 100644 --- a/moto/iam/responses.py +++ b/moto/iam/responses.py @@ -1386,10 +1386,6 @@ GET_ACCOUNT_AUTHORIZATION_DETAILS_TEMPLATE = """ {% endfor %} - - EXAMPLEkakv9BCuUNFDtxWSyfzetYwEx2ADc8dnzfvERF5S6YMvXKx41t6gCl/eeaCX3Jo94/ - bKqezEAg8TEVS99EKFLxm3jtbpl25FDWEXAMPLE - {% for group in groups %} From 8e909f580a13bc87e6281cea8d44c8ccf32cc2ac Mon Sep 17 00:00:00 2001 From: Jordan Guymon Date: Thu, 6 Sep 2018 15:15:27 -0700 Subject: [PATCH 66/72] MockAWS implementation using botocore event hooks --- moto/apigateway/models.py | 4 +- moto/awslambda/responses.py | 6 +-- moto/core/models.py | 86 ++++++++++++++++++++++++++++++++++++- moto/core/utils.py | 11 +++++ moto/s3/responses.py | 8 +++- requirements-dev.txt | 2 +- setup.py | 4 +- 7 files changed, 110 insertions(+), 11 deletions(-) diff --git a/moto/apigateway/models.py b/moto/apigateway/models.py index db4746a0..41a49e36 100644 --- a/moto/apigateway/models.py +++ b/moto/apigateway/models.py @@ -10,6 +10,7 @@ from boto3.session import Session import responses from moto.core import BaseBackend, BaseModel from .utils import create_id +from moto.core.utils import path_url from .exceptions import StageNotFoundException, ApiKeyNotFoundException STAGE_URL = "https://{api_id}.execute-api.{region_name}.amazonaws.com/{stage_name}" @@ -372,7 +373,8 @@ class RestAPI(BaseModel): # TODO deal with no matching resource def resource_callback(self, request): - path_after_stage_name = '/'.join(request.path_url.split("/")[2:]) + path = path_url(request.url) + path_after_stage_name = '/'.join(path.split("/")[2:]) if not path_after_stage_name: path_after_stage_name = '/' diff --git a/moto/awslambda/responses.py b/moto/awslambda/responses.py index 2c8a5452..1a9a4df8 100644 --- a/moto/awslambda/responses.py +++ b/moto/awslambda/responses.py @@ -7,7 +7,7 @@ try: except ImportError: from urllib.parse import unquote -from moto.core.utils import amz_crc32, amzn_request_id +from moto.core.utils import amz_crc32, amzn_request_id, path_url from moto.core.responses import BaseResponse from .models import lambda_backends @@ -94,7 +94,7 @@ class LambdaResponse(BaseResponse): return self._add_policy(request, full_url, headers) def _add_policy(self, request, full_url, headers): - path = request.path if hasattr(request, 'path') else request.path_url + path = request.path if hasattr(request, 'path') else path_url(request.url) function_name = path.split('/')[-2] if self.lambda_backend.get_function(function_name): policy = request.body.decode('utf8') @@ -104,7 +104,7 @@ class LambdaResponse(BaseResponse): return 404, {}, "{}" def _get_policy(self, request, full_url, headers): - path = request.path if hasattr(request, 'path') else request.path_url + path = request.path if hasattr(request, 'path') else path_url(request.url) function_name = path.split('/')[-2] if self.lambda_backend.get_function(function_name): lambda_function = self.lambda_backend.get_function(function_name) diff --git a/moto/core/models.py b/moto/core/models.py index adc06a9c..b12374fd 100644 --- a/moto/core/models.py +++ b/moto/core/models.py @@ -2,11 +2,14 @@ from __future__ import unicode_literals from __future__ import absolute_import -from collections import defaultdict import functools import inspect import re import six +from io import BytesIO +from collections import defaultdict +from botocore.handlers import BUILTIN_HANDLERS +from botocore.awsrequest import AWSResponse from moto import settings import responses @@ -233,7 +236,86 @@ class ResponsesMockAWS(BaseMockAWS): pass -MockAWS = ResponsesMockAWS +BOTOCORE_HTTP_METHODS = [ + 'GET', 'DELETE', 'HEAD', 'OPTIONS', 'PATCH', 'POST', 'PUT' +] + + +class MockRawResponse(BytesIO): + def __init__(self, input): + if isinstance(input, six.text_type): + input = input.encode('utf-8') + super(MockRawResponse, self).__init__(input) + + def stream(self, **kwargs): + contents = self.read() + while contents: + yield contents + contents = self.read() + + +class BotocoreStubber(object): + def __init__(self): + self.enabled = False + self.methods = defaultdict(list) + + def reset(self): + self.methods.clear() + + def register_response(self, method, pattern, response): + matchers = self.methods[method] + matchers.append((pattern, response)) + + def __call__(self, event_name, request, **kwargs): + if not self.enabled: + return None + + response = None + response_callback = None + found_index = None + matchers = self.methods.get(request.method) + + base_url = request.url.split('?', 1)[0] + for i, (pattern, callback) in enumerate(matchers): + if pattern.match(base_url): + if found_index is None: + found_index = i + response_callback = callback + else: + matchers.pop(found_index) + break + + if response_callback is not None: + for header, value in request.headers.items(): + if isinstance(value, six.binary_type): + request.headers[header] = value.decode('utf-8') + status, headers, body = response_callback(request, request.url, request.headers) + body = MockRawResponse(body) + response = AWSResponse(request.url, status, headers, body) + + return response + +botocore_stubber = BotocoreStubber() +BUILTIN_HANDLERS.append(('before-send', botocore_stubber)) + +class BotocoreEventMockAWS(BaseMockAWS): + def reset(self): + botocore_stubber.reset() + + def enable_patching(self): + botocore_stubber.enabled = True + for method in BOTOCORE_HTTP_METHODS: + for backend in self.backends_for_urls.values(): + for key, value in backend.urls.items(): + pattern = re.compile(key) + botocore_stubber.register_response(method, pattern, value) + + def disable_patching(self): + botocore_stubber.enabled = False + self.reset() + + +MockAWS = BotocoreEventMockAWS class ServerModeMockAWS(BaseMockAWS): diff --git a/moto/core/utils.py b/moto/core/utils.py index 86e7632b..777a0375 100644 --- a/moto/core/utils.py +++ b/moto/core/utils.py @@ -8,6 +8,7 @@ import random import re import six import string +from six.moves.urllib.parse import urlparse REQUEST_ID_LONG = string.digits + string.ascii_uppercase @@ -286,3 +287,13 @@ def amzn_request_id(f): return status, headers, body return _wrapper + + +def path_url(url): + parsed_url = urlparse(url) + path = parsed_url.path + if not path: + path = '/' + if parsed_url.query: + path = path + '?' + parsed_url.query + return path diff --git a/moto/s3/responses.py b/moto/s3/responses.py index 962025cb..13e5f87d 100755 --- a/moto/s3/responses.py +++ b/moto/s3/responses.py @@ -10,6 +10,7 @@ import xmltodict from moto.packages.httpretty.core import HTTPrettyRequest from moto.core.responses import _TemplateEnvironmentMixin +from moto.core.utils import path_url from moto.s3bucket_path.utils import bucket_name_from_url as bucketpath_bucket_name_from_url, \ parse_key_name as bucketpath_parse_key_name, is_delete_keys as bucketpath_is_delete_keys @@ -487,7 +488,7 @@ class ResponseObject(_TemplateEnvironmentMixin): if isinstance(request, HTTPrettyRequest): path = request.path else: - path = request.full_path if hasattr(request, 'full_path') else request.path_url + path = request.full_path if hasattr(request, 'full_path') else path_url(request.url) if self.is_delete_keys(request, path, bucket_name): return self._bucket_response_delete_keys(request, body, bucket_name, headers) @@ -708,7 +709,10 @@ class ResponseObject(_TemplateEnvironmentMixin): # Copy key # you can have a quoted ?version=abc with a version Id, so work on # we need to parse the unquoted string first - src_key_parsed = urlparse(request.headers.get("x-amz-copy-source")) + src_key = request.headers.get("x-amz-copy-source") + if isinstance(src_key, six.binary_type): + src_key = src_key.decode('utf-8') + src_key_parsed = urlparse(src_key) src_bucket, src_key = unquote(src_key_parsed.path).\ lstrip("/").split("/", 1) src_version_id = parse_qs(src_key_parsed.query).get( diff --git a/requirements-dev.txt b/requirements-dev.txt index 111cd5f3..f87ab3db 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -8,7 +8,7 @@ freezegun flask boto>=2.45.0 boto3>=1.4.4 -botocore>=1.8.36 +botocore>=1.12.13 six>=1.9 prompt-toolkit==1.0.14 click==6.7 diff --git a/setup.py b/setup.py index 98780dd5..66dba0f2 100755 --- a/setup.py +++ b/setup.py @@ -8,8 +8,8 @@ import sys install_requires = [ "Jinja2>=2.7.3", "boto>=2.36.0", - "boto3>=1.6.16,<1.8", - "botocore>=1.9.16,<1.11", + "boto3>=1.6.16", + "botocore>=1.12.13", "cryptography>=2.3.0", "requests>=2.5", "xmltodict", From fd4e5248559545a9c559cc42b263a9b61970c9c1 Mon Sep 17 00:00:00 2001 From: Jordan Guymon Date: Mon, 1 Oct 2018 09:45:12 -0700 Subject: [PATCH 67/72] Use env credentials for all tests --- .travis.yml | 4 ++-- moto/core/models.py | 2 ++ 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index de22818b..3a5de0fa 100644 --- a/.travis.yml +++ b/.travis.yml @@ -23,6 +23,8 @@ matrix: sudo: true before_install: - export BOTO_CONFIG=/dev/null + - export AWS_SECRET_ACCESS_KEY=foobar_secret + - export AWS_ACCESS_KEY_ID=foobar_key install: # We build moto first so the docker container doesn't try to compile it as well, also note we don't use # -d for docker run so the logs show up in travis @@ -32,8 +34,6 @@ install: if [ "$TEST_SERVER_MODE" = "true" ]; then docker run --rm -t --name motoserver -e TEST_SERVER_MODE=true -e AWS_SECRET_ACCESS_KEY=server_secret -e AWS_ACCESS_KEY_ID=server_key -v `pwd`:/moto -p 5000:5000 -v /var/run/docker.sock:/var/run/docker.sock python:${TRAVIS_PYTHON_VERSION}-stretch /moto/travis_moto_server.sh & - export AWS_SECRET_ACCESS_KEY=foobar_secret - export AWS_ACCESS_KEY_ID=foobar_key fi travis_retry pip install boto==2.45.0 travis_retry pip install boto3 diff --git a/moto/core/models.py b/moto/core/models.py index b12374fd..d5f7b842 100644 --- a/moto/core/models.py +++ b/moto/core/models.py @@ -295,9 +295,11 @@ class BotocoreStubber(object): return response + botocore_stubber = BotocoreStubber() BUILTIN_HANDLERS.append(('before-send', botocore_stubber)) + class BotocoreEventMockAWS(BaseMockAWS): def reset(self): botocore_stubber.reset() From b20e190995ed469244da3a7e556d76aeaf0cdfa5 Mon Sep 17 00:00:00 2001 From: Lorenz Hufnagel Date: Sun, 14 Oct 2018 19:58:56 +0200 Subject: [PATCH 68/72] Try to get tests running --- moto/core/models.py | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/moto/core/models.py b/moto/core/models.py index d5f7b842..19267ca0 100644 --- a/moto/core/models.py +++ b/moto/core/models.py @@ -303,6 +303,7 @@ BUILTIN_HANDLERS.append(('before-send', botocore_stubber)) class BotocoreEventMockAWS(BaseMockAWS): def reset(self): botocore_stubber.reset() + responses_mock.reset() def enable_patching(self): botocore_stubber.enabled = True @@ -312,10 +313,32 @@ class BotocoreEventMockAWS(BaseMockAWS): pattern = re.compile(key) botocore_stubber.register_response(method, pattern, value) + if not hasattr(responses_mock, '_patcher') or not hasattr(responses_mock._patcher, 'target'): + responses_mock.start() + + for method in RESPONSES_METHODS: + # for backend in default_backends.values(): + for backend in self.backends_for_urls.values(): + for key, value in backend.urls.items(): + responses_mock.add( + CallbackResponse( + method=method, + url=re.compile(key), + callback=convert_flask_to_responses_response(value), + stream=True, + match_querystring=False, + ) + ) + def disable_patching(self): botocore_stubber.enabled = False self.reset() + try: + responses_mock.stop() + except RuntimeError: + pass + MockAWS = BotocoreEventMockAWS From 75f2c56a3643b053fbddf458e6e48e5a8e5e6214 Mon Sep 17 00:00:00 2001 From: Steve Pulec Date: Tue, 30 Oct 2018 22:03:09 -0400 Subject: [PATCH 69/72] Fix ecs error response to be json. --- moto/ecs/exceptions.py | 4 +++- tests/test_ecs/test_ecs_boto3.py | 17 ++++++++++++++++- 2 files changed, 19 insertions(+), 2 deletions(-) diff --git a/moto/ecs/exceptions.py b/moto/ecs/exceptions.py index c23d6fd1..a780dc7c 100644 --- a/moto/ecs/exceptions.py +++ b/moto/ecs/exceptions.py @@ -8,4 +8,6 @@ class ServiceNotFoundException(RESTError): def __init__(self, service_name): super(ServiceNotFoundException, self).__init__( error_type="ServiceNotFoundException", - message="The service {0} does not exist".format(service_name)) + message="The service {0} does not exist".format(service_name), + template='error_json', + ) diff --git a/tests/test_ecs/test_ecs_boto3.py b/tests/test_ecs/test_ecs_boto3.py index 70c1463e..a0e8318d 100644 --- a/tests/test_ecs/test_ecs_boto3.py +++ b/tests/test_ecs/test_ecs_boto3.py @@ -631,7 +631,22 @@ def test_delete_service(): response['service']['schedulingStrategy'].should.equal('REPLICA') response['service']['taskDefinition'].should.equal( 'arn:aws:ecs:us-east-1:012345678910:task-definition/test_ecs_task:1') - + + +@mock_ecs +def test_update_non_existant_service(): + client = boto3.client('ecs', region_name='us-east-1') + try: + client.update_service( + cluster="my-clustet", + service="my-service", + desiredCount=0, + ) + except ClientError as exc: + error_code = exc.response['Error']['Code'] + error_code.should.equal('ServiceNotFoundException') + else: + raise Exception("Didn't raise ClientError") @mock_ec2 From a8bc7a608e7b7de47b1f8ad1f9dec8ceecf2dd1c Mon Sep 17 00:00:00 2001 From: Steve Pulec Date: Tue, 30 Oct 2018 22:09:47 -0400 Subject: [PATCH 70/72] Lint. --- moto/ecs/exceptions.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/moto/ecs/exceptions.py b/moto/ecs/exceptions.py index a780dc7c..bb7e685c 100644 --- a/moto/ecs/exceptions.py +++ b/moto/ecs/exceptions.py @@ -10,4 +10,4 @@ class ServiceNotFoundException(RESTError): error_type="ServiceNotFoundException", message="The service {0} does not exist".format(service_name), template='error_json', - ) + ) From 90a62b56400251620684a094d85e458bc3abb17e Mon Sep 17 00:00:00 2001 From: Steve Pulec Date: Sun, 4 Nov 2018 17:30:44 -0500 Subject: [PATCH 71/72] 1.3.7 --- CHANGELOG.md | 5 +++++ moto/__init__.py | 2 +- setup.py | 2 +- 3 files changed, 7 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7f7ee444..f42619b3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,11 @@ Moto Changelog =================== +1.3.7 +----- + + * Switch from mocking requests to using before-send for AWS calls + 1.3.6 ----- diff --git a/moto/__init__.py b/moto/__init__.py index 6992c535..dd3593d5 100644 --- a/moto/__init__.py +++ b/moto/__init__.py @@ -3,7 +3,7 @@ import logging # logging.getLogger('boto').setLevel(logging.CRITICAL) __title__ = 'moto' -__version__ = '1.3.6' +__version__ = '1.3.7' from .acm import mock_acm # flake8: noqa from .apigateway import mock_apigateway, mock_apigateway_deprecated # flake8: noqa diff --git a/setup.py b/setup.py index 66dba0f2..046ecf6e 100755 --- a/setup.py +++ b/setup.py @@ -40,7 +40,7 @@ else: setup( name='moto', - version='1.3.6', + version='1.3.7', description='A library that allows your python tests to easily' ' mock out the boto library', author='Steve Pulec', From ed861ecae1039a048a6350a4ff832ef094cdf2c2 Mon Sep 17 00:00:00 2001 From: Niko Eckerskorn Date: Thu, 15 Nov 2018 18:29:05 +1100 Subject: [PATCH 72/72] Loosen aws-xray-sdk requirements (#1948) --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 046ecf6e..a1b8c5da 100755 --- a/setup.py +++ b/setup.py @@ -22,7 +22,7 @@ install_requires = [ "mock", "docker>=2.5.1", "jsondiff==1.1.1", - "aws-xray-sdk<0.96,>=0.93", + "aws-xray-sdk!=0.96,>=0.93", "responses>=0.9.0", ]