Iam cloudformation update, singificant cloudformation refactoring (#3218)

* IAM User Cloudformation Enhancements: update, delete, getatt.

* AWS::IAM::Policy Support

* Added unit tests for AWS:IAM:Policy for roles and groups.  Fixed bug related to groups.

* AWS:IAM:AccessKey CloudFormation support.

* Refactor of CloudFormation parsing.py methods to simplify and standardize how they call to the models.  Adjusted some models accordingly.

* Further model CloudFormation support changes to align with revised CloudFormation logic.  Mostly avoidance of getting resoure name from properties.

* Support for Kinesis Stream RetentionPeriodHours param.

* Kinesis Stream Cloudformation Tag Support.

* Added omitted 'region' param to boto3.client() calls in new tests.

Co-authored-by: Joseph Weitekamp <jweite@amazon.com>

tests/test_cloudformation/test_cloudformation_stack_crud_boto3.py

@@ -592,7 +592,7 @@ def test_boto3_create_stack_set_with_yaml():
 @mock_cloudformation
 @mock_s3
 def test_create_stack_set_from_s3_url():
-    s3 = boto3.client("s3")
+    s3 = boto3.client("s3", region_name="us-east-1")
     s3_conn = boto3.resource("s3", region_name="us-east-1")
     s3_conn.create_bucket(Bucket="foobar")
 
@@ -704,7 +704,7 @@ def test_boto3_create_stack_with_short_form_func_yaml():
 @mock_s3
 @mock_cloudformation
 def test_get_template_summary():
-    s3 = boto3.client("s3")
+    s3 = boto3.client("s3", region_name="us-east-1")
     s3_conn = boto3.resource("s3", region_name="us-east-1")
 
     conn = boto3.client("cloudformation", region_name="us-east-1")
@@ -802,7 +802,7 @@ def test_create_stack_with_role_arn():
 @mock_cloudformation
 @mock_s3
 def test_create_stack_from_s3_url():
-    s3 = boto3.client("s3")
+    s3 = boto3.client("s3", region_name="us-east-1")
     s3_conn = boto3.resource("s3", region_name="us-east-1")
     s3_conn.create_bucket(Bucket="foobar")
 
@@ -857,7 +857,7 @@ def test_update_stack_with_previous_value():
 @mock_s3
 @mock_ec2
 def test_update_stack_from_s3_url():
-    s3 = boto3.client("s3")
+    s3 = boto3.client("s3", region_name="us-east-1")
     s3_conn = boto3.resource("s3", region_name="us-east-1")
 
     cf_conn = boto3.client("cloudformation", region_name="us-east-1")
@@ -886,7 +886,7 @@ def test_update_stack_from_s3_url():
 @mock_cloudformation
 @mock_s3
 def test_create_change_set_from_s3_url():
-    s3 = boto3.client("s3")
+    s3 = boto3.client("s3", region_name="us-east-1")
     s3_conn = boto3.resource("s3", region_name="us-east-1")
     s3_conn.create_bucket(Bucket="foobar")
 

tests/test_cloudformation/test_validate.py

@@ -118,7 +118,7 @@ def test_boto3_yaml_validate_successful():
 @mock_cloudformation
 @mock_s3
 def test_boto3_yaml_validate_template_url_successful():
-    s3 = boto3.client("s3")
+    s3 = boto3.client("s3", region_name="us-east-1")
     s3_conn = boto3.resource("s3", region_name="us-east-1")
     s3_conn.create_bucket(Bucket="foobar")
 

tests/test_iam/test_iam.py

@@ -5,12 +5,9 @@ import json
 import boto
 import boto3
 import csv
-import os
 import sure  # noqa
-import sys
 from boto.exception import BotoServerError
 from botocore.exceptions import ClientError
-from dateutil.tz import tzutc
 
 from moto import mock_iam, mock_iam_deprecated, settings
 from moto.core import ACCOUNT_ID

tests/test_kinesis/test_kinesis_cloudformation.py

@@ -73,6 +73,12 @@ Resources:
     Properties:
       Name: MyStream
       ShardCount: 4
+      RetentionPeriodHours: 48
+      Tags:
+      - Key: TagKey1
+        Value: TagValue1
+      - Key: TagKey2
+        Value: TagValue2
 """.strip()
 
     cf_conn.create_stack(StackName=stack_name, TemplateBody=template)
@@ -83,6 +89,14 @@ Resources:
     stream_description = kinesis_conn.describe_stream(StreamName="MyStream")[
         "StreamDescription"
     ]
+    stream_description["RetentionPeriodHours"].should.equal(48)
+
+    tags = kinesis_conn.list_tags_for_stream(StreamName="MyStream")["Tags"]
+    tag1_value = [tag for tag in tags if tag["Key"] == "TagKey1"][0]["Value"]
+    tag2_value = [tag for tag in tags if tag["Key"] == "TagKey2"][0]["Value"]
+    tag1_value.should.equal("TagValue1")
+    tag2_value.should.equal("TagValue2")
+
     shards_provisioned = len(
         [
             shard
@@ -98,12 +112,27 @@ Resources:
         Type: AWS::Kinesis::Stream
         Properties:
           ShardCount: 6
+          RetentionPeriodHours: 24
+          Tags:
+          - Key: TagKey1
+            Value: TagValue1a
+          - Key: TagKey2
+            Value: TagValue2a
+
     """.strip()
     cf_conn.update_stack(StackName=stack_name, TemplateBody=template)
 
     stream_description = kinesis_conn.describe_stream(StreamName="MyStream")[
         "StreamDescription"
     ]
+    stream_description["RetentionPeriodHours"].should.equal(24)
+
+    tags = kinesis_conn.list_tags_for_stream(StreamName="MyStream")["Tags"]
+    tag1_value = [tag for tag in tags if tag["Key"] == "TagKey1"][0]["Value"]
+    tag2_value = [tag for tag in tags if tag["Key"] == "TagKey2"][0]["Value"]
+    tag1_value.should.equal("TagValue1a")
+    tag2_value.should.equal("TagValue2a")
+
     shards_provisioned = len(
         [
             shard

tests/test_s3/test_s3.py

@@ -2,7 +2,6 @@
 from __future__ import unicode_literals
 
 import datetime
-import os
 import sys
 
 from boto3 import Session
@@ -11,7 +10,6 @@ from six.moves.urllib.error import HTTPError
 from functools import wraps
 from gzip import GzipFile
 from io import BytesIO
-import mimetypes
 import zlib
 import pickle
 import uuid
@@ -36,7 +34,7 @@ from nose.tools import assert_raises
 
 import sure  # noqa
 
-from moto import settings, mock_s3, mock_s3_deprecated, mock_config, mock_cloudformation
+from moto import settings, mock_s3, mock_s3_deprecated, mock_config
 import moto.s3.models as s3model
 from moto.core.exceptions import InvalidNextTokenException
 from moto.core.utils import py2_strip_unicode_keys
@@ -4686,142 +4684,3 @@ def test_presigned_put_url_with_custom_headers():
 
     s3.delete_object(Bucket=bucket, Key=key)
     s3.delete_bucket(Bucket=bucket)
-
-
-@mock_s3
-@mock_cloudformation
-def test_s3_bucket_cloudformation_basic():
-    s3 = boto3.client("s3", region_name="us-east-1")
-    cf = boto3.client("cloudformation", region_name="us-east-1")
-
-    template = {
-        "AWSTemplateFormatVersion": "2010-09-09",
-        "Resources": {"testInstance": {"Type": "AWS::S3::Bucket", "Properties": {},}},
-        "Outputs": {"Bucket": {"Value": {"Ref": "testInstance"}}},
-    }
-    template_json = json.dumps(template)
-    stack_id = cf.create_stack(StackName="test_stack", TemplateBody=template_json)[
-        "StackId"
-    ]
-    stack_description = cf.describe_stacks(StackName="test_stack")["Stacks"][0]
-
-    s3.head_bucket(Bucket=stack_description["Outputs"][0]["OutputValue"])
-
-
-@mock_s3
-@mock_cloudformation
-def test_s3_bucket_cloudformation_with_properties():
-    s3 = boto3.client("s3", region_name="us-east-1")
-    cf = boto3.client("cloudformation", region_name="us-east-1")
-
-    bucket_name = "MyBucket"
-    template = {
-        "AWSTemplateFormatVersion": "2010-09-09",
-        "Resources": {
-            "testInstance": {
-                "Type": "AWS::S3::Bucket",
-                "Properties": {
-                    "BucketName": bucket_name,
-                    "BucketEncryption": {
-                        "ServerSideEncryptionConfiguration": [
-                            {
-                                "ServerSideEncryptionByDefault": {
-                                    "SSEAlgorithm": "AES256"
-                                }
-                            }
-                        ]
-                    },
-                },
-            }
-        },
-        "Outputs": {"Bucket": {"Value": {"Ref": "testInstance"}}},
-    }
-    template_json = json.dumps(template)
-    stack_id = cf.create_stack(StackName="test_stack", TemplateBody=template_json)[
-        "StackId"
-    ]
-    stack_description = cf.describe_stacks(StackName="test_stack")["Stacks"][0]
-    s3.head_bucket(Bucket=bucket_name)
-
-    encryption = s3.get_bucket_encryption(Bucket=bucket_name)
-    encryption["ServerSideEncryptionConfiguration"]["Rules"][0][
-        "ApplyServerSideEncryptionByDefault"
-    ]["SSEAlgorithm"].should.equal("AES256")
-
-
-@mock_s3
-@mock_cloudformation
-def test_s3_bucket_cloudformation_update_no_interruption():
-    s3 = boto3.client("s3", region_name="us-east-1")
-    cf = boto3.client("cloudformation", region_name="us-east-1")
-
-    template = {
-        "AWSTemplateFormatVersion": "2010-09-09",
-        "Resources": {"testInstance": {"Type": "AWS::S3::Bucket"}},
-        "Outputs": {"Bucket": {"Value": {"Ref": "testInstance"}}},
-    }
-    template_json = json.dumps(template)
-    cf.create_stack(StackName="test_stack", TemplateBody=template_json)
-    stack_description = cf.describe_stacks(StackName="test_stack")["Stacks"][0]
-    s3.head_bucket(Bucket=stack_description["Outputs"][0]["OutputValue"])
-
-    template = {
-        "AWSTemplateFormatVersion": "2010-09-09",
-        "Resources": {
-            "testInstance": {
-                "Type": "AWS::S3::Bucket",
-                "Properties": {
-                    "BucketEncryption": {
-                        "ServerSideEncryptionConfiguration": [
-                            {
-                                "ServerSideEncryptionByDefault": {
-                                    "SSEAlgorithm": "AES256"
-                                }
-                            }
-                        ]
-                    }
-                },
-            }
-        },
-        "Outputs": {"Bucket": {"Value": {"Ref": "testInstance"}}},
-    }
-    template_json = json.dumps(template)
-    cf.update_stack(StackName="test_stack", TemplateBody=template_json)
-    encryption = s3.get_bucket_encryption(
-        Bucket=stack_description["Outputs"][0]["OutputValue"]
-    )
-    encryption["ServerSideEncryptionConfiguration"]["Rules"][0][
-        "ApplyServerSideEncryptionByDefault"
-    ]["SSEAlgorithm"].should.equal("AES256")
-
-
-@mock_s3
-@mock_cloudformation
-def test_s3_bucket_cloudformation_update_replacement():
-    s3 = boto3.client("s3", region_name="us-east-1")
-    cf = boto3.client("cloudformation", region_name="us-east-1")
-
-    template = {
-        "AWSTemplateFormatVersion": "2010-09-09",
-        "Resources": {"testInstance": {"Type": "AWS::S3::Bucket"}},
-        "Outputs": {"Bucket": {"Value": {"Ref": "testInstance"}}},
-    }
-    template_json = json.dumps(template)
-    cf.create_stack(StackName="test_stack", TemplateBody=template_json)
-    stack_description = cf.describe_stacks(StackName="test_stack")["Stacks"][0]
-    s3.head_bucket(Bucket=stack_description["Outputs"][0]["OutputValue"])
-
-    template = {
-        "AWSTemplateFormatVersion": "2010-09-09",
-        "Resources": {
-            "testInstance": {
-                "Type": "AWS::S3::Bucket",
-                "Properties": {"BucketName": "MyNewBucketName"},
-            }
-        },
-        "Outputs": {"Bucket": {"Value": {"Ref": "testInstance"}}},
-    }
-    template_json = json.dumps(template)
-    cf.update_stack(StackName="test_stack", TemplateBody=template_json)
-    stack_description = cf.describe_stacks(StackName="test_stack")["Stacks"][0]
-    s3.head_bucket(Bucket=stack_description["Outputs"][0]["OutputValue"])

tests/test_s3/test_s3_cloudformation.py

@@ -0,0 +1,145 @@
+import json
+import boto3
+
+import sure  # noqa
+
+from moto import mock_s3, mock_cloudformation
+
+
+@mock_s3
+@mock_cloudformation
+def test_s3_bucket_cloudformation_basic():
+    s3 = boto3.client("s3", region_name="us-east-1")
+    cf = boto3.client("cloudformation", region_name="us-east-1")
+
+    template = {
+        "AWSTemplateFormatVersion": "2010-09-09",
+        "Resources": {"testInstance": {"Type": "AWS::S3::Bucket", "Properties": {},}},
+        "Outputs": {"Bucket": {"Value": {"Ref": "testInstance"}}},
+    }
+    template_json = json.dumps(template)
+    stack_id = cf.create_stack(StackName="test_stack", TemplateBody=template_json)[
+        "StackId"
+    ]
+    stack_description = cf.describe_stacks(StackName="test_stack")["Stacks"][0]
+
+    s3.head_bucket(Bucket=stack_description["Outputs"][0]["OutputValue"])
+
+
+@mock_s3
+@mock_cloudformation
+def test_s3_bucket_cloudformation_with_properties():
+    s3 = boto3.client("s3", region_name="us-east-1")
+    cf = boto3.client("cloudformation", region_name="us-east-1")
+
+    bucket_name = "MyBucket"
+    template = {
+        "AWSTemplateFormatVersion": "2010-09-09",
+        "Resources": {
+            "testInstance": {
+                "Type": "AWS::S3::Bucket",
+                "Properties": {
+                    "BucketName": bucket_name,
+                    "BucketEncryption": {
+                        "ServerSideEncryptionConfiguration": [
+                            {
+                                "ServerSideEncryptionByDefault": {
+                                    "SSEAlgorithm": "AES256"
+                                }
+                            }
+                        ]
+                    },
+                },
+            }
+        },
+        "Outputs": {"Bucket": {"Value": {"Ref": "testInstance"}}},
+    }
+    template_json = json.dumps(template)
+    stack_id = cf.create_stack(StackName="test_stack", TemplateBody=template_json)[
+        "StackId"
+    ]
+    stack_description = cf.describe_stacks(StackName="test_stack")["Stacks"][0]
+    s3.head_bucket(Bucket=bucket_name)
+
+    encryption = s3.get_bucket_encryption(Bucket=bucket_name)
+    encryption["ServerSideEncryptionConfiguration"]["Rules"][0][
+        "ApplyServerSideEncryptionByDefault"
+    ]["SSEAlgorithm"].should.equal("AES256")
+
+
+@mock_s3
+@mock_cloudformation
+def test_s3_bucket_cloudformation_update_no_interruption():
+    s3 = boto3.client("s3", region_name="us-east-1")
+    cf = boto3.client("cloudformation", region_name="us-east-1")
+
+    template = {
+        "AWSTemplateFormatVersion": "2010-09-09",
+        "Resources": {"testInstance": {"Type": "AWS::S3::Bucket"}},
+        "Outputs": {"Bucket": {"Value": {"Ref": "testInstance"}}},
+    }
+    template_json = json.dumps(template)
+    cf.create_stack(StackName="test_stack", TemplateBody=template_json)
+    stack_description = cf.describe_stacks(StackName="test_stack")["Stacks"][0]
+    s3.head_bucket(Bucket=stack_description["Outputs"][0]["OutputValue"])
+
+    template = {
+        "AWSTemplateFormatVersion": "2010-09-09",
+        "Resources": {
+            "testInstance": {
+                "Type": "AWS::S3::Bucket",
+                "Properties": {
+                    "BucketEncryption": {
+                        "ServerSideEncryptionConfiguration": [
+                            {
+                                "ServerSideEncryptionByDefault": {
+                                    "SSEAlgorithm": "AES256"
+                                }
+                            }
+                        ]
+                    }
+                },
+            }
+        },
+        "Outputs": {"Bucket": {"Value": {"Ref": "testInstance"}}},
+    }
+    template_json = json.dumps(template)
+    cf.update_stack(StackName="test_stack", TemplateBody=template_json)
+    encryption = s3.get_bucket_encryption(
+        Bucket=stack_description["Outputs"][0]["OutputValue"]
+    )
+    encryption["ServerSideEncryptionConfiguration"]["Rules"][0][
+        "ApplyServerSideEncryptionByDefault"
+    ]["SSEAlgorithm"].should.equal("AES256")
+
+
+@mock_s3
+@mock_cloudformation
+def test_s3_bucket_cloudformation_update_replacement():
+    s3 = boto3.client("s3", region_name="us-east-1")
+    cf = boto3.client("cloudformation", region_name="us-east-1")
+
+    template = {
+        "AWSTemplateFormatVersion": "2010-09-09",
+        "Resources": {"testInstance": {"Type": "AWS::S3::Bucket"}},
+        "Outputs": {"Bucket": {"Value": {"Ref": "testInstance"}}},
+    }
+    template_json = json.dumps(template)
+    cf.create_stack(StackName="test_stack", TemplateBody=template_json)
+    stack_description = cf.describe_stacks(StackName="test_stack")["Stacks"][0]
+    s3.head_bucket(Bucket=stack_description["Outputs"][0]["OutputValue"])
+
+    template = {
+        "AWSTemplateFormatVersion": "2010-09-09",
+        "Resources": {
+            "testInstance": {
+                "Type": "AWS::S3::Bucket",
+                "Properties": {"BucketName": "MyNewBucketName"},
+            }
+        },
+        "Outputs": {"Bucket": {"Value": {"Ref": "testInstance"}}},
+    }
+    template_json = json.dumps(template)
+    cf.update_stack(StackName="test_stack", TemplateBody=template_json)
+    stack_description = cf.describe_stacks(StackName="test_stack")["Stacks"][0]
+    s3.head_bucket(Bucket=stack_description["Outputs"][0]["OutputValue"])