Merge branch 'master' into add_instance_type_offerings

tests/test_cloudformation/test_stack_parsing.py

@@ -15,7 +15,7 @@ from moto.cloudformation.parsing import (
 from moto.sqs.models import Queue
 from moto.s3.models import FakeBucket
 from moto.cloudformation.utils import yaml_tag_constructor
-from boto.cloudformation.stack import Output
+from moto.packages.boto.cloudformation.stack import Output
 
 
 dummy_template = {

tests/test_cognitoidp/test_cognitoidp.py

@@ -1840,6 +1840,31 @@ def test_admin_set_user_password():
     result["UserStatus"].should.equal("CONFIRMED")
 
 
+@mock_cognitoidp
+def test_change_password_with_invalid_token_raises_error():
+    client = boto3.client("cognito-idp", "us-west-2")
+    with pytest.raises(ClientError) as ex:
+        client.change_password(
+            AccessToken=str(uuid.uuid4()),
+            PreviousPassword="previous_password",
+            ProposedPassword="newer_password",
+        )
+    ex.value.response["Error"]["Code"].should.equal("NotAuthorizedException")
+
+
+@mock_cognitoidp
+def test_confirm_forgot_password_with_non_existent_client_id_raises_error():
+    client = boto3.client("cognito-idp", "us-west-2")
+    with pytest.raises(ClientError) as ex:
+        client.confirm_forgot_password(
+            ClientId="non-existent-client-id",
+            Username="not-existent-username",
+            ConfirmationCode=str(uuid.uuid4()),
+            Password=str(uuid.uuid4()),
+        )
+    ex.value.response["Error"]["Code"].should.equal("ResourceNotFoundException")
+
+
 # Test will retrieve public key from cognito.amazonaws.com/.well-known/jwks.json,
 # which isnt mocked in ServerMode
 if not settings.TEST_SERVER_MODE:

tests/test_dynamodb2/test_dynamodb.py

@@ -5523,6 +5523,61 @@ def test_gsi_projection_type_keys_only():
     )
 
 
+@mock_dynamodb2
+def test_gsi_projection_type_include():
+    table_schema = {
+        "KeySchema": [{"AttributeName": "partitionKey", "KeyType": "HASH"}],
+        "GlobalSecondaryIndexes": [
+            {
+                "IndexName": "GSI-INC",
+                "KeySchema": [
+                    {"AttributeName": "gsiK1PartitionKey", "KeyType": "HASH"},
+                    {"AttributeName": "gsiK1SortKey", "KeyType": "RANGE"},
+                ],
+                "Projection": {
+                    "ProjectionType": "INCLUDE",
+                    "NonKeyAttributes": ["projectedAttribute"],
+                },
+            }
+        ],
+        "AttributeDefinitions": [
+            {"AttributeName": "partitionKey", "AttributeType": "S"},
+            {"AttributeName": "gsiK1PartitionKey", "AttributeType": "S"},
+            {"AttributeName": "gsiK1SortKey", "AttributeType": "S"},
+        ],
+    }
+
+    item = {
+        "partitionKey": "pk-1",
+        "gsiK1PartitionKey": "gsi-pk",
+        "gsiK1SortKey": "gsi-sk",
+        "projectedAttribute": "lore ipsum",
+        "nonProjectedAttribute": "dolor sit amet",
+    }
+
+    dynamodb = boto3.resource("dynamodb", region_name="us-east-1")
+    dynamodb.create_table(
+        TableName="test-table", BillingMode="PAY_PER_REQUEST", **table_schema
+    )
+    table = dynamodb.Table("test-table")
+    table.put_item(Item=item)
+
+    items = table.query(
+        KeyConditionExpression=Key("gsiK1PartitionKey").eq("gsi-pk"),
+        IndexName="GSI-INC",
+    )["Items"]
+    items.should.have.length_of(1)
+    # Item should only include keys and additionally projected attributes only
+    items[0].should.equal(
+        {
+            "gsiK1PartitionKey": "gsi-pk",
+            "gsiK1SortKey": "gsi-sk",
+            "partitionKey": "pk-1",
+            "projectedAttribute": "lore ipsum",
+        }
+    )
+
+
 @mock_dynamodb2
 def test_lsi_projection_type_keys_only():
     table_schema = {

tests/test_ec2/test_ec2_cloudformation.py

@@ -2,6 +2,9 @@ from moto import mock_cloudformation_deprecated, mock_ec2_deprecated
 from moto import mock_cloudformation, mock_ec2
 from tests.test_cloudformation.fixtures import vpc_eni
 import boto
+import boto.ec2
+import boto.cloudformation
+import boto.vpc
 import boto3
 import json
 import sure  # noqa

tests/test_emr/test_emr_boto3.py

@@ -636,7 +636,7 @@ def test_run_job_flow_with_custom_ami():
 
     args = deepcopy(run_job_flow_args)
     args["CustomAmiId"] = "MyEmrCustomAmi"
-    args["ReleaseLabel"] = "emr-5.7.0"
+    args["ReleaseLabel"] = "emr-5.31.0"
     cluster_id = client.run_job_flow(**args)["JobFlowId"]
     resp = client.describe_cluster(ClusterId=cluster_id)
     resp["Cluster"]["CustomAmiId"].should.equal("MyEmrCustomAmi")

tests/test_emr/test_utils.py

@@ -0,0 +1,49 @@
+import pytest
+
+from moto.emr.utils import ReleaseLabel
+
+
+def test_invalid_release_labels_raise_exception():
+    invalid_releases = [
+        "",
+        "0",
+        "1.0",
+        "emr-2.0",
+    ]
+    for invalid_release in invalid_releases:
+        with pytest.raises(ValueError):
+            ReleaseLabel(invalid_release)
+
+
+def test_release_label_comparisons():
+    assert str(ReleaseLabel("emr-5.1.2")) == "emr-5.1.2"
+
+    assert ReleaseLabel("emr-5.0.0") != ReleaseLabel("emr-5.0.1")
+    assert ReleaseLabel("emr-5.0.0") == ReleaseLabel("emr-5.0.0")
+
+    assert ReleaseLabel("emr-5.31.0") > ReleaseLabel("emr-5.7.0")
+    assert ReleaseLabel("emr-6.0.0") > ReleaseLabel("emr-5.7.0")
+
+    assert ReleaseLabel("emr-5.7.0") < ReleaseLabel("emr-5.10.0")
+    assert ReleaseLabel("emr-5.10.0") < ReleaseLabel("emr-5.10.1")
+
+    assert ReleaseLabel("emr-5.60.0") >= ReleaseLabel("emr-5.7.0")
+    assert ReleaseLabel("emr-6.0.0") >= ReleaseLabel("emr-6.0.0")
+
+    assert ReleaseLabel("emr-5.7.0") <= ReleaseLabel("emr-5.17.0")
+    assert ReleaseLabel("emr-5.7.0") <= ReleaseLabel("emr-5.7.0")
+
+    releases_unsorted = [
+        ReleaseLabel("emr-5.60.2"),
+        ReleaseLabel("emr-4.0.1"),
+        ReleaseLabel("emr-4.0.0"),
+        ReleaseLabel("emr-5.7.3"),
+    ]
+    releases_sorted = [str(label) for label in sorted(releases_unsorted)]
+    expected = [
+        "emr-4.0.0",
+        "emr-4.0.1",
+        "emr-5.7.3",
+        "emr-5.60.2",
+    ]
+    assert releases_sorted == expected

tests/test_iam/test_iam_policies.py

@@ -3,6 +3,7 @@ import json
 import boto3
 from botocore.exceptions import ClientError
 import pytest
+import sure  # noqa
 
 from moto import mock_iam
 
@@ -1611,31 +1612,25 @@ valid_policy_documents = [
 ]
 
 
-def test_create_policy_with_invalid_policy_documents():
-    for test_case in invalid_policy_document_test_cases:
-        yield check_create_policy_with_invalid_policy_document, test_case
-
-
-def test_create_policy_with_valid_policy_documents():
-    for valid_policy_document in valid_policy_documents:
-        yield check_create_policy_with_valid_policy_document, valid_policy_document
-
-
+@pytest.mark.parametrize("invalid_policy_document", invalid_policy_document_test_cases)
 @mock_iam
-def check_create_policy_with_invalid_policy_document(test_case):
+def test_create_policy_with_invalid_policy_document(invalid_policy_document):
     conn = boto3.client("iam", region_name="us-east-1")
     with pytest.raises(ClientError) as ex:
         conn.create_policy(
             PolicyName="TestCreatePolicy",
-            PolicyDocument=json.dumps(test_case["document"]),
+            PolicyDocument=json.dumps(invalid_policy_document["document"]),
         )
     ex.value.response["Error"]["Code"].should.equal("MalformedPolicyDocument")
     ex.value.response["ResponseMetadata"]["HTTPStatusCode"].should.equal(400)
-    ex.value.response["Error"]["Message"].should.equal(test_case["error_message"])
+    ex.value.response["Error"]["Message"].should.equal(
+        invalid_policy_document["error_message"]
+    )
 
 
+@pytest.mark.parametrize("valid_policy_document", valid_policy_documents)
 @mock_iam
-def check_create_policy_with_valid_policy_document(valid_policy_document):
+def test_create_policy_with_valid_policy_document(valid_policy_document):
     conn = boto3.client("iam", region_name="us-east-1")
     conn.create_policy(
         PolicyName="TestCreatePolicy", PolicyDocument=json.dumps(valid_policy_document)

tests/test_rds2/test_rds2.py

@@ -4,6 +4,7 @@ from botocore.exceptions import ClientError, ParamValidationError
 import boto3
 import sure  # noqa
 from moto import mock_ec2, mock_kms, mock_rds2
+from moto.core import ACCOUNT_ID
 
 
 @mock_rds2
@@ -1504,7 +1505,9 @@ def test_create_database_with_encrypted_storage():
 
 @mock_rds2
 def test_create_db_parameter_group():
-    conn = boto3.client("rds", region_name="us-west-2")
+    region = "us-west-2"
+    pg_name = "test"
+    conn = boto3.client("rds", region_name=region)
     db_parameter_group = conn.create_db_parameter_group(
         DBParameterGroupName="test",
         DBParameterGroupFamily="mysql5.6",
@@ -1518,6 +1521,9 @@ def test_create_db_parameter_group():
     db_parameter_group["DBParameterGroup"]["Description"].should.equal(
         "test parameter group"
     )
+    db_parameter_group["DBParameterGroup"]["DBParameterGroupArn"].should.equal(
+        "arn:aws:rds:{0}:{1}:pg:{2}".format(region, ACCOUNT_ID, pg_name)
+    )
 
 
 @mock_rds2
@@ -1629,9 +1635,11 @@ def test_create_db_parameter_group_duplicate():
 
 @mock_rds2
 def test_describe_db_parameter_group():
-    conn = boto3.client("rds", region_name="us-west-2")
+    region = "us-west-2"
+    pg_name = "test"
+    conn = boto3.client("rds", region_name=region)
     conn.create_db_parameter_group(
-        DBParameterGroupName="test",
+        DBParameterGroupName=pg_name,
         DBParameterGroupFamily="mysql5.6",
         Description="test parameter group",
     )
@@ -1639,6 +1647,9 @@ def test_describe_db_parameter_group():
     db_parameter_groups["DBParameterGroups"][0]["DBParameterGroupName"].should.equal(
         "test"
     )
+    db_parameter_groups["DBParameterGroups"][0]["DBParameterGroupArn"].should.equal(
+        "arn:aws:rds:{0}:{1}:pg:{2}".format(region, ACCOUNT_ID, pg_name)
+    )
 
 
 @mock_rds2

tests/test_redshift/test_redshift.py

@@ -43,7 +43,7 @@ def test_create_cluster_boto3():
 
 
 @mock_redshift
-def test_create_cluster_boto3():
+def test_create_cluster_with_enhanced_vpc_routing_enabled():
     client = boto3.client("redshift", region_name="us-east-1")
     response = client.create_cluster(
         DBName="test",
@@ -76,7 +76,7 @@ def test_create_snapshot_copy_grant():
 
     client.describe_snapshot_copy_grants.when.called_with(
         SnapshotCopyGrantName="test-us-east-1"
-    ).should.throw(Exception)
+    ).should.throw(ClientError)
 
 
 @mock_redshift
@@ -424,7 +424,7 @@ def test_delete_cluster():
     )
 
     conn.delete_cluster.when.called_with(cluster_identifier, False).should.throw(
-        AttributeError
+        boto.exception.JSONResponseError
     )
 
     clusters = conn.describe_clusters()["DescribeClustersResponse"][
@@ -826,12 +826,11 @@ def test_describe_cluster_snapshots():
 @mock_redshift
 def test_describe_cluster_snapshots_not_found_error():
     client = boto3.client("redshift", region_name="us-east-1")
-    cluster_identifier = "my_cluster"
-    snapshot_identifier = "my_snapshot"
+    cluster_identifier = "non-existent-cluster-id"
+    snapshot_identifier = "non-existent-snapshot-id"
 
-    client.describe_cluster_snapshots.when.called_with(
-        ClusterIdentifier=cluster_identifier
-    ).should.throw(ClientError, "Cluster {} not found.".format(cluster_identifier))
+    resp = client.describe_cluster_snapshots(ClusterIdentifier=cluster_identifier)
+    resp["Snapshots"].should.have.length_of(0)
 
     client.describe_cluster_snapshots.when.called_with(
         SnapshotIdentifier=snapshot_identifier
@@ -867,8 +866,8 @@ def test_delete_cluster_snapshot():
 
     # Delete invalid id
     client.delete_cluster_snapshot.when.called_with(
-        SnapshotIdentifier="not-a-snapshot"
-    ).should.throw(ClientError)
+        SnapshotIdentifier="non-existent"
+    ).should.throw(ClientError, "Snapshot non-existent not found.")
 
 
 @mock_redshift
@@ -892,7 +891,7 @@ def test_cluster_snapshot_already_exists():
 
     client.create_cluster_snapshot.when.called_with(
         SnapshotIdentifier=snapshot_identifier, ClusterIdentifier=cluster_identifier
-    ).should.throw(ClientError)
+    ).should.throw(ClientError, "{} already exists".format(snapshot_identifier))
 
 
 @mock_redshift
@@ -1269,6 +1268,15 @@ def test_enable_snapshot_copy():
     ex.value.response["Error"]["Message"].should.contain(
         "SnapshotCopyGrantName is required for Snapshot Copy on KMS encrypted clusters."
     )
+    with pytest.raises(ClientError) as ex:
+        client.enable_snapshot_copy(
+            ClusterIdentifier="test",
+            DestinationRegion="us-east-1",
+            RetentionPeriod=3,
+            SnapshotCopyGrantName="invalid-us-east-1-to-us-east-1",
+        )
+    ex.value.response["Error"]["Code"].should.equal("UnknownSnapshotCopyRegionFault")
+    ex.value.response["Error"]["Message"].should.contain("Invalid region us-east-1")
     client.enable_snapshot_copy(
         ClusterIdentifier="test",
         DestinationRegion="us-west-2",
@@ -1364,3 +1372,74 @@ def test_create_duplicate_cluster_fails():
     client.create_cluster.when.called_with(**kwargs).should.throw(
         ClientError, "ClusterAlreadyExists"
     )
+
+
+@mock_redshift
+def test_delete_cluster_with_final_snapshot():
+    client = boto3.client("redshift", region_name="us-east-1")
+
+    with pytest.raises(ClientError) as ex:
+        client.delete_cluster(ClusterIdentifier="non-existent")
+    ex.value.response["Error"]["Code"].should.equal("ClusterNotFound")
+    ex.value.response["Error"]["Message"].should.match(r"Cluster .+ not found.")
+
+    cluster_identifier = "my_cluster"
+    client.create_cluster(
+        ClusterIdentifier=cluster_identifier,
+        ClusterType="single-node",
+        DBName="test",
+        MasterUsername="user",
+        MasterUserPassword="password",
+        NodeType="ds2.xlarge",
+    )
+
+    with pytest.raises(ClientError) as ex:
+        client.delete_cluster(
+            ClusterIdentifier=cluster_identifier, SkipFinalClusterSnapshot=False
+        )
+    ex.value.response["Error"]["Code"].should.equal("InvalidParameterCombination")
+    ex.value.response["Error"]["Message"].should.contain(
+        "FinalClusterSnapshotIdentifier is required unless SkipFinalClusterSnapshot is specified."
+    )
+
+    snapshot_identifier = "my_snapshot"
+    client.delete_cluster(
+        ClusterIdentifier=cluster_identifier,
+        SkipFinalClusterSnapshot=False,
+        FinalClusterSnapshotIdentifier=snapshot_identifier,
+    )
+
+    resp = client.describe_cluster_snapshots(ClusterIdentifier=cluster_identifier)
+    resp["Snapshots"].should.have.length_of(1)
+    resp["Snapshots"][0]["SnapshotIdentifier"].should.equal(snapshot_identifier)
+    resp["Snapshots"][0]["SnapshotType"].should.equal("manual")
+
+    with pytest.raises(ClientError) as ex:
+        client.describe_clusters(ClusterIdentifier=cluster_identifier)
+    ex.value.response["Error"]["Code"].should.equal("ClusterNotFound")
+    ex.value.response["Error"]["Message"].should.match(r"Cluster .+ not found.")
+
+
+@mock_redshift
+def test_delete_cluster_without_final_snapshot():
+    client = boto3.client("redshift", region_name="us-east-1")
+    cluster_identifier = "my_cluster"
+    client.create_cluster(
+        ClusterIdentifier=cluster_identifier,
+        ClusterType="single-node",
+        DBName="test",
+        MasterUsername="user",
+        MasterUserPassword="password",
+        NodeType="ds2.xlarge",
+    )
+    client.delete_cluster(
+        ClusterIdentifier=cluster_identifier, SkipFinalClusterSnapshot=True
+    )
+
+    resp = client.describe_cluster_snapshots(ClusterIdentifier=cluster_identifier)
+    resp["Snapshots"].should.have.length_of(0)
+
+    with pytest.raises(ClientError) as ex:
+        client.describe_clusters(ClusterIdentifier=cluster_identifier)
+    ex.value.response["Error"]["Code"].should.equal("ClusterNotFound")
+    ex.value.response["Error"]["Message"].should.match(r"Cluster .+ not found.")

tests/test_redshift/test_server.py

@@ -1,6 +1,5 @@
 from __future__ import unicode_literals
 
-import json
 import sure  # noqa
 
 import moto.server as server
@@ -20,3 +19,14 @@ def test_describe_clusters():
 
     result = res.data.decode("utf-8")
     result.should.contain("<Clusters></Clusters>")
+
+
+@mock_redshift
+def test_describe_clusters_with_json_content_type():
+    backend = server.create_backend_app("redshift")
+    test_client = backend.test_client()
+
+    res = test_client.get("/?Action=DescribeClusters&ContentType=JSON")
+
+    result = res.data.decode("utf-8")
+    result.should.contain('{"Clusters": []}')