Merge branch 'master' into add-organizations-tags

This commit is contained in:
Anton Grübel 2019-11-22 16:35:20 +01:00 committed by GitHub
commit 57c97e367e
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
24 changed files with 819 additions and 86 deletions

View file

@ -46,8 +46,9 @@ try:
except ImportError:
from backports.tempfile import TemporaryDirectory
_stderr_regex = re.compile(r"START|END|REPORT RequestId: .*")
# The lambci container is returning a special escape character for the "RequestID" fields. Unicode 033:
# _stderr_regex = re.compile(r"START|END|REPORT RequestId: .*")
_stderr_regex = re.compile(r"\033\[\d+.*")
_orig_adapter_send = requests.adapters.HTTPAdapter.send
docker_3 = docker.__version__[0] >= "3"
@ -444,7 +445,7 @@ class LambdaFunction(BaseModel):
if exit_code != 0:
raise Exception("lambda invoke failed output: {}".format(output))
# strip out RequestId lines
# strip out RequestId lines (TODO: This will return an additional '\n' in the response)
output = os.linesep.join(
[
line

View file

@ -624,7 +624,7 @@ class BatchBackend(BaseBackend):
def get_job_definition(self, identifier):
"""
Get job defintiion by name or ARN
Get job definitions by name or ARN
:param identifier: Name or ARN
:type identifier: str
@ -643,7 +643,7 @@ class BatchBackend(BaseBackend):
def get_job_definitions(self, identifier):
"""
Get job defintiion by name or ARN
Get job definitions by name or ARN
:param identifier: Name or ARN
:type identifier: str
@ -934,7 +934,7 @@ class BatchBackend(BaseBackend):
self.ecs_backend.delete_cluster(compute_env.ecs_name)
if compute_env.env_type == "MANAGED":
# Delete compute envrionment
# Delete compute environment
instance_ids = [instance.id for instance in compute_env.instances]
self.ec2_backend.terminate_instances(instance_ids)
@ -1195,7 +1195,7 @@ class BatchBackend(BaseBackend):
depends_on=None,
container_overrides=None,
):
# TODO parameters, retries (which is a dict raw from request), job dependancies and container overrides are ignored for now
# TODO parameters, retries (which is a dict raw from request), job dependencies and container overrides are ignored for now
# Look for job definition
job_def = self.get_job_definition(job_def_id)

View file

@ -44,6 +44,7 @@ class BaseMockAWS(object):
"AWS_ACCESS_KEY_ID": "foobar_key",
"AWS_SECRET_ACCESS_KEY": "foobar_secret",
}
self.default_session_mock = mock.patch("boto3.DEFAULT_SESSION", None)
self.env_variables_mocks = mock.patch.dict(os.environ, FAKE_KEYS)
if self.__class__.nested_count == 0:
@ -62,6 +63,7 @@ class BaseMockAWS(object):
self.stop()
def start(self, reset=True):
self.default_session_mock.start()
self.env_variables_mocks.start()
self.__class__.nested_count += 1
@ -72,6 +74,7 @@ class BaseMockAWS(object):
self.enable_patching()
def stop(self):
self.default_session_mock.stop()
self.env_variables_mocks.stop()
self.__class__.nested_count -= 1

View file

@ -307,7 +307,7 @@ class BaseResponse(_TemplateEnvironmentMixin, ActionAuthenticatorMixin):
def _convert(elem, is_last):
if not re.match("^{.*}$", elem):
return elem
name = elem.replace("{", "").replace("}", "")
name = elem.replace("{", "").replace("}", "").replace("+", "")
if is_last:
return "(?P<%s>[^/]*)" % name
return "(?P<%s>.*)" % name

View file

@ -153,7 +153,7 @@ class DataSyncResponse(BaseResponse):
task_execution_arn = self._get_param("TaskExecutionArn")
task_execution = self.datasync_backend._get_task_execution(task_execution_arn)
result = json.dumps(
{"TaskExecutionArn": task_execution.arn, "Status": task_execution.status,}
{"TaskExecutionArn": task_execution.arn, "Status": task_execution.status}
)
if task_execution.status == "SUCCESS":
self.datasync_backend.tasks[task_execution.task_arn].status = "AVAILABLE"

View file

@ -4,6 +4,4 @@ from .responses import DataSyncResponse
url_bases = ["https?://(.*?)(datasync)(.*?).amazonaws.com"]
url_paths = {
"{0}/$": DataSyncResponse.dispatch,
}
url_paths = {"{0}/$": DataSyncResponse.dispatch}

View file

@ -316,8 +316,7 @@ class EventsBackend(BaseBackend):
if not event_bus:
raise JsonRESTError(
"ResourceNotFoundException",
"Event bus {} does not exist.".format(name),
"ResourceNotFoundException", "Event bus {} does not exist.".format(name)
)
return event_bus

View file

@ -261,10 +261,7 @@ class EventsHandler(BaseResponse):
name = self._get_param("Name")
event_bus = self.events_backend.describe_event_bus(name)
response = {
"Name": event_bus.name,
"Arn": event_bus.arn,
}
response = {"Name": event_bus.name, "Arn": event_bus.arn}
if event_bus.policy:
response["Policy"] = event_bus.policy
@ -285,10 +282,7 @@ class EventsHandler(BaseResponse):
response = []
for event_bus in self.events_backend.list_event_buses(name_prefix):
event_bus_response = {
"Name": event_bus.name,
"Arn": event_bus.arn,
}
event_bus_response = {"Name": event_bus.name, "Arn": event_bus.arn}
if event_bus.policy:
event_bus_response["Policy"] = event_bus.policy

View file

@ -1,5 +1,6 @@
from __future__ import unicode_literals
import base64
import hashlib
import os
import random
import string
@ -475,6 +476,20 @@ class AccessKey(BaseModel):
raise UnformattedGetAttTemplateException()
class SshPublicKey(BaseModel):
def __init__(self, user_name, ssh_public_key_body):
self.user_name = user_name
self.ssh_public_key_body = ssh_public_key_body
self.ssh_public_key_id = "APKA" + random_access_key()
self.fingerprint = hashlib.md5(ssh_public_key_body.encode()).hexdigest()
self.status = "Active"
self.upload_date = datetime.utcnow()
@property
def uploaded_iso_8601(self):
return iso_8601_datetime_without_milliseconds(self.upload_date)
class Group(BaseModel):
def __init__(self, name, path="/"):
self.name = name
@ -536,6 +551,7 @@ class User(BaseModel):
self.policies = {}
self.managed_policies = {}
self.access_keys = []
self.ssh_public_keys = []
self.password = None
self.password_reset_required = False
self.signing_certificates = {}
@ -605,6 +621,33 @@ class User(BaseModel):
"The Access Key with id {0} cannot be found".format(access_key_id)
)
def upload_ssh_public_key(self, ssh_public_key_body):
pubkey = SshPublicKey(self.name, ssh_public_key_body)
self.ssh_public_keys.append(pubkey)
return pubkey
def get_ssh_public_key(self, ssh_public_key_id):
for key in self.ssh_public_keys:
if key.ssh_public_key_id == ssh_public_key_id:
return key
else:
raise IAMNotFoundException(
"The SSH Public Key with id {0} cannot be found".format(
ssh_public_key_id
)
)
def get_all_ssh_public_keys(self):
return self.ssh_public_keys
def update_ssh_public_key(self, ssh_public_key_id, status):
key = self.get_ssh_public_key(ssh_public_key_id)
key.status = status
def delete_ssh_public_key(self, ssh_public_key_id):
key = self.get_ssh_public_key(ssh_public_key_id)
self.ssh_public_keys.remove(key)
def get_cfn_attribute(self, attribute_name):
from moto.cloudformation.exceptions import UnformattedGetAttTemplateException
@ -736,6 +779,134 @@ class AccountPasswordPolicy(BaseModel):
)
class AccountSummary(BaseModel):
def __init__(self, iam_backend):
self._iam_backend = iam_backend
self._group_policy_size_quota = 5120
self._instance_profiles_quota = 1000
self._groups_per_user_quota = 10
self._attached_policies_per_user_quota = 10
self._policies_quota = 1500
self._account_mfa_enabled = 0 # Haven't found any information being able to activate MFA for the root account programmatically
self._access_keys_per_user_quota = 2
self._assume_role_policy_size_quota = 2048
self._policy_versions_in_use_quota = 10000
self._global_endpoint_token_version = (
1 # ToDo: Implement set_security_token_service_preferences()
)
self._versions_per_policy_quota = 5
self._attached_policies_per_group_quota = 10
self._policy_size_quota = 6144
self._account_signing_certificates_present = 0 # valid values: 0 | 1
self._users_quota = 5000
self._server_certificates_quota = 20
self._user_policy_size_quota = 2048
self._roles_quota = 1000
self._signing_certificates_per_user_quota = 2
self._role_policy_size_quota = 10240
self._attached_policies_per_role_quota = 10
self._account_access_keys_present = 0 # valid values: 0 | 1
self._groups_quota = 300
@property
def summary_map(self):
return {
"GroupPolicySizeQuota": self._group_policy_size_quota,
"InstanceProfilesQuota": self._instance_profiles_quota,
"Policies": self._policies,
"GroupsPerUserQuota": self._groups_per_user_quota,
"InstanceProfiles": self._instance_profiles,
"AttachedPoliciesPerUserQuota": self._attached_policies_per_user_quota,
"Users": self._users,
"PoliciesQuota": self._policies_quota,
"Providers": self._providers,
"AccountMFAEnabled": self._account_mfa_enabled,
"AccessKeysPerUserQuota": self._access_keys_per_user_quota,
"AssumeRolePolicySizeQuota": self._assume_role_policy_size_quota,
"PolicyVersionsInUseQuota": self._policy_versions_in_use_quota,
"GlobalEndpointTokenVersion": self._global_endpoint_token_version,
"VersionsPerPolicyQuota": self._versions_per_policy_quota,
"AttachedPoliciesPerGroupQuota": self._attached_policies_per_group_quota,
"PolicySizeQuota": self._policy_size_quota,
"Groups": self._groups,
"AccountSigningCertificatesPresent": self._account_signing_certificates_present,
"UsersQuota": self._users_quota,
"ServerCertificatesQuota": self._server_certificates_quota,
"MFADevices": self._mfa_devices,
"UserPolicySizeQuota": self._user_policy_size_quota,
"PolicyVersionsInUse": self._policy_versions_in_use,
"ServerCertificates": self._server_certificates,
"Roles": self._roles,
"RolesQuota": self._roles_quota,
"SigningCertificatesPerUserQuota": self._signing_certificates_per_user_quota,
"MFADevicesInUse": self._mfa_devices_in_use,
"RolePolicySizeQuota": self._role_policy_size_quota,
"AttachedPoliciesPerRoleQuota": self._attached_policies_per_role_quota,
"AccountAccessKeysPresent": self._account_access_keys_present,
"GroupsQuota": self._groups_quota,
}
@property
def _groups(self):
return len(self._iam_backend.groups)
@property
def _instance_profiles(self):
return len(self._iam_backend.instance_profiles)
@property
def _mfa_devices(self):
# Don't know, if hardware devices are also counted here
return len(self._iam_backend.virtual_mfa_devices)
@property
def _mfa_devices_in_use(self):
devices = 0
for user in self._iam_backend.users.values():
devices += len(user.mfa_devices)
return devices
@property
def _policies(self):
customer_policies = [
policy
for policy in self._iam_backend.managed_policies
if not policy.startswith("arn:aws:iam::aws:policy")
]
return len(customer_policies)
@property
def _policy_versions_in_use(self):
attachments = 0
for policy in self._iam_backend.managed_policies.values():
attachments += policy.attachment_count
return attachments
@property
def _providers(self):
providers = len(self._iam_backend.saml_providers) + len(
self._iam_backend.open_id_providers
)
return providers
@property
def _roles(self):
return len(self._iam_backend.roles)
@property
def _server_certificates(self):
return len(self._iam_backend.certificates)
@property
def _users(self):
return len(self._iam_backend.users)
class IAMBackend(BaseBackend):
def __init__(self):
self.instance_profiles = {}
@ -751,6 +922,7 @@ class IAMBackend(BaseBackend):
self.policy_arn_regex = re.compile(r"^arn:aws:iam::[0-9]*:policy/.*$")
self.virtual_mfa_devices = {}
self.account_password_policy = None
self.account_summary = AccountSummary(self)
super(IAMBackend, self).__init__()
def _init_managed_policies(self):
@ -1162,7 +1334,7 @@ class IAMBackend(BaseBackend):
def get_all_server_certs(self, marker=None):
return self.certificates.values()
def upload_server_cert(
def upload_server_certificate(
self, cert_name, cert_body, private_key, cert_chain=None, path=None
):
certificate_id = random_resource_id()
@ -1455,6 +1627,26 @@ class IAMBackend(BaseBackend):
user = self.get_user(user_name)
user.delete_access_key(access_key_id)
def upload_ssh_public_key(self, user_name, ssh_public_key_body):
user = self.get_user(user_name)
return user.upload_ssh_public_key(ssh_public_key_body)
def get_ssh_public_key(self, user_name, ssh_public_key_id):
user = self.get_user(user_name)
return user.get_ssh_public_key(ssh_public_key_id)
def get_all_ssh_public_keys(self, user_name):
user = self.get_user(user_name)
return user.get_all_ssh_public_keys()
def update_ssh_public_key(self, user_name, ssh_public_key_id, status):
user = self.get_user(user_name)
return user.update_ssh_public_key(ssh_public_key_id, status)
def delete_ssh_public_key(self, user_name, ssh_public_key_id):
user = self.get_user(user_name)
return user.delete_ssh_public_key(ssh_public_key_id)
def enable_mfa_device(
self, user_name, serial_number, authentication_code_1, authentication_code_2
):
@ -1741,5 +1933,8 @@ class IAMBackend(BaseBackend):
self.account_password_policy = None
def get_account_summary(self):
return self.account_summary
iam_backend = IAMBackend()

View file

@ -351,7 +351,7 @@ class IamResponse(BaseResponse):
private_key = self._get_param("PrivateKey")
cert_chain = self._get_param("CertificateName")
cert = iam_backend.upload_server_cert(
cert = iam_backend.upload_server_certificate(
cert_name, cert_body, private_key, cert_chain=cert_chain, path=path
)
template = self.response_template(UPLOAD_CERT_TEMPLATE)
@ -590,6 +590,46 @@ class IamResponse(BaseResponse):
template = self.response_template(GENERIC_EMPTY_TEMPLATE)
return template.render(name="DeleteAccessKey")
def upload_ssh_public_key(self):
user_name = self._get_param("UserName")
ssh_public_key_body = self._get_param("SSHPublicKeyBody")
key = iam_backend.upload_ssh_public_key(user_name, ssh_public_key_body)
template = self.response_template(UPLOAD_SSH_PUBLIC_KEY_TEMPLATE)
return template.render(key=key)
def get_ssh_public_key(self):
user_name = self._get_param("UserName")
ssh_public_key_id = self._get_param("SSHPublicKeyId")
key = iam_backend.get_ssh_public_key(user_name, ssh_public_key_id)
template = self.response_template(GET_SSH_PUBLIC_KEY_TEMPLATE)
return template.render(key=key)
def list_ssh_public_keys(self):
user_name = self._get_param("UserName")
keys = iam_backend.get_all_ssh_public_keys(user_name)
template = self.response_template(LIST_SSH_PUBLIC_KEYS_TEMPLATE)
return template.render(keys=keys)
def update_ssh_public_key(self):
user_name = self._get_param("UserName")
ssh_public_key_id = self._get_param("SSHPublicKeyId")
status = self._get_param("Status")
iam_backend.update_ssh_public_key(user_name, ssh_public_key_id, status)
template = self.response_template(UPDATE_SSH_PUBLIC_KEY_TEMPLATE)
return template.render()
def delete_ssh_public_key(self):
user_name = self._get_param("UserName")
ssh_public_key_id = self._get_param("SSHPublicKeyId")
iam_backend.delete_ssh_public_key(user_name, ssh_public_key_id)
template = self.response_template(DELETE_SSH_PUBLIC_KEY_TEMPLATE)
return template.render()
def deactivate_mfa_device(self):
user_name = self._get_param("UserName")
serial_number = self._get_param("SerialNumber")
@ -888,6 +928,12 @@ class IamResponse(BaseResponse):
template = self.response_template(DELETE_ACCOUNT_PASSWORD_POLICY_TEMPLATE)
return template.render()
def get_account_summary(self):
account_summary = iam_backend.get_account_summary()
template = self.response_template(GET_ACCOUNT_SUMMARY_TEMPLATE)
return template.render(summary_map=account_summary.summary_map)
LIST_ENTITIES_FOR_POLICY_TEMPLATE = """<ListEntitiesForPolicyResponse>
<ListEntitiesForPolicyResult>
@ -1690,6 +1736,73 @@ GET_ACCESS_KEY_LAST_USED_TEMPLATE = """
</GetAccessKeyLastUsedResponse>
"""
UPLOAD_SSH_PUBLIC_KEY_TEMPLATE = """<UploadSSHPublicKeyResponse>
<UploadSSHPublicKeyResult>
<SSHPublicKey>
<UserName>{{ key.user_name }}</UserName>
<SSHPublicKeyBody>{{ key.ssh_public_key_body }}</SSHPublicKeyBody>
<SSHPublicKeyId>{{ key.ssh_public_key_id }}</SSHPublicKeyId>
<Fingerprint>{{ key.fingerprint }}</Fingerprint>
<Status>{{ key.status }}</Status>
<UploadDate>{{ key.uploaded_iso_8601 }}</UploadDate>
</SSHPublicKey>
</UploadSSHPublicKeyResult>
<ResponseMetadata>
<RequestId>7a62c49f-347e-4fc4-9331-6e8eEXAMPLE</RequestId>
</ResponseMetadata>
</UploadSSHPublicKeyResponse>"""
GET_SSH_PUBLIC_KEY_TEMPLATE = """<GetSSHPublicKeyResponse>
<GetSSHPublicKeyResult>
<SSHPublicKey>
<UserName>{{ key.user_name }}</UserName>
<SSHPublicKeyBody>{{ key.ssh_public_key_body }}</SSHPublicKeyBody>
<SSHPublicKeyId>{{ key.ssh_public_key_id }}</SSHPublicKeyId>
<Fingerprint>{{ key.fingerprint }}</Fingerprint>
<Status>{{ key.status }}</Status>
<UploadDate>{{ key.uploaded_iso_8601 }}</UploadDate>
</SSHPublicKey>
</GetSSHPublicKeyResult>
<ResponseMetadata>
<RequestId>7a62c49f-347e-4fc4-9331-6e8eEXAMPLE</RequestId>
</ResponseMetadata>
</GetSSHPublicKeyResponse>"""
LIST_SSH_PUBLIC_KEYS_TEMPLATE = """<ListSSHPublicKeysResponse>
<ListSSHPublicKeysResult>
<SSHPublicKeys>
{% for key in keys %}
<member>
<UserName>{{ key.user_name }}</UserName>
<SSHPublicKeyId>{{ key.ssh_public_key_id }}</SSHPublicKeyId>
<Status>{{ key.status }}</Status>
<UploadDate>{{ key.uploaded_iso_8601 }}</UploadDate>
</member>
{% endfor %}
</SSHPublicKeys>
<IsTruncated>false</IsTruncated>
</ListSSHPublicKeysResult>
<ResponseMetadata>
<RequestId>7a62c49f-347e-4fc4-9331-6e8eEXAMPLE</RequestId>
</ResponseMetadata>
</ListSSHPublicKeysResponse>"""
UPDATE_SSH_PUBLIC_KEY_TEMPLATE = """<UpdateSSHPublicKeyResponse>
<UpdateSSHPublicKeyResult>
</UpdateSSHPublicKeyResult>
<ResponseMetadata>
<RequestId>7a62c49f-347e-4fc4-9331-6e8eEXAMPLE</RequestId>
</ResponseMetadata>
</UpdateSSHPublicKeyResponse>"""
DELETE_SSH_PUBLIC_KEY_TEMPLATE = """<DeleteSSHPublicKeyResponse>
<DeleteSSHPublicKeyResult>
</DeleteSSHPublicKeyResult>
<ResponseMetadata>
<RequestId>7a62c49f-347e-4fc4-9331-6e8eEXAMPLE</RequestId>
</ResponseMetadata>
</DeleteSSHPublicKeyResponse>"""
CREDENTIAL_REPORT_GENERATING = """
<GenerateCredentialReportResponse>
<GenerateCredentialReportResult>
@ -2261,3 +2374,20 @@ DELETE_ACCOUNT_PASSWORD_POLICY_TEMPLATE = """<DeleteAccountPasswordPolicyRespons
<RequestId>7a62c49f-347e-4fc4-9331-6e8eEXAMPLE</RequestId>
</ResponseMetadata>
</DeleteAccountPasswordPolicyResponse>"""
GET_ACCOUNT_SUMMARY_TEMPLATE = """<GetAccountSummaryResponse xmlns="https://iam.amazonaws.com/doc/2010-05-08/">
<GetAccountSummaryResult>
<SummaryMap>
{% for key, value in summary_map.items() %}
<entry>
<key>{{ key }}</key>
<value>{{ value }}</value>
</entry>
{% endfor %}
</SummaryMap>
</GetAccountSummaryResult>
<ResponseMetadata>
<RequestId>85cb9b90-ac28-11e4-a88d-97964EXAMPLE</RequestId>
</ResponseMetadata>
</GetAccountSummaryResponse>"""

View file

@ -125,7 +125,7 @@ class HTTPrettyRequest(BaseHTTPRequestHandler, BaseClass):
internal `parse_request` method.
It also replaces the `rfile` and `wfile` attributes with StringIO
instances so that we garantee that it won't make any I/O, neighter
instances so that we guarantee that it won't make any I/O, neighter
for writing nor reading.
It has some convenience attributes:

View file

@ -190,7 +190,7 @@ def create_backend_app(service):
index = 2
while endpoint in backend_app.view_functions:
# HACK: Sometimes we map the same view to multiple url_paths. Flask
# requries us to have different names.
# requires us to have different names.
endpoint = original_endpoint + str(index)
index += 1

View file

@ -147,7 +147,7 @@ class SESBackend(BaseBackend):
def __type_of_message__(self, destinations):
"""Checks the destination for any special address that could indicate delivery,
complaint or bounce like in SES simualtor"""
complaint or bounce like in SES simulator"""
alladdress = (
destinations.get("ToAddresses", [])
+ destinations.get("CcAddresses", [])

View file

@ -227,7 +227,7 @@ class Subscription(BaseModel):
return False
for attribute_values in attribute_values:
# Even the offical documentation states a 5 digits of accuracy after the decimal point for numerics, in reality it is 6
# Even the official documentation states a 5 digits of accuracy after the decimal point for numerics, in reality it is 6
# https://docs.aws.amazon.com/sns/latest/dg/sns-subscription-filter-policies.html#subscription-filter-policy-constraints
if int(attribute_values * 1000000) == int(rule * 1000000):
return True
@ -573,7 +573,7 @@ class SNSBackend(BaseBackend):
combinations = 1
for rules in six.itervalues(value):
combinations *= len(rules)
# Even the offical documentation states the total combination of values must not exceed 100, in reality it is 150
# Even the official documentation states the total combination of values must not exceed 100, in reality it is 150
# https://docs.aws.amazon.com/sns/latest/dg/sns-subscription-filter-policies.html#subscription-filter-policy-constraints
if combinations > 150:
raise SNSInvalidParameter(

View file

@ -761,7 +761,7 @@ class SQSBackend(BaseBackend):
new_messages = []
for message in queue._messages:
# Only delete message if it is not visible and the reciept_handle
# Only delete message if it is not visible and the receipt_handle
# matches.
if message.receipt_handle == receipt_handle:
queue.pending_messages.remove(message)

View file

@ -430,7 +430,7 @@ class WorkflowExecution(BaseModel):
)
def fail(self, event_id, details=None, reason=None):
# TODO: implement lenght constraints on details/reason
# TODO: implement length constraints on details/reason
self.execution_status = "CLOSED"
self.close_status = "FAILED"
self.close_timestamp = unix_time()