Merge pull request #2751 from mikegrima/s3control
Implemented S3 Account-level public access block.
This commit is contained in:
Steve Pulec
GitHub
commit
a1ffb47ae4
11 changed files with 745 additions and 50 deletions
|
|
@ -1,8 +1,13 @@
|
|||
import datetime
|
||||
import json
|
||||
import time
|
||||
|
||||
from boto3 import Session
|
||||
|
||||
from moto.core.exceptions import InvalidNextTokenException
|
||||
from moto.core.models import ConfigQueryModel
|
||||
from moto.s3 import s3_backends
|
||||
from moto.s3.models import get_moto_s3_account_id
|
||||
|
||||
|
||||
class S3ConfigQuery(ConfigQueryModel):
|
||||
|
|
@ -118,4 +123,146 @@ class S3ConfigQuery(ConfigQueryModel):
|
|||
return config_data
|
||||
|
||||
|
||||
class S3AccountPublicAccessBlockConfigQuery(ConfigQueryModel):
|
||||
def list_config_service_resources(
|
||||
self,
|
||||
resource_ids,
|
||||
resource_name,
|
||||
limit,
|
||||
next_token,
|
||||
backend_region=None,
|
||||
resource_region=None,
|
||||
):
|
||||
# For the Account Public Access Block, they are the same for all regions. The resource ID is the AWS account ID
|
||||
# There is no resource name -- it should be a blank string "" if provided.
|
||||
|
||||
# The resource name can only ever be None or an empty string:
|
||||
if resource_name is not None and resource_name != "":
|
||||
return [], None
|
||||
|
||||
pab = None
|
||||
account_id = get_moto_s3_account_id()
|
||||
regions = [region for region in Session().get_available_regions("config")]
|
||||
|
||||
# If a resource ID was passed in, then filter accordingly:
|
||||
if resource_ids:
|
||||
for id in resource_ids:
|
||||
if account_id == id:
|
||||
pab = self.backends["global"].account_public_access_block
|
||||
break
|
||||
|
||||
# Otherwise, just grab the one from the backend:
|
||||
if not resource_ids:
|
||||
pab = self.backends["global"].account_public_access_block
|
||||
|
||||
# If it's not present, then return nothing
|
||||
if not pab:
|
||||
return [], None
|
||||
|
||||
# Filter on regions (and paginate on them as well):
|
||||
if backend_region:
|
||||
pab_list = [backend_region]
|
||||
elif resource_region:
|
||||
# Invalid region?
|
||||
if resource_region not in regions:
|
||||
return [], None
|
||||
|
||||
pab_list = [resource_region]
|
||||
|
||||
# Aggregated query where no regions were supplied so return them all:
|
||||
else:
|
||||
pab_list = regions
|
||||
|
||||
# Pagination logic:
|
||||
sorted_regions = sorted(pab_list)
|
||||
new_token = None
|
||||
|
||||
# Get the start:
|
||||
if not next_token:
|
||||
start = 0
|
||||
else:
|
||||
# Tokens for this moto feature is just the region-name:
|
||||
# For OTHER non-global resource types, it's the region concatenated with the resource ID.
|
||||
if next_token not in sorted_regions:
|
||||
raise InvalidNextTokenException()
|
||||
|
||||
start = sorted_regions.index(next_token)
|
||||
|
||||
# Get the list of items to collect:
|
||||
pab_list = sorted_regions[start : (start + limit)]
|
||||
|
||||
if len(sorted_regions) > (start + limit):
|
||||
new_token = sorted_regions[start + limit]
|
||||
|
||||
return (
|
||||
[
|
||||
{
|
||||
"type": "AWS::S3::AccountPublicAccessBlock",
|
||||
"id": account_id,
|
||||
"region": region,
|
||||
}
|
||||
for region in pab_list
|
||||
],
|
||||
new_token,
|
||||
)
|
||||
|
||||
def get_config_resource(
|
||||
self, resource_id, resource_name=None, backend_region=None, resource_region=None
|
||||
):
|
||||
# Do we even have this defined?
|
||||
if not self.backends["global"].account_public_access_block:
|
||||
return None
|
||||
|
||||
# Resource name can only ever be "" if it's supplied:
|
||||
if resource_name is not None and resource_name != "":
|
||||
return None
|
||||
|
||||
# Are we filtering based on region?
|
||||
account_id = get_moto_s3_account_id()
|
||||
regions = [region for region in Session().get_available_regions("config")]
|
||||
|
||||
# Is the resource ID correct?:
|
||||
if account_id == resource_id:
|
||||
if backend_region:
|
||||
pab_region = backend_region
|
||||
|
||||
# Invalid region?
|
||||
elif resource_region not in regions:
|
||||
return None
|
||||
|
||||
else:
|
||||
pab_region = resource_region
|
||||
|
||||
else:
|
||||
return None
|
||||
|
||||
# Format the PAB to the AWS Config format:
|
||||
creation_time = datetime.datetime.utcnow()
|
||||
config_data = {
|
||||
"version": "1.3",
|
||||
"accountId": account_id,
|
||||
"configurationItemCaptureTime": str(creation_time),
|
||||
"configurationItemStatus": "OK",
|
||||
"configurationStateId": str(
|
||||
int(time.mktime(creation_time.timetuple()))
|
||||
), # PY2 and 3 compatible
|
||||
"resourceType": "AWS::S3::AccountPublicAccessBlock",
|
||||
"resourceId": account_id,
|
||||
"awsRegion": pab_region,
|
||||
"availabilityZone": "Not Applicable",
|
||||
"configuration": self.backends[
|
||||
"global"
|
||||
].account_public_access_block.to_config_dict(),
|
||||
"supplementaryConfiguration": {},
|
||||
}
|
||||
|
||||
# The 'configuration' field is also a JSON string:
|
||||
config_data["configuration"] = json.dumps(config_data["configuration"])
|
||||
|
||||
return config_data
|
||||
|
||||
|
||||
s3_config_query = S3ConfigQuery(s3_backends)
|
||||
s3_account_public_access_block_query = S3AccountPublicAccessBlockConfigQuery(
|
||||
s3_backends
|
||||
)
|
||||
|
|
|
|||
|
|
@ -359,3 +359,12 @@ class InvalidPublicAccessBlockConfiguration(S3ClientError):
|
|||
*args,
|
||||
**kwargs
|
||||
)
|
||||
|
||||
|
||||
class WrongPublicAccessBlockAccountIdError(S3ClientError):
|
||||
code = 403
|
||||
|
||||
def __init__(self):
|
||||
super(WrongPublicAccessBlockAccountIdError, self).__init__(
|
||||
"AccessDenied", "Access Denied"
|
||||
)
|
||||
|
|
|
|||
|
|
@ -19,7 +19,7 @@ import uuid
|
|||
import six
|
||||
|
||||
from bisect import insort
|
||||
from moto.core import BaseBackend, BaseModel
|
||||
from moto.core import ACCOUNT_ID, BaseBackend, BaseModel
|
||||
from moto.core.utils import iso_8601_datetime_with_milliseconds, rfc_1123_datetime
|
||||
from .exceptions import (
|
||||
BucketAlreadyExists,
|
||||
|
|
@ -37,6 +37,7 @@ from .exceptions import (
|
|||
CrossLocationLoggingProhibitted,
|
||||
NoSuchPublicAccessBlockConfiguration,
|
||||
InvalidPublicAccessBlockConfiguration,
|
||||
WrongPublicAccessBlockAccountIdError,
|
||||
)
|
||||
from .utils import clean_key_name, _VersionedKeyStore
|
||||
|
||||
|
|
@ -58,6 +59,13 @@ DEFAULT_TEXT_ENCODING = sys.getdefaultencoding()
|
|||
OWNER = "75aa57f09aa0c8caeab4f8c24e99d10f8e7faeebf76c078efc7c6caea54ba06a"
|
||||
|
||||
|
||||
def get_moto_s3_account_id():
|
||||
"""This makes it easy for mocking AWS Account IDs when using AWS Config
|
||||
-- Simply mock.patch the ACCOUNT_ID here, and Config gets it for free.
|
||||
"""
|
||||
return ACCOUNT_ID
|
||||
|
||||
|
||||
class FakeDeleteMarker(BaseModel):
|
||||
def __init__(self, key):
|
||||
self.key = key
|
||||
|
|
@ -1163,6 +1171,7 @@ class FakeBucket(BaseModel):
|
|||
class S3Backend(BaseBackend):
|
||||
def __init__(self):
|
||||
self.buckets = {}
|
||||
self.account_public_access_block = None
|
||||
|
||||
def create_bucket(self, bucket_name, region_name):
|
||||
if bucket_name in self.buckets:
|
||||
|
|
@ -1264,6 +1273,16 @@ class S3Backend(BaseBackend):
|
|||
|
||||
return bucket.public_access_block
|
||||
|
||||
def get_account_public_access_block(self, account_id):
|
||||
# The account ID should equal the account id that is set for Moto:
|
||||
if account_id != ACCOUNT_ID:
|
||||
raise WrongPublicAccessBlockAccountIdError()
|
||||
|
||||
if not self.account_public_access_block:
|
||||
raise NoSuchPublicAccessBlockConfiguration()
|
||||
|
||||
return self.account_public_access_block
|
||||
|
||||
def set_key(
|
||||
self, bucket_name, key_name, value, storage=None, etag=None, multipart=None
|
||||
):
|
||||
|
|
@ -1356,6 +1375,13 @@ class S3Backend(BaseBackend):
|
|||
bucket = self.get_bucket(bucket_name)
|
||||
bucket.public_access_block = None
|
||||
|
||||
def delete_account_public_access_block(self, account_id):
|
||||
# The account ID should equal the account id that is set for Moto:
|
||||
if account_id != ACCOUNT_ID:
|
||||
raise WrongPublicAccessBlockAccountIdError()
|
||||
|
||||
self.account_public_access_block = None
|
||||
|
||||
def put_bucket_notification_configuration(self, bucket_name, notification_config):
|
||||
bucket = self.get_bucket(bucket_name)
|
||||
bucket.set_notification_configuration(notification_config)
|
||||
|
|
@ -1384,6 +1410,21 @@ class S3Backend(BaseBackend):
|
|||
pub_block_config.get("RestrictPublicBuckets"),
|
||||
)
|
||||
|
||||
def put_account_public_access_block(self, account_id, pub_block_config):
|
||||
# The account ID should equal the account id that is set for Moto:
|
||||
if account_id != ACCOUNT_ID:
|
||||
raise WrongPublicAccessBlockAccountIdError()
|
||||
|
||||
if not pub_block_config:
|
||||
raise InvalidPublicAccessBlockConfiguration()
|
||||
|
||||
self.account_public_access_block = PublicAccessBlock(
|
||||
pub_block_config.get("BlockPublicAcls"),
|
||||
pub_block_config.get("IgnorePublicAcls"),
|
||||
pub_block_config.get("BlockPublicPolicy"),
|
||||
pub_block_config.get("RestrictPublicBuckets"),
|
||||
)
|
||||
|
||||
def initiate_multipart(self, bucket_name, key_name, metadata):
|
||||
bucket = self.get_bucket(bucket_name)
|
||||
new_multipart = FakeMultipart(key_name, metadata)
|
||||
|
|
|
|||
|
|
@ -4,6 +4,7 @@ import re
|
|||
import sys
|
||||
|
||||
import six
|
||||
from botocore.awsrequest import AWSPreparedRequest
|
||||
|
||||
from moto.core.utils import str_to_rfc_1123_datetime, py2_strip_unicode_keys
|
||||
from six.moves.urllib.parse import parse_qs, urlparse, unquote
|
||||
|
|
@ -123,6 +124,11 @@ ACTION_MAP = {
|
|||
"uploadId": "PutObject",
|
||||
},
|
||||
},
|
||||
"CONTROL": {
|
||||
"GET": {"publicAccessBlock": "GetPublicAccessBlock"},
|
||||
"PUT": {"publicAccessBlock": "PutPublicAccessBlock"},
|
||||
"DELETE": {"publicAccessBlock": "DeletePublicAccessBlock"},
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
|
|
@ -220,7 +226,7 @@ class ResponseObject(_TemplateEnvironmentMixin, ActionAuthenticatorMixin):
|
|||
# Depending on which calling format the client is using, we don't know
|
||||
# if this is a bucket or key request so we have to check
|
||||
if self.subdomain_based_buckets(request):
|
||||
return self.key_response(request, full_url, headers)
|
||||
return self.key_or_control_response(request, full_url, headers)
|
||||
else:
|
||||
# Using path-based buckets
|
||||
return self.bucket_response(request, full_url, headers)
|
||||
|
|
@ -287,7 +293,7 @@ class ResponseObject(_TemplateEnvironmentMixin, ActionAuthenticatorMixin):
|
|||
return self._bucket_response_post(request, body, bucket_name)
|
||||
else:
|
||||
raise NotImplementedError(
|
||||
"Method {0} has not been impelemented in the S3 backend yet".format(
|
||||
"Method {0} has not been implemented in the S3 backend yet".format(
|
||||
method
|
||||
)
|
||||
)
|
||||
|
|
@ -595,6 +601,20 @@ class ResponseObject(_TemplateEnvironmentMixin, ActionAuthenticatorMixin):
|
|||
pass
|
||||
return False
|
||||
|
||||
def _parse_pab_config(self, body):
|
||||
parsed_xml = xmltodict.parse(body)
|
||||
parsed_xml["PublicAccessBlockConfiguration"].pop("@xmlns", None)
|
||||
|
||||
# If Python 2, fix the unicode strings:
|
||||
if sys.version_info[0] < 3:
|
||||
parsed_xml = {
|
||||
"PublicAccessBlockConfiguration": py2_strip_unicode_keys(
|
||||
dict(parsed_xml["PublicAccessBlockConfiguration"])
|
||||
)
|
||||
}
|
||||
|
||||
return parsed_xml
|
||||
|
||||
def _bucket_response_put(
|
||||
self, request, body, region_name, bucket_name, querystring
|
||||
):
|
||||
|
|
@ -673,19 +693,9 @@ class ResponseObject(_TemplateEnvironmentMixin, ActionAuthenticatorMixin):
|
|||
raise e
|
||||
|
||||
elif "publicAccessBlock" in querystring:
|
||||
parsed_xml = xmltodict.parse(body)
|
||||
parsed_xml["PublicAccessBlockConfiguration"].pop("@xmlns", None)
|
||||
|
||||
# If Python 2, fix the unicode strings:
|
||||
if sys.version_info[0] < 3:
|
||||
parsed_xml = {
|
||||
"PublicAccessBlockConfiguration": py2_strip_unicode_keys(
|
||||
dict(parsed_xml["PublicAccessBlockConfiguration"])
|
||||
)
|
||||
}
|
||||
|
||||
pab_config = self._parse_pab_config(body)
|
||||
self.backend.put_bucket_public_access_block(
|
||||
bucket_name, parsed_xml["PublicAccessBlockConfiguration"]
|
||||
bucket_name, pab_config["PublicAccessBlockConfiguration"]
|
||||
)
|
||||
return ""
|
||||
|
||||
|
|
@ -870,15 +880,21 @@ class ResponseObject(_TemplateEnvironmentMixin, ActionAuthenticatorMixin):
|
|||
)
|
||||
return 206, response_headers, response_content[begin : end + 1]
|
||||
|
||||
def key_response(self, request, full_url, headers):
|
||||
def key_or_control_response(self, request, full_url, headers):
|
||||
# Key and Control are lumped in because splitting out the regex is too much of a pain :/
|
||||
self.method = request.method
|
||||
self.path = self._get_path(request)
|
||||
self.headers = request.headers
|
||||
if "host" not in self.headers:
|
||||
self.headers["host"] = urlparse(full_url).netloc
|
||||
response_headers = {}
|
||||
|
||||
try:
|
||||
response = self._key_response(request, full_url, headers)
|
||||
# Is this an S3 control response?
|
||||
if isinstance(request, AWSPreparedRequest) and "s3-control" in request.url:
|
||||
response = self._control_response(request, full_url, headers)
|
||||
else:
|
||||
response = self._key_response(request, full_url, headers)
|
||||
except S3ClientError as s3error:
|
||||
response = s3error.code, {}, s3error.description
|
||||
|
||||
|
|
@ -894,6 +910,94 @@ class ResponseObject(_TemplateEnvironmentMixin, ActionAuthenticatorMixin):
|
|||
)
|
||||
return status_code, response_headers, response_content
|
||||
|
||||
def _control_response(self, request, full_url, headers):
|
||||
parsed_url = urlparse(full_url)
|
||||
query = parse_qs(parsed_url.query, keep_blank_values=True)
|
||||
method = request.method
|
||||
|
||||
if hasattr(request, "body"):
|
||||
# Boto
|
||||
body = request.body
|
||||
if hasattr(body, "read"):
|
||||
body = body.read()
|
||||
else:
|
||||
# Flask server
|
||||
body = request.data
|
||||
if body is None:
|
||||
body = b""
|
||||
|
||||
if method == "GET":
|
||||
return self._control_response_get(request, query, headers)
|
||||
elif method == "PUT":
|
||||
return self._control_response_put(request, body, query, headers)
|
||||
elif method == "DELETE":
|
||||
return self._control_response_delete(request, query, headers)
|
||||
else:
|
||||
raise NotImplementedError(
|
||||
"Method {0} has not been implemented in the S3 backend yet".format(
|
||||
method
|
||||
)
|
||||
)
|
||||
|
||||
def _control_response_get(self, request, query, headers):
|
||||
action = self.path.split("?")[0].split("/")[
|
||||
-1
|
||||
] # Gets the action out of the URL sans query params.
|
||||
self._set_action("CONTROL", "GET", action)
|
||||
self._authenticate_and_authorize_s3_action()
|
||||
|
||||
response_headers = {}
|
||||
if "publicAccessBlock" in action:
|
||||
public_block_config = self.backend.get_account_public_access_block(
|
||||
headers["x-amz-account-id"]
|
||||
)
|
||||
template = self.response_template(S3_PUBLIC_ACCESS_BLOCK_CONFIGURATION)
|
||||
return (
|
||||
200,
|
||||
response_headers,
|
||||
template.render(public_block_config=public_block_config),
|
||||
)
|
||||
|
||||
raise NotImplementedError(
|
||||
"Method {0} has not been implemented in the S3 backend yet".format(action)
|
||||
)
|
||||
|
||||
def _control_response_put(self, request, body, query, headers):
|
||||
action = self.path.split("?")[0].split("/")[
|
||||
-1
|
||||
] # Gets the action out of the URL sans query params.
|
||||
self._set_action("CONTROL", "PUT", action)
|
||||
self._authenticate_and_authorize_s3_action()
|
||||
|
||||
response_headers = {}
|
||||
if "publicAccessBlock" in action:
|
||||
pab_config = self._parse_pab_config(body)
|
||||
self.backend.put_account_public_access_block(
|
||||
headers["x-amz-account-id"],
|
||||
pab_config["PublicAccessBlockConfiguration"],
|
||||
)
|
||||
return 200, response_headers, ""
|
||||
|
||||
raise NotImplementedError(
|
||||
"Method {0} has not been implemented in the S3 backend yet".format(action)
|
||||
)
|
||||
|
||||
def _control_response_delete(self, request, query, headers):
|
||||
action = self.path.split("?")[0].split("/")[
|
||||
-1
|
||||
] # Gets the action out of the URL sans query params.
|
||||
self._set_action("CONTROL", "DELETE", action)
|
||||
self._authenticate_and_authorize_s3_action()
|
||||
|
||||
response_headers = {}
|
||||
if "publicAccessBlock" in action:
|
||||
self.backend.delete_account_public_access_block(headers["x-amz-account-id"])
|
||||
return 200, response_headers, ""
|
||||
|
||||
raise NotImplementedError(
|
||||
"Method {0} has not been implemented in the S3 backend yet".format(action)
|
||||
)
|
||||
|
||||
def _key_response(self, request, full_url, headers):
|
||||
parsed_url = urlparse(full_url)
|
||||
query = parse_qs(parsed_url.query, keep_blank_values=True)
|
||||
|
|
|
|||
|
|
@ -13,7 +13,7 @@ url_paths = {
|
|||
# subdomain key of path-based bucket
|
||||
"{0}/(?P<key_or_bucket_name>[^/]+)/?$": S3ResponseInstance.ambiguous_response,
|
||||
# path-based bucket + key
|
||||
"{0}/(?P<bucket_name_path>[^/]+)/(?P<key_name>.+)": S3ResponseInstance.key_response,
|
||||
"{0}/(?P<bucket_name_path>[^/]+)/(?P<key_name>.+)": S3ResponseInstance.key_or_control_response,
|
||||
# subdomain bucket + key with empty first part of path
|
||||
"{0}//(?P<key_name>.*)$": S3ResponseInstance.key_response,
|
||||
"{0}//(?P<key_name>.*)$": S3ResponseInstance.key_or_control_response,
|
||||
}
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue