catalin/moto
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

Add support for RDS resource filtering (#3669)

Browse source 
* Add support for RDS resource filtering

* Extensive testing was performed against real AWS endpoints in order to
  nail down the filter behavior under various scenarios, ensuring that
  `moto` returns the proper response or error.
* Full test coverage of all utility functions as well as several
  filter/parameter combinations.

* Split up filter tests, per PR feedback

* Remove unused import

* Fix pytest teardown failure on Python 2.7
This commit is contained in:
Brian Pandola 2021-02-10 01:06:54 -08:00 • committed by GitHub
commit 8557fb8439
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
6 changed files with 788 additions and 23 deletions
Download patch file Download diff file Expand all files Collapse all files

16
moto/rds2/exceptions.py
View file

@ -29,10 +29,10 @@ class DBInstanceNotFoundError(RDSClientError):
class DBSnapshotNotFoundError(RDSClientError):
def __init__(self):
def __init__(self, snapshot_identifier):
super(DBSnapshotNotFoundError, self).__init__(
"DBSnapshotNotFound",
"DBSnapshotIdentifier does not refer to an existing DB snapshot.",
"DBSnapshot {} not found.".format(snapshot_identifier),
)
@ -107,3 +107,15 @@ class DBSnapshotAlreadyExistsError(RDSClientError):
database_snapshot_identifier
),
)
class InvalidParameterValue(RDSClientError):
def __init__(self, message):
super(InvalidParameterValue, self).__init__("InvalidParameterValue", message)
class InvalidParameterCombination(RDSClientError):
def __init__(self, message):
super(InvalidParameterCombination, self).__init__(
"InvalidParameterCombination", message
)

84
moto/rds2/models.py
View file

@ -25,10 +25,24 @@ from .exceptions import (
InvalidDBInstanceStateError,
SnapshotQuotaExceededError,
DBSnapshotAlreadyExistsError,
InvalidParameterValue,
InvalidParameterCombination,
)
from .utils import FilterDef, apply_filter, merge_filters, validate_filters
class Database(CloudFormationModel):
SUPPORTED_FILTERS = {
"db-cluster-id": FilterDef(None, "DB Cluster Identifiers"),
"db-instance-id": FilterDef(
["db_instance_arn", "db_instance_identifier"], "DB Instance Identifiers"
),
"dbi-resource-id": FilterDef(["dbi_resource_id"], "Dbi Resource Ids"),
"domain": FilterDef(None, ""),
"engine": FilterDef(["engine"], "Engine Names"),
}
def __init__(self, **kwargs):
self.status = "available"
self.is_replica = False
@ -517,6 +531,18 @@ class Database(CloudFormationModel):
class Snapshot(BaseModel):
SUPPORTED_FILTERS = {
"db-instance-id": FilterDef(
["database.db_instance_arn", "database.db_instance_identifier"],
"DB Instance Identifiers",
),
"db-snapshot-id": FilterDef(["snapshot_id"], "DB Snapshot Identifiers"),
"dbi-resource-id": FilterDef(["database.dbi_resource_id"], "Dbi Resource Ids"),
"snapshot-type": FilterDef(None, "Snapshot Types"),
"engine": FilterDef(["database.engine"], "Engine Names"),
}
def __init__(self, database, snapshot_id, tags):
self.database = database
self.snapshot_id = snapshot_id
@ -534,6 +560,7 @@ class Snapshot(BaseModel):
"""<DBSnapshot>
<DBSnapshotIdentifier>{{ snapshot.snapshot_id }}</DBSnapshotIdentifier>
<DBInstanceIdentifier>{{ database.db_instance_identifier }}</DBInstanceIdentifier>
<DbiResourceId>{{ database.dbi_resource_id }}</DbiResourceId>
<SnapshotCreateTime>{{ snapshot.created_at }}</SnapshotCreateTime>
<Engine>{{ database.engine }}</Engine>
<AllocatedStorage>{{ database.allocated_storage }}</AllocatedStorage>
@ -839,7 +866,7 @@ class RDS2Backend(BaseBackend):
def delete_snapshot(self, db_snapshot_identifier):
if db_snapshot_identifier not in self.snapshots:
raise DBSnapshotNotFoundError()
raise DBSnapshotNotFoundError(db_snapshot_identifier)
return self.snapshots.pop(db_snapshot_identifier)
@ -858,28 +885,35 @@ class RDS2Backend(BaseBackend):
primary.add_replica(replica)
return replica
def describe_databases(self, db_instance_identifier=None):
def describe_databases(self, db_instance_identifier=None, filters=None):
databases = self.databases
if db_instance_identifier:
if db_instance_identifier in self.databases:
return [self.databases[db_instance_identifier]]
else:
raise DBInstanceNotFoundError(db_instance_identifier)
return self.databases.values()
filters = merge_filters(
filters, {"db-instance-id": [db_instance_identifier]}
)
if filters:
databases = self._filter_resources(databases, filters, Database)
if db_instance_identifier and not databases:
raise DBInstanceNotFoundError(db_instance_identifier)
return list(databases.values())
def describe_snapshots(self, db_instance_identifier, db_snapshot_identifier):
def describe_snapshots(
self, db_instance_identifier, db_snapshot_identifier, filters=None
):
snapshots = self.snapshots
if db_instance_identifier:
db_instance_snapshots = []
for snapshot in self.snapshots.values():
if snapshot.database.db_instance_identifier == db_instance_identifier:
db_instance_snapshots.append(snapshot)
return db_instance_snapshots
filters = merge_filters(
filters, {"db-instance-id": [db_instance_identifier]}
)
if db_snapshot_identifier:
if db_snapshot_identifier in self.snapshots:
return [self.snapshots[db_snapshot_identifier]]
raise DBSnapshotNotFoundError()
return self.snapshots.values()
filters = merge_filters(
filters, {"db-snapshot-id": [db_snapshot_identifier]}
)
if filters:
snapshots = self._filter_resources(snapshots, filters, Snapshot)
if db_snapshot_identifier and not snapshots and not db_instance_identifier:
raise DBSnapshotNotFoundError(db_snapshot_identifier)
return list(snapshots.values())
def modify_database(self, db_instance_identifier, db_kwargs):
database = self.describe_databases(db_instance_identifier)[0]
@ -1322,6 +1356,18 @@ class RDS2Backend(BaseBackend):
"InvalidParameterValue", "Invalid resource name: {0}".format(arn)
)
@staticmethod
def _filter_resources(resources, filters, resource_class):
try:
filter_defs = resource_class.SUPPORTED_FILTERS
validate_filters(filters, filter_defs)
return apply_filter(resources, filters, filter_defs)
except KeyError as e:
# https://stackoverflow.com/questions/24998968/why-does-strkeyerror-add-extra-quotes
raise InvalidParameterValue(e.args[0])
except ValueError as e:
raise InvalidParameterCombination(str(e))
class OptionGroup(object):
def __init__(self, name, engine_name, major_engine_version, description=None):

9
moto/rds2/responses.py
View file

@ -5,6 +5,7 @@ from moto.core.responses import BaseResponse
from moto.ec2.models import ec2_backends
from .models import rds2_backends
from .exceptions import DBParameterGroupNotFoundError
from .utils import filters_from_querystring
class RDS2Response(BaseResponse):
@ -122,7 +123,10 @@ class RDS2Response(BaseResponse):
def describe_db_instances(self):
db_instance_identifier = self._get_param("DBInstanceIdentifier")
all_instances = list(self.backend.describe_databases(db_instance_identifier))
filters = filters_from_querystring(self.querystring)
all_instances = list(
self.backend.describe_databases(db_instance_identifier, filters=filters)
)
marker = self._get_param("Marker")
all_ids = [instance.db_instance_identifier for instance in all_instances]
if marker:
@ -178,8 +182,9 @@ class RDS2Response(BaseResponse):
def describe_db_snapshots(self):
db_instance_identifier = self._get_param("DBInstanceIdentifier")
db_snapshot_identifier = self._get_param("DBSnapshotIdentifier")
filters = filters_from_querystring(self.querystring)
snapshots = self.backend.describe_snapshots(
db_instance_identifier, db_snapshot_identifier
db_instance_identifier, db_snapshot_identifier, filters
)
template = self.response_template(DESCRIBE_SNAPSHOTS_TEMPLATE)
return template.render(snapshots=snapshots)

169
moto/rds2/utils.py Normal file
View file

@ -0,0 +1,169 @@
from __future__ import unicode_literals
import re
from collections import namedtuple
from botocore.utils import merge_dicts
from moto.compat import OrderedDict
FilterDef = namedtuple(
"FilterDef",
[
# A list of object attributes to check against the filter values.
# Set to None if filter is not yet implemented in `moto`.
"attrs_to_check",
# Description of the filter, e.g. 'Object Identifiers'.
# Used in filter error messaging.
"description",
],
)
def filters_from_querystring(querystring):
"""Parses filters out of the query string computed by the
moto.core.responses.BaseResponse class.
:param dict[str, list[str]] querystring:
The `moto`-processed URL query string dictionary.
:returns:
Dict mapping filter names to filter values.
:rtype:
dict[str, list[str]]
"""
response_values = {}
for key, value in sorted(querystring.items()):
match = re.search(r"Filters.Filter.(\d).Name", key)
if match:
filter_index = match.groups()[0]
value_prefix = "Filters.Filter.{0}.Value".format(filter_index)
filter_values = [
filter_value[0]
for filter_key, filter_value in querystring.items()
if filter_key.startswith(value_prefix)
]
# The AWS query protocol serializes empty lists as an empty string.
if filter_values == [""]:
filter_values = []
response_values[value[0]] = filter_values
return response_values
def get_object_value(obj, attr):
"""Retrieves an arbitrary attribute value from an object.
Nested attributes can be specified using dot notation,
e.g. 'parent.child'.
:param object obj:
A valid Python object.
:param str attr:
The attribute name of the value to retrieve from the object.
:returns:
The attribute value, if it exists, or None.
:rtype:
any
"""
keys = attr.split(".")
val = obj
for key in keys:
if hasattr(val, key):
val = getattr(val, key)
else:
return None
return val
def merge_filters(filters_to_update, filters_to_merge):
"""Given two groups of filters, merge the second into the first.
List values are appended instead of overwritten:
>>> merge_filters({'filter-name': ['value1']}, {'filter-name':['value2']})
>>> {'filter-name': ['value1', 'value2']}
:param filters_to_update:
The filters to update.
:type filters_to_update:
dict[str, list] or None
:param filters_to_merge:
The filters to merge.
:type filters_to_merge:
dict[str, list] or None
:returns:
The updated filters.
:rtype:
dict[str, list]
"""
if filters_to_update is None:
filters_to_update = {}
if filters_to_merge is None:
filters_to_merge = {}
merge_dicts(filters_to_update, filters_to_merge, append_lists=True)
return filters_to_update
def validate_filters(filters, filter_defs):
"""Validates filters against a set of filter definitions.
Raises standard Python exceptions which should be caught
and translated to an appropriate AWS/Moto exception higher
up the call stack.
:param dict[str, list] filters:
The filters to validate.
:param dict[str, FilterDef] filter_defs:
The filter definitions to validate against.
:returns: None
:rtype: None
:raises KeyError:
if filter name not found in the filter definitions.
:raises ValueError:
if filter values is an empty list.
:raises NotImplementedError:
if `moto` does not yet support this filter.
"""
for filter_name, filter_values in filters.items():
filter_def = filter_defs.get(filter_name)
if filter_def is None:
raise KeyError("Unrecognized filter name: {}".format(filter_name))
if not filter_values:
raise ValueError(
"The list of {} must not be empty.".format(filter_def.description)
)
if filter_def.attrs_to_check is None:
raise NotImplementedError(
"{} filter has not been implemented in Moto yet.".format(filter_name)
)
def apply_filter(resources, filters, filter_defs):
"""Apply an arbitrary filter to a group of resources.
:param dict[str, object] resources:
A dictionary mapping resource identifiers to resource objects.
:param dict[str, list] filters:
The filters to apply.
:param dict[str, FilterDef] filter_defs:
The supported filter definitions for the resource type.
:returns:
The filtered collection of resources.
:rtype:
dict[str, object]
"""
resources_filtered = OrderedDict()
for identifier, obj in resources.items():
matches_filter = False
for filter_name, filter_values in filters.items():
filter_def = filter_defs.get(filter_name)
for attr in filter_def.attrs_to_check:
if get_object_value(obj, attr) in filter_values:
matches_filter = True
break
else:
matches_filter = False
if not matches_filter:
break
if matches_filter:
resources_filtered[identifier] = obj
return resources_filtered