catalin/moto
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

Run black on moto & test directories.

Browse source
This commit is contained in:
Asher Foa 2019-10-31 08:44:26 -07:00
commit 96e5b1993d
507 changed files with 52541 additions and 47814 deletions
Download patch file Download diff file Expand all files Collapse all files

9
moto/logs/exceptions.py
View file

@ -10,8 +10,7 @@ class ResourceNotFoundException(LogsClientError):
def __init__(self):
self.code = 400
super(ResourceNotFoundException, self).__init__(
"ResourceNotFoundException",
"The specified resource does not exist"
"ResourceNotFoundException", "The specified resource does not exist"
)
@ -19,8 +18,7 @@ class InvalidParameterException(LogsClientError):
def __init__(self, msg=None):
self.code = 400
super(InvalidParameterException, self).__init__(
"InvalidParameterException",
msg or "A parameter is specified incorrectly."
"InvalidParameterException", msg or "A parameter is specified incorrectly."
)
@ -28,6 +26,5 @@ class ResourceAlreadyExistsException(LogsClientError):
def __init__(self):
self.code = 400
super(ResourceAlreadyExistsException, self).__init__(
'ResourceAlreadyExistsException',
'The specified log group already exists'
"ResourceAlreadyExistsException", "The specified log group already exists"
)

280
moto/logs/models.py
View file

@ -1,10 +1,7 @@
from moto.core import BaseBackend
import boto.logs
from moto.core.utils import unix_time_millis
from .exceptions import (
ResourceNotFoundException,
ResourceAlreadyExistsException
)
from .exceptions import ResourceNotFoundException, ResourceAlreadyExistsException
class LogEvent:
@ -13,7 +10,7 @@ class LogEvent:
def __init__(self, ingestion_time, log_event):
self.ingestionTime = ingestion_time
self.timestamp = log_event["timestamp"]
self.message = log_event['message']
self.message = log_event["message"]
self.eventId = self.__class__._event_id
self.__class__._event_id += 1
@ -23,14 +20,14 @@ class LogEvent:
"ingestionTime": self.ingestionTime,
# "logStreamName":
"message": self.message,
"timestamp": self.timestamp
"timestamp": self.timestamp,
}
def to_response_dict(self):
return {
"ingestionTime": self.ingestionTime,
"message": self.message,
"timestamp": self.timestamp
"timestamp": self.timestamp,
}
@ -40,22 +37,32 @@ class LogStream:
def __init__(self, region, log_group, name):
self.region = region
self.arn = "arn:aws:logs:{region}:{id}:log-group:{log_group}:log-stream:{log_stream}".format(
region=region, id=self.__class__._log_ids, log_group=log_group, log_stream=name)
region=region,
id=self.__class__._log_ids,
log_group=log_group,
log_stream=name,
)
self.creationTime = int(unix_time_millis())
self.firstEventTimestamp = None
self.lastEventTimestamp = None
self.lastIngestionTime = None
self.logStreamName = name
self.storedBytes = 0
self.uploadSequenceToken = 0 # I'm guessing this is token needed for sequenceToken by put_events
self.uploadSequenceToken = (
0 # I'm guessing this is token needed for sequenceToken by put_events
)
self.events = []
self.__class__._log_ids += 1
def _update(self):
# events can be empty when stream is described soon after creation
self.firstEventTimestamp = min([x.timestamp for x in self.events]) if self.events else None
self.lastEventTimestamp = max([x.timestamp for x in self.events]) if self.events else None
self.firstEventTimestamp = (
min([x.timestamp for x in self.events]) if self.events else None
)
self.lastEventTimestamp = (
max([x.timestamp for x in self.events]) if self.events else None
)
def to_describe_dict(self):
# Compute start and end times
@ -77,18 +84,31 @@ class LogStream:
res.update(rest)
return res
def put_log_events(self, log_group_name, log_stream_name, log_events, sequence_token):
def put_log_events(
self, log_group_name, log_stream_name, log_events, sequence_token
):
# TODO: ensure sequence_token
# TODO: to be thread safe this would need a lock
self.lastIngestionTime = int(unix_time_millis())
# TODO: make this match AWS if possible
self.storedBytes += sum([len(log_event["message"]) for log_event in log_events])
self.events += [LogEvent(self.lastIngestionTime, log_event) for log_event in log_events]
self.events += [
LogEvent(self.lastIngestionTime, log_event) for log_event in log_events
]
self.uploadSequenceToken += 1
return '{:056d}'.format(self.uploadSequenceToken)
return "{:056d}".format(self.uploadSequenceToken)
def get_log_events(self, log_group_name, log_stream_name, start_time, end_time, limit, next_token, start_from_head):
def get_log_events(
self,
log_group_name,
log_stream_name,
start_time,
end_time,
limit,
next_token,
start_from_head,
):
def filter_func(event):
if start_time and event.timestamp < start_time:
return False
@ -108,11 +128,18 @@ class LogStream:
return int(token[2:])
return 0
events = sorted(filter(filter_func, self.events), key=lambda event: event.timestamp, reverse=start_from_head)
events = sorted(
filter(filter_func, self.events),
key=lambda event: event.timestamp,
reverse=start_from_head,
)
next_index = get_index_from_paging_token(next_token)
back_index = next_index
events_page = [event.to_response_dict() for event in events[next_index: next_index + limit]]
events_page = [
event.to_response_dict()
for event in events[next_index : next_index + limit]
]
if next_index + limit < len(self.events):
next_index += limit
else:
@ -122,11 +149,25 @@ class LogStream:
if back_index <= 0:
back_index = 0
return events_page, get_paging_token_from_index(back_index, True), get_paging_token_from_index(next_index)
return (
events_page,
get_paging_token_from_index(back_index, True),
get_paging_token_from_index(next_index),
)
def filter_log_events(self, log_group_name, log_stream_names, start_time, end_time, limit, next_token, filter_pattern, interleaved):
def filter_log_events(
self,
log_group_name,
log_stream_names,
start_time,
end_time,
limit,
next_token,
filter_pattern,
interleaved,
):
if filter_pattern:
raise NotImplementedError('filter_pattern is not yet implemented')
raise NotImplementedError("filter_pattern is not yet implemented")
def filter_func(event):
if start_time and event.timestamp < start_time:
@ -138,9 +179,11 @@ class LogStream:
return True
events = []
for event in sorted(filter(filter_func, self.events), key=lambda x: x.timestamp):
for event in sorted(
filter(filter_func, self.events), key=lambda x: x.timestamp
):
event_obj = event.to_filter_dict()
event_obj['logStreamName'] = self.logStreamName
event_obj["logStreamName"] = self.logStreamName
events.append(event_obj)
return events
@ -150,72 +193,140 @@ class LogGroup:
self.name = name
self.region = region
self.arn = "arn:aws:logs:{region}:1:log-group:{log_group}".format(
region=region, log_group=name)
region=region, log_group=name
)
self.creationTime = int(unix_time_millis())
self.tags = tags
self.streams = dict() # {name: LogStream}
self.retentionInDays = None # AWS defaults to Never Expire for log group retention
self.retentionInDays = (
None # AWS defaults to Never Expire for log group retention
)
def create_log_stream(self, log_stream_name):
if log_stream_name in self.streams:
raise ResourceAlreadyExistsException()
self.streams[log_stream_name] = LogStream(self.region, self.name, log_stream_name)
self.streams[log_stream_name] = LogStream(
self.region, self.name, log_stream_name
)
def delete_log_stream(self, log_stream_name):
if log_stream_name not in self.streams:
raise ResourceNotFoundException()
del self.streams[log_stream_name]
def describe_log_streams(self, descending, limit, log_group_name, log_stream_name_prefix, next_token, order_by):
def describe_log_streams(
self,
descending,
limit,
log_group_name,
log_stream_name_prefix,
next_token,
order_by,
):
# responses only logStreamName, creationTime, arn, storedBytes when no events are stored.
log_streams = [(name, stream.to_describe_dict()) for name, stream in self.streams.items() if name.startswith(log_stream_name_prefix)]
log_streams = [
(name, stream.to_describe_dict())
for name, stream in self.streams.items()
if name.startswith(log_stream_name_prefix)
]
def sorter(item):
return item[0] if order_by == 'logStreamName' else item[1].get('lastEventTimestamp', 0)
return (
item[0]
if order_by == "logStreamName"
else item[1].get("lastEventTimestamp", 0)
)
if next_token is None:
next_token = 0
log_streams = sorted(log_streams, key=sorter, reverse=descending)
new_token = next_token + limit
log_streams_page = [x[1] for x in log_streams[next_token: new_token]]
log_streams_page = [x[1] for x in log_streams[next_token:new_token]]
if new_token >= len(log_streams):
new_token = None
return log_streams_page, new_token
def put_log_events(self, log_group_name, log_stream_name, log_events, sequence_token):
def put_log_events(
self, log_group_name, log_stream_name, log_events, sequence_token
):
if log_stream_name not in self.streams:
raise ResourceNotFoundException()
stream = self.streams[log_stream_name]
return stream.put_log_events(log_group_name, log_stream_name, log_events, sequence_token)
return stream.put_log_events(
log_group_name, log_stream_name, log_events, sequence_token
)
def get_log_events(self, log_group_name, log_stream_name, start_time, end_time, limit, next_token, start_from_head):
def get_log_events(
self,
log_group_name,
log_stream_name,
start_time,
end_time,
limit,
next_token,
start_from_head,
):
if log_stream_name not in self.streams:
raise ResourceNotFoundException()
stream = self.streams[log_stream_name]
return stream.get_log_events(log_group_name, log_stream_name, start_time, end_time, limit, next_token, start_from_head)
return stream.get_log_events(
log_group_name,
log_stream_name,
start_time,
end_time,
limit,
next_token,
start_from_head,
)
def filter_log_events(self, log_group_name, log_stream_names, start_time, end_time, limit, next_token, filter_pattern, interleaved):
streams = [stream for name, stream in self.streams.items() if not log_stream_names or name in log_stream_names]
def filter_log_events(
self,
log_group_name,
log_stream_names,
start_time,
end_time,
limit,
next_token,
filter_pattern,
interleaved,
):
streams = [
stream
for name, stream in self.streams.items()
if not log_stream_names or name in log_stream_names
]
events = []
for stream in streams:
events += stream.filter_log_events(log_group_name, log_stream_names, start_time, end_time, limit, next_token, filter_pattern, interleaved)
events += stream.filter_log_events(
log_group_name,
log_stream_names,
start_time,
end_time,
limit,
next_token,
filter_pattern,
interleaved,
)
if interleaved:
events = sorted(events, key=lambda event: event['timestamp'])
events = sorted(events, key=lambda event: event["timestamp"])
if next_token is None:
next_token = 0
events_page = events[next_token: next_token + limit]
events_page = events[next_token : next_token + limit]
next_token += limit
if next_token >= len(events):
next_token = None
searched_streams = [{"logStreamName": stream.logStreamName, "searchedCompletely": True} for stream in streams]
searched_streams = [
{"logStreamName": stream.logStreamName, "searchedCompletely": True}
for stream in streams
]
return events_page, next_token, searched_streams
def to_describe_dict(self):
@ -245,7 +356,9 @@ class LogGroup:
def untag(self, tags_to_remove):
if self.tags:
self.tags = {k: v for (k, v) in self.tags.items() if k not in tags_to_remove}
self.tags = {
k: v for (k, v) in self.tags.items() if k not in tags_to_remove
}
class LogsBackend(BaseBackend):
@ -275,13 +388,17 @@ class LogsBackend(BaseBackend):
def describe_log_groups(self, limit, log_group_name_prefix, next_token):
if log_group_name_prefix is None:
log_group_name_prefix = ''
log_group_name_prefix = ""
if next_token is None:
next_token = 0
groups = [group.to_describe_dict() for name, group in self.groups.items() if name.startswith(log_group_name_prefix)]
groups = sorted(groups, key=lambda x: x['creationTime'], reverse=True)
groups_page = groups[next_token:next_token + limit]
groups = [
group.to_describe_dict()
for name, group in self.groups.items()
if name.startswith(log_group_name_prefix)
]
groups = sorted(groups, key=lambda x: x["creationTime"], reverse=True)
groups_page = groups[next_token : next_token + limit]
next_token += limit
if next_token >= len(groups):
@ -301,30 +418,85 @@ class LogsBackend(BaseBackend):
log_group = self.groups[log_group_name]
return log_group.delete_log_stream(log_stream_name)
def describe_log_streams(self, descending, limit, log_group_name, log_stream_name_prefix, next_token, order_by):
def describe_log_streams(
self,
descending,
limit,
log_group_name,
log_stream_name_prefix,
next_token,
order_by,
):
if log_group_name not in self.groups:
raise ResourceNotFoundException()
log_group = self.groups[log_group_name]
return log_group.describe_log_streams(descending, limit, log_group_name, log_stream_name_prefix, next_token, order_by)
return log_group.describe_log_streams(
descending,
limit,
log_group_name,
log_stream_name_prefix,
next_token,
order_by,
)
def put_log_events(self, log_group_name, log_stream_name, log_events, sequence_token):
def put_log_events(
self, log_group_name, log_stream_name, log_events, sequence_token
):
# TODO: add support for sequence_tokens
if log_group_name not in self.groups:
raise ResourceNotFoundException()
log_group = self.groups[log_group_name]
return log_group.put_log_events(log_group_name, log_stream_name, log_events, sequence_token)
return log_group.put_log_events(
log_group_name, log_stream_name, log_events, sequence_token
)
def get_log_events(self, log_group_name, log_stream_name, start_time, end_time, limit, next_token, start_from_head):
def get_log_events(
self,
log_group_name,
log_stream_name,
start_time,
end_time,
limit,
next_token,
start_from_head,
):
if log_group_name not in self.groups:
raise ResourceNotFoundException()
log_group = self.groups[log_group_name]
return log_group.get_log_events(log_group_name, log_stream_name, start_time, end_time, limit, next_token, start_from_head)
return log_group.get_log_events(
log_group_name,
log_stream_name,
start_time,
end_time,
limit,
next_token,
start_from_head,
)
def filter_log_events(self, log_group_name, log_stream_names, start_time, end_time, limit, next_token, filter_pattern, interleaved):
def filter_log_events(
self,
log_group_name,
log_stream_names,
start_time,
end_time,
limit,
next_token,
filter_pattern,
interleaved,
):
if log_group_name not in self.groups:
raise ResourceNotFoundException()
log_group = self.groups[log_group_name]
return log_group.filter_log_events(log_group_name, log_stream_names, start_time, end_time, limit, next_token, filter_pattern, interleaved)
return log_group.filter_log_events(
log_group_name,
log_stream_names,
start_time,
end_time,
limit,
next_token,
filter_pattern,
interleaved,
)
def put_retention_policy(self, log_group_name, retention_in_days):
if log_group_name not in self.groups:
@ -357,4 +529,6 @@ class LogsBackend(BaseBackend):
log_group.untag(tags)
logs_backends = {region.name: LogsBackend(region.name) for region in boto.logs.regions()}
logs_backends = {
region.name: LogsBackend(region.name) for region in boto.logs.regions()
}

185
moto/logs/responses.py
View file

@ -5,6 +5,7 @@ import json
# See http://docs.aws.amazon.com/AmazonCloudWatchLogs/latest/APIReference/Welcome.html
class LogsResponse(BaseResponse):
@property
def logs_backend(self):
@ -21,135 +22,159 @@ class LogsResponse(BaseResponse):
return self.request_params.get(param, if_none)
def create_log_group(self):
log_group_name = self._get_param('logGroupName')
tags = self._get_param('tags')
log_group_name = self._get_param("logGroupName")
tags = self._get_param("tags")
assert 1 <= len(log_group_name) <= 512 # TODO: assert pattern
self.logs_backend.create_log_group(log_group_name, tags)
return ''
return ""
def delete_log_group(self):
log_group_name = self._get_param('logGroupName')
log_group_name = self._get_param("logGroupName")
self.logs_backend.delete_log_group(log_group_name)
return ''
return ""
def describe_log_groups(self):
log_group_name_prefix = self._get_param('logGroupNamePrefix')
next_token = self._get_param('nextToken')
limit = self._get_param('limit', 50)
log_group_name_prefix = self._get_param("logGroupNamePrefix")
next_token = self._get_param("nextToken")
limit = self._get_param("limit", 50)
assert limit <= 50
groups, next_token = self.logs_backend.describe_log_groups(
limit, log_group_name_prefix, next_token)
return json.dumps({
"logGroups": groups,
"nextToken": next_token
})
limit, log_group_name_prefix, next_token
)
return json.dumps({"logGroups": groups, "nextToken": next_token})
def create_log_stream(self):
log_group_name = self._get_param('logGroupName')
log_stream_name = self._get_param('logStreamName')
log_group_name = self._get_param("logGroupName")
log_stream_name = self._get_param("logStreamName")
self.logs_backend.create_log_stream(log_group_name, log_stream_name)
return ''
return ""
def delete_log_stream(self):
log_group_name = self._get_param('logGroupName')
log_stream_name = self._get_param('logStreamName')
log_group_name = self._get_param("logGroupName")
log_stream_name = self._get_param("logStreamName")
self.logs_backend.delete_log_stream(log_group_name, log_stream_name)
return ''
return ""
def describe_log_streams(self):
log_group_name = self._get_param('logGroupName')
log_stream_name_prefix = self._get_param('logStreamNamePrefix', '')
descending = self._get_param('descending', False)
limit = self._get_param('limit', 50)
log_group_name = self._get_param("logGroupName")
log_stream_name_prefix = self._get_param("logStreamNamePrefix", "")
descending = self._get_param("descending", False)
limit = self._get_param("limit", 50)
assert limit <= 50
next_token = self._get_param('nextToken')
order_by = self._get_param('orderBy', 'LogStreamName')
assert order_by in {'LogStreamName', 'LastEventTime'}
next_token = self._get_param("nextToken")
order_by = self._get_param("orderBy", "LogStreamName")
assert order_by in {"LogStreamName", "LastEventTime"}
if order_by == 'LastEventTime':
if order_by == "LastEventTime":
assert not log_stream_name_prefix
streams, next_token = self.logs_backend.describe_log_streams(
descending, limit, log_group_name, log_stream_name_prefix,
next_token, order_by)
return json.dumps({
"logStreams": streams,
"nextToken": next_token
})
descending,
limit,
log_group_name,
log_stream_name_prefix,
next_token,
order_by,
)
return json.dumps({"logStreams": streams, "nextToken": next_token})
def put_log_events(self):
log_group_name = self._get_param('logGroupName')
log_stream_name = self._get_param('logStreamName')
log_events = self._get_param('logEvents')
sequence_token = self._get_param('sequenceToken')
log_group_name = self._get_param("logGroupName")
log_stream_name = self._get_param("logStreamName")
log_events = self._get_param("logEvents")
sequence_token = self._get_param("sequenceToken")
next_sequence_token = self.logs_backend.put_log_events(log_group_name, log_stream_name, log_events, sequence_token)
return json.dumps({'nextSequenceToken': next_sequence_token})
next_sequence_token = self.logs_backend.put_log_events(
log_group_name, log_stream_name, log_events, sequence_token
)
return json.dumps({"nextSequenceToken": next_sequence_token})
def get_log_events(self):
log_group_name = self._get_param('logGroupName')
log_stream_name = self._get_param('logStreamName')
start_time = self._get_param('startTime')
log_group_name = self._get_param("logGroupName")
log_stream_name = self._get_param("logStreamName")
start_time = self._get_param("startTime")
end_time = self._get_param("endTime")
limit = self._get_param('limit', 10000)
limit = self._get_param("limit", 10000)
assert limit <= 10000
next_token = self._get_param('nextToken')
start_from_head = self._get_param('startFromHead', False)
next_token = self._get_param("nextToken")
start_from_head = self._get_param("startFromHead", False)
events, next_backward_token, next_foward_token = \
self.logs_backend.get_log_events(log_group_name, log_stream_name, start_time, end_time, limit, next_token, start_from_head)
return json.dumps({
"events": events,
"nextBackwardToken": next_backward_token,
"nextForwardToken": next_foward_token
})
(
events,
next_backward_token,
next_foward_token,
) = self.logs_backend.get_log_events(
log_group_name,
log_stream_name,
start_time,
end_time,
limit,
next_token,
start_from_head,
)
return json.dumps(
{
"events": events,
"nextBackwardToken": next_backward_token,
"nextForwardToken": next_foward_token,
}
)
def filter_log_events(self):
log_group_name = self._get_param('logGroupName')
log_stream_names = self._get_param('logStreamNames', [])
start_time = self._get_param('startTime')
log_group_name = self._get_param("logGroupName")
log_stream_names = self._get_param("logStreamNames", [])
start_time = self._get_param("startTime")
# impl, see: http://docs.aws.amazon.com/AmazonCloudWatch/latest/logs/FilterAndPatternSyntax.html
filter_pattern = self._get_param('filterPattern')
interleaved = self._get_param('interleaved', False)
filter_pattern = self._get_param("filterPattern")
interleaved = self._get_param("interleaved", False)
end_time = self._get_param("endTime")
limit = self._get_param('limit', 10000)
limit = self._get_param("limit", 10000)
assert limit <= 10000
next_token = self._get_param('nextToken')
next_token = self._get_param("nextToken")
events, next_token, searched_streams = self.logs_backend.filter_log_events(log_group_name, log_stream_names, start_time, end_time, limit, next_token, filter_pattern, interleaved)
return json.dumps({
"events": events,
"nextToken": next_token,
"searchedLogStreams": searched_streams
})
events, next_token, searched_streams = self.logs_backend.filter_log_events(
log_group_name,
log_stream_names,
start_time,
end_time,
limit,
next_token,
filter_pattern,
interleaved,
)
return json.dumps(
{
"events": events,
"nextToken": next_token,
"searchedLogStreams": searched_streams,
}
)
def put_retention_policy(self):
log_group_name = self._get_param('logGroupName')
retention_in_days = self._get_param('retentionInDays')
log_group_name = self._get_param("logGroupName")
retention_in_days = self._get_param("retentionInDays")
self.logs_backend.put_retention_policy(log_group_name, retention_in_days)
return ''
return ""
def delete_retention_policy(self):
log_group_name = self._get_param('logGroupName')
log_group_name = self._get_param("logGroupName")
self.logs_backend.delete_retention_policy(log_group_name)
return ''
return ""
def list_tags_log_group(self):
log_group_name = self._get_param('logGroupName')
log_group_name = self._get_param("logGroupName")
tags = self.logs_backend.list_tags_log_group(log_group_name)
return json.dumps({
'tags': tags
})
return json.dumps({"tags": tags})
def tag_log_group(self):
log_group_name = self._get_param('logGroupName')
tags = self._get_param('tags')
log_group_name = self._get_param("logGroupName")
tags = self._get_param("tags")
self.logs_backend.tag_log_group(log_group_name, tags)
return ''
return ""
def untag_log_group(self):
log_group_name = self._get_param('logGroupName')
tags = self._get_param('tags')
log_group_name = self._get_param("logGroupName")
tags = self._get_param("tags")
self.logs_backend.untag_log_group(log_group_name, tags)
return ''
return ""

8
moto/logs/urls.py
View file

@ -1,9 +1,5 @@
from .responses import LogsResponse
url_bases = [
"https?://logs.(.+).amazonaws.com",
]
url_bases = ["https?://logs.(.+).amazonaws.com"]
url_paths = {
'{0}/$': LogsResponse.dispatch,
}
url_paths = {"{0}/$": LogsResponse.dispatch}