catalin/moto
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

Supporting more modern Firehose features

Browse source 
This implements ExtendedS3DestinationConfiguration, allowing
serialization to Parquet and ORC formats as part of a Firehose
destination.
This commit is contained in:
Jack Danger 2019-10-10 12:00:31 -07:00
commit 0723fcf6e7
3 changed files with 117 additions and 35 deletions
Download patch file Download diff file Expand all files Collapse all files

29
moto/kinesis/models.py
View file

@ -222,12 +222,8 @@ class DeliveryStream(BaseModel):
self.redshift_role_arn = stream_kwargs.get('redshift_role_arn')
self.redshift_copy_command = stream_kwargs.get('redshift_copy_command')
self.s3_role_arn = stream_kwargs.get('s3_role_arn')
self.s3_bucket_arn = stream_kwargs.get('s3_bucket_arn')
self.s3_prefix = stream_kwargs.get('s3_prefix')
self.s3_compression_format = stream_kwargs.get(
's3_compression_format', 'UNCOMPRESSED')
self.s3_buffering_hings = stream_kwargs.get('s3_buffering_hings')
self.s3_config = stream_kwargs.get('s3_config')
self.extended_s3_config = stream_kwargs.get('extended_s3_config')
self.redshift_s3_role_arn = stream_kwargs.get('redshift_s3_role_arn')
self.redshift_s3_bucket_arn = stream_kwargs.get(
@ -235,8 +231,8 @@ class DeliveryStream(BaseModel):
self.redshift_s3_prefix = stream_kwargs.get('redshift_s3_prefix')
self.redshift_s3_compression_format = stream_kwargs.get(
'redshift_s3_compression_format', 'UNCOMPRESSED')
self.redshift_s3_buffering_hings = stream_kwargs.get(
'redshift_s3_buffering_hings')
self.redshift_s3_buffering_hints = stream_kwargs.get(
'redshift_s3_buffering_hints')
self.records = []
self.status = 'ACTIVE'
@ -248,16 +244,15 @@ class DeliveryStream(BaseModel):
return 'arn:aws:firehose:us-east-1:123456789012:deliverystream/{0}'.format(self.name)
def destinations_to_dict(self):
if self.s3_role_arn:
if self.s3_config:
return [{
'DestinationId': 'string',
'S3DestinationDescription': {
'RoleARN': self.s3_role_arn,
'BucketARN': self.s3_bucket_arn,
'Prefix': self.s3_prefix,
'BufferingHints': self.s3_buffering_hings,
'CompressionFormat': self.s3_compression_format,
}
'S3DestinationDescription': self.s3_config,
}]
elif self.extended_s3_config:
return [{
'DestinationId': 'string',
'ExtendedS3DestinationDescription': self.extended_s3_config,
}]
else:
return [{
@ -268,7 +263,7 @@ class DeliveryStream(BaseModel):
"RoleARN": self.redshift_role_arn,
"S3DestinationDescription": {
"BucketARN": self.redshift_s3_bucket_arn,
"BufferingHints": self.redshift_s3_buffering_hings,
"BufferingHints": self.redshift_s3_buffering_hints,
"CompressionFormat": self.redshift_s3_compression_format,
"Prefix": self.redshift_s3_prefix,
"RoleARN": self.redshift_s3_role_arn

21
moto/kinesis/responses.py
View file

@ -149,6 +149,10 @@ class KinesisResponse(BaseResponse):
stream_name = self.parameters['DeliveryStreamName']
redshift_config = self.parameters.get(
'RedshiftDestinationConfiguration')
s3_config = self.parameters.get(
'S3DestinationConfiguration')
extended_s3_config = self.parameters.get(
'ExtendedS3DestinationConfiguration')
if redshift_config:
redshift_s3_config = redshift_config['S3Configuration']
@ -163,18 +167,13 @@ class KinesisResponse(BaseResponse):
'redshift_s3_bucket_arn': redshift_s3_config['BucketARN'],
'redshift_s3_prefix': redshift_s3_config['Prefix'],
'redshift_s3_compression_format': redshift_s3_config.get('CompressionFormat'),
'redshift_s3_buffering_hings': redshift_s3_config['BufferingHints'],
}
else:
# S3 Config
s3_config = self.parameters['S3DestinationConfiguration']
stream_kwargs = {
's3_role_arn': s3_config['RoleARN'],
's3_bucket_arn': s3_config['BucketARN'],
's3_prefix': s3_config['Prefix'],
's3_compression_format': s3_config.get('CompressionFormat'),
's3_buffering_hings': s3_config['BufferingHints'],
'redshift_s3_buffering_hints': redshift_s3_config['BufferingHints'],
}
elif s3_config:
stream_kwargs = {'s3_config': s3_config}
elif extended_s3_config:
stream_kwargs = {'extended_s3_config': extended_s3_config}
stream = self.kinesis_backend.create_delivery_stream(
stream_name, **stream_kwargs)
return json.dumps({