diff --git a/data-collection/deploy/deploy-data-collection.yaml b/data-collection/deploy/deploy-data-collection.yaml index 261ba010..24e688b2 100644 --- a/data-collection/deploy/deploy-data-collection.yaml +++ b/data-collection/deploy/deploy-data-collection.yaml @@ -109,7 +109,7 @@ Mappings: us-west-1: {CodeBucket: aws-managed-cost-intelligence-dashboards-us-west-1 } us-west-2: {CodeBucket: aws-managed-cost-intelligence-dashboards-us-west-2 } StepFunctionCode: - main-v3: {TemplatePath: cfn/data-collection/source/step-functions/main-state-machine-v3.json} + main-v4: {TemplatePath: cfn/data-collection/source/step-functions/main-state-machine-v3.json} crawler-v1: {TemplatePath: cfn/data-collection/source/step-functions/crawler-state-machine-v1.json} standalone-v1: {TemplatePath: cfn/data-collection/source/step-functions/awsfeeds-state-machine-v1.json} @@ -893,7 +893,7 @@ Resources: LambdaAnalyticsARN: !GetAtt LambdaAnalytics.Arn AccountCollectorLambdaARN: !Sub "${AccountCollector.Outputs.LambdaFunctionARN}" CodeBucket: !If [ ProdCFNTemplateUsed, !FindInMap [RegionMap, !Ref "AWS::Region", CodeBucket], !Ref CFNSourceBucket ] - StepFunctionTemplate: !FindInMap [StepFunctionCode, main-v3, TemplatePath] + StepFunctionTemplate: !FindInMap [StepFunctionCode, main-v4, TemplatePath] StepFunctionExecutionRoleARN: !GetAtt StepFunctionExecutionRole.Arn SchedulerExecutionRoleARN: !GetAtt SchedulerExecutionRole.Arn @@ -913,7 +913,7 @@ Resources: LambdaAnalyticsARN: !GetAtt LambdaAnalytics.Arn AccountCollectorLambdaARN: !Sub "${AccountCollector.Outputs.LambdaFunctionARN}" CodeBucket: !If [ ProdCFNTemplateUsed, !FindInMap [RegionMap, !Ref "AWS::Region", CodeBucket], !Ref CFNSourceBucket ] - StepFunctionTemplate: !FindInMap [StepFunctionCode, main-v3, TemplatePath] + StepFunctionTemplate: !FindInMap [StepFunctionCode, main-v4, TemplatePath] StepFunctionExecutionRoleARN: !GetAtt StepFunctionExecutionRole.Arn SchedulerExecutionRoleARN: !GetAtt SchedulerExecutionRole.Arn @@ -933,7 +933,7 @@ Resources: LambdaAnalyticsARN: !GetAtt LambdaAnalytics.Arn AccountCollectorLambdaARN: !Sub "${AccountCollector.Outputs.LambdaFunctionARN}" CodeBucket: !If [ ProdCFNTemplateUsed, !FindInMap [RegionMap, !Ref "AWS::Region", CodeBucket], !Ref CFNSourceBucket ] - StepFunctionTemplate: !FindInMap [StepFunctionCode, main-v3, TemplatePath] + StepFunctionTemplate: !FindInMap [StepFunctionCode, main-v4, TemplatePath] StepFunctionExecutionRoleARN: !GetAtt StepFunctionExecutionRole.Arn LambdaManageGlueTableARN: !GetAtt LambdaManageGlueTable.Arn SchedulerExecutionRoleARN: !GetAtt SchedulerExecutionRole.Arn @@ -954,7 +954,7 @@ Resources: LambdaAnalyticsARN: !GetAtt LambdaAnalytics.Arn AccountCollectorLambdaARN: !Sub "${AccountCollector.Outputs.LambdaFunctionARN}" CodeBucket: !If [ ProdCFNTemplateUsed, !FindInMap [RegionMap, !Ref "AWS::Region", CodeBucket], !Ref CFNSourceBucket ] - StepFunctionTemplate: !FindInMap [StepFunctionCode, main-v3, TemplatePath] + StepFunctionTemplate: !FindInMap [StepFunctionCode, main-v4, TemplatePath] StepFunctionExecutionRoleARN: !GetAtt StepFunctionExecutionRole.Arn SchedulerExecutionRoleARN: !GetAtt SchedulerExecutionRole.Arn @@ -974,7 +974,7 @@ Resources: LambdaAnalyticsARN: !GetAtt LambdaAnalytics.Arn AccountCollectorLambdaARN: !Sub "${AccountCollector.Outputs.LambdaFunctionARN}" CodeBucket: !If [ ProdCFNTemplateUsed, !FindInMap [RegionMap, !Ref "AWS::Region", CodeBucket], !Ref CFNSourceBucket ] - StepFunctionTemplate: !FindInMap [StepFunctionCode, main-v3, TemplatePath] + StepFunctionTemplate: !FindInMap [StepFunctionCode, main-v4, TemplatePath] StepFunctionExecutionRoleARN: !GetAtt StepFunctionExecutionRole.Arn SchedulerExecutionRoleARN: !GetAtt SchedulerExecutionRole.Arn @@ -994,7 +994,7 @@ Resources: LambdaAnalyticsARN: !GetAtt LambdaAnalytics.Arn AccountCollectorLambdaARN: !Sub "${AccountCollector.Outputs.LambdaFunctionARN}" CodeBucket: !If [ ProdCFNTemplateUsed, !FindInMap [RegionMap, !Ref "AWS::Region", CodeBucket], !Ref CFNSourceBucket ] - StepFunctionTemplate: !FindInMap [StepFunctionCode, main-v3, TemplatePath] + StepFunctionTemplate: !FindInMap [StepFunctionCode, main-v4, TemplatePath] StepFunctionExecutionRoleARN: !GetAtt StepFunctionExecutionRole.Arn LambdaManageGlueTableARN: !GetAtt LambdaManageGlueTable.Arn SchedulerExecutionRoleARN: !GetAtt SchedulerExecutionRole.Arn @@ -1046,7 +1046,7 @@ Resources: LambdaAnalyticsARN: !GetAtt LambdaAnalytics.Arn AccountCollectorLambdaARN: !Sub "${AccountCollector.Outputs.LambdaFunctionARN}" CodeBucket: !If [ ProdCFNTemplateUsed, !FindInMap [RegionMap, !Ref "AWS::Region", CodeBucket], !Ref CFNSourceBucket ] - StepFunctionTemplate: !FindInMap [StepFunctionCode, main-v3, TemplatePath] + StepFunctionTemplate: !FindInMap [StepFunctionCode, main-v4, TemplatePath] StepFunctionExecutionRoleARN: !GetAtt StepFunctionExecutionRole.Arn SchedulerExecutionRoleARN: !GetAtt SchedulerExecutionRole.Arn @@ -1066,7 +1066,7 @@ Resources: LambdaAnalyticsARN: !GetAtt LambdaAnalytics.Arn AccountCollectorLambdaARN: !Sub "${AccountCollector.Outputs.LambdaFunctionARN}" CodeBucket: !If [ ProdCFNTemplateUsed, !FindInMap [RegionMap, !Ref "AWS::Region", CodeBucket], !Ref CFNSourceBucket ] - StepFunctionTemplate: !FindInMap [StepFunctionCode, main-v3, TemplatePath] + StepFunctionTemplate: !FindInMap [StepFunctionCode, main-v4, TemplatePath] StepFunctionExecutionRoleARN: !GetAtt StepFunctionExecutionRole.Arn SchedulerExecutionRoleARN: !GetAtt SchedulerExecutionRole.Arn RegionsInScope: @@ -1091,7 +1091,7 @@ Resources: LambdaAnalyticsARN: !GetAtt LambdaAnalytics.Arn AccountCollectorLambdaARN: !Sub "${AccountCollector.Outputs.LambdaFunctionARN}" CodeBucket: !If [ ProdCFNTemplateUsed, !FindInMap [RegionMap, !Ref "AWS::Region", CodeBucket], !Ref CFNSourceBucket ] - StepFunctionTemplate: !FindInMap [StepFunctionCode, main-v3, TemplatePath] + StepFunctionTemplate: !FindInMap [StepFunctionCode, main-v4, TemplatePath] StepFunctionExecutionRoleARN: !GetAtt StepFunctionExecutionRole.Arn SchedulerExecutionRoleARN: !GetAtt SchedulerExecutionRole.Arn RegionsInScope: @@ -1116,7 +1116,7 @@ Resources: LambdaAnalyticsARN: !GetAtt LambdaAnalytics.Arn AccountCollectorLambdaARN: !Sub "${AccountCollector.Outputs.LambdaFunctionARN}" CodeBucket: !If [ ProdCFNTemplateUsed, !FindInMap [RegionMap, !Ref "AWS::Region", CodeBucket], !Ref CFNSourceBucket ] - StepFunctionTemplate: !FindInMap [StepFunctionCode, main-v3, TemplatePath] + StepFunctionTemplate: !FindInMap [StepFunctionCode, main-v4, TemplatePath] StepFunctionExecutionRoleARN: !GetAtt StepFunctionExecutionRole.Arn SchedulerExecutionRoleARN: !GetAtt SchedulerExecutionRole.Arn @@ -1136,7 +1136,7 @@ Resources: LambdaAnalyticsARN: !GetAtt LambdaAnalytics.Arn AccountCollectorLambdaARN: !Sub "${AccountCollector.Outputs.LambdaFunctionARN}" CodeBucket: !If [ ProdCFNTemplateUsed, !FindInMap [RegionMap, !Ref "AWS::Region", CodeBucket], !Ref CFNSourceBucket ] - StepFunctionTemplate: !FindInMap [StepFunctionCode, main-v3, TemplatePath] + StepFunctionTemplate: !FindInMap [StepFunctionCode, main-v4, TemplatePath] StepFunctionExecutionRoleARN: !GetAtt StepFunctionExecutionRole.Arn SchedulerExecutionRoleARN: !GetAtt SchedulerExecutionRole.Arn @@ -1156,7 +1156,7 @@ Resources: LambdaAnalyticsARN: !GetAtt LambdaAnalytics.Arn AccountCollectorLambdaARN: !Sub "${AccountCollector.Outputs.LambdaFunctionARN}" CodeBucket: !If [ ProdCFNTemplateUsed, !FindInMap [RegionMap, !Ref "AWS::Region", CodeBucket], !Ref CFNSourceBucket ] - StepFunctionTemplate: !FindInMap [StepFunctionCode, main-v3, TemplatePath] + StepFunctionTemplate: !FindInMap [StepFunctionCode, main-v4, TemplatePath] StepFunctionExecutionRoleARN: !GetAtt StepFunctionExecutionRole.Arn SchedulerExecutionRoleARN: !GetAtt SchedulerExecutionRole.Arn RegionsInScope: @@ -1199,7 +1199,7 @@ Resources: LambdaAnalyticsARN: !GetAtt LambdaAnalytics.Arn AccountCollectorLambdaARN: !Sub "${AccountCollector.Outputs.LambdaFunctionARN}" CodeBucket: !If [ ProdCFNTemplateUsed, !FindInMap [RegionMap, !Ref "AWS::Region", CodeBucket], !Ref CFNSourceBucket ] - StepFunctionTemplate: !FindInMap [StepFunctionCode, main-v3, TemplatePath] + StepFunctionTemplate: !FindInMap [StepFunctionCode, main-v4, TemplatePath] StepFunctionExecutionRoleARN: !GetAtt StepFunctionExecutionRole.Arn SchedulerExecutionRoleARN: !GetAtt SchedulerExecutionRole.Arn @@ -1219,7 +1219,7 @@ Resources: LambdaAnalyticsARN: !GetAtt LambdaAnalytics.Arn AccountCollectorLambdaARN: !Sub "${AccountCollector.Outputs.LambdaFunctionARN}" CodeBucket: !If [ ProdCFNTemplateUsed, !FindInMap [RegionMap, !Ref "AWS::Region", CodeBucket], !Ref CFNSourceBucket ] - StepFunctionTemplate: !FindInMap [StepFunctionCode, main-v3, TemplatePath] + StepFunctionTemplate: !FindInMap [StepFunctionCode, main-v4, TemplatePath] StepFunctionExecutionRoleARN: !GetAtt StepFunctionExecutionRole.Arn SchedulerExecutionRoleARN: !GetAtt SchedulerExecutionRole.Arn diff --git a/data-collection/deploy/source/step-functions/main-state-machine-v4.json b/data-collection/deploy/source/step-functions/main-state-machine-v4.json new file mode 100644 index 00000000..4811c3ad --- /dev/null +++ b/data-collection/deploy/source/step-functions/main-state-machine-v4.json @@ -0,0 +1,94 @@ +{ + "Comment": "Orchestrate the collection of ${Module} data", + "StartAt": "AccountCollectorInvoke", + "States": { + "AccountCollectorInvoke": { + "Type": "Task", + "Resource": "arn:aws:states:::lambda:invoke", + "Parameters": { + "Payload": { + "Type": "${CollectionType}" + }, + "FunctionName": "${AccountCollectorLambdaARN}" + }, + "Retry": [ + { + "ErrorEquals": [ + "Lambda.ServiceException", + "Lambda.AWSLambdaException", + "Lambda.SdkClientException", + "Lambda.TooManyRequestsException" + ], + "IntervalSeconds": 2, + "MaxAttempts": 6, + "BackoffRate": 2 + } + ], + "Next": "AccountMap", + "ResultPath": "$.accountLambdaOutput" + }, + "AccountMap": { + "Type": "Map", + "ItemProcessor": { + "ProcessorConfig": { + "Mode": "DISTRIBUTED", + "ExecutionType": "STANDARD" + }, + "StartAt": "InvokeModuleLambda", + "States": { + "InvokeModuleLambda": { + "Type": "Task", + "Resource": "arn:aws:states:${DeployRegion}:${Account}:lambda:invoke", + "OutputPath": "$.Payload", + "Parameters": { + "Payload": { + "account.$": "$.account", + "params": "${Params}" + }, + "FunctionName": "${ModuleLambdaARN}" + }, + "Retry": [ + { + "ErrorEquals": [ + "Lambda.ServiceException", + "Lambda.AWSLambdaException", + "Lambda.SdkClientException", + "Lambda.TooManyRequestsException" + ], + "IntervalSeconds": 2, + "MaxAttempts": 6, + "BackoffRate": 2 + } + ], + "End": true + } + } + }, + "MaxConcurrency": 60, + "ToleratedFailurePercentage": 10, + "ItemReader": { + "Resource": "arn:aws:states:::s3:getObject", + "ReaderConfig": { + "InputType": "JSON" + }, + "Parameters": { + "Bucket.$": "$.accountLambdaOutput.Payload.bucket", + "Key.$": "$.accountLambdaOutput.Payload.accountList" + } + }, + "Next": "CrawlerStepFunctionStartExecution" + }, + "CrawlerStepFunctionStartExecution": { + "Type": "Task", + "Resource": "arn:aws:states:::states:startExecution.sync:2", + "Parameters": { + "StateMachineArn": "arn:aws:states:${DeployRegion}:${Account}:stateMachine:${Prefix}CrawlerExecution-StateMachine", + "Input": { + "crawlers": ${Crawlers} + } + }, + "End": true + } + }, + "TimeoutSeconds": 10800 + }