Data Processing Pipeline (Terraform)

Source: examples/data-pipeline.yaml

To check and price this model from a clone of the repository:

infra-cost-model validate examples/data-pipeline.yaml
infra-cost-model compute --time-basis monthly examples/data-pipeline.yaml

# Example: Data Processing Pipeline (Terraform)
# Demonstrates: multi-workflow model, mixed time units (perDay),
# provisioned RDS (flatOverride), batch processing, data volume metrics.
#
# Issue: https://github.com/elecnix/infra-cost-model/issues/6

version: "1.0"

workflows:
  - name: data-pipeline
    entry: aws_s3_bucket.uploads
    frequency:
      unit: perDay
      value: 1000

  - name: daily-analytics
    entry: aws_eventbridge_rule.daily_schedule
    frequency:
      unit: perDay
      value: 1

nodes:
  aws_s3_bucket.uploads:
    nodeType: storage
    resourceAddress: aws_s3_bucket.uploads
    provider: aws
    region: us-east-1
    service: AmazonS3
    usageMetrics:
      putRequests: { unit: requests, value: 1 }
      storageGb: { unit: GB-Mo, value: 1500, fixed: true }

  aws_eventbridge_rule.upload_trigger:
    nodeType: routing
    resourceAddress: aws_eventbridge_rule.upload_trigger
    provider: aws
    region: us-east-1
    service: AmazonEventBridge
    usageMetrics:
      eventsPublished: { unit: events, value: 1 }

  aws_lambda_function.extractor:
    nodeType: compute
    resourceAddress: aws_lambda_function.extractor
    provider: aws
    region: us-east-1
    service: AWSLambda
    usageMetrics:
      invocations: { unit: requests, value: 1 }
      avgDurationMs: { unit: ms, value: 5000 }
      memoryMb: { unit: MB, value: 512 }

  aws_lambda_function.transformer:
    nodeType: compute
    resourceAddress: aws_lambda_function.transformer
    provider: aws
    region: us-east-1
    service: AWSLambda
    usageMetrics:
      invocations: { unit: requests, value: 1 }
      avgDurationMs: { unit: ms, value: 2000 }
      memoryMb: { unit: MB, value: 1024 }

  aws_dynamodb_table.data:
    nodeType: storage
    resourceAddress: aws_dynamodb_table.data
    provider: aws
    region: us-east-1
    service: AmazonDynamoDB
    usageMetrics:
      writeRequests: { unit: requests, value: 1 }
      storageGb: { unit: GB-Mo, value: 3, fixed: true }    # 3GB data stored

  aws_eventbridge_rule.daily_schedule:
    nodeType: routing
    resourceAddress: aws_eventbridge_rule.daily_schedule
    provider: aws
    region: us-east-1
    service: AmazonEventBridge
    usageMetrics:
      scheduledInvocations: { unit: invocations, value: 1 }

  aws_lambda_function.aggregator:
    nodeType: compute
    resourceAddress: aws_lambda_function.aggregator
    provider: aws
    region: us-east-1
    service: AWSLambda
    usageMetrics:
      invocations: { unit: requests, value: 1 }
      avgDurationMs: { unit: ms, value: 300000 }
      memoryMb: { unit: MB, value: 2048 }

  aws_db_instance.analytics:
    nodeType: storage
    resourceAddress: aws_db_instance.analytics
    provider: aws
    region: us-east-1
    service: AmazonRDS
    flatOverride: true              # RDS is provisioned — flat monthly cost
    usageMetrics:
      instanceHours: { unit: hours, value: 730 }    # 730 hours/month

  aws_s3_bucket.reports:
    nodeType: storage
    resourceAddress: aws_s3_bucket.reports
    provider: aws
    region: us-east-1
    service: AmazonS3
    usageMetrics:
      putRequests: { unit: requests, value: 1 }
      storageGb: { unit: GB-Mo, value: 0.3, fixed: true }    # 300MB reports

edges:
  # -- data-pipeline edges --
  - from: aws_s3_bucket.uploads
    to: aws_eventbridge_rule.upload_trigger
    rate: 1
  - from: aws_eventbridge_rule.upload_trigger
    to: aws_lambda_function.extractor
    rate: 1
  - from: aws_lambda_function.extractor
    to: aws_lambda_function.transformer
    rate: 1
    dataSize:
      unit: kB
      average: 100                    # 100KB CSV data per extraction

  - from: aws_lambda_function.transformer
    to: aws_dynamodb_table.data
    rate: 1
    type: write

  # -- daily-analytics edges --
  - from: aws_eventbridge_rule.daily_schedule
    to: aws_lambda_function.aggregator
    rate: 1
  - from: aws_lambda_function.aggregator
    to: aws_db_instance.analytics
    rate: 1
    type: read
  - from: aws_lambda_function.aggregator
    to: aws_s3_bucket.reports
    rate: 1
    dataSize:
      unit: MB
      average: 10                     # 10MB report per day