Merge branch 'develop' into fix/adjust-function-nesting

Author: dhutchison

.github/workflows/release.yml

@@ -22,7 +22,7 @@ jobs:
           fetch-depth: 0
 
       - name: Setup Python 3.12
-        uses: actions/setup-python@v5.3.0
+        uses: actions/setup-python@v5.4.0
         with:
           python-version: '3.12'
           architecture: x64
@@ -75,7 +75,7 @@ jobs:
         uses: actions/[email protected]
 
       - name: Setup Python 3.12
-        uses: actions/setup-python@v5.3.0
+        uses: actions/setup-python@v5.4.0
         with:
           python-version: '3.12'
           architecture: x64

.github/workflows/test.yml

@@ -18,14 +18,14 @@ jobs:
         uses: actions/[email protected]
 
       - name: Setup Latest Python
-        uses: actions/setup-python@v5.3.0
+        uses: actions/setup-python@v5.4.0
         with:
           python-version: 3.12
           architecture: x64
 
       - name: Setup Poetry
         run: |
-          pip install poetry==1.4.2
+          pip install poetry
           poetry install
 
       - name: Setup Coverage
@@ -56,7 +56,7 @@ jobs:
         uses: actions/[email protected]
 
       - name: Setup Python ${{ matrix.python-version }}
-        uses: actions/setup-python@v5.3.0
+        uses: actions/setup-python@v5.4.0
         with:
           python-version: ${{ matrix.python-version }}
           architecture: x64

.scripts/requirements.txt

@@ -1,2 +1,2 @@
 requests==2.32.3
-semver==3.0.2
+semver==3.0.4

README.md

@@ -169,8 +169,8 @@ The default values for pseudo parameters:
 | **NoValue**      | ""              |
 | **Partition**    | "aws"           |
 | Region           | "us-east-1"     |
-| **StackId**      | ""              |
-| **StackName**    | ""              |
+| StackId          | (generated based on other values)              |
+| StackName    | "my-cloud-radar-stack"              |
 | **URLSuffix**    | "amazonaws.com" |
 _Note: Bold variables are not fully implemented yet see the [Roadmap](#roadmap)_
 
@@ -211,6 +211,24 @@ dynamic_references = {
 template = Template(template_content, dynamic_references=dynamic_references)
 ```
 
+There are cases where the default behaviour of our `GetAtt` implementation may not be sufficient and you need a more accurate returned value. When unit testing there are no real AWS resources created, and cloud-radar does not attempt to realistically generate attribute values - a string is always returned. This works good enough most of the time, but there are some cases where if you are attempting to apply intrinsic functions against the attribute value it needs to be more correct. When this occurs, you can add Metadata to the template to provide test values to use.
+
+```
+Resources:
+  MediaPackageV2Channel:
+    Type: AWS::MediaPackageV2::Channel
+    Metadata:
+      Cloud-Radar:
+        attribute-values:
+        # Default behaviour of a string is not good enough here, the attribute value is expected to be a List.
+          IngestEndpointUrls:
+            - http://one.example.com
+            - http://two.example.com
+    Properties:
+      ChannelGroupName: dev_video_1
+      ChannelName: !Sub ${AWS::StackName}-MediaPackageChannel
+```
+
 A real unit testing example using Pytest can be seen [here](./tests/test_cf/test_examples/test_unit.py)
 
 </details>
@@ -307,7 +325,6 @@ A real functional testing example using Pytest can be seen [here](./tests/test_c
 ### Unit
 - Add full functionality to pseudo variables.
   * Variables like `Partition`, `URLSuffix` should change if the region changes.
-  * Variables like `StackName` and `StackId` should have a better default than ""
 - Handle References to resources that shouldn't exist.
   * It's currently possible that a `!Ref` to a Resource stays in the final template even if that resource is later removed because of a conditional.
 

poetry.lock

@@ -1,31 +1,36 @@
-[tool.poetry]
+[project]
 name = "cloud-radar"
 version = "0.0.0"
 description = "Run functional tests on cloudformation stacks."
 readme = "README.md"
-authors = ["Levi Blaney <[email protected]>"]
+authors = [
+    { name = "Levi Blaney", email = "[email protected]" },
+]
 license = "Apache-2.0"
-repository = "https://github.com/DontShaveTheYak/cloud-radar"
 keywords = ["aws", "cloudformation", "cloud-radar", "testing", "taskcat", "cloud", "radar"]
+requires-python = ">=3.9,<4.0"
+dynamic = [ "classifiers" ]
+dependencies = [
+  "taskcat >=0.9.41, <1.0.0",
+  "cfn-flip >=1.3.0, <2.0.0",
+  "botocore >=1.35.36, <2.0.0",
+]
+[project.urls]
+Repository = "https://github.com/DontShaveTheYak/cloud-radar"
+Issues = "https://github.com/DontShaveTheYak/cloud-radar/issues"
+Changelog = "https://github.com/DontShaveTheYak/cloud-radar/releases"
+
+
+[tool.poetry]
+requires-poetry = ">=2.0"
 classifiers = [
   "Development Status :: 2 - Pre-Alpha",
-  "License :: OSI Approved :: Apache Software License",
   "Operating System :: OS Independent",
-  "Programming Language :: Python :: 3",
-  "Programming Language :: Python :: 3.9",
-  "Programming Language :: Python :: 3.10",
-  "Programming Language :: Python :: 3.11",
-  "Programming Language :: Python :: 3.12",
-  "Programming Language :: Python :: 3.13",
   "Topic :: Software Development :: Libraries",
   "Topic :: Software Development :: Testing"
 ]
 
-[tool.poetry.dependencies]
-python = ">=3.9,<3.14"
-taskcat = "^0.9.41"
-cfn-flip = "^1.3.0"
-botocore = {version = ">=1.35.36", python = ">=3.13"}
+
 
 [tool.poetry.group.dev.dependencies]
 pytest = "^8.0.0"
@@ -41,9 +46,12 @@ flake8-bugbear = "^24.0.0"
 mypy = "^1.0.0"
 types-requests = "^2.28.11"
 types-PyYAML = "^6.0.12"
-cfn-lint = "1.22.3"
+cfn-lint = "1.24.0"
 setuptools = {version = "75.4.0", python = ">=3.12"}
 
+[tool.poetry.requires-plugins]
+poetry-plugin-export = ">=1.8"
+
 [tool.coverage.paths]
 source = ["src", "*/site-packages"]
 

pyproject.toml

@@ -2,6 +2,7 @@
 
 import json
 import re
+import uuid
 from pathlib import Path
 from typing import Any, Callable, Dict, Generator, List, Optional, Tuple, Union
 
@@ -26,8 +27,8 @@ class Template:
     NoValue: str = ""  # Not yet implemented
     Partition: str = "aws"  # Other regions not implemented
     Region: str = "us-east-1"
-    StackId: str = ""  # Not yet implemented
-    StackName: str = ""  # Not yet implemented
+    StackId: str = ""  # If left blank this will be generated
+    StackName: str = "my-cloud-radar-stack"
     URLSuffix: str = "amazonaws.com"  # Other regions not implemented
 
     def __init__(
@@ -310,6 +311,19 @@ def remove_condtional_resources(self, template: Dict[str, Any]) -> Dict[str, Any
 
         return template
 
+    # If the StackId variable is not set, generate a value for it
+    def _get_populated_stack_id(self) -> str:
+        if not Template.StackId:
+            # Not explicitly set, generate a value
+            unique_uuid = uuid.uuid4()
+
+            return (
+                f"arn:{Template.Partition}:cloudformation:{self.Region}:"
+                f"{Template.AccountId}:stack/{Template.StackName}/{unique_uuid}"
+            )
+
+        return Template.StackId
+
     def create_stack(
         self,
         params: Optional[Dict[str, str]] = None,
@@ -318,6 +332,7 @@ def create_stack(
     ):
         if region:
             self.Region = region
+        self.StackId = self._get_populated_stack_id()
 
         self.render(params, parameters_file=parameters_file)
 

src/cloud_radar/cf/unit/_template.py

@@ -455,7 +455,18 @@ def get_att(template: "Template", values: Any) -> str:
     if resource_name not in template.template["Resources"]:
         raise KeyError(f"Fn::GetAtt - Resource {resource_name} not found in template.")
 
-    return f"{resource_name}.{att_name}"
+    # Get the resource definition
+    resource = template.template["Resources"][resource_name]
+
+    # Check if there is a value in the resource Metadata for this attribute.
+    # If the attribute requested is in the metadata, return it.
+    # Otherwise use the string value of "{resource_name}.{att_name}"
+
+    metadata = resource.get("Metadata", {})
+    cloud_radar_metadata = metadata.get("Cloud-Radar", {})
+    attribute_values = cloud_radar_metadata.get("attribute-values", {})
+
+    return attribute_values.get(att_name, f"{resource_name}.{att_name}")
 
 
 def get_azs(_t: "Template", region: Any) -> List[str]:

src/cloud_radar/cf/unit/functions.py

@@ -0,0 +1,36 @@
+AWSTemplateFormatVersion: 2010-09-09
+Description: "Basic template to check GetAtt behaviours"
+
+Resources:
+  MediaPackageV2Channel:
+    Type: AWS::MediaPackageV2::Channel
+    Metadata:
+      Cloud-Radar:
+        attribute-values:
+        # When unit testing there are no real AWS resources created, and cloud-radar
+        # does not attempt to realistically generate attribute values - a string is always
+        # returned. This works good enough most of the time, but there are some cases where
+        # if you are attempting to apply intrinsic functions against the attribute value
+        # it needs to be more correct.
+        #
+        # In this case, the attribute value is expected to be a List, not a string.
+          IngestEndpointUrls:
+            - http://one.example.com
+            - http://two.example.com
+    Properties:
+      ChannelGroupName: dev_video_1
+      ChannelName: !Sub ${AWS::StackName}-MediaPackageChannel
+
+Outputs:
+  ChannelArn:
+    Description: The ARN of the MediaPackageV2 Channel.
+    Value: !GetAtt MediaPackageV2Channel.Arn
+  ChannelCreatedAt:
+    Description: The creation timestamp of the MediaPackageV2 Channel.
+    Value: !GetAtt MediaPackageV2Channel.CreatedAt
+  ChannelIngestEndpointUrl1:
+    Description: The first IngestEndpointUrl of the MediaPackageV2 Channel.
+    Value: !Select [0, !GetAtt MediaPackageV2Channel.IngestEndpointUrls]
+  ChannelIngestEndpointUrl2:
+    Description: The second IngestEndpointUrl of the MediaPackageV2 Channel.
+    Value: !Select [1, !GetAtt MediaPackageV2Channel.IngestEndpointUrls]

tests/templates/test_media_getatt.yaml

@@ -0,0 +1,16 @@
+AWSTemplateFormatVersion: 2010-09-09
+Description: "Creates an S3 bucket to store logs."
+
+Resources:
+  UniqueBucket:
+    Type: AWS::S3::Bucket
+    Properties:
+      BucketName: !Sub
+          - 'my-test-${stack_region}-${uniqifier}-bucket'
+          - # AWS::StackId has this format
+            # arn:aws:cloudformation:us-west-2:123456789012:stack/teststack/51af3dc0-da77-11e4-872e-1234567db123
+            # Trying to capture the last piece after the '-'
+            # As stack name could contain "-"s, split on the "/"s first
+            uniqifier: !Select [ 4, !Split [ "-",  !Select [ 2, !Split [ "/", !Ref AWS::StackId ] ] ] ]
+            # Usually you would refer to AWS:::Region, but trying to test StackId creation works as expected
+            stack_region: !Select [ 3, !Split [":", !Ref AWS::StackId]]

tests/templates/test_stackid.yaml

@@ -27,9 +27,8 @@ def test_constructor(template_dir, default_params):
 
     assert stack.config.config.project.regions[0] == "us-east-1"
 
-    assert (
-        not stack.config.config.project.parameters
-    ) or stack.config.config.project.parameters == {}
+    # Assert either empty or None at the start
+    assert not stack.config.config.project.parameters
 
     stack = Stack(str(template))
 

tests/test_cf/test_e2e/test_stack.py

@@ -0,0 +1,29 @@
+from pathlib import Path
+
+import pytest
+
+from cloud_radar.cf.unit._template import Template
+
+"""Tests that the GetAtt function can use attribute values defined in a template."""
+
+
+@pytest.fixture
+def template():
+    template_path = Path(__file__).parent / "../../templates/test_media_getatt.yaml"
+
+    return Template.from_yaml(template_path.resolve(), {})
+
+
+def test_outputs(template: Template):
+    stack = template.create_stack()
+
+    # These two outputs are expected to use values which came from the metadata override
+    stack.get_output("ChannelIngestEndpointUrl1").assert_value_is(
+        "http://one.example.com"
+    )
+    stack.get_output("ChannelIngestEndpointUrl2").assert_value_is(
+        "http://two.example.com"
+    )
+
+    # This attribute will use the default format
+    stack.get_output("ChannelArn").assert_value_is("MediaPackageV2Channel.Arn")

tests/test_cf/test_unit/test_functions_get_att.py

@@ -0,0 +1,73 @@
+# Test case that verifies that generation of the value for AWS::StackId works as expected
+
+from pathlib import Path
+
+import pytest
+
+from cloud_radar.cf.unit._template import Template
+
+
+@pytest.fixture
+def template():
+    template_path = Path(__file__).parent / "../../templates/test_stackid.yaml"
+
+    return Template.from_yaml(template_path.resolve(), {})
+
+
+def test_function_populated_var(template):
+    expected_value = "arn:aws:cloudformation:us-west-2:123456789012:stack/teststack/51af3dc0-da77-11e4-872e-1234567db123"
+    Template.StackId = expected_value
+
+    actual_value = template._get_populated_stack_id()
+    assert actual_value == expected_value
+
+
+def test_function_blank_var(template):
+    Template.StackId = ""
+
+    actual_value = template._get_populated_stack_id()
+    # Check all except the UUID
+    assert actual_value.startswith(
+        f"arn:{Template.Partition}:cloudformation:{Template.Region}:{Template.AccountId}:stack/{Template.StackName}/"
+    )
+
+    # Check the UUID part looks UUID like
+    unique_uuid = actual_value.split("/")[2]
+    assert 5 == len(unique_uuid.split("-"))
+
+
+def test_template_blank_var_stack_region(template):
+    Template.StackId = ""
+
+    stack = template.create_stack({}, region="eu-west-1")
+
+    bucket = stack.get_resource("UniqueBucket")
+    bucket_name = bucket.get_property_value("BucketName")
+
+    assert len(bucket_name) == 37
+    assert bucket_name[:18] == "my-test-eu-west-1-"
+    assert bucket_name[30:] == "-bucket"
+
+
+def test_template_blank_var_global_region(template):
+    Template.StackId = ""
+
+    stack = template.create_stack({})
+
+    bucket = stack.get_resource("UniqueBucket")
+    bucket_name = bucket.get_property_value("BucketName")
+
+    assert len(bucket_name) == 37
+    assert bucket_name[:18] == "my-test-us-east-1-"
+    assert bucket_name[30:] == "-bucket"
+
+
+def test_template_populated_var(template):
+    Template.StackId = "arn:aws:cloudformation:us-west-2:123456789012:stack/teststack/51af3dc0-da77-11e4-872e-1234567db123"
+
+    stack = template.create_stack({})
+
+    bucket = stack.get_resource("UniqueBucket")
+    bucket_name = bucket.get_property_value("BucketName")
+
+    assert "my-test-us-west-2-1234567db123-bucket" == bucket_name