[components] Move dagster_components.lib.test -> dagster_components_t…

…ests.components
dagster-io · Feb 27, 2025 · 56af59b · 56af59b
1 parent 53cba88
commit 56af59b
Show file tree

Hide file tree

Showing 22 changed files with 126 additions and 95 deletions.
diff --git a/python_modules/dagster-test/dagster_test/components/__init__.py b/python_modules/dagster-test/dagster_test/components/__init__.py
@@ -0,0 +1,17 @@
+import importlib.util
+
+_has_dagster_components = importlib.util.find_spec("dagster_components") is not None
+
+if not _has_dagster_components:
+    raise Exception("dagster-components is not installed")
+
+from dagster_test.components.all_metadata_empty_asset import (
+    AllMetadataEmptyComponent as AllMetadataEmptyComponent,
+)
+from dagster_test.components.complex_schema_asset import (
+    ComplexAssetComponent as ComplexAssetComponent,
+)
+from dagster_test.components.simple_asset import SimpleAssetComponent as SimpleAssetComponent
+from dagster_test.components.simple_pipes_script_asset import (
+    SimplePipesScriptComponent as SimplePipesScriptComponent,
+)
diff --git a/...ents/lib/test/all_metadata_empty_asset.py → ...st/components/all_metadata_empty_asset.py b/...ents/lib/test/all_metadata_empty_asset.py → ...st/components/all_metadata_empty_asset.py
@@ -1,7 +1,6 @@
 from dagster._core.definitions.decorators.asset_decorator import asset
 from dagster._core.definitions.definitions_class import Definitions
 from dagster._core.execution.context.asset_execution_context import AssetExecutionContext
-
 from dagster_components import Component, ComponentLoadContext
 from dagster_components.core.component_scaffolder import DefaultComponentScaffolder
 

diff --git a/...mponents/lib/test/complex_schema_asset.py → ...r_test/components/complex_schema_asset.py b/...mponents/lib/test/complex_schema_asset.py → ...r_test/components/complex_schema_asset.py
@@ -4,8 +4,6 @@
 from dagster._core.definitions.decorators.asset_decorator import asset
 from dagster._core.definitions.definitions_class import Definitions
 from dagster._core.execution.context.asset_execution_context import AssetExecutionContext
-from pydantic import Field
-
 from dagster_components import Component, ComponentLoadContext
 from dagster_components.core.component_scaffolder import DefaultComponentScaffolder
 from dagster_components.core.schema.base import ResolvableSchema
@@ -16,6 +14,7 @@
     OpSpecSchema,
     PostProcessorFn,
 )
+from pydantic import Field
 
 
 class ComplexAssetSchema(ResolvableSchema["ComplexAssetComponent"]):

diff --git a/...gster_components/lib/test/simple_asset.py → ...t/dagster_test/components/simple_asset.py b/...gster_components/lib/test/simple_asset.py → ...t/dagster_test/components/simple_asset.py
@@ -2,13 +2,12 @@
 from dagster._core.definitions.decorators.asset_decorator import asset
 from dagster._core.definitions.definitions_class import Definitions
 from dagster._core.execution.context.asset_execution_context import AssetExecutionContext
-from pydantic import BaseModel
-
 from dagster_components import Component, ComponentLoadContext
 from dagster_components.core.component_scaffolder import (
     ComponentScaffolder,
     DefaultComponentScaffolder,
 )
+from pydantic import BaseModel
 
 
 class SimpleAssetSchema(BaseModel):

diff --git a/...nts/lib/test/simple_pipes_script_asset.py → ...t/components/simple_pipes_script_asset.py b/...nts/lib/test/simple_pipes_script_asset.py → ...t/components/simple_pipes_script_asset.py
@@ -6,14 +6,13 @@
 from dagster._core.definitions.definitions_class import Definitions
 from dagster._core.execution.context.asset_execution_context import AssetExecutionContext
 from dagster._core.pipes.subprocess import PipesSubprocessClient
-from pydantic import BaseModel
-
 from dagster_components import Component, ComponentLoadContext
 from dagster_components.core.component_scaffolder import (
     ComponentScaffolder,
     ComponentScaffoldRequest,
 )
 from dagster_components.scaffold import scaffold_component_yaml
+from pydantic import BaseModel
 
 
 # Same schema used for file generation and defs generation

diff --git a/python_modules/dagster-test/dagster_test/toys/pyspark_assets/pyspark_assets_job.py b/python_modules/dagster-test/dagster_test/toys/pyspark_assets/pyspark_assets_job.py
@@ -1,6 +1,12 @@
+import importlib.util
 import os
 
 from dagster import Field, In, String, graph, op, resource
+
+_has_pyspark = importlib.util.find_spec("pyspark") is not None
+if not _has_pyspark:
+    raise Exception("This example requires the pyspark package")
+
 from pyspark.sql import SparkSession, Window
 from pyspark.sql.functions import (
     col,

diff --git a/python_modules/dagster-test/setup.py b/python_modules/dagster-test/setup.py
@@ -19,8 +19,13 @@
     python_requires=">=3.9,<3.13",
     install_requires=[
         "dagster",
-        "pyspark",
+        "dagster-components",
         "rich",
     ],
+    extras_require={
+        # Eventually we sholud remove this as an extra and just include it in the main package
+        "components": ["dagster-components"],
+        "pyspark": ["pyspark"],
+    },
     zip_safe=False,
 )
diff --git a/python_modules/dagster-test/tox.ini b/python_modules/dagster-test/tox.ini
@@ -13,6 +13,7 @@ deps =
   -e ../dagster[test]
   -e ../dagster-pipes
   -e ../dagster-graphql
+  -e ../libraries/dagster-components
   -e ../libraries/dagster-pandas
   -e ../libraries/dagster-aws
   -e ../libraries/dagster-gcp
@@ -24,7 +25,7 @@ deps =
   -e ../libraries/dagster-celery-k8s
   -e ../libraries/dagster-celery-docker
   -e ../libraries/dagstermill
-  -e .
+  -e .[pyspark]
 allowlist_externals =
   /bin/bash
   uv

diff --git a/python_modules/libraries/dagster-components/dagster_components/lib/test/__init__.py b/python_modules/libraries/dagster-components/dagster_components/lib/test/__init__.py
diff --git a/..._modules/libraries/dagster-components/dagster_components_tests/cli_tests/test_commands.py b/..._modules/libraries/dagster-components/dagster_components_tests/cli_tests/test_commands.py
@@ -33,22 +33,22 @@ def test_list_components_types_from_module():
     # Now check what we get when we load directly from the test component library. This has stable
     # results.
     result = runner.invoke(
-        cli, ["list", "component-types", "--no-entry-points", "dagster_components.lib.test"]
+        cli, ["list", "component-types", "--no-entry-points", "dagster_test.components"]
     )
     assert result.exit_code == 0
     result = json.loads(result.output)
     assert len(result) > 1
 
     assert list(result.keys()) == [
-        "dagster_components.lib.test.AllMetadataEmptyComponent",
-        "dagster_components.lib.test.ComplexAssetComponent",
-        "dagster_components.lib.test.SimpleAssetComponent",
-        "dagster_components.lib.test.SimplePipesScriptComponent",
+        "dagster_test.components.AllMetadataEmptyComponent",
+        "dagster_test.components.ComplexAssetComponent",
+        "dagster_test.components.SimpleAssetComponent",
+        "dagster_test.components.SimplePipesScriptComponent",
     ]
 
-    assert result["dagster_components.lib.test.SimpleAssetComponent"] == {
+    assert result["dagster_test.components.SimpleAssetComponent"] == {
         "name": "SimpleAssetComponent",
-        "namespace": "dagster_components.lib.test",
+        "namespace": "dagster_test.components",
         "summary": "A simple asset that returns a constant string value.",
         "description": "A simple asset that returns a constant string value.",
         "scaffold_params_schema": None,
@@ -73,9 +73,9 @@ def test_list_components_types_from_module():
         "type": "object",
     }
 
-    assert result["dagster_components.lib.test.SimplePipesScriptComponent"] == {
+    assert result["dagster_test.components.SimplePipesScriptComponent"] == {
         "name": "SimplePipesScriptComponent",
-        "namespace": "dagster_components.lib.test",
+        "namespace": "dagster_test.components",
         "summary": "A simple asset that runs a Python script with the Pipes subprocess client.",
         "description": "A simple asset that runs a Python script with the Pipes subprocess client.\n\nBecause it is a pipes asset, no value is returned.",
         "scaffold_params_schema": pipes_script_params_schema,
@@ -156,7 +156,13 @@ def test_all_components_schema_command():
     runner = CliRunner()
 
     result = runner.invoke(
-        cli, ["list", "all-components-schema", "--no-entry-points", "dagster_components.lib.test"]
+        cli,
+        [
+            "list",
+            "all-components-schema",
+            "--no-entry-points",
+            "dagster_test.components",
+        ],
     )
     assert_runner_result(result)
     result = json.loads(result.output)
@@ -178,25 +184,25 @@ def test_all_components_schema_command():
         assert "type" in component_type_schema_def["properties"]
         assert (
             component_type_schema_def["properties"]["type"]["default"]
-            == f"dagster_components.lib.test.{component_type_key}"
+            == f"dagster_test.components.{component_type_key}"
         )
         assert (
             component_type_schema_def["properties"]["type"]["const"]
-            == f"dagster_components.lib.test.{component_type_key}"
+            == f"dagster_test.components.{component_type_key}"
         )
         assert "attributes" in component_type_schema_def["properties"]
 
     top_level_component_validator = Draft202012Validator(schema=result)
     top_level_component_validator.validate(
         {
-            "type": "dagster_components.lib.test.SimpleAssetComponent",
+            "type": "dagster_test.components.SimpleAssetComponent",
             "attributes": {"asset_key": "my_asset", "value": "my_value"},
         }
     )
     with pytest.raises(ValidationError):
         top_level_component_validator.validate(
             {
-                "type": "dagster_components.lib.test.SimpleAssetComponent",
+                "type": "dagster_test.components.SimpleAssetComponent",
                 "attributes": {"asset_key": "my_asset", "value": "my_value"},
                 "extra_key": "extra_value",
             }
@@ -212,7 +218,7 @@ def test_scaffold_component_command():
             [
                 "scaffold",
                 "component",
-                "dagster_components.lib.test.SimplePipesScriptComponent",
+                "dagster_test.components.SimplePipesScriptComponent",
                 "bar/components/qux",
                 "--json-params",
                 '{"asset_key": "my_asset", "filename": "my_asset.py"}',

diff --git a/...s_tests/integration_tests/components/validation/simple_asset_invalid_value/component.yaml b/...s_tests/integration_tests/components/validation/simple_asset_invalid_value/component.yaml
@@ -1,4 +1,4 @@
-type: dagster_components.lib.test.SimpleAssetComponent
+type: dagster_components_tests.lib.SimpleAssetComponent
 
 attributes:
   asset_key: "test"

diff --git a/python_modules/libraries/dagster-components/dagster_components_tests/test_integrity.py b/python_modules/libraries/dagster-components/dagster_components_tests/test_integrity.py
@@ -4,7 +4,7 @@
 
 _COMPONENT_LIBRARY_MODULES = [
     "dagster_components.lib",
-    "dagster_components.lib.test",
+    "dagster_test.components",
 ]
 
 

diff --git a/python_modules/libraries/dagster-components/setup.py b/python_modules/libraries/dagster-components/setup.py
@@ -47,6 +47,6 @@ def get_version() -> str:
     extras_require={
         "sling": ["dagster-sling"],
         "dbt": ["dagster-dbt"],
-        "test": ["dbt-duckdb", "dagster-dg", "tomlkit", "jsonschema"],
+        "test": ["dagster-test", "dbt-duckdb", "dagster-dg", "tomlkit", "jsonschema"],
     },
 )
diff --git a/python_modules/libraries/dagster-dg/dagster_dg/context.py b/python_modules/libraries/dagster-dg/dagster_dg/context.py
@@ -4,7 +4,7 @@
 from collections.abc import Iterable, Mapping
 from functools import cached_property
 from pathlib import Path
-from typing import Final, Optional
+from typing import Final, Optional, Union
 
 import tomlkit
 import tomlkit.items
@@ -205,7 +205,8 @@ def cache(self) -> DgCache:
     def has_cache(self) -> bool:
         return self._cache is not None
 
-    def get_cache_key(self, data_type: CachableDataType) -> tuple[str, str, str]:
+    # Allowing open-ended str data_type for now so we can do module names
+    def get_cache_key(self, data_type: Union[CachableDataType, str]) -> tuple[str, str, str]:
         path_parts = [str(part) for part in self.root_path.parts if part != self.root_path.anchor]
         paths_to_hash = [
             self.root_path / "uv.lock",
@@ -215,10 +216,13 @@ def get_cache_key(self, data_type: CachableDataType) -> tuple[str, str, str]:
         return ("_".join(path_parts), env_hash, data_type)
 
     def get_cache_key_for_module(self, module_name: str) -> tuple[str, str, str]:
-        path = self.get_path_for_module(module_name)
-        env_hash = hash_paths([path], includes=["*.py"])
-        path_parts = [str(part) for part in path.parts if part != "/"]
-        return ("_".join(path_parts), env_hash, "local_component_registry")
+        if module_name.startswith(self.root_module_name):
+            path = self.get_path_for_module(module_name)
+            env_hash = hash_paths([path], includes=["*.py"])
+            path_parts = [str(part) for part in path.parts if part != "/"]
+            return ("_".join(path_parts), env_hash, "local_component_registry")
+        else:
+            return self.get_cache_key(module_name)
 
     # ########################
     # ##### WORKSPACE METHODS

diff --git a/python_modules/libraries/dagster-dg/dagster_dg/scaffold.py b/python_modules/libraries/dagster-dg/dagster_dg/scaffold.py
@@ -24,6 +24,7 @@
     "dagster",
     "dagster-pipes",
     "dagster-components",
+    "dagster-test[components]",  # we include dagster-test for testing purposes
 )
 EDITABLE_DAGSTER_DEV_DEPENDENCIES = ("dagster-webserver", "dagster-graphql")
 PYPI_DAGSTER_DEPENDENCIES = ("dagster-components",)

diff --git a/python_modules/libraries/dagster-dg/dagster_dg_tests/cli_tests/test_custom_help_format.py b/python_modules/libraries/dagster-dg/dagster_dg_tests/cli_tests/test_custom_help_format.py
@@ -180,7 +180,7 @@ def test_dynamic_subcommand_help_message():
             result = runner.invoke(
                 "scaffold",
                 "component",
-                "dagster_components.lib.test.SimplePipesScriptComponent",
+                "dagster_test.components.SimplePipesScriptComponent",
                 "--help",
             )
             assert_runner_result(result)
@@ -189,8 +189,9 @@ def test_dynamic_subcommand_help_message():
         assert match_terminal_box_output(
             output.strip(),
             textwrap.dedent("""
-                 Usage: dg scaffold component [GLOBAL OPTIONS] dagster_components.lib.test.SimplePipesScriptComponent                   
-                            [OPTIONS] COMPONENT_INSTANCE_NAME  
+                 Usage: dg scaffold component [GLOBAL OPTIONS] dagster_test.components.SimplePipesScriptComponent [OPTIONS]             
+                                                                                                      COMPONENT_INSTANCE_NA 
+                 ME
 
                 ╭─ Arguments ──────────────────────────────────────────────────────────────────────────────────────────────────────────╮
                 │ *    component_instance_name      TEXT  [required]                                                                   │

diff --git a/python_modules/libraries/dagster-dg/dagster_dg_tests/cli_tests/test_docs_commands.py b/python_modules/libraries/dagster-dg/dagster_dg_tests/cli_tests/test_docs_commands.py
@@ -12,7 +12,7 @@
 def test_docs_component_type_success():
     with ProxyRunner.test() as runner, isolated_components_venv(runner):
         result = runner.invoke(
-            "docs", "component-type", "dagster_components.lib.test.SimpleAssetComponent"
+            "docs", "component-type", "dagster_test.components.SimpleAssetComponent"
         )
         assert_runner_result(result)
 
@@ -30,7 +30,7 @@ def test_docs_component_type_success_output_console():
         result = runner.invoke(
             "docs",
             "component-type",
-            "dagster_components.lib.test.ComplexAssetComponent",
+            "dagster_test.components.ComplexAssetComponent",
             "--output",
             "cli",
         )

diff --git a/...on_modules/libraries/dagster-dg/dagster_dg_tests/cli_tests/test_environment_validation.py b/...on_modules/libraries/dagster-dg/dagster_dg_tests/cli_tests/test_environment_validation.py
@@ -31,7 +31,7 @@ def to_cli_args(self) -> tuple[str, ...]:
         return (*self.command, *self.args)
 
 
-DEFAULT_COMPONENT_TYPE = "dagster_components.lib.test.SimpleAssetComponent"
+DEFAULT_COMPONENT_TYPE = "dagster_test.components.SimpleAssetComponent"
 
 NO_REQUIRED_CONTEXT_COMMANDS = [
     CommandSpec(("scaffold", "project"), "foo"),