From 1e06ab0745c58e67020fcc616475f3f45df81225 Mon Sep 17 00:00:00 2001 From: Sean Mackesey Date: Thu, 27 Feb 2025 12:19:47 -0500 Subject: [PATCH] [components] Move dagster_components.lib.test -> dagster_components_tests.components --- python_modules/dagster-test/Dockerfile | 1 + .../dagster_test/components/__init__.py | 17 +++++++++ .../components}/all_metadata_empty_asset.py | 1 - .../components}/complex_schema_asset.py | 3 +- .../dagster_test/components}/simple_asset.py | 3 +- .../components}/simple_pipes_script_asset.py | 3 +- .../dagster_test/test_project/build.sh | 1 + .../toys/pyspark_assets/pyspark_assets_job.py | 6 +++ python_modules/dagster-test/setup.py | 7 +++- python_modules/dagster-test/tox.ini | 3 +- .../dagster_components/lib/test/__init__.py | 10 ----- .../cli_tests/test_commands.py | 36 ++++++++++-------- .../simple_asset_invalid_value/component.yaml | 2 +- .../test_integrity.py | 2 +- .../libraries/dagster-components/setup.py | 2 +- .../dagster-dg/dagster_dg/context.py | 16 +++++--- .../dagster-dg/dagster_dg/scaffold.py | 1 + .../cli_tests/test_custom_help_format.py | 7 ++-- .../cli_tests/test_docs_commands.py | 4 +- .../cli_tests/test_environment_validation.py | 2 +- .../cli_tests/test_info_commands.py | 16 ++++---- .../cli_tests/test_list_commands.py | 30 +++++++-------- .../cli_tests/test_scaffold_commands.py | 37 ++++++++++--------- .../dagster-dg/dagster_dg_tests/utils.py | 13 ++++--- 24 files changed, 128 insertions(+), 95 deletions(-) create mode 100644 python_modules/dagster-test/dagster_test/components/__init__.py rename python_modules/{libraries/dagster-components/dagster_components/lib/test => dagster-test/dagster_test/components}/all_metadata_empty_asset.py (99%) rename python_modules/{libraries/dagster-components/dagster_components/lib/test => dagster-test/dagster_test/components}/complex_schema_asset.py (99%) rename python_modules/{libraries/dagster-components/dagster_components/lib/test => dagster-test/dagster_test/components}/simple_asset.py (99%) rename python_modules/{libraries/dagster-components/dagster_components/lib/test => dagster-test/dagster_test/components}/simple_pipes_script_asset.py (99%) delete mode 100644 python_modules/libraries/dagster-components/dagster_components/lib/test/__init__.py diff --git a/python_modules/dagster-test/Dockerfile b/python_modules/dagster-test/Dockerfile index 6b8b2b2a68446..187f10f6c7428 100644 --- a/python_modules/dagster-test/Dockerfile +++ b/python_modules/dagster-test/Dockerfile @@ -26,6 +26,7 @@ RUN python -m uv pip install \ -e modules/dagster-k8s \ -e modules/dagster-celery-k8s \ -e modules/dagster-celery-docker \ + -e modules/dagster-components \ -e modules/dagster-docker \ -e modules/dagster-airflow \ -e modules/dagstermill \ diff --git a/python_modules/dagster-test/dagster_test/components/__init__.py b/python_modules/dagster-test/dagster_test/components/__init__.py new file mode 100644 index 0000000000000..cb93258365e74 --- /dev/null +++ b/python_modules/dagster-test/dagster_test/components/__init__.py @@ -0,0 +1,17 @@ +import importlib.util + +_has_dagster_components = importlib.util.find_spec("dagster_components") is not None + +if not _has_dagster_components: + raise Exception("dagster-components is not installed") + +from dagster_test.components.all_metadata_empty_asset import ( + AllMetadataEmptyComponent as AllMetadataEmptyComponent, +) +from dagster_test.components.complex_schema_asset import ( + ComplexAssetComponent as ComplexAssetComponent, +) +from dagster_test.components.simple_asset import SimpleAssetComponent as SimpleAssetComponent +from dagster_test.components.simple_pipes_script_asset import ( + SimplePipesScriptComponent as SimplePipesScriptComponent, +) diff --git a/python_modules/libraries/dagster-components/dagster_components/lib/test/all_metadata_empty_asset.py b/python_modules/dagster-test/dagster_test/components/all_metadata_empty_asset.py similarity index 99% rename from python_modules/libraries/dagster-components/dagster_components/lib/test/all_metadata_empty_asset.py rename to python_modules/dagster-test/dagster_test/components/all_metadata_empty_asset.py index 245112fe7ea04..7c8b44cf16073 100644 --- a/python_modules/libraries/dagster-components/dagster_components/lib/test/all_metadata_empty_asset.py +++ b/python_modules/dagster-test/dagster_test/components/all_metadata_empty_asset.py @@ -1,7 +1,6 @@ from dagster._core.definitions.decorators.asset_decorator import asset from dagster._core.definitions.definitions_class import Definitions from dagster._core.execution.context.asset_execution_context import AssetExecutionContext - from dagster_components import Component, ComponentLoadContext from dagster_components.core.component_scaffolder import DefaultComponentScaffolder diff --git a/python_modules/libraries/dagster-components/dagster_components/lib/test/complex_schema_asset.py b/python_modules/dagster-test/dagster_test/components/complex_schema_asset.py similarity index 99% rename from python_modules/libraries/dagster-components/dagster_components/lib/test/complex_schema_asset.py rename to python_modules/dagster-test/dagster_test/components/complex_schema_asset.py index 764729453a34e..25e18efd111fc 100644 --- a/python_modules/libraries/dagster-components/dagster_components/lib/test/complex_schema_asset.py +++ b/python_modules/dagster-test/dagster_test/components/complex_schema_asset.py @@ -4,8 +4,6 @@ from dagster._core.definitions.decorators.asset_decorator import asset from dagster._core.definitions.definitions_class import Definitions from dagster._core.execution.context.asset_execution_context import AssetExecutionContext -from pydantic import Field - from dagster_components import Component, ComponentLoadContext from dagster_components.core.component_scaffolder import DefaultComponentScaffolder from dagster_components.core.schema.base import ResolvableSchema @@ -16,6 +14,7 @@ OpSpecSchema, PostProcessorFn, ) +from pydantic import Field class ComplexAssetSchema(ResolvableSchema["ComplexAssetComponent"]): diff --git a/python_modules/libraries/dagster-components/dagster_components/lib/test/simple_asset.py b/python_modules/dagster-test/dagster_test/components/simple_asset.py similarity index 99% rename from python_modules/libraries/dagster-components/dagster_components/lib/test/simple_asset.py rename to python_modules/dagster-test/dagster_test/components/simple_asset.py index 67dfad309c288..382414c492f26 100644 --- a/python_modules/libraries/dagster-components/dagster_components/lib/test/simple_asset.py +++ b/python_modules/dagster-test/dagster_test/components/simple_asset.py @@ -2,13 +2,12 @@ from dagster._core.definitions.decorators.asset_decorator import asset from dagster._core.definitions.definitions_class import Definitions from dagster._core.execution.context.asset_execution_context import AssetExecutionContext -from pydantic import BaseModel - from dagster_components import Component, ComponentLoadContext from dagster_components.core.component_scaffolder import ( ComponentScaffolder, DefaultComponentScaffolder, ) +from pydantic import BaseModel class SimpleAssetSchema(BaseModel): diff --git a/python_modules/libraries/dagster-components/dagster_components/lib/test/simple_pipes_script_asset.py b/python_modules/dagster-test/dagster_test/components/simple_pipes_script_asset.py similarity index 99% rename from python_modules/libraries/dagster-components/dagster_components/lib/test/simple_pipes_script_asset.py rename to python_modules/dagster-test/dagster_test/components/simple_pipes_script_asset.py index 8e40944a2a197..1b6eda46d2f51 100644 --- a/python_modules/libraries/dagster-components/dagster_components/lib/test/simple_pipes_script_asset.py +++ b/python_modules/dagster-test/dagster_test/components/simple_pipes_script_asset.py @@ -6,14 +6,13 @@ from dagster._core.definitions.definitions_class import Definitions from dagster._core.execution.context.asset_execution_context import AssetExecutionContext from dagster._core.pipes.subprocess import PipesSubprocessClient -from pydantic import BaseModel - from dagster_components import Component, ComponentLoadContext from dagster_components.core.component_scaffolder import ( ComponentScaffolder, ComponentScaffoldRequest, ) from dagster_components.scaffold import scaffold_component_yaml +from pydantic import BaseModel # Same schema used for file generation and defs generation diff --git a/python_modules/dagster-test/dagster_test/test_project/build.sh b/python_modules/dagster-test/dagster_test/test_project/build.sh index 60827f691cc26..37dc961134c1f 100755 --- a/python_modules/dagster-test/dagster_test/test_project/build.sh +++ b/python_modules/dagster-test/dagster_test/test_project/build.sh @@ -54,6 +54,7 @@ copy_py $ROOT/python_modules/dagster \ $ROOT/python_modules/libraries/dagster-celery \ $ROOT/python_modules/libraries/dagster-celery-k8s \ $ROOT/python_modules/libraries/dagster-celery-docker \ + $ROOT/python_modules/libraries/dagster-components \ $ROOT/python_modules/libraries/dagster-docker \ $ROOT/python_modules/libraries/dagster-pandas \ $ROOT/python_modules/libraries/dagster-postgres \ diff --git a/python_modules/dagster-test/dagster_test/toys/pyspark_assets/pyspark_assets_job.py b/python_modules/dagster-test/dagster_test/toys/pyspark_assets/pyspark_assets_job.py index 047c28760ddc3..e172731b53023 100644 --- a/python_modules/dagster-test/dagster_test/toys/pyspark_assets/pyspark_assets_job.py +++ b/python_modules/dagster-test/dagster_test/toys/pyspark_assets/pyspark_assets_job.py @@ -1,6 +1,12 @@ +import importlib.util import os from dagster import Field, In, String, graph, op, resource + +_has_pyspark = importlib.util.find_spec("pyspark") is not None +if not _has_pyspark: + raise Exception("This example requires the pyspark package") + from pyspark.sql import SparkSession, Window from pyspark.sql.functions import ( col, diff --git a/python_modules/dagster-test/setup.py b/python_modules/dagster-test/setup.py index d57b68e41f14d..54342f69923e6 100644 --- a/python_modules/dagster-test/setup.py +++ b/python_modules/dagster-test/setup.py @@ -19,8 +19,13 @@ python_requires=">=3.9,<3.13", install_requires=[ "dagster", - "pyspark", + "dagster-components", "rich", ], + extras_require={ + # Eventually we sholud remove this as an extra and just include it in the main package + "components": ["dagster-components"], + "pyspark": ["pyspark"], + }, zip_safe=False, ) diff --git a/python_modules/dagster-test/tox.ini b/python_modules/dagster-test/tox.ini index 6b74e23480418..6cb1580d163ed 100644 --- a/python_modules/dagster-test/tox.ini +++ b/python_modules/dagster-test/tox.ini @@ -13,6 +13,7 @@ deps = -e ../dagster[test] -e ../dagster-pipes -e ../dagster-graphql + -e ../libraries/dagster-components -e ../libraries/dagster-pandas -e ../libraries/dagster-aws -e ../libraries/dagster-gcp @@ -24,7 +25,7 @@ deps = -e ../libraries/dagster-celery-k8s -e ../libraries/dagster-celery-docker -e ../libraries/dagstermill - -e . + -e .[pyspark] allowlist_externals = /bin/bash uv diff --git a/python_modules/libraries/dagster-components/dagster_components/lib/test/__init__.py b/python_modules/libraries/dagster-components/dagster_components/lib/test/__init__.py deleted file mode 100644 index 9f0ac4bc38db3..0000000000000 --- a/python_modules/libraries/dagster-components/dagster_components/lib/test/__init__.py +++ /dev/null @@ -1,10 +0,0 @@ -from dagster_components.lib.test.all_metadata_empty_asset import ( - AllMetadataEmptyComponent as AllMetadataEmptyComponent, -) -from dagster_components.lib.test.complex_schema_asset import ( - ComplexAssetComponent as ComplexAssetComponent, -) -from dagster_components.lib.test.simple_asset import SimpleAssetComponent as SimpleAssetComponent -from dagster_components.lib.test.simple_pipes_script_asset import ( - SimplePipesScriptComponent as SimplePipesScriptComponent, -) diff --git a/python_modules/libraries/dagster-components/dagster_components_tests/cli_tests/test_commands.py b/python_modules/libraries/dagster-components/dagster_components_tests/cli_tests/test_commands.py index 8322c624ce57b..7900636e9553a 100644 --- a/python_modules/libraries/dagster-components/dagster_components_tests/cli_tests/test_commands.py +++ b/python_modules/libraries/dagster-components/dagster_components_tests/cli_tests/test_commands.py @@ -33,22 +33,22 @@ def test_list_components_types_from_module(): # Now check what we get when we load directly from the test component library. This has stable # results. result = runner.invoke( - cli, ["list", "component-types", "--no-entry-points", "dagster_components.lib.test"] + cli, ["list", "component-types", "--no-entry-points", "dagster_test.components"] ) assert result.exit_code == 0 result = json.loads(result.output) assert len(result) > 1 assert list(result.keys()) == [ - "dagster_components.lib.test.AllMetadataEmptyComponent", - "dagster_components.lib.test.ComplexAssetComponent", - "dagster_components.lib.test.SimpleAssetComponent", - "dagster_components.lib.test.SimplePipesScriptComponent", + "dagster_test.components.AllMetadataEmptyComponent", + "dagster_test.components.ComplexAssetComponent", + "dagster_test.components.SimpleAssetComponent", + "dagster_test.components.SimplePipesScriptComponent", ] - assert result["dagster_components.lib.test.SimpleAssetComponent"] == { + assert result["dagster_test.components.SimpleAssetComponent"] == { "name": "SimpleAssetComponent", - "namespace": "dagster_components.lib.test", + "namespace": "dagster_test.components", "summary": "A simple asset that returns a constant string value.", "description": "A simple asset that returns a constant string value.", "scaffold_params_schema": None, @@ -73,9 +73,9 @@ def test_list_components_types_from_module(): "type": "object", } - assert result["dagster_components.lib.test.SimplePipesScriptComponent"] == { + assert result["dagster_test.components.SimplePipesScriptComponent"] == { "name": "SimplePipesScriptComponent", - "namespace": "dagster_components.lib.test", + "namespace": "dagster_test.components", "summary": "A simple asset that runs a Python script with the Pipes subprocess client.", "description": "A simple asset that runs a Python script with the Pipes subprocess client.\n\nBecause it is a pipes asset, no value is returned.", "scaffold_params_schema": pipes_script_params_schema, @@ -156,7 +156,13 @@ def test_all_components_schema_command(): runner = CliRunner() result = runner.invoke( - cli, ["list", "all-components-schema", "--no-entry-points", "dagster_components.lib.test"] + cli, + [ + "list", + "all-components-schema", + "--no-entry-points", + "dagster_test.components", + ], ) assert_runner_result(result) result = json.loads(result.output) @@ -178,25 +184,25 @@ def test_all_components_schema_command(): assert "type" in component_type_schema_def["properties"] assert ( component_type_schema_def["properties"]["type"]["default"] - == f"dagster_components.lib.test.{component_type_key}" + == f"dagster_test.components.{component_type_key}" ) assert ( component_type_schema_def["properties"]["type"]["const"] - == f"dagster_components.lib.test.{component_type_key}" + == f"dagster_test.components.{component_type_key}" ) assert "attributes" in component_type_schema_def["properties"] top_level_component_validator = Draft202012Validator(schema=result) top_level_component_validator.validate( { - "type": "dagster_components.lib.test.SimpleAssetComponent", + "type": "dagster_test.components.SimpleAssetComponent", "attributes": {"asset_key": "my_asset", "value": "my_value"}, } ) with pytest.raises(ValidationError): top_level_component_validator.validate( { - "type": "dagster_components.lib.test.SimpleAssetComponent", + "type": "dagster_test.components.SimpleAssetComponent", "attributes": {"asset_key": "my_asset", "value": "my_value"}, "extra_key": "extra_value", } @@ -212,7 +218,7 @@ def test_scaffold_component_command(): [ "scaffold", "component", - "dagster_components.lib.test.SimplePipesScriptComponent", + "dagster_test.components.SimplePipesScriptComponent", "bar/components/qux", "--json-params", '{"asset_key": "my_asset", "filename": "my_asset.py"}', diff --git a/python_modules/libraries/dagster-components/dagster_components_tests/integration_tests/components/validation/simple_asset_invalid_value/component.yaml b/python_modules/libraries/dagster-components/dagster_components_tests/integration_tests/components/validation/simple_asset_invalid_value/component.yaml index d0a6eb25edb02..6e5ac207c2e76 100644 --- a/python_modules/libraries/dagster-components/dagster_components_tests/integration_tests/components/validation/simple_asset_invalid_value/component.yaml +++ b/python_modules/libraries/dagster-components/dagster_components_tests/integration_tests/components/validation/simple_asset_invalid_value/component.yaml @@ -1,4 +1,4 @@ -type: dagster_components.lib.test.SimpleAssetComponent +type: dagster_components_tests.lib.SimpleAssetComponent attributes: asset_key: "test" diff --git a/python_modules/libraries/dagster-components/dagster_components_tests/test_integrity.py b/python_modules/libraries/dagster-components/dagster_components_tests/test_integrity.py index 0775b27f715ba..51326267a8e41 100644 --- a/python_modules/libraries/dagster-components/dagster_components_tests/test_integrity.py +++ b/python_modules/libraries/dagster-components/dagster_components_tests/test_integrity.py @@ -4,7 +4,7 @@ _COMPONENT_LIBRARY_MODULES = [ "dagster_components.lib", - "dagster_components.lib.test", + "dagster_test.components", ] diff --git a/python_modules/libraries/dagster-components/setup.py b/python_modules/libraries/dagster-components/setup.py index 95be152a6c229..828a82185c6d2 100644 --- a/python_modules/libraries/dagster-components/setup.py +++ b/python_modules/libraries/dagster-components/setup.py @@ -47,6 +47,6 @@ def get_version() -> str: extras_require={ "sling": ["dagster-sling"], "dbt": ["dagster-dbt"], - "test": ["dbt-duckdb", "dagster-dg", "tomlkit", "jsonschema"], + "test": ["dagster-test", "dbt-duckdb", "dagster-dg", "tomlkit", "jsonschema"], }, ) diff --git a/python_modules/libraries/dagster-dg/dagster_dg/context.py b/python_modules/libraries/dagster-dg/dagster_dg/context.py index 5f61e41541b15..eff32dd42df93 100644 --- a/python_modules/libraries/dagster-dg/dagster_dg/context.py +++ b/python_modules/libraries/dagster-dg/dagster_dg/context.py @@ -4,7 +4,7 @@ from collections.abc import Iterable, Mapping from functools import cached_property from pathlib import Path -from typing import Final, Optional +from typing import Final, Optional, Union import tomlkit import tomlkit.items @@ -205,7 +205,8 @@ def cache(self) -> DgCache: def has_cache(self) -> bool: return self._cache is not None - def get_cache_key(self, data_type: CachableDataType) -> tuple[str, str, str]: + # Allowing open-ended str data_type for now so we can do module names + def get_cache_key(self, data_type: Union[CachableDataType, str]) -> tuple[str, str, str]: path_parts = [str(part) for part in self.root_path.parts if part != self.root_path.anchor] paths_to_hash = [ self.root_path / "uv.lock", @@ -215,10 +216,13 @@ def get_cache_key(self, data_type: CachableDataType) -> tuple[str, str, str]: return ("_".join(path_parts), env_hash, data_type) def get_cache_key_for_module(self, module_name: str) -> tuple[str, str, str]: - path = self.get_path_for_module(module_name) - env_hash = hash_paths([path], includes=["*.py"]) - path_parts = [str(part) for part in path.parts if part != "/"] - return ("_".join(path_parts), env_hash, "local_component_registry") + if module_name.startswith(self.root_module_name): + path = self.get_path_for_module(module_name) + env_hash = hash_paths([path], includes=["*.py"]) + path_parts = [str(part) for part in path.parts if part != "/"] + return ("_".join(path_parts), env_hash, "local_component_registry") + else: + return self.get_cache_key(module_name) # ######################## # ##### WORKSPACE METHODS diff --git a/python_modules/libraries/dagster-dg/dagster_dg/scaffold.py b/python_modules/libraries/dagster-dg/dagster_dg/scaffold.py index 2ced2223e0e97..583156d005325 100644 --- a/python_modules/libraries/dagster-dg/dagster_dg/scaffold.py +++ b/python_modules/libraries/dagster-dg/dagster_dg/scaffold.py @@ -24,6 +24,7 @@ "dagster", "dagster-pipes", "dagster-components", + "dagster-test[components]", # we include dagster-test for testing purposes ) EDITABLE_DAGSTER_DEV_DEPENDENCIES = ("dagster-webserver", "dagster-graphql") PYPI_DAGSTER_DEPENDENCIES = ("dagster-components",) diff --git a/python_modules/libraries/dagster-dg/dagster_dg_tests/cli_tests/test_custom_help_format.py b/python_modules/libraries/dagster-dg/dagster_dg_tests/cli_tests/test_custom_help_format.py index 83b0b848c0e18..707c581d5e0f4 100644 --- a/python_modules/libraries/dagster-dg/dagster_dg_tests/cli_tests/test_custom_help_format.py +++ b/python_modules/libraries/dagster-dg/dagster_dg_tests/cli_tests/test_custom_help_format.py @@ -180,7 +180,7 @@ def test_dynamic_subcommand_help_message(): result = runner.invoke( "scaffold", "component", - "dagster_components.lib.test.SimplePipesScriptComponent", + "dagster_test.components.SimplePipesScriptComponent", "--help", ) assert_runner_result(result) @@ -189,8 +189,9 @@ def test_dynamic_subcommand_help_message(): assert match_terminal_box_output( output.strip(), textwrap.dedent(""" - Usage: dg scaffold component [GLOBAL OPTIONS] dagster_components.lib.test.SimplePipesScriptComponent - [OPTIONS] COMPONENT_INSTANCE_NAME + Usage: dg scaffold component [GLOBAL OPTIONS] dagster_test.components.SimplePipesScriptComponent [OPTIONS] + COMPONENT_INSTANCE_NA + ME ╭─ Arguments ──────────────────────────────────────────────────────────────────────────────────────────────────────────╮ │ * component_instance_name TEXT [required] │ diff --git a/python_modules/libraries/dagster-dg/dagster_dg_tests/cli_tests/test_docs_commands.py b/python_modules/libraries/dagster-dg/dagster_dg_tests/cli_tests/test_docs_commands.py index 967a6f2e53630..bce0368be7e6e 100644 --- a/python_modules/libraries/dagster-dg/dagster_dg_tests/cli_tests/test_docs_commands.py +++ b/python_modules/libraries/dagster-dg/dagster_dg_tests/cli_tests/test_docs_commands.py @@ -12,7 +12,7 @@ def test_docs_component_type_success(): with ProxyRunner.test() as runner, isolated_components_venv(runner): result = runner.invoke( - "docs", "component-type", "dagster_components.lib.test.SimpleAssetComponent" + "docs", "component-type", "dagster_test.components.SimpleAssetComponent" ) assert_runner_result(result) @@ -30,7 +30,7 @@ def test_docs_component_type_success_output_console(): result = runner.invoke( "docs", "component-type", - "dagster_components.lib.test.ComplexAssetComponent", + "dagster_test.components.ComplexAssetComponent", "--output", "cli", ) diff --git a/python_modules/libraries/dagster-dg/dagster_dg_tests/cli_tests/test_environment_validation.py b/python_modules/libraries/dagster-dg/dagster_dg_tests/cli_tests/test_environment_validation.py index a17ee7cd4216a..0116d8607570d 100644 --- a/python_modules/libraries/dagster-dg/dagster_dg_tests/cli_tests/test_environment_validation.py +++ b/python_modules/libraries/dagster-dg/dagster_dg_tests/cli_tests/test_environment_validation.py @@ -31,7 +31,7 @@ def to_cli_args(self) -> tuple[str, ...]: return (*self.command, *self.args) -DEFAULT_COMPONENT_TYPE = "dagster_components.lib.test.SimpleAssetComponent" +DEFAULT_COMPONENT_TYPE = "dagster_test.components.SimpleAssetComponent" NO_REQUIRED_CONTEXT_COMMANDS = [ CommandSpec(("scaffold", "project"), "foo"), diff --git a/python_modules/libraries/dagster-dg/dagster_dg_tests/cli_tests/test_info_commands.py b/python_modules/libraries/dagster-dg/dagster_dg_tests/cli_tests/test_info_commands.py index cadf0b809411e..37511ecc276d6 100644 --- a/python_modules/libraries/dagster-dg/dagster_dg_tests/cli_tests/test_info_commands.py +++ b/python_modules/libraries/dagster-dg/dagster_dg_tests/cli_tests/test_info_commands.py @@ -11,7 +11,7 @@ # ######################## _EXPECTED_INSPECT_COMPONENT_TYPE_FULL = textwrap.dedent(""" - dagster_components.lib.test.SimplePipesScriptComponent + dagster_test.components.SimplePipesScriptComponent Description: @@ -68,7 +68,7 @@ def test_inspect_component_type_all_metadata_success(): result = runner.invoke( "utils", "inspect-component-type", - "dagster_components.lib.test.SimplePipesScriptComponent", + "dagster_test.components.SimplePipesScriptComponent", ) assert_runner_result(result) assert result.output.strip().endswith(_EXPECTED_INSPECT_COMPONENT_TYPE_FULL) @@ -79,12 +79,12 @@ def test_inspect_component_type_all_metadata_empty_success(): result = runner.invoke( "utils", "inspect-component-type", - "dagster_components.lib.test.AllMetadataEmptyComponent", + "dagster_test.components.AllMetadataEmptyComponent", ) assert_runner_result(result) assert result.output.strip().endswith( textwrap.dedent(""" - dagster_components.lib.test.AllMetadataEmptyComponent + dagster_test.components.AllMetadataEmptyComponent """).strip() ) @@ -94,7 +94,7 @@ def test_inspect_component_type_flag_fields_success(): result = runner.invoke( "utils", "inspect-component-type", - "dagster_components.lib.test.SimplePipesScriptComponent", + "dagster_test.components.SimplePipesScriptComponent", "--description", ) assert_runner_result(result) @@ -109,7 +109,7 @@ def test_inspect_component_type_flag_fields_success(): result = runner.invoke( "utils", "inspect-component-type", - "dagster_components.lib.test.SimplePipesScriptComponent", + "dagster_test.components.SimplePipesScriptComponent", "--scaffold-params-schema", ) assert_runner_result(result) @@ -139,7 +139,7 @@ def test_inspect_component_type_flag_fields_success(): result = runner.invoke( "utils", "inspect-component-type", - "dagster_components.lib.test.SimplePipesScriptComponent", + "dagster_test.components.SimplePipesScriptComponent", "--component-schema", ) assert_runner_result(result) @@ -172,7 +172,7 @@ def test_inspect_component_type_multiple_flags_fails() -> None: result = runner.invoke( "utils", "inspect-component-type", - "dagster_components.lib.test.SimplePipesScriptComponent", + "dagster_test.components.SimplePipesScriptComponent", "--description", "--scaffold-params-schema", ) diff --git a/python_modules/libraries/dagster-dg/dagster_dg_tests/cli_tests/test_list_commands.py b/python_modules/libraries/dagster-dg/dagster_dg_tests/cli_tests/test_list_commands.py index 6035531e84fe2..c9aed4e69b565 100644 --- a/python_modules/libraries/dagster-dg/dagster_dg_tests/cli_tests/test_list_commands.py +++ b/python_modules/libraries/dagster-dg/dagster_dg_tests/cli_tests/test_list_commands.py @@ -43,11 +43,11 @@ def test_list_project_success(): def test_list_components_succeeds(): - with ProxyRunner.test() as runner, isolated_example_project_foo_bar(runner): + with ProxyRunner.test() as runner, isolated_example_project_foo_bar(runner, in_workspace=False): result = runner.invoke( "scaffold", "component", - "dagster_components.lib.test.AllMetadataEmptyComponent", + "dagster_test.components.AllMetadataEmptyComponent", "qux", ) assert_runner_result(result) @@ -66,33 +66,33 @@ def test_list_components_succeeds(): # ######################## _EXPECTED_COMPONENT_TYPES = textwrap.dedent(""" - ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓ - ┃ Component Type ┃ Summary ┃ - ┡━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┩ - │ dagster_components.lib.test.AllMetadataEmptyComponent │ │ - │ dagster_components.lib.test.ComplexAssetComponent │ An asset that has a complex schema. │ - │ dagster_components.lib.test.SimpleAssetComponent │ A simple asset that returns a constant string value. │ - │ dagster_components.lib.test.SimplePipesScriptComponent │ A simple asset that runs a Python script with the Pipes │ - │ │ subprocess client. │ - └────────────────────────────────────────────────────────┴─────────────────────────────────────────────────────────────┘ + ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓ + ┃ Component Type ┃ Summary ┃ + ┡━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┩ + │ dagster_test.components.AllMetadataEmptyComponent │ │ + │ dagster_test.components.ComplexAssetComponent │ An asset that has a complex schema. │ + │ dagster_test.components.SimpleAssetComponent │ A simple asset that returns a constant string value. │ + │ dagster_test.components.SimplePipesScriptComponent │ A simple asset that runs a Python script with the Pipes │ + │ │ subprocess client. │ + └────────────────────────────────────────────────────┴─────────────────────────────────────────────────────────────────┘ """).strip() _EXPECTED_COMPONENT_TYPES_JSON = textwrap.dedent(""" [ { - "key": "dagster_components.lib.test.AllMetadataEmptyComponent", + "key": "dagster_test.components.AllMetadataEmptyComponent", "summary": null }, { - "key": "dagster_components.lib.test.ComplexAssetComponent", + "key": "dagster_test.components.ComplexAssetComponent", "summary": "An asset that has a complex schema." }, { - "key": "dagster_components.lib.test.SimpleAssetComponent", + "key": "dagster_test.components.SimpleAssetComponent", "summary": "A simple asset that returns a constant string value." }, { - "key": "dagster_components.lib.test.SimplePipesScriptComponent", + "key": "dagster_test.components.SimplePipesScriptComponent", "summary": "A simple asset that runs a Python script with the Pipes subprocess client." } ] diff --git a/python_modules/libraries/dagster-dg/dagster_dg_tests/cli_tests/test_scaffold_commands.py b/python_modules/libraries/dagster-dg/dagster_dg_tests/cli_tests/test_scaffold_commands.py index 8542dc8b40cd8..2a55ea499487a 100644 --- a/python_modules/libraries/dagster-dg/dagster_dg_tests/cli_tests/test_scaffold_commands.py +++ b/python_modules/libraries/dagster-dg/dagster_dg_tests/cli_tests/test_scaffold_commands.py @@ -280,10 +280,10 @@ def test_scaffold_component_dynamic_subcommand_generation() -> None: # These are wrapped in a table so it's hard to check exact output. for line in [ "╭─ Commands", - "│ dagster_components.lib.test.AllMetadataEmptyComponent", - "│ dagster_components.lib.test.ComplexAssetComponent", - "│ dagster_components.lib.test.SimpleAssetComponent", - "│ dagster_components.lib.test.SimplePipesScriptComponent", + "│ dagster_test.components.AllMetadataEmptyComponent", + "│ dagster_test.components.ComplexAssetComponent", + "│ dagster_test.components.SimpleAssetComponent", + "│ dagster_test.components.SimplePipesScriptComponent", ]: assert standardize_box_characters(line) in normalized_output @@ -297,7 +297,7 @@ def test_scaffold_component_no_params_success(in_workspace: bool) -> None: result = runner.invoke( "scaffold", "component", - "dagster_components.lib.test.AllMetadataEmptyComponent", + "dagster_test.components.AllMetadataEmptyComponent", "qux", ) assert_runner_result(result) @@ -305,7 +305,7 @@ def test_scaffold_component_no_params_success(in_workspace: bool) -> None: component_yaml_path = Path("foo_bar/components/qux/component.yaml") assert component_yaml_path.exists() assert ( - "type: dagster_components.lib.test.AllMetadataEmptyComponent" + "type: dagster_test.components.AllMetadataEmptyComponent" in component_yaml_path.read_text() ) @@ -319,7 +319,7 @@ def test_scaffold_component_json_params_success(in_workspace: bool) -> None: result = runner.invoke( "scaffold", "component", - "dagster_components.lib.test.SimplePipesScriptComponent", + "dagster_test.components.SimplePipesScriptComponent", "qux", "--json-params", '{"asset_key": "foo", "filename": "hello.py"}', @@ -330,7 +330,7 @@ def test_scaffold_component_json_params_success(in_workspace: bool) -> None: component_yaml_path = Path("foo_bar/components/qux/component.yaml") assert component_yaml_path.exists() assert ( - "type: dagster_components.lib.test.SimplePipesScriptComponent" + "type: dagster_test.components.SimplePipesScriptComponent" in component_yaml_path.read_text() ) @@ -344,7 +344,7 @@ def test_scaffold_component_key_value_params_success(in_workspace: bool) -> None result = runner.invoke( "scaffold", "component", - "dagster_components.lib.test.SimplePipesScriptComponent", + "dagster_test.components.SimplePipesScriptComponent", "qux", "--asset-key=foo", "--filename=hello.py", @@ -355,7 +355,7 @@ def test_scaffold_component_key_value_params_success(in_workspace: bool) -> None component_yaml_path = Path("foo_bar/components/qux/component.yaml") assert component_yaml_path.exists() assert ( - "type: dagster_components.lib.test.SimplePipesScriptComponent" + "type: dagster_test.components.SimplePipesScriptComponent" in component_yaml_path.read_text() ) @@ -365,7 +365,7 @@ def test_scaffold_component_json_params_and_key_value_params_fails() -> None: result = runner.invoke( "scaffold", "component", - "dagster_components.lib.test.SimplePipesScriptComponent", + "dagster_test.components.SimplePipesScriptComponent", "qux", "--json-params", '{"filename": "hello.py"}', @@ -391,7 +391,10 @@ def test_scaffold_component_command_with_non_matching_module_name(): python_module.rename("module_not_same_as_project") result = runner.invoke( - "scaffold", "component", "dagster_components.lib.test.AllMetadataEmptyComponent", "qux" + "scaffold", + "component", + "dagster_test.components.AllMetadataEmptyComponent", + "qux", ) assert_runner_result(result, exit_0=False) assert "Module `foo_bar` is not installed" in str(result.exception) @@ -406,14 +409,14 @@ def test_scaffold_component_already_exists_fails(in_workspace: bool) -> None: result = runner.invoke( "scaffold", "component", - "dagster_components.lib.test.AllMetadataEmptyComponent", + "dagster_test.components.AllMetadataEmptyComponent", "qux", ) assert_runner_result(result) result = runner.invoke( "scaffold", "component", - "dagster_components.lib.test.AllMetadataEmptyComponent", + "dagster_test.components.AllMetadataEmptyComponent", "qux", ) assert_runner_result(result, exit_0=False) @@ -431,7 +434,7 @@ def test_scaffold_component_succeeds_non_default_component_package() -> None: result = runner.invoke( "scaffold", "component", - "dagster_components.lib.test.AllMetadataEmptyComponent", + "dagster_test.components.AllMetadataEmptyComponent", "qux", ) assert_runner_result(result) @@ -439,7 +442,7 @@ def test_scaffold_component_succeeds_non_default_component_package() -> None: component_yaml_path = Path("foo_bar/_components/qux/component.yaml") assert component_yaml_path.exists() assert ( - "type: dagster_components.lib.test.AllMetadataEmptyComponent" + "type: dagster_test.components.AllMetadataEmptyComponent" in component_yaml_path.read_text() ) @@ -453,7 +456,7 @@ def test_scaffold_component_fails_components_package_does_not_exist() -> None: result = runner.invoke( "scaffold", "component", - "dagster_components.lib.test.AllMetadataEmptyComponent", + "dagster_test.components.AllMetadataEmptyComponent", "qux", ) assert_runner_result(result, exit_0=False) diff --git a/python_modules/libraries/dagster-dg/dagster_dg_tests/utils.py b/python_modules/libraries/dagster-dg/dagster_dg_tests/utils.py index adb94452ddd65..3b40ee14c652c 100644 --- a/python_modules/libraries/dagster-dg/dagster_dg_tests/utils.py +++ b/python_modules/libraries/dagster-dg/dagster_dg_tests/utils.py @@ -28,7 +28,7 @@ ) from typing_extensions import Self -STANDARD_TEST_COMPONENT_MODULE = "dagster_components.lib.test" +STANDARD_TEST_COMPONENT_MODULE = "dagster_test.components" def _install_libraries_to_venv(venv_path: Path, libraries_rel_paths: Sequence[str]) -> None: @@ -46,7 +46,7 @@ def isolated_components_venv(runner: Union[CliRunner, "ProxyRunner"]) -> Iterato subprocess.run(["uv", "venv", ".venv"], check=True) venv_path = Path.cwd() / ".venv" _install_libraries_to_venv( - venv_path, ["dagster", "libraries/dagster-components", "dagster-pipes"] + venv_path, ["dagster", "libraries/dagster-components", "dagster-pipes", "dagster-test"] ) venv_exec_path = get_venv_executable(venv_path).parent @@ -83,7 +83,7 @@ def isolated_example_workspace( subprocess.run(["uv", "venv", ".venv"], check=True) venv_path = Path.cwd() / ".venv" _install_libraries_to_venv( - venv_path, ["dagster", "dagster-webserver", "dagster-graphql"] + venv_path, ["dagster", "dagster-webserver", "dagster-graphql", "dagster-test"] ) yield @@ -95,7 +95,7 @@ def isolated_example_project_foo_bar( runner: Union[CliRunner, "ProxyRunner"], in_workspace: bool = True, skip_venv: bool = False, - populate_cache: bool = True, + populate_cache: bool = False, component_dirs: Sequence[Path] = [], ) -> Iterator[None]: """Scaffold a project named foo_bar in an isolated filesystem. @@ -126,6 +126,7 @@ def isolated_example_project_foo_bar( ) assert_runner_result(result) with clear_module_from_cache("foo_bar"), pushd(project_path): + # _install_libraries_to_venv(Path(".venv"), ["dagster-test"]) for src_dir in component_dirs: component_name = src_dir.name components_dir = Path.cwd() / "foo_bar" / "components" / component_name @@ -339,11 +340,11 @@ def normalize_windows_path(path: str) -> str: # ######################## -# NOTE: This class sets up a runner that by default targets only the `dagster_components.lib.test` +# NOTE: This class sets up a runner that by default targets only the `dagster_test.components` # components in the remote environment. This is to provide stability against the set of components # we are testing against. Tests that are testing against scaffolded components need to set # `use_entry_points=True` in order to detect scaffolded components (since they aren't part of -# `dagster_components.lib.test`). +# `dagster_test.components`). @dataclass class ProxyRunner: original: CliRunner