From 980534caccf504f1888734f76d34fa613bee0cf3 Mon Sep 17 00:00:00 2001 From: Sean Mackesey Date: Wed, 26 Feb 2025 21:09:18 -0500 Subject: [PATCH] [components] Add *Component suffix to all component classes [INTERNAL_BRANCH=sean/components/add-component-suffix] --- .../customizing-components.md | 2 +- .../custom-subclass/basic-subclass.py | 4 +-- .../custom-subclass/custom-scope.py | 4 +-- .../components/custom-subclass/debug-mode.py | 4 +-- .../8-dg-list-component-types.txt | 22 ++++++------ .../guides/components/index/11-component.yaml | 2 +- .../index/18-dg-list-component-types.txt | 34 +++++++++++-------- .../index/7-dg-list-component-types.txt | 22 ++++++------ .../index/8-dg-list-component-types.txt | 29 +++++++++------- .../index/9-dg-scaffold-sling-replication.txt | 2 +- .../3-dg-list-component-types.txt | 27 ++++++++------- .../workspace/5-component-type-list.txt | 22 ++++++------ .../workspace/6-component-type-list.txt | 29 +++++++++------- .../workspace/9-component-type-list.txt | 22 ++++++------ .../guides/components/test_components_docs.py | 2 +- .../src/assets/AssetWipeDialog.oss.tsx | 2 +- .../dagster_components/lib/__init__.py | 4 +-- .../lib/pipes_subprocess_script_collection.py | 10 +++--- .../sling_replication_collection/component.py | 4 +-- .../dagster_components/lib/test/__init__.py | 8 ++--- .../lib/test/all_metadata_empty_asset.py | 2 +- .../lib/test/complex_schema_asset.py | 4 +-- .../lib/test/simple_asset.py | 2 +- .../lib/test/simple_pipes_script_asset.py | 16 ++++----- .../cli_tests/test_commands.py | 30 ++++++++-------- .../script_python_decl/component.py | 6 ++-- .../components/scripts/component.yaml | 2 +- .../components/ingest/component.yaml | 2 +- .../simple_asset_invalid_value/component.yaml | 2 +- .../test_sling_integration_test.py | 24 +++++++------ .../registry_tests/test_registry.py | 12 +++---- .../test_integrity.py | 17 ++++++++++ .../cli_tests/test_custom_help_format.py | 8 ++--- .../cli_tests/test_docs_commands.py | 6 ++-- .../cli_tests/test_environment_validation.py | 2 +- .../cli_tests/test_info_commands.py | 24 ++++++------- .../cli_tests/test_list_commands.py | 28 +++++++-------- .../cli_tests/test_scaffold_commands.py | 34 +++++++++---------- 38 files changed, 258 insertions(+), 218 deletions(-) create mode 100644 python_modules/libraries/dagster-components/dagster_components_tests/test_integrity.py diff --git a/docs/docs/guides/labs/components/building-pipelines-with-components/customizing-components.md b/docs/docs/guides/labs/components/building-pipelines-with-components/customizing-components.md index f453a962bd54a..df8fc33a77d2a 100644 --- a/docs/docs/guides/labs/components/building-pipelines-with-components/customizing-components.md +++ b/docs/docs/guides/labs/components/building-pipelines-with-components/customizing-components.md @@ -28,7 +28,7 @@ attributes: By convention, most library components have an `execute()` method that defines the core runtime behavior of the component. This can be overridden by subclasses of the component to customize this behavior. -For example, we can create a subclass of the `SlingReplicationCollectioncomponent` that adds a debug log message during execution: +For example, we can create a subclass of the `SlingReplicationCollectionComponent` that adds a debug log message during execution: diff --git a/examples/docs_beta_snippets/docs_beta_snippets/guides/components/custom-subclass/basic-subclass.py b/examples/docs_beta_snippets/docs_beta_snippets/guides/components/custom-subclass/basic-subclass.py index c8c1d5989b7ec..dc0a2893e3b3a 100644 --- a/examples/docs_beta_snippets/docs_beta_snippets/guides/components/custom-subclass/basic-subclass.py +++ b/examples/docs_beta_snippets/docs_beta_snippets/guides/components/custom-subclass/basic-subclass.py @@ -1,4 +1,4 @@ -from dagster_components.lib import SlingReplicationCollection +from dagster_components.lib import SlingReplicationCollectionComponent -class CustomSubclass(SlingReplicationCollection): ... +class CustomSubclass(SlingReplicationCollectionComponent): ... diff --git a/examples/docs_beta_snippets/docs_beta_snippets/guides/components/custom-subclass/custom-scope.py b/examples/docs_beta_snippets/docs_beta_snippets/guides/components/custom-subclass/custom-scope.py index 2f7596a27eb68..ad1ac822bac98 100644 --- a/examples/docs_beta_snippets/docs_beta_snippets/guides/components/custom-subclass/custom-scope.py +++ b/examples/docs_beta_snippets/docs_beta_snippets/guides/components/custom-subclass/custom-scope.py @@ -1,12 +1,12 @@ from collections.abc import Mapping from typing import Any -from dagster_components.lib import SlingReplicationCollection +from dagster_components.lib import SlingReplicationCollectionComponent import dagster as dg -class SubclassWithScope(SlingReplicationCollection): +class SubclassWithScope(SlingReplicationCollectionComponent): def get_additional_scope(self) -> Mapping[str, Any]: def _custom_cron(cron_schedule: str) -> dg.AutomationCondition: return ( diff --git a/examples/docs_beta_snippets/docs_beta_snippets/guides/components/custom-subclass/debug-mode.py b/examples/docs_beta_snippets/docs_beta_snippets/guides/components/custom-subclass/debug-mode.py index 2a7a718d39928..9886e3e3a0b38 100644 --- a/examples/docs_beta_snippets/docs_beta_snippets/guides/components/custom-subclass/debug-mode.py +++ b/examples/docs_beta_snippets/docs_beta_snippets/guides/components/custom-subclass/debug-mode.py @@ -1,12 +1,12 @@ from collections.abc import Iterator -from dagster_components.lib import SlingReplicationCollection +from dagster_components.lib import SlingReplicationCollectionComponent from dagster_sling import SlingResource import dagster as dg -class DebugSlingReplicationComponent(SlingReplicationCollection): +class DebugSlingReplicationComponent(SlingReplicationCollectionComponent): def execute( self, context: dg.AssetExecutionContext, sling: SlingResource ) -> Iterator: diff --git a/examples/docs_beta_snippets/docs_beta_snippets/guides/components/existing-project/8-dg-list-component-types.txt b/examples/docs_beta_snippets/docs_beta_snippets/guides/components/existing-project/8-dg-list-component-types.txt index 4610ba5af6c53..26138080604e0 100644 --- a/examples/docs_beta_snippets/docs_beta_snippets/guides/components/existing-project/8-dg-list-component-types.txt +++ b/examples/docs_beta_snippets/docs_beta_snippets/guides/components/existing-project/8-dg-list-component-types.txt @@ -1,13 +1,15 @@ dg list component-type Using /.../my-existing-project/.venv/bin/dagster-components -┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓ -┃ Component Type ┃ Summary ┃ -┡━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┩ -│ dagster_components.lib.DefinitionsComponent │ Wraps an arbitrary set of │ -│ │ Dagster definitions. │ -│ dagster_components.lib.PipesSubprocessScriptCollection │ Assets that wrap Python │ -│ │ scripts executed with │ -│ │ Dagster's │ -│ │ PipesSubprocessClient. │ -└────────────────────────────────────────────────────────┴───────────────────────────────┘ +┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━┓ +┃ Component Type ┃ Summary ┃ +┡━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━┩ +│ dagster_components.lib.DefinitionsComponent │ Wraps an arbitrary │ +│ │ set of Dagster │ +│ │ definitions. │ +│ dagster_components.lib.PipesSubprocessScriptCollectionComponent │ Assets that wrap │ +│ │ Python scripts │ +│ │ executed with │ +│ │ Dagster's │ +│ │ PipesSubprocessClie… │ +└─────────────────────────────────────────────────────────────────┴──────────────────────┘ diff --git a/examples/docs_beta_snippets/docs_beta_snippets/guides/components/index/11-component.yaml b/examples/docs_beta_snippets/docs_beta_snippets/guides/components/index/11-component.yaml index 8263561201864..51b11785c809c 100644 --- a/examples/docs_beta_snippets/docs_beta_snippets/guides/components/index/11-component.yaml +++ b/examples/docs_beta_snippets/docs_beta_snippets/guides/components/index/11-component.yaml @@ -1,4 +1,4 @@ -type: dagster_components.lib.SlingReplicationCollection +type: dagster_components.lib.SlingReplicationCollectionComponent attributes: replications: diff --git a/examples/docs_beta_snippets/docs_beta_snippets/guides/components/index/18-dg-list-component-types.txt b/examples/docs_beta_snippets/docs_beta_snippets/guides/components/index/18-dg-list-component-types.txt index b4aa604831723..65dedfc0441b2 100644 --- a/examples/docs_beta_snippets/docs_beta_snippets/guides/components/index/18-dg-list-component-types.txt +++ b/examples/docs_beta_snippets/docs_beta_snippets/guides/components/index/18-dg-list-component-types.txt @@ -1,18 +1,22 @@ dg list component-type Using /.../jaffle-platform/.venv/bin/dagster-components -┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓ -┃ Component Type ┃ Summary ┃ -┡━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┩ -│ dagster_components.lib.DbtProjectComponent │ Expose a DBT project to │ -│ │ Dagster as a set of assets. │ -│ dagster_components.lib.DefinitionsComponent │ Wraps an arbitrary set of │ -│ │ Dagster definitions. │ -│ dagster_components.lib.PipesSubprocessScriptCollection │ Assets that wrap Python │ -│ │ scripts executed with │ -│ │ Dagster's │ -│ │ PipesSubprocessClient. │ -│ dagster_components.lib.SlingReplicationCollection │ Expose one or more Sling │ -│ │ replications to Dagster as │ -│ │ assets. │ -└────────────────────────────────────────────────────────┴───────────────────────────────┘ +┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━┓ +┃ Component Type ┃ Summary ┃ +┡━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━┩ +│ dagster_components.lib.DbtProjectComponent │ Expose a DBT project │ +│ │ to Dagster as a set │ +│ │ of assets. │ +│ dagster_components.lib.DefinitionsComponent │ Wraps an arbitrary │ +│ │ set of Dagster │ +│ │ definitions. │ +│ dagster_components.lib.PipesSubprocessScriptCollectionComponent │ Assets that wrap │ +│ │ Python scripts │ +│ │ executed with │ +│ │ Dagster's │ +│ │ PipesSubprocessClie… │ +│ dagster_components.lib.SlingReplicationCollectionComponent │ Expose one or more │ +│ │ Sling replications │ +│ │ to Dagster as │ +│ │ assets. │ +└─────────────────────────────────────────────────────────────────┴──────────────────────┘ diff --git a/examples/docs_beta_snippets/docs_beta_snippets/guides/components/index/7-dg-list-component-types.txt b/examples/docs_beta_snippets/docs_beta_snippets/guides/components/index/7-dg-list-component-types.txt index ef07a7e5993ac..4139ba4915572 100644 --- a/examples/docs_beta_snippets/docs_beta_snippets/guides/components/index/7-dg-list-component-types.txt +++ b/examples/docs_beta_snippets/docs_beta_snippets/guides/components/index/7-dg-list-component-types.txt @@ -1,12 +1,14 @@ dg list component-type -┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓ -┃ Component Type ┃ Summary ┃ -┡━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┩ -│ dagster_components.lib.DefinitionsComponent │ Wraps an arbitrary set of │ -│ │ Dagster definitions. │ -│ dagster_components.lib.PipesSubprocessScriptCollection │ Assets that wrap Python │ -│ │ scripts executed with │ -│ │ Dagster's │ -│ │ PipesSubprocessClient. │ -└────────────────────────────────────────────────────────┴───────────────────────────────┘ +┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━┓ +┃ Component Type ┃ Summary ┃ +┡━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━┩ +│ dagster_components.lib.DefinitionsComponent │ Wraps an arbitrary │ +│ │ set of Dagster │ +│ │ definitions. │ +│ dagster_components.lib.PipesSubprocessScriptCollectionComponent │ Assets that wrap │ +│ │ Python scripts │ +│ │ executed with │ +│ │ Dagster's │ +│ │ PipesSubprocessClie… │ +└─────────────────────────────────────────────────────────────────┴──────────────────────┘ diff --git a/examples/docs_beta_snippets/docs_beta_snippets/guides/components/index/8-dg-list-component-types.txt b/examples/docs_beta_snippets/docs_beta_snippets/guides/components/index/8-dg-list-component-types.txt index cf835532e1241..6d35f4a52b390 100644 --- a/examples/docs_beta_snippets/docs_beta_snippets/guides/components/index/8-dg-list-component-types.txt +++ b/examples/docs_beta_snippets/docs_beta_snippets/guides/components/index/8-dg-list-component-types.txt @@ -1,16 +1,19 @@ dg list component-type Using /.../jaffle-platform/.venv/bin/dagster-components -┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓ -┃ Component Type ┃ Summary ┃ -┡━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┩ -│ dagster_components.lib.DefinitionsComponent │ Wraps an arbitrary set of │ -│ │ Dagster definitions. │ -│ dagster_components.lib.PipesSubprocessScriptCollection │ Assets that wrap Python │ -│ │ scripts executed with │ -│ │ Dagster's │ -│ │ PipesSubprocessClient. │ -│ dagster_components.lib.SlingReplicationCollection │ Expose one or more Sling │ -│ │ replications to Dagster as │ -│ │ assets. │ -└────────────────────────────────────────────────────────┴───────────────────────────────┘ +┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━┓ +┃ Component Type ┃ Summary ┃ +┡━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━┩ +│ dagster_components.lib.DefinitionsComponent │ Wraps an arbitrary │ +│ │ set of Dagster │ +│ │ definitions. │ +│ dagster_components.lib.PipesSubprocessScriptCollectionComponent │ Assets that wrap │ +│ │ Python scripts │ +│ │ executed with │ +│ │ Dagster's │ +│ │ PipesSubprocessClie… │ +│ dagster_components.lib.SlingReplicationCollectionComponent │ Expose one or more │ +│ │ Sling replications │ +│ │ to Dagster as │ +│ │ assets. │ +└─────────────────────────────────────────────────────────────────┴──────────────────────┘ diff --git a/examples/docs_beta_snippets/docs_beta_snippets/guides/components/index/9-dg-scaffold-sling-replication.txt b/examples/docs_beta_snippets/docs_beta_snippets/guides/components/index/9-dg-scaffold-sling-replication.txt index 4f458fc059b38..aece333f7f79f 100644 --- a/examples/docs_beta_snippets/docs_beta_snippets/guides/components/index/9-dg-scaffold-sling-replication.txt +++ b/examples/docs_beta_snippets/docs_beta_snippets/guides/components/index/9-dg-scaffold-sling-replication.txt @@ -1,4 +1,4 @@ -dg scaffold component 'dagster_components.lib.SlingReplicationCollection' ingest_files +dg scaffold component 'dagster_components.lib.SlingReplicationCollectionComponent' ingest_files Creating a Dagster component instance folder at /.../jaffle-platform/jaffle_platform/components/ingest_files. Using /.../jaffle-platform/.venv/bin/dagster-components diff --git a/examples/docs_beta_snippets/docs_beta_snippets/guides/components/shell-script-component/3-dg-list-component-types.txt b/examples/docs_beta_snippets/docs_beta_snippets/guides/components/shell-script-component/3-dg-list-component-types.txt index 0a7329c27149a..fa7c7f6beb687 100644 --- a/examples/docs_beta_snippets/docs_beta_snippets/guides/components/shell-script-component/3-dg-list-component-types.txt +++ b/examples/docs_beta_snippets/docs_beta_snippets/guides/components/shell-script-component/3-dg-list-component-types.txt @@ -1,15 +1,18 @@ dg list component-type Using /.../my-component-library/.venv/bin/dagster-components -┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓ -┃ Component Type ┃ Summary ┃ -┡━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┩ -│ dagster_components.lib.DefinitionsComponent │ Wraps an arbitrary set of │ -│ │ Dagster definitions. │ -│ dagster_components.lib.PipesSubprocessScriptCollection │ Assets that wrap Python │ -│ │ scripts executed with │ -│ │ Dagster's │ -│ │ PipesSubprocessClient. │ -│ my_component_library.lib.ShellCommand │ Models a shell script as a │ -│ │ Dagster asset. │ -└────────────────────────────────────────────────────────┴───────────────────────────────┘ +┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━┓ +┃ Component Type ┃ Summary ┃ +┡━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━┩ +│ dagster_components.lib.DefinitionsComponent │ Wraps an arbitrary │ +│ │ set of Dagster │ +│ │ definitions. │ +│ dagster_components.lib.PipesSubprocessScriptCollectionComponent │ Assets that wrap │ +│ │ Python scripts │ +│ │ executed with │ +│ │ Dagster's │ +│ │ PipesSubprocessClie… │ +│ my_component_library.lib.ShellCommand │ Models a shell │ +│ │ script as a Dagster │ +│ │ asset. │ +└─────────────────────────────────────────────────────────────────┴──────────────────────┘ diff --git a/examples/docs_beta_snippets/docs_beta_snippets/guides/components/workspace/5-component-type-list.txt b/examples/docs_beta_snippets/docs_beta_snippets/guides/components/workspace/5-component-type-list.txt index ee4c2d3aa3b7d..15fdc181cfe96 100644 --- a/examples/docs_beta_snippets/docs_beta_snippets/guides/components/workspace/5-component-type-list.txt +++ b/examples/docs_beta_snippets/docs_beta_snippets/guides/components/workspace/5-component-type-list.txt @@ -1,12 +1,14 @@ cd projects/project-1 && dg list component-type -┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓ -┃ Component Type ┃ Summary ┃ -┡━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┩ -│ dagster_components.lib.DefinitionsComponent │ Wraps an arbitrary set of │ -│ │ Dagster definitions. │ -│ dagster_components.lib.PipesSubprocessScriptCollection │ Assets that wrap Python │ -│ │ scripts executed with │ -│ │ Dagster's │ -│ │ PipesSubprocessClient. │ -└────────────────────────────────────────────────────────┴───────────────────────────────┘ +┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━┓ +┃ Component Type ┃ Summary ┃ +┡━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━┩ +│ dagster_components.lib.DefinitionsComponent │ Wraps an arbitrary │ +│ │ set of Dagster │ +│ │ definitions. │ +│ dagster_components.lib.PipesSubprocessScriptCollectionComponent │ Assets that wrap │ +│ │ Python scripts │ +│ │ executed with │ +│ │ Dagster's │ +│ │ PipesSubprocessClie… │ +└─────────────────────────────────────────────────────────────────┴──────────────────────┘ diff --git a/examples/docs_beta_snippets/docs_beta_snippets/guides/components/workspace/6-component-type-list.txt b/examples/docs_beta_snippets/docs_beta_snippets/guides/components/workspace/6-component-type-list.txt index 5ced210f59a80..507315c5b4ad6 100644 --- a/examples/docs_beta_snippets/docs_beta_snippets/guides/components/workspace/6-component-type-list.txt +++ b/examples/docs_beta_snippets/docs_beta_snippets/guides/components/workspace/6-component-type-list.txt @@ -1,16 +1,19 @@ dg list component-type Using /.../dagster-workspace/projects/project-1/.venv/bin/dagster-components -┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓ -┃ Component Type ┃ Summary ┃ -┡━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┩ -│ dagster_components.lib.DefinitionsComponent │ Wraps an arbitrary set of │ -│ │ Dagster definitions. │ -│ dagster_components.lib.PipesSubprocessScriptCollection │ Assets that wrap Python │ -│ │ scripts executed with │ -│ │ Dagster's │ -│ │ PipesSubprocessClient. │ -│ dagster_components.lib.SlingReplicationCollection │ Expose one or more Sling │ -│ │ replications to Dagster as │ -│ │ assets. │ -└────────────────────────────────────────────────────────┴───────────────────────────────┘ +┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━┓ +┃ Component Type ┃ Summary ┃ +┡━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━┩ +│ dagster_components.lib.DefinitionsComponent │ Wraps an arbitrary │ +│ │ set of Dagster │ +│ │ definitions. │ +│ dagster_components.lib.PipesSubprocessScriptCollectionComponent │ Assets that wrap │ +│ │ Python scripts │ +│ │ executed with │ +│ │ Dagster's │ +│ │ PipesSubprocessClie… │ +│ dagster_components.lib.SlingReplicationCollectionComponent │ Expose one or more │ +│ │ Sling replications │ +│ │ to Dagster as │ +│ │ assets. │ +└─────────────────────────────────────────────────────────────────┴──────────────────────┘ diff --git a/examples/docs_beta_snippets/docs_beta_snippets/guides/components/workspace/9-component-type-list.txt b/examples/docs_beta_snippets/docs_beta_snippets/guides/components/workspace/9-component-type-list.txt index 23c914a3dacf9..c42fe6b9d0f11 100644 --- a/examples/docs_beta_snippets/docs_beta_snippets/guides/components/workspace/9-component-type-list.txt +++ b/examples/docs_beta_snippets/docs_beta_snippets/guides/components/workspace/9-component-type-list.txt @@ -1,12 +1,14 @@ cd projects/project-2 && dg list component-type -┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓ -┃ Component Type ┃ Summary ┃ -┡━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┩ -│ dagster_components.lib.DefinitionsComponent │ Wraps an arbitrary set of │ -│ │ Dagster definitions. │ -│ dagster_components.lib.PipesSubprocessScriptCollection │ Assets that wrap Python │ -│ │ scripts executed with │ -│ │ Dagster's │ -│ │ PipesSubprocessClient. │ -└────────────────────────────────────────────────────────┴───────────────────────────────┘ +┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━┓ +┃ Component Type ┃ Summary ┃ +┡━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━┩ +│ dagster_components.lib.DefinitionsComponent │ Wraps an arbitrary │ +│ │ set of Dagster │ +│ │ definitions. │ +│ dagster_components.lib.PipesSubprocessScriptCollectionComponent │ Assets that wrap │ +│ │ Python scripts │ +│ │ executed with │ +│ │ Dagster's │ +│ │ PipesSubprocessClie… │ +└─────────────────────────────────────────────────────────────────┴──────────────────────┘ diff --git a/examples/docs_beta_snippets/docs_beta_snippets_tests/snippet_checks/guides/components/test_components_docs.py b/examples/docs_beta_snippets/docs_beta_snippets_tests/snippet_checks/guides/components/test_components_docs.py index 25e374d12bf3a..a06f08cb6b572 100644 --- a/examples/docs_beta_snippets/docs_beta_snippets_tests/snippet_checks/guides/components/test_components_docs.py +++ b/examples/docs_beta_snippets/docs_beta_snippets_tests/snippet_checks/guides/components/test_components_docs.py @@ -119,7 +119,7 @@ def next_snip_no(): # Scaffold new ingestion, validate new files run_command_and_snippet_output( - cmd="dg scaffold component 'dagster_components.lib.SlingReplicationCollection' ingest_files", + cmd="dg scaffold component 'dagster_components.lib.SlingReplicationCollectionComponent' ingest_files", snippet_path=COMPONENTS_SNIPPETS_DIR / f"{next_snip_no()}-dg-scaffold-sling-replication.txt", update_snippets=update_snippets, diff --git a/js_modules/dagster-ui/packages/ui-core/src/assets/AssetWipeDialog.oss.tsx b/js_modules/dagster-ui/packages/ui-core/src/assets/AssetWipeDialog.oss.tsx index ba3919a3c05b2..082ab56b3fb5c 100644 --- a/js_modules/dagster-ui/packages/ui-core/src/assets/AssetWipeDialog.oss.tsx +++ b/js_modules/dagster-ui/packages/ui-core/src/assets/AssetWipeDialog.oss.tsx @@ -12,10 +12,10 @@ import { } from '@dagster-io/ui-components'; import {memo, useMemo} from 'react'; +import {RefetchQueriesFunction} from '../apollo-client'; import {VirtualizedSimpleAssetKeyList} from './VirtualizedSimpleAssetKeyList'; import {asAssetPartitionRangeInput} from './asInput'; import {useWipeAssets} from './useWipeAssets'; -import {RefetchQueriesFunction} from '../apollo-client'; import {AssetKeyInput} from '../graphql/types'; import {NavigationBlock} from '../runs/NavigationBlock'; import {numberFormatter} from '../ui/formatters'; diff --git a/python_modules/libraries/dagster-components/dagster_components/lib/__init__.py b/python_modules/libraries/dagster-components/dagster_components/lib/__init__.py index e41363f55342e..75e877a133786 100644 --- a/python_modules/libraries/dagster-components/dagster_components/lib/__init__.py +++ b/python_modules/libraries/dagster-components/dagster_components/lib/__init__.py @@ -10,12 +10,12 @@ if _has_dagster_sling: from dagster_components.lib.sling_replication_collection.component import ( - SlingReplicationCollection as SlingReplicationCollection, + SlingReplicationCollectionComponent as SlingReplicationCollectionComponent, ) from dagster_components.lib.definitions_component.component import ( DefinitionsComponent as DefinitionsComponent, ) from dagster_components.lib.pipes_subprocess_script_collection import ( - PipesSubprocessScriptCollection as PipesSubprocessScriptCollection, + PipesSubprocessScriptCollectionComponent as PipesSubprocessScriptCollectionComponent, ) diff --git a/python_modules/libraries/dagster-components/dagster_components/lib/pipes_subprocess_script_collection.py b/python_modules/libraries/dagster-components/dagster_components/lib/pipes_subprocess_script_collection.py index cae7e2aef4d87..5622b8ff34815 100644 --- a/python_modules/libraries/dagster-components/dagster_components/lib/pipes_subprocess_script_collection.py +++ b/python_modules/libraries/dagster-components/dagster_components/lib/pipes_subprocess_script_collection.py @@ -33,7 +33,9 @@ class PipesSubprocessScriptSchema(ResolvableSchema[PipesSubprocessScriptSpec]): assets: Sequence[AssetSpecSchema] -class PipesSubprocessScriptCollectionSchema(ResolvableSchema["PipesSubprocessScriptCollection"]): +class PipesSubprocessScriptCollectionSchema( + ResolvableSchema["PipesSubprocessScriptCollectionComponent"] +): scripts: Sequence[PipesSubprocessScriptSchema] @@ -44,7 +46,7 @@ def resolve_specs_by_path( @dataclass -class PipesSubprocessScriptCollection(Component): +class PipesSubprocessScriptCollectionComponent(Component): """Assets that wrap Python scripts executed with Dagster's PipesSubprocessClient.""" specs_by_path: Annotated[ @@ -55,9 +57,9 @@ class PipesSubprocessScriptCollection(Component): ) @staticmethod - def introspect_from_path(path: Path) -> "PipesSubprocessScriptCollection": + def introspect_from_path(path: Path) -> "PipesSubprocessScriptCollectionComponent": path_specs = {str(path): [AssetSpec(path.stem)] for path in list(path.rglob("*.py"))} - return PipesSubprocessScriptCollection(specs_by_path=path_specs) + return PipesSubprocessScriptCollectionComponent(specs_by_path=path_specs) @classmethod def get_schema(cls) -> type[PipesSubprocessScriptCollectionSchema]: diff --git a/python_modules/libraries/dagster-components/dagster_components/lib/sling_replication_collection/component.py b/python_modules/libraries/dagster-components/dagster_components/lib/sling_replication_collection/component.py index 4a8f980a19b3e..e876bb6513425 100644 --- a/python_modules/libraries/dagster-components/dagster_components/lib/sling_replication_collection/component.py +++ b/python_modules/libraries/dagster-components/dagster_components/lib/sling_replication_collection/component.py @@ -70,7 +70,7 @@ class SlingReplicationSchema(ResolvableSchema[SlingReplicationSpec]): ) -class SlingReplicationCollectionSchema(ResolvableSchema["SlingReplicationCollection"]): +class SlingReplicationCollectionSchema(ResolvableSchema["SlingReplicationCollectionComponent"]): sling: Optional[SlingResource] = None replications: Sequence[SlingReplicationSchema] asset_post_processors: Optional[Sequence[AssetPostProcessorSchema]] = None @@ -87,7 +87,7 @@ def resolve_resource( @dataclass -class SlingReplicationCollection(Component): +class SlingReplicationCollectionComponent(Component): """Expose one or more Sling replications to Dagster as assets.""" resource: Annotated[SlingResource, FieldResolver(resolve_resource)] = Field( diff --git a/python_modules/libraries/dagster-components/dagster_components/lib/test/__init__.py b/python_modules/libraries/dagster-components/dagster_components/lib/test/__init__.py index 26d34cad225bf..9f0ac4bc38db3 100644 --- a/python_modules/libraries/dagster-components/dagster_components/lib/test/__init__.py +++ b/python_modules/libraries/dagster-components/dagster_components/lib/test/__init__.py @@ -1,10 +1,10 @@ from dagster_components.lib.test.all_metadata_empty_asset import ( - AllMetadataEmptyAsset as AllMetadataEmptyAsset, + AllMetadataEmptyComponent as AllMetadataEmptyComponent, ) from dagster_components.lib.test.complex_schema_asset import ( - ComplexSchemaAsset as ComplexSchemaAsset, + ComplexAssetComponent as ComplexAssetComponent, ) -from dagster_components.lib.test.simple_asset import SimpleAsset as SimpleAsset +from dagster_components.lib.test.simple_asset import SimpleAssetComponent as SimpleAssetComponent from dagster_components.lib.test.simple_pipes_script_asset import ( - SimplePipesScriptAsset as SimplePipesScriptAsset, + SimplePipesScriptComponent as SimplePipesScriptComponent, ) diff --git a/python_modules/libraries/dagster-components/dagster_components/lib/test/all_metadata_empty_asset.py b/python_modules/libraries/dagster-components/dagster_components/lib/test/all_metadata_empty_asset.py index f9d35f4b167d8..245112fe7ea04 100644 --- a/python_modules/libraries/dagster-components/dagster_components/lib/test/all_metadata_empty_asset.py +++ b/python_modules/libraries/dagster-components/dagster_components/lib/test/all_metadata_empty_asset.py @@ -6,7 +6,7 @@ from dagster_components.core.component_scaffolder import DefaultComponentScaffolder -class AllMetadataEmptyAsset(Component): +class AllMetadataEmptyComponent(Component): @classmethod def get_scaffolder(cls) -> DefaultComponentScaffolder: return DefaultComponentScaffolder() diff --git a/python_modules/libraries/dagster-components/dagster_components/lib/test/complex_schema_asset.py b/python_modules/libraries/dagster-components/dagster_components/lib/test/complex_schema_asset.py index d42d452f629ae..764729453a34e 100644 --- a/python_modules/libraries/dagster-components/dagster_components/lib/test/complex_schema_asset.py +++ b/python_modules/libraries/dagster-components/dagster_components/lib/test/complex_schema_asset.py @@ -18,7 +18,7 @@ ) -class ComplexAssetSchema(ResolvableSchema["ComplexSchemaAsset"]): +class ComplexAssetSchema(ResolvableSchema["ComplexAssetComponent"]): value: str = Field(..., examples=["example_for_value"]) list_value: list[str] = Field( ..., examples=[["example_for_list_value_1", "example_for_list_value_2"]] @@ -31,7 +31,7 @@ class ComplexAssetSchema(ResolvableSchema["ComplexSchemaAsset"]): asset_post_processors: Optional[Sequence[AssetPostProcessorSchema]] = None -class ComplexSchemaAsset(Component): +class ComplexAssetComponent(Component): """An asset that has a complex schema.""" @classmethod diff --git a/python_modules/libraries/dagster-components/dagster_components/lib/test/simple_asset.py b/python_modules/libraries/dagster-components/dagster_components/lib/test/simple_asset.py index e4855c464fcac..67dfad309c288 100644 --- a/python_modules/libraries/dagster-components/dagster_components/lib/test/simple_asset.py +++ b/python_modules/libraries/dagster-components/dagster_components/lib/test/simple_asset.py @@ -16,7 +16,7 @@ class SimpleAssetSchema(BaseModel): value: str -class SimpleAsset(Component): +class SimpleAssetComponent(Component): """A simple asset that returns a constant string value.""" @classmethod diff --git a/python_modules/libraries/dagster-components/dagster_components/lib/test/simple_pipes_script_asset.py b/python_modules/libraries/dagster-components/dagster_components/lib/test/simple_pipes_script_asset.py index aae19b920ee09..8e40944a2a197 100644 --- a/python_modules/libraries/dagster-components/dagster_components/lib/test/simple_pipes_script_asset.py +++ b/python_modules/libraries/dagster-components/dagster_components/lib/test/simple_pipes_script_asset.py @@ -17,19 +17,17 @@ # Same schema used for file generation and defs generation -class SimplePipesScriptAssetSchema(BaseModel): +class SimplePipesScriptSchema(BaseModel): asset_key: str filename: str -class SimplePipesScriptAssetScaffolder(ComponentScaffolder): +class SimplePipesScriptScaffolder(ComponentScaffolder): @classmethod def get_schema(cls): - return SimplePipesScriptAssetSchema + return SimplePipesScriptSchema - def scaffold( - self, request: ComponentScaffoldRequest, params: SimplePipesScriptAssetSchema - ) -> None: + def scaffold(self, request: ComponentScaffoldRequest, params: SimplePipesScriptSchema) -> None: scaffold_component_yaml(request, params.model_dump()) Path(request.component_instance_root_path, params.filename).write_text( _SCRIPT_TEMPLATE.format(asset_key=params.asset_key) @@ -46,7 +44,7 @@ def scaffold( """ -class SimplePipesScriptAsset(Component): +class SimplePipesScriptComponent(Component): """A simple asset that runs a Python script with the Pipes subprocess client. Because it is a pipes asset, no value is returned. @@ -54,11 +52,11 @@ class SimplePipesScriptAsset(Component): @classmethod def get_scaffolder(cls) -> ComponentScaffolder: - return SimplePipesScriptAssetScaffolder() + return SimplePipesScriptScaffolder() @classmethod def get_schema(cls): - return SimplePipesScriptAssetSchema + return SimplePipesScriptSchema def __init__(self, asset_key: AssetKey, script_path: Path): self._asset_key = asset_key diff --git a/python_modules/libraries/dagster-components/dagster_components_tests/cli_tests/test_commands.py b/python_modules/libraries/dagster-components/dagster_components_tests/cli_tests/test_commands.py index 1000876207218..002e2dcfc8042 100644 --- a/python_modules/libraries/dagster-components/dagster_components_tests/cli_tests/test_commands.py +++ b/python_modules/libraries/dagster-components/dagster_components_tests/cli_tests/test_commands.py @@ -43,14 +43,14 @@ def test_list_component_types_command(): result = json.loads(result.output) assert list(result.keys()) == [ - "dagster_components.lib.test.AllMetadataEmptyAsset", - "dagster_components.lib.test.ComplexSchemaAsset", - "dagster_components.lib.test.SimpleAsset", - "dagster_components.lib.test.SimplePipesScriptAsset", + "dagster_components.lib.test.AllMetadataEmptyComponent", + "dagster_components.lib.test.ComplexAssetComponent", + "dagster_components.lib.test.SimpleAssetComponent", + "dagster_components.lib.test.SimplePipesScriptComponent", ] - assert result["dagster_components.lib.test.SimpleAsset"] == { - "name": "SimpleAsset", + assert result["dagster_components.lib.test.SimpleAssetComponent"] == { + "name": "SimpleAssetComponent", "namespace": "dagster_components.lib.test", "summary": "A simple asset that returns a constant string value.", "description": "A simple asset that returns a constant string value.", @@ -72,12 +72,12 @@ def test_list_component_types_command(): "filename": {"title": "Filename", "type": "string"}, }, "required": ["asset_key", "filename"], - "title": "SimplePipesScriptAssetSchema", + "title": "SimplePipesScriptSchema", "type": "object", } - assert result["dagster_components.lib.test.SimplePipesScriptAsset"] == { - "name": "SimplePipesScriptAsset", + assert result["dagster_components.lib.test.SimplePipesScriptComponent"] == { + "name": "SimplePipesScriptComponent", "namespace": "dagster_components.lib.test", "summary": "A simple asset that runs a Python script with the Pipes subprocess client.", "description": "A simple asset that runs a Python script with the Pipes subprocess client.\n\nBecause it is a pipes asset, no value is returned.", @@ -170,9 +170,9 @@ def test_all_components_schema_command(): result = json.loads(result.output) component_type_keys = [ - "ComplexSchemaAsset", - "SimpleAsset", - "SimplePipesScriptAsset", + "ComplexAssetComponent", + "SimpleAssetComponent", + "SimplePipesScriptComponent", ] assert result["anyOf"] == [ @@ -197,14 +197,14 @@ def test_all_components_schema_command(): top_level_component_validator = Draft202012Validator(schema=result) top_level_component_validator.validate( { - "type": "dagster_components.lib.test.SimpleAsset", + "type": "dagster_components.lib.test.SimpleAssetComponent", "attributes": {"asset_key": "my_asset", "value": "my_value"}, } ) with pytest.raises(ValidationError): top_level_component_validator.validate( { - "type": "dagster_components.lib.test.SimpleAsset", + "type": "dagster_components.lib.test.SimpleAssetComponent", "attributes": {"asset_key": "my_asset", "value": "my_value"}, "extra_key": "extra_value", } @@ -222,7 +222,7 @@ def test_scaffold_component_command(): "dagster_components.test", "scaffold", "component", - "dagster_components.lib.test.SimplePipesScriptAsset", + "dagster_components.lib.test.SimplePipesScriptComponent", "bar/components/qux", "--json-params", '{"asset_key": "my_asset", "filename": "my_asset.py"}', diff --git a/python_modules/libraries/dagster-components/dagster_components_tests/code_locations/python_script_location/components/script_python_decl/component.py b/python_modules/libraries/dagster-components/dagster_components_tests/code_locations/python_script_location/components/script_python_decl/component.py index 3e384841320a7..8500165d97bcd 100644 --- a/python_modules/libraries/dagster-components/dagster_components_tests/code_locations/python_script_location/components/script_python_decl/component.py +++ b/python_modules/libraries/dagster-components/dagster_components_tests/code_locations/python_script_location/components/script_python_decl/component.py @@ -1,7 +1,7 @@ from dagster_components import ComponentLoadContext from dagster_components.core.component import component from dagster_components.core.schema.objects import AssetSpecSchema -from dagster_components.lib import PipesSubprocessScriptCollection +from dagster_components.lib import PipesSubprocessScriptCollectionComponent from dagster_components.lib.pipes_subprocess_script_collection import ( PipesSubprocessScriptCollectionSchema, PipesSubprocessScriptSchema, @@ -9,7 +9,7 @@ @component -def load(context: ComponentLoadContext) -> PipesSubprocessScriptCollection: +def load(context: ComponentLoadContext) -> PipesSubprocessScriptCollectionComponent: attributes = PipesSubprocessScriptCollectionSchema( scripts=[ PipesSubprocessScriptSchema( @@ -23,4 +23,4 @@ def load(context: ComponentLoadContext) -> PipesSubprocessScriptCollection: ), ] ) - return PipesSubprocessScriptCollection.load(attributes=attributes, context=context) + return PipesSubprocessScriptCollectionComponent.load(attributes=attributes, context=context) diff --git a/python_modules/libraries/dagster-components/dagster_components_tests/code_locations/python_script_location/components/scripts/component.yaml b/python_modules/libraries/dagster-components/dagster_components_tests/code_locations/python_script_location/components/scripts/component.yaml index 0e3b15d8e8641..cf94e9e132488 100644 --- a/python_modules/libraries/dagster-components/dagster_components_tests/code_locations/python_script_location/components/scripts/component.yaml +++ b/python_modules/libraries/dagster-components/dagster_components_tests/code_locations/python_script_location/components/scripts/component.yaml @@ -1,4 +1,4 @@ -type: dagster_components.lib.PipesSubprocessScriptCollection +type: dagster_components.lib.PipesSubprocessScriptCollectionComponent attributes: scripts: diff --git a/python_modules/libraries/dagster-components/dagster_components_tests/code_locations/sling_location/components/ingest/component.yaml b/python_modules/libraries/dagster-components/dagster_components_tests/code_locations/sling_location/components/ingest/component.yaml index 87e432ab66ff2..b73cebd48a1b7 100644 --- a/python_modules/libraries/dagster-components/dagster_components_tests/code_locations/sling_location/components/ingest/component.yaml +++ b/python_modules/libraries/dagster-components/dagster_components_tests/code_locations/sling_location/components/ingest/component.yaml @@ -1,4 +1,4 @@ -type: dagster_components.lib.SlingReplicationCollection +type: dagster_components.lib.SlingReplicationCollectionComponent attributes: replications: diff --git a/python_modules/libraries/dagster-components/dagster_components_tests/integration_tests/components/validation/simple_asset_invalid_value/component.yaml b/python_modules/libraries/dagster-components/dagster_components_tests/integration_tests/components/validation/simple_asset_invalid_value/component.yaml index 97daf9abf0fa0..d0a6eb25edb02 100644 --- a/python_modules/libraries/dagster-components/dagster_components_tests/integration_tests/components/validation/simple_asset_invalid_value/component.yaml +++ b/python_modules/libraries/dagster-components/dagster_components_tests/integration_tests/components/validation/simple_asset_invalid_value/component.yaml @@ -1,4 +1,4 @@ -type: dagster_components.lib.test.SimpleAsset +type: dagster_components.lib.test.SimpleAssetComponent attributes: asset_key: "test" diff --git a/python_modules/libraries/dagster-components/dagster_components_tests/integration_tests/test_sling_integration_test.py b/python_modules/libraries/dagster-components/dagster_components_tests/integration_tests/test_sling_integration_test.py index 32cdc2369037a..b5884a5442471 100644 --- a/python_modules/libraries/dagster-components/dagster_components_tests/integration_tests/test_sling_integration_test.py +++ b/python_modules/libraries/dagster-components/dagster_components_tests/integration_tests/test_sling_integration_test.py @@ -15,7 +15,9 @@ from dagster._utils.env import environ from dagster_components.core.component_decl_builder import ComponentFileModel from dagster_components.core.component_defs_builder import YamlComponentDecl, build_component_defs -from dagster_components.lib.sling_replication_collection.component import SlingReplicationCollection +from dagster_components.lib.sling_replication_collection.component import ( + SlingReplicationCollectionComponent, +) from dagster_sling import SlingResource from dagster_components_tests.utils import script_load_context @@ -63,8 +65,8 @@ def temp_sling_component_instance( def test_python_attributes() -> None: with temp_sling_component_instance([{"path": "./replication.yaml"}]) as decl_node: context = script_load_context(decl_node) - attributes = decl_node.get_attributes(SlingReplicationCollection.get_schema()) - component = SlingReplicationCollection.load(attributes, context) + attributes = decl_node.get_attributes(SlingReplicationCollectionComponent.get_schema()) + component = SlingReplicationCollectionComponent.load(attributes, context) replications = component.replications assert len(replications) == 1 @@ -87,8 +89,8 @@ def test_python_attributes_op_name() -> None: ] ) as decl_node: context = script_load_context(decl_node) - attributes = decl_node.get_attributes(SlingReplicationCollection.get_schema()) - component = SlingReplicationCollection.load(attributes, context=context) + attributes = decl_node.get_attributes(SlingReplicationCollectionComponent.get_schema()) + component = SlingReplicationCollectionComponent.load(attributes, context=context) replications = component.replications assert len(replications) == 1 op = replications[0].op @@ -109,8 +111,8 @@ def test_python_attributes_op_tags() -> None: ] ) as decl_node: context = script_load_context(decl_node) - attributes = decl_node.get_attributes(SlingReplicationCollection.get_schema()) - component = SlingReplicationCollection.load(attributes=attributes, context=context) + attributes = decl_node.get_attributes(SlingReplicationCollectionComponent.get_schema()) + component = SlingReplicationCollectionComponent.load(attributes=attributes, context=context) replications = component.replications assert len(replications) == 1 op = replications[0].op @@ -127,8 +129,8 @@ def test_python_params_include_metadata() -> None: ] ) as decl_node: context = script_load_context(decl_node) - attributes = decl_node.get_attributes(SlingReplicationCollection.get_schema()) - component = SlingReplicationCollection.load(attributes=attributes, context=context) + attributes = decl_node.get_attributes(SlingReplicationCollectionComponent.get_schema()) + component = SlingReplicationCollectionComponent.load(attributes=attributes, context=context) replications = component.replications assert len(replications) == 1 include_metadata = replications[0].include_metadata @@ -152,7 +154,7 @@ def test_load_from_path() -> None: components = decl_node.load(context) assert len(components) == 1 component = components[0] - assert isinstance(component, SlingReplicationCollection) + assert isinstance(component, SlingReplicationCollectionComponent) resource = getattr(component, "resource") assert isinstance(resource, SlingResource) @@ -169,7 +171,7 @@ def test_load_from_path() -> None: def test_sling_subclass() -> None: - class DebugSlingReplicationComponent(SlingReplicationCollection): + class DebugSlingReplicationComponent(SlingReplicationCollectionComponent): def execute( self, context: AssetExecutionContext, sling: SlingResource ) -> Iterator[Union[AssetMaterialization, MaterializeResult]]: diff --git a/python_modules/libraries/dagster-components/dagster_components_tests/registry_tests/test_registry.py b/python_modules/libraries/dagster-components/dagster_components_tests/registry_tests/test_registry.py index 37bb6b1506aec..777e8b39dceab 100644 --- a/python_modules/libraries/dagster-components/dagster_components_tests/registry_tests/test_registry.py +++ b/python_modules/libraries/dagster-components/dagster_components_tests/registry_tests/test_registry.py @@ -102,25 +102,25 @@ def test_components_from_dagster(): # No extras with _temp_venv([*common_deps, "-e", components_root]) as python_executable: component_types = _get_component_types_in_python_environment(python_executable) - assert "dagster_components.lib.PipesSubprocessScriptCollection" in component_types + assert "dagster_components.lib.PipesSubprocessScriptCollectionComponent" in component_types assert "dagster_components.lib.DbtProjectComponent" not in component_types - assert "dagster_components.lib.SlingReplicationCollection" not in component_types + assert "dagster_components.lib.SlingReplicationCollectionComponent" not in component_types with _temp_venv( [*common_deps, "-e", f"{components_root}[dbt]", "-e", dbt_root] ) as python_executable: component_types = _get_component_types_in_python_environment(python_executable) - assert "dagster_components.lib.PipesSubprocessScriptCollection" in component_types + assert "dagster_components.lib.PipesSubprocessScriptCollectionComponent" in component_types assert "dagster_components.lib.DbtProjectComponent" in component_types - assert "dagster_components.lib.SlingReplicationCollection" not in component_types + assert "dagster_components.lib.SlingReplicationCollectionComponent" not in component_types with _temp_venv( [*common_deps, "-e", f"{components_root}[sling]", "-e", sling_root] ) as python_executable: component_types = _get_component_types_in_python_environment(python_executable) - assert "dagster_components.lib.PipesSubprocessScriptCollection" in component_types + assert "dagster_components.lib.PipesSubprocessScriptCollectionComponent" in component_types assert "dagster_components.lib.DbtProjectComponent" not in component_types - assert "dagster_components.lib.SlingReplicationCollection" in component_types + assert "dagster_components.lib.SlingReplicationCollectionComponent" in component_types def test_all_dagster_components_have_defined_summary(): diff --git a/python_modules/libraries/dagster-components/dagster_components_tests/test_integrity.py b/python_modules/libraries/dagster-components/dagster_components_tests/test_integrity.py new file mode 100644 index 0000000000000..0775b27f715ba --- /dev/null +++ b/python_modules/libraries/dagster-components/dagster_components_tests/test_integrity.py @@ -0,0 +1,17 @@ +import importlib + +from dagster_components.core.component import get_component_types_in_module + +_COMPONENT_LIBRARY_MODULES = [ + "dagster_components.lib", + "dagster_components.lib.test", +] + + +def test_all_components_have_component_suffix(): + for module_name in _COMPONENT_LIBRARY_MODULES: + module = importlib.import_module(module_name) + for name, _ in get_component_types_in_module(module): + assert name.endswith( + "Component" + ), f"Component {name} in module {module_name} does not have 'Component' suffix" diff --git a/python_modules/libraries/dagster-dg/dagster_dg_tests/cli_tests/test_custom_help_format.py b/python_modules/libraries/dagster-dg/dagster_dg_tests/cli_tests/test_custom_help_format.py index 4c080256a9fdc..cf2037ac15ae0 100644 --- a/python_modules/libraries/dagster-dg/dagster_dg_tests/cli_tests/test_custom_help_format.py +++ b/python_modules/libraries/dagster-dg/dagster_dg_tests/cli_tests/test_custom_help_format.py @@ -180,7 +180,7 @@ def test_dynamic_subcommand_help_message(): result = runner.invoke( "scaffold", "component", - "dagster_components.lib.test.SimplePipesScriptAsset", + "dagster_components.lib.test.SimplePipesScriptComponent", "--help", ) assert_runner_result(result) @@ -189,10 +189,8 @@ def test_dynamic_subcommand_help_message(): assert match_terminal_box_output( output.strip(), textwrap.dedent(""" - - Usage: dg scaffold component [GLOBAL OPTIONS] dagster_components.lib.test.SimplePipesScriptAsset [OPTIONS] - COMPONENT_INSTANCE_NA - ME + Usage: dg scaffold component [GLOBAL OPTIONS] dagster_components.lib.test.SimplePipesScriptComponent + [OPTIONS] COMPONENT_INSTANCE_NAME ╭─ Arguments ──────────────────────────────────────────────────────────────────────────────────────────────────────────╮ │ * component_instance_name TEXT [required] │ diff --git a/python_modules/libraries/dagster-dg/dagster_dg_tests/cli_tests/test_docs_commands.py b/python_modules/libraries/dagster-dg/dagster_dg_tests/cli_tests/test_docs_commands.py index b3179b17cfd64..967a6f2e53630 100644 --- a/python_modules/libraries/dagster-dg/dagster_dg_tests/cli_tests/test_docs_commands.py +++ b/python_modules/libraries/dagster-dg/dagster_dg_tests/cli_tests/test_docs_commands.py @@ -11,7 +11,9 @@ def test_docs_component_type_success(): with ProxyRunner.test() as runner, isolated_components_venv(runner): - result = runner.invoke("docs", "component-type", "dagster_components.lib.test.SimpleAsset") + result = runner.invoke( + "docs", "component-type", "dagster_components.lib.test.SimpleAssetComponent" + ) assert_runner_result(result) @@ -28,7 +30,7 @@ def test_docs_component_type_success_output_console(): result = runner.invoke( "docs", "component-type", - "dagster_components.lib.test.ComplexSchemaAsset", + "dagster_components.lib.test.ComplexAssetComponent", "--output", "cli", ) diff --git a/python_modules/libraries/dagster-dg/dagster_dg_tests/cli_tests/test_environment_validation.py b/python_modules/libraries/dagster-dg/dagster_dg_tests/cli_tests/test_environment_validation.py index 1111592009455..a17ee7cd4216a 100644 --- a/python_modules/libraries/dagster-dg/dagster_dg_tests/cli_tests/test_environment_validation.py +++ b/python_modules/libraries/dagster-dg/dagster_dg_tests/cli_tests/test_environment_validation.py @@ -31,7 +31,7 @@ def to_cli_args(self) -> tuple[str, ...]: return (*self.command, *self.args) -DEFAULT_COMPONENT_TYPE = "dagster_components.lib.test.SimpleAsset" +DEFAULT_COMPONENT_TYPE = "dagster_components.lib.test.SimpleAssetComponent" NO_REQUIRED_CONTEXT_COMMANDS = [ CommandSpec(("scaffold", "project"), "foo"), diff --git a/python_modules/libraries/dagster-dg/dagster_dg_tests/cli_tests/test_info_commands.py b/python_modules/libraries/dagster-dg/dagster_dg_tests/cli_tests/test_info_commands.py index 0e381b949e29a..cadf0b809411e 100644 --- a/python_modules/libraries/dagster-dg/dagster_dg_tests/cli_tests/test_info_commands.py +++ b/python_modules/libraries/dagster-dg/dagster_dg_tests/cli_tests/test_info_commands.py @@ -11,7 +11,7 @@ # ######################## _EXPECTED_INSPECT_COMPONENT_TYPE_FULL = textwrap.dedent(""" - dagster_components.lib.test.SimplePipesScriptAsset + dagster_components.lib.test.SimplePipesScriptComponent Description: @@ -36,7 +36,7 @@ "asset_key", "filename" ], - "title": "SimplePipesScriptAssetSchema", + "title": "SimplePipesScriptSchema", "type": "object" } @@ -57,7 +57,7 @@ "asset_key", "filename" ], - "title": "SimplePipesScriptAssetSchema", + "title": "SimplePipesScriptSchema", "type": "object" } """).strip() @@ -68,7 +68,7 @@ def test_inspect_component_type_all_metadata_success(): result = runner.invoke( "utils", "inspect-component-type", - "dagster_components.lib.test.SimplePipesScriptAsset", + "dagster_components.lib.test.SimplePipesScriptComponent", ) assert_runner_result(result) assert result.output.strip().endswith(_EXPECTED_INSPECT_COMPONENT_TYPE_FULL) @@ -79,12 +79,12 @@ def test_inspect_component_type_all_metadata_empty_success(): result = runner.invoke( "utils", "inspect-component-type", - "dagster_components.lib.test.AllMetadataEmptyAsset", + "dagster_components.lib.test.AllMetadataEmptyComponent", ) assert_runner_result(result) assert result.output.strip().endswith( textwrap.dedent(""" - dagster_components.lib.test.AllMetadataEmptyAsset + dagster_components.lib.test.AllMetadataEmptyComponent """).strip() ) @@ -94,7 +94,7 @@ def test_inspect_component_type_flag_fields_success(): result = runner.invoke( "utils", "inspect-component-type", - "dagster_components.lib.test.SimplePipesScriptAsset", + "dagster_components.lib.test.SimplePipesScriptComponent", "--description", ) assert_runner_result(result) @@ -109,7 +109,7 @@ def test_inspect_component_type_flag_fields_success(): result = runner.invoke( "utils", "inspect-component-type", - "dagster_components.lib.test.SimplePipesScriptAsset", + "dagster_components.lib.test.SimplePipesScriptComponent", "--scaffold-params-schema", ) assert_runner_result(result) @@ -130,7 +130,7 @@ def test_inspect_component_type_flag_fields_success(): "asset_key", "filename" ], - "title": "SimplePipesScriptAssetSchema", + "title": "SimplePipesScriptSchema", "type": "object" } """).strip() @@ -139,7 +139,7 @@ def test_inspect_component_type_flag_fields_success(): result = runner.invoke( "utils", "inspect-component-type", - "dagster_components.lib.test.SimplePipesScriptAsset", + "dagster_components.lib.test.SimplePipesScriptComponent", "--component-schema", ) assert_runner_result(result) @@ -160,7 +160,7 @@ def test_inspect_component_type_flag_fields_success(): "asset_key", "filename" ], - "title": "SimplePipesScriptAssetSchema", + "title": "SimplePipesScriptSchema", "type": "object" } """).strip() @@ -172,7 +172,7 @@ def test_inspect_component_type_multiple_flags_fails() -> None: result = runner.invoke( "utils", "inspect-component-type", - "dagster_components.lib.test.SimplePipesScriptAsset", + "dagster_components.lib.test.SimplePipesScriptComponent", "--description", "--scaffold-params-schema", ) diff --git a/python_modules/libraries/dagster-dg/dagster_dg_tests/cli_tests/test_list_commands.py b/python_modules/libraries/dagster-dg/dagster_dg_tests/cli_tests/test_list_commands.py index 98a98402ab56a..4a1d8d79eac51 100644 --- a/python_modules/libraries/dagster-dg/dagster_dg_tests/cli_tests/test_list_commands.py +++ b/python_modules/libraries/dagster-dg/dagster_dg_tests/cli_tests/test_list_commands.py @@ -47,7 +47,7 @@ def test_list_components_succeeds(): result = runner.invoke( "scaffold", "component", - "dagster_components.lib.test.AllMetadataEmptyAsset", + "dagster_components.lib.test.AllMetadataEmptyComponent", "qux", ) assert_runner_result(result) @@ -66,33 +66,33 @@ def test_list_components_succeeds(): # ######################## _EXPECTED_COMPONENT_TYPES = textwrap.dedent(""" - ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓ - ┃ Component Type ┃ Summary ┃ - ┡━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┩ - │ dagster_components.lib.test.AllMetadataEmptyAsset │ │ - │ dagster_components.lib.test.ComplexSchemaAsset │ An asset that has a complex schema. │ - │ dagster_components.lib.test.SimpleAsset │ A simple asset that returns a constant string value. │ - │ dagster_components.lib.test.SimplePipesScriptAsset │ A simple asset that runs a Python script with the Pipes │ - │ │ subprocess client. │ - └────────────────────────────────────────────────────┴─────────────────────────────────────────────────────────────────┘ + ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓ + ┃ Component Type ┃ Summary ┃ + ┡━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┩ + │ dagster_components.lib.test.AllMetadataEmptyComponent │ │ + │ dagster_components.lib.test.ComplexAssetComponent │ An asset that has a complex schema. │ + │ dagster_components.lib.test.SimpleAssetComponent │ A simple asset that returns a constant string value. │ + │ dagster_components.lib.test.SimplePipesScriptComponent │ A simple asset that runs a Python script with the Pipes │ + │ │ subprocess client. │ + └────────────────────────────────────────────────────────┴─────────────────────────────────────────────────────────────┘ """).strip() _EXPECTED_COMPONENT_TYPES_JSON = textwrap.dedent(""" [ { - "key": "dagster_components.lib.test.AllMetadataEmptyAsset", + "key": "dagster_components.lib.test.AllMetadataEmptyComponent", "summary": null }, { - "key": "dagster_components.lib.test.ComplexSchemaAsset", + "key": "dagster_components.lib.test.ComplexAssetComponent", "summary": "An asset that has a complex schema." }, { - "key": "dagster_components.lib.test.SimpleAsset", + "key": "dagster_components.lib.test.SimpleAssetComponent", "summary": "A simple asset that returns a constant string value." }, { - "key": "dagster_components.lib.test.SimplePipesScriptAsset", + "key": "dagster_components.lib.test.SimplePipesScriptComponent", "summary": "A simple asset that runs a Python script with the Pipes subprocess client." } ] diff --git a/python_modules/libraries/dagster-dg/dagster_dg_tests/cli_tests/test_scaffold_commands.py b/python_modules/libraries/dagster-dg/dagster_dg_tests/cli_tests/test_scaffold_commands.py index fc34616f8b274..d40ff02d491c6 100644 --- a/python_modules/libraries/dagster-dg/dagster_dg_tests/cli_tests/test_scaffold_commands.py +++ b/python_modules/libraries/dagster-dg/dagster_dg_tests/cli_tests/test_scaffold_commands.py @@ -280,10 +280,10 @@ def test_scaffold_component_dynamic_subcommand_generation() -> None: # These are wrapped in a table so it's hard to check exact output. for line in [ "╭─ Commands", - "│ dagster_components.lib.test.AllMetadataEmptyAsset", - "│ dagster_components.lib.test.ComplexSchemaAsset", - "│ dagster_components.lib.test.SimpleAsset", - "│ dagster_components.lib.test.SimplePipesScriptAsset", + "│ dagster_components.lib.test.AllMetadataEmptyComponent", + "│ dagster_components.lib.test.ComplexAssetComponent", + "│ dagster_components.lib.test.SimpleAssetComponent", + "│ dagster_components.lib.test.SimplePipesScriptComponent", ]: assert standardize_box_characters(line) in normalized_output @@ -297,7 +297,7 @@ def test_scaffold_component_no_params_success(in_workspace: bool) -> None: result = runner.invoke( "scaffold", "component", - "dagster_components.lib.test.AllMetadataEmptyAsset", + "dagster_components.lib.test.AllMetadataEmptyComponent", "qux", ) assert_runner_result(result) @@ -305,7 +305,7 @@ def test_scaffold_component_no_params_success(in_workspace: bool) -> None: component_yaml_path = Path("foo_bar/components/qux/component.yaml") assert component_yaml_path.exists() assert ( - "type: dagster_components.lib.test.AllMetadataEmptyAsset" + "type: dagster_components.lib.test.AllMetadataEmptyComponent" in component_yaml_path.read_text() ) @@ -319,7 +319,7 @@ def test_scaffold_component_json_params_success(in_workspace: bool) -> None: result = runner.invoke( "scaffold", "component", - "dagster_components.lib.test.SimplePipesScriptAsset", + "dagster_components.lib.test.SimplePipesScriptComponent", "qux", "--json-params", '{"asset_key": "foo", "filename": "hello.py"}', @@ -330,7 +330,7 @@ def test_scaffold_component_json_params_success(in_workspace: bool) -> None: component_yaml_path = Path("foo_bar/components/qux/component.yaml") assert component_yaml_path.exists() assert ( - "type: dagster_components.lib.test.SimplePipesScriptAsset" + "type: dagster_components.lib.test.SimplePipesScriptComponent" in component_yaml_path.read_text() ) @@ -344,7 +344,7 @@ def test_scaffold_component_key_value_params_success(in_workspace: bool) -> None result = runner.invoke( "scaffold", "component", - "dagster_components.lib.test.SimplePipesScriptAsset", + "dagster_components.lib.test.SimplePipesScriptComponent", "qux", "--asset-key=foo", "--filename=hello.py", @@ -355,7 +355,7 @@ def test_scaffold_component_key_value_params_success(in_workspace: bool) -> None component_yaml_path = Path("foo_bar/components/qux/component.yaml") assert component_yaml_path.exists() assert ( - "type: dagster_components.lib.test.SimplePipesScriptAsset" + "type: dagster_components.lib.test.SimplePipesScriptComponent" in component_yaml_path.read_text() ) @@ -365,7 +365,7 @@ def test_scaffold_component_json_params_and_key_value_params_fails() -> None: result = runner.invoke( "scaffold", "component", - "dagster_components.lib.test.SimplePipesScriptAsset", + "dagster_components.lib.test.SimplePipesScriptComponent", "qux", "--json-params", '{"filename": "hello.py"}', @@ -391,7 +391,7 @@ def test_scaffold_component_command_with_non_matching_module_name(): python_module.rename("module_not_same_as_project") result = runner.invoke( - "scaffold", "component", "dagster_components.lib.test.AllMetadataEmptyAsset", "qux" + "scaffold", "component", "dagster_components.lib.test.AllMetadataEmptyComponent", "qux" ) assert_runner_result(result, exit_0=False) assert "Module `foo_bar` is not installed" in str(result.exception) @@ -406,14 +406,14 @@ def test_scaffold_component_already_exists_fails(in_workspace: bool) -> None: result = runner.invoke( "scaffold", "component", - "dagster_components.lib.test.AllMetadataEmptyAsset", + "dagster_components.lib.test.AllMetadataEmptyComponent", "qux", ) assert_runner_result(result) result = runner.invoke( "scaffold", "component", - "dagster_components.lib.test.AllMetadataEmptyAsset", + "dagster_components.lib.test.AllMetadataEmptyComponent", "qux", ) assert_runner_result(result, exit_0=False) @@ -431,7 +431,7 @@ def test_scaffold_component_succeeds_non_default_component_package() -> None: result = runner.invoke( "scaffold", "component", - "dagster_components.lib.test.AllMetadataEmptyAsset", + "dagster_components.lib.test.AllMetadataEmptyComponent", "qux", ) assert_runner_result(result) @@ -439,7 +439,7 @@ def test_scaffold_component_succeeds_non_default_component_package() -> None: component_yaml_path = Path("foo_bar/_components/qux/component.yaml") assert component_yaml_path.exists() assert ( - "type: dagster_components.lib.test.AllMetadataEmptyAsset" + "type: dagster_components.lib.test.AllMetadataEmptyComponent" in component_yaml_path.read_text() ) @@ -453,7 +453,7 @@ def test_scaffold_component_fails_components_package_does_not_exist() -> None: result = runner.invoke( "scaffold", "component", - "dagster_components.lib.test.AllMetadataEmptyAsset", + "dagster_components.lib.test.AllMetadataEmptyComponent", "qux", ) assert_runner_result(result, exit_0=False)