From eede618991094062b2d10098d4a0628bd9ab0b59 Mon Sep 17 00:00:00 2001 From: Sean Mackesey Date: Thu, 27 Feb 2025 12:55:19 -0500 Subject: [PATCH] [components] Move dagster_components.lib components to top-level --- .../custom-subclass/basic-subclass.py | 2 +- .../custom-subclass/custom-scope.py | 2 +- .../components/custom-subclass/debug-mode.py | 2 +- .../8-dg-list-component-types.txt | 22 ++++++------ .../guides/components/index/11-component.yaml | 2 +- .../index/18-dg-list-component-types.txt | 35 +++++++++---------- .../components/index/19-dg-scaffold-jdbt.txt | 2 +- .../components/index/20-component-jdbt.yaml | 2 +- .../components/index/23-project-jdbt.yaml | 2 +- .../index/7-dg-list-component-types.txt | 22 ++++++------ .../index/8-dg-list-component-types.txt | 29 +++++++-------- .../index/9-dg-scaffold-sling-replication.txt | 2 +- .../migrating-definitions/3-scaffold.txt | 2 +- .../6-component-yaml.txt | 2 +- .../components/python-components/component.py | 3 +- .../3-dg-list-component-types.txt | 27 +++++++------- .../workspace/5-component-type-list.txt | 22 ++++++------ .../workspace/6-component-type-list.txt | 29 +++++++-------- .../workspace/9-component-type-list.txt | 22 ++++++------ .../guides/components/test_components_docs.py | 10 +++--- ...st_components_docs_creating_a_component.py | 2 +- .../test_migrating_definitions.py | 8 ++--- .../dagster_components/__init__.py | 26 ++++++++++++++ .../dbt_project => components}/__init__.py | 0 .../dbt_project}/__init__.py | 0 .../dbt_project/component.py | 2 +- .../dbt_project/scaffolder.py | 0 .../definitions_component}/__init__.py | 0 .../definitions_component/component.py | 4 ++- .../definitions_component/scaffolder.py | 0 .../pipes_subprocess_script_collection.py | 0 .../sling_replication_collection/__init__.py | 0 .../sling_replication_collection/component.py | 2 +- .../scaffolder.py | 0 .../dagster_components/lib/__init__.py | 21 ----------- .../components/jaffle_shop_dbt/component.yaml | 2 +- .../jaffle_shop_dbt/component.yaml | 2 +- .../script_python_decl/component.py | 8 ++--- .../components/scripts/component.yaml | 2 +- .../components/ingest/component.yaml | 2 +- .../components/jaffle_shop_dbt/component.yaml | 2 +- .../definitions/default_file/component.yaml | 2 +- .../definitions/explicit_file/component.yaml | 2 +- .../component.yaml | 2 +- .../component.yaml | 2 +- .../component.yaml | 2 +- .../validation_error_file/component.yaml | 2 +- .../integration_tests/test_dbt_project.py | 2 +- .../test_sling_integration_test.py | 8 ++--- .../test_templated_custom_keys_dbt_project.py | 2 +- .../registry_tests/test_registry.py | 18 +++++----- .../test_integrity.py | 2 +- .../libraries/dagster-components/setup.py | 2 +- .../cli_tests/test_scaffold_commands.py | 4 +-- 54 files changed, 180 insertions(+), 194 deletions(-) rename python_modules/libraries/dagster-components/dagster_components/{lib/dbt_project => components}/__init__.py (100%) rename python_modules/libraries/dagster-components/dagster_components/{lib/definitions_component => components/dbt_project}/__init__.py (100%) rename python_modules/libraries/dagster-components/dagster_components/{lib => components}/dbt_project/component.py (98%) rename python_modules/libraries/dagster-components/dagster_components/{lib => components}/dbt_project/scaffolder.py (100%) rename python_modules/libraries/dagster-components/dagster_components/{lib/sling_replication_collection => components/definitions_component}/__init__.py (100%) rename python_modules/libraries/dagster-components/dagster_components/{lib => components}/definitions_component/component.py (91%) rename python_modules/libraries/dagster-components/dagster_components/{lib => components}/definitions_component/scaffolder.py (100%) rename python_modules/libraries/dagster-components/dagster_components/{lib => components}/pipes_subprocess_script_collection.py (100%) create mode 100644 python_modules/libraries/dagster-components/dagster_components/components/sling_replication_collection/__init__.py rename python_modules/libraries/dagster-components/dagster_components/{lib => components}/sling_replication_collection/component.py (98%) rename python_modules/libraries/dagster-components/dagster_components/{lib => components}/sling_replication_collection/scaffolder.py (100%) delete mode 100644 python_modules/libraries/dagster-components/dagster_components/lib/__init__.py diff --git a/examples/docs_beta_snippets/docs_beta_snippets/guides/components/custom-subclass/basic-subclass.py b/examples/docs_beta_snippets/docs_beta_snippets/guides/components/custom-subclass/basic-subclass.py index dc0a2893e3b3a..b1fdbd64e22ab 100644 --- a/examples/docs_beta_snippets/docs_beta_snippets/guides/components/custom-subclass/basic-subclass.py +++ b/examples/docs_beta_snippets/docs_beta_snippets/guides/components/custom-subclass/basic-subclass.py @@ -1,4 +1,4 @@ -from dagster_components.lib import SlingReplicationCollectionComponent +from dagster_components import SlingReplicationCollectionComponent class CustomSubclass(SlingReplicationCollectionComponent): ... diff --git a/examples/docs_beta_snippets/docs_beta_snippets/guides/components/custom-subclass/custom-scope.py b/examples/docs_beta_snippets/docs_beta_snippets/guides/components/custom-subclass/custom-scope.py index ad1ac822bac98..56fd3219e6f15 100644 --- a/examples/docs_beta_snippets/docs_beta_snippets/guides/components/custom-subclass/custom-scope.py +++ b/examples/docs_beta_snippets/docs_beta_snippets/guides/components/custom-subclass/custom-scope.py @@ -1,7 +1,7 @@ from collections.abc import Mapping from typing import Any -from dagster_components.lib import SlingReplicationCollectionComponent +from dagster_components import SlingReplicationCollectionComponent import dagster as dg diff --git a/examples/docs_beta_snippets/docs_beta_snippets/guides/components/custom-subclass/debug-mode.py b/examples/docs_beta_snippets/docs_beta_snippets/guides/components/custom-subclass/debug-mode.py index 9886e3e3a0b38..3ecc1f72e1190 100644 --- a/examples/docs_beta_snippets/docs_beta_snippets/guides/components/custom-subclass/debug-mode.py +++ b/examples/docs_beta_snippets/docs_beta_snippets/guides/components/custom-subclass/debug-mode.py @@ -1,6 +1,6 @@ from collections.abc import Iterator -from dagster_components.lib import SlingReplicationCollectionComponent +from dagster_components import SlingReplicationCollectionComponent from dagster_sling import SlingResource import dagster as dg diff --git a/examples/docs_beta_snippets/docs_beta_snippets/guides/components/existing-project/8-dg-list-component-types.txt b/examples/docs_beta_snippets/docs_beta_snippets/guides/components/existing-project/8-dg-list-component-types.txt index 26138080604e0..353f2bc4f708e 100644 --- a/examples/docs_beta_snippets/docs_beta_snippets/guides/components/existing-project/8-dg-list-component-types.txt +++ b/examples/docs_beta_snippets/docs_beta_snippets/guides/components/existing-project/8-dg-list-component-types.txt @@ -1,15 +1,13 @@ dg list component-type Using /.../my-existing-project/.venv/bin/dagster-components -┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━┓ -┃ Component Type ┃ Summary ┃ -┡━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━┩ -│ dagster_components.lib.DefinitionsComponent │ Wraps an arbitrary │ -│ │ set of Dagster │ -│ │ definitions. │ -│ dagster_components.lib.PipesSubprocessScriptCollectionComponent │ Assets that wrap │ -│ │ Python scripts │ -│ │ executed with │ -│ │ Dagster's │ -│ │ PipesSubprocessClie… │ -└─────────────────────────────────────────────────────────────────┴──────────────────────┘ +┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━━━┓ +┃ Component Type ┃ Summary ┃ +┡━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━━━┩ +│ dagster_components.DefinitionsComponent │ Wraps an arbitrary set │ +│ │ of Dagster definitions. │ +│ dagster_components.PipesSubprocessScriptCollectionComponent │ Assets that wrap Python │ +│ │ scripts executed with │ +│ │ Dagster's │ +│ │ PipesSubprocessClient. │ +└─────────────────────────────────────────────────────────────┴──────────────────────────┘ diff --git a/examples/docs_beta_snippets/docs_beta_snippets/guides/components/index/11-component.yaml b/examples/docs_beta_snippets/docs_beta_snippets/guides/components/index/11-component.yaml index 51b11785c809c..47e0100198dd8 100644 --- a/examples/docs_beta_snippets/docs_beta_snippets/guides/components/index/11-component.yaml +++ b/examples/docs_beta_snippets/docs_beta_snippets/guides/components/index/11-component.yaml @@ -1,4 +1,4 @@ -type: dagster_components.lib.SlingReplicationCollectionComponent +type: dagster_components.SlingReplicationCollectionComponent attributes: replications: diff --git a/examples/docs_beta_snippets/docs_beta_snippets/guides/components/index/18-dg-list-component-types.txt b/examples/docs_beta_snippets/docs_beta_snippets/guides/components/index/18-dg-list-component-types.txt index 65dedfc0441b2..aaaf7a527cfb9 100644 --- a/examples/docs_beta_snippets/docs_beta_snippets/guides/components/index/18-dg-list-component-types.txt +++ b/examples/docs_beta_snippets/docs_beta_snippets/guides/components/index/18-dg-list-component-types.txt @@ -1,22 +1,19 @@ dg list component-type Using /.../jaffle-platform/.venv/bin/dagster-components -┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━┓ -┃ Component Type ┃ Summary ┃ -┡━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━┩ -│ dagster_components.lib.DbtProjectComponent │ Expose a DBT project │ -│ │ to Dagster as a set │ -│ │ of assets. │ -│ dagster_components.lib.DefinitionsComponent │ Wraps an arbitrary │ -│ │ set of Dagster │ -│ │ definitions. │ -│ dagster_components.lib.PipesSubprocessScriptCollectionComponent │ Assets that wrap │ -│ │ Python scripts │ -│ │ executed with │ -│ │ Dagster's │ -│ │ PipesSubprocessClie… │ -│ dagster_components.lib.SlingReplicationCollectionComponent │ Expose one or more │ -│ │ Sling replications │ -│ │ to Dagster as │ -│ │ assets. │ -└─────────────────────────────────────────────────────────────────┴──────────────────────┘ +┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━━━┓ +┃ Component Type ┃ Summary ┃ +┡━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━━━┩ +│ dagster_components.DbtProjectComponent │ Expose a DBT project to │ +│ │ Dagster as a set of │ +│ │ assets. │ +│ dagster_components.DefinitionsComponent │ Wraps an arbitrary set │ +│ │ of Dagster definitions. │ +│ dagster_components.PipesSubprocessScriptCollectionComponent │ Assets that wrap Python │ +│ │ scripts executed with │ +│ │ Dagster's │ +│ │ PipesSubprocessClient. │ +│ dagster_components.SlingReplicationCollectionComponent │ Expose one or more Sling │ +│ │ replications to Dagster │ +│ │ as assets. │ +└─────────────────────────────────────────────────────────────┴──────────────────────────┘ diff --git a/examples/docs_beta_snippets/docs_beta_snippets/guides/components/index/19-dg-scaffold-jdbt.txt b/examples/docs_beta_snippets/docs_beta_snippets/guides/components/index/19-dg-scaffold-jdbt.txt index 9a3e6da741aff..3a0a914ca89cd 100644 --- a/examples/docs_beta_snippets/docs_beta_snippets/guides/components/index/19-dg-scaffold-jdbt.txt +++ b/examples/docs_beta_snippets/docs_beta_snippets/guides/components/index/19-dg-scaffold-jdbt.txt @@ -1,4 +1,4 @@ -dg scaffold component dagster_components.lib.DbtProjectComponent jdbt --project-path dbt/jdbt +dg scaffold component dagster_components.DbtProjectComponent jdbt --project-path dbt/jdbt Creating a Dagster component instance folder at /.../jaffle-platform/jaffle_platform/components/jdbt. Using /.../jaffle-platform/.venv/bin/dagster-components diff --git a/examples/docs_beta_snippets/docs_beta_snippets/guides/components/index/20-component-jdbt.yaml b/examples/docs_beta_snippets/docs_beta_snippets/guides/components/index/20-component-jdbt.yaml index a6b88cacdd06a..20cf3dbfa8980 100644 --- a/examples/docs_beta_snippets/docs_beta_snippets/guides/components/index/20-component-jdbt.yaml +++ b/examples/docs_beta_snippets/docs_beta_snippets/guides/components/index/20-component-jdbt.yaml @@ -1,4 +1,4 @@ -type: dagster_components.lib.DbtProjectComponent +type: dagster_components.DbtProjectComponent attributes: dbt: diff --git a/examples/docs_beta_snippets/docs_beta_snippets/guides/components/index/23-project-jdbt.yaml b/examples/docs_beta_snippets/docs_beta_snippets/guides/components/index/23-project-jdbt.yaml index 3c7cd1243ad97..b8661ad495158 100644 --- a/examples/docs_beta_snippets/docs_beta_snippets/guides/components/index/23-project-jdbt.yaml +++ b/examples/docs_beta_snippets/docs_beta_snippets/guides/components/index/23-project-jdbt.yaml @@ -1,4 +1,4 @@ -type: dagster_components.lib.DbtProjectComponent +type: dagster_components.DbtProjectComponent attributes: dbt: diff --git a/examples/docs_beta_snippets/docs_beta_snippets/guides/components/index/7-dg-list-component-types.txt b/examples/docs_beta_snippets/docs_beta_snippets/guides/components/index/7-dg-list-component-types.txt index 4139ba4915572..1bba26255e279 100644 --- a/examples/docs_beta_snippets/docs_beta_snippets/guides/components/index/7-dg-list-component-types.txt +++ b/examples/docs_beta_snippets/docs_beta_snippets/guides/components/index/7-dg-list-component-types.txt @@ -1,14 +1,12 @@ dg list component-type -┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━┓ -┃ Component Type ┃ Summary ┃ -┡━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━┩ -│ dagster_components.lib.DefinitionsComponent │ Wraps an arbitrary │ -│ │ set of Dagster │ -│ │ definitions. │ -│ dagster_components.lib.PipesSubprocessScriptCollectionComponent │ Assets that wrap │ -│ │ Python scripts │ -│ │ executed with │ -│ │ Dagster's │ -│ │ PipesSubprocessClie… │ -└─────────────────────────────────────────────────────────────────┴──────────────────────┘ +┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━━━┓ +┃ Component Type ┃ Summary ┃ +┡━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━━━┩ +│ dagster_components.DefinitionsComponent │ Wraps an arbitrary set │ +│ │ of Dagster definitions. │ +│ dagster_components.PipesSubprocessScriptCollectionComponent │ Assets that wrap Python │ +│ │ scripts executed with │ +│ │ Dagster's │ +│ │ PipesSubprocessClient. │ +└─────────────────────────────────────────────────────────────┴──────────────────────────┘ diff --git a/examples/docs_beta_snippets/docs_beta_snippets/guides/components/index/8-dg-list-component-types.txt b/examples/docs_beta_snippets/docs_beta_snippets/guides/components/index/8-dg-list-component-types.txt index 6d35f4a52b390..ed65a805abda3 100644 --- a/examples/docs_beta_snippets/docs_beta_snippets/guides/components/index/8-dg-list-component-types.txt +++ b/examples/docs_beta_snippets/docs_beta_snippets/guides/components/index/8-dg-list-component-types.txt @@ -1,19 +1,16 @@ dg list component-type Using /.../jaffle-platform/.venv/bin/dagster-components -┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━┓ -┃ Component Type ┃ Summary ┃ -┡━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━┩ -│ dagster_components.lib.DefinitionsComponent │ Wraps an arbitrary │ -│ │ set of Dagster │ -│ │ definitions. │ -│ dagster_components.lib.PipesSubprocessScriptCollectionComponent │ Assets that wrap │ -│ │ Python scripts │ -│ │ executed with │ -│ │ Dagster's │ -│ │ PipesSubprocessClie… │ -│ dagster_components.lib.SlingReplicationCollectionComponent │ Expose one or more │ -│ │ Sling replications │ -│ │ to Dagster as │ -│ │ assets. │ -└─────────────────────────────────────────────────────────────────┴──────────────────────┘ +┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━━━┓ +┃ Component Type ┃ Summary ┃ +┡━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━━━┩ +│ dagster_components.DefinitionsComponent │ Wraps an arbitrary set │ +│ │ of Dagster definitions. │ +│ dagster_components.PipesSubprocessScriptCollectionComponent │ Assets that wrap Python │ +│ │ scripts executed with │ +│ │ Dagster's │ +│ │ PipesSubprocessClient. │ +│ dagster_components.SlingReplicationCollectionComponent │ Expose one or more Sling │ +│ │ replications to Dagster │ +│ │ as assets. │ +└─────────────────────────────────────────────────────────────┴──────────────────────────┘ diff --git a/examples/docs_beta_snippets/docs_beta_snippets/guides/components/index/9-dg-scaffold-sling-replication.txt b/examples/docs_beta_snippets/docs_beta_snippets/guides/components/index/9-dg-scaffold-sling-replication.txt index aece333f7f79f..76e9b21708eeb 100644 --- a/examples/docs_beta_snippets/docs_beta_snippets/guides/components/index/9-dg-scaffold-sling-replication.txt +++ b/examples/docs_beta_snippets/docs_beta_snippets/guides/components/index/9-dg-scaffold-sling-replication.txt @@ -1,4 +1,4 @@ -dg scaffold component 'dagster_components.lib.SlingReplicationCollectionComponent' ingest_files +dg scaffold component 'dagster_components.SlingReplicationCollectionComponent' ingest_files Creating a Dagster component instance folder at /.../jaffle-platform/jaffle_platform/components/ingest_files. Using /.../jaffle-platform/.venv/bin/dagster-components diff --git a/examples/docs_beta_snippets/docs_beta_snippets/guides/components/migrating-definitions/3-scaffold.txt b/examples/docs_beta_snippets/docs_beta_snippets/guides/components/migrating-definitions/3-scaffold.txt index 81c744d80af5a..c8a71d4e68e38 100644 --- a/examples/docs_beta_snippets/docs_beta_snippets/guides/components/migrating-definitions/3-scaffold.txt +++ b/examples/docs_beta_snippets/docs_beta_snippets/guides/components/migrating-definitions/3-scaffold.txt @@ -1,4 +1,4 @@ -dg scaffold component 'dagster_components.lib.DefinitionsComponent' elt-definitions +dg scaffold component 'dagster_components.DefinitionsComponent' elt-definitions Using /.../my-existing-project/.venv/bin/dagster-components Creating a Dagster component instance folder at /.../my-existing-project/my_existing_project/components/elt-definitions. diff --git a/examples/docs_beta_snippets/docs_beta_snippets/guides/components/migrating-definitions/6-component-yaml.txt b/examples/docs_beta_snippets/docs_beta_snippets/guides/components/migrating-definitions/6-component-yaml.txt index 060ada0f37752..7a6fa97ae4862 100644 --- a/examples/docs_beta_snippets/docs_beta_snippets/guides/components/migrating-definitions/6-component-yaml.txt +++ b/examples/docs_beta_snippets/docs_beta_snippets/guides/components/migrating-definitions/6-component-yaml.txt @@ -1,4 +1,4 @@ -type: dagster_components.lib.DefinitionsComponent +type: dagster_components.DefinitionsComponent attributes: definitions_path: definitions.py diff --git a/examples/docs_beta_snippets/docs_beta_snippets/guides/components/python-components/component.py b/examples/docs_beta_snippets/docs_beta_snippets/guides/components/python-components/component.py index defb3e123b4c8..102169a85a17a 100644 --- a/examples/docs_beta_snippets/docs_beta_snippets/guides/components/python-components/component.py +++ b/examples/docs_beta_snippets/docs_beta_snippets/guides/components/python-components/component.py @@ -1,5 +1,4 @@ -from dagster_components import ComponentLoadContext, component -from dagster_components.lib import DbtProjectComponent +from dagster_components import ComponentLoadContext, DbtProjectComponent, component from dagster_dbt import DagsterDbtTranslator, DbtCliResource diff --git a/examples/docs_beta_snippets/docs_beta_snippets/guides/components/shell-script-component/3-dg-list-component-types.txt b/examples/docs_beta_snippets/docs_beta_snippets/guides/components/shell-script-component/3-dg-list-component-types.txt index fa7c7f6beb687..8284b17e9de38 100644 --- a/examples/docs_beta_snippets/docs_beta_snippets/guides/components/shell-script-component/3-dg-list-component-types.txt +++ b/examples/docs_beta_snippets/docs_beta_snippets/guides/components/shell-script-component/3-dg-list-component-types.txt @@ -1,18 +1,15 @@ dg list component-type Using /.../my-component-library/.venv/bin/dagster-components -┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━┓ -┃ Component Type ┃ Summary ┃ -┡━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━┩ -│ dagster_components.lib.DefinitionsComponent │ Wraps an arbitrary │ -│ │ set of Dagster │ -│ │ definitions. │ -│ dagster_components.lib.PipesSubprocessScriptCollectionComponent │ Assets that wrap │ -│ │ Python scripts │ -│ │ executed with │ -│ │ Dagster's │ -│ │ PipesSubprocessClie… │ -│ my_component_library.lib.ShellCommand │ Models a shell │ -│ │ script as a Dagster │ -│ │ asset. │ -└─────────────────────────────────────────────────────────────────┴──────────────────────┘ +┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━━━┓ +┃ Component Type ┃ Summary ┃ +┡━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━━━┩ +│ dagster_components.DefinitionsComponent │ Wraps an arbitrary set │ +│ │ of Dagster definitions. │ +│ dagster_components.PipesSubprocessScriptCollectionComponent │ Assets that wrap Python │ +│ │ scripts executed with │ +│ │ Dagster's │ +│ │ PipesSubprocessClient. │ +│ my_component_library.lib.ShellCommand │ Models a shell script as │ +│ │ a Dagster asset. │ +└─────────────────────────────────────────────────────────────┴──────────────────────────┘ diff --git a/examples/docs_beta_snippets/docs_beta_snippets/guides/components/workspace/5-component-type-list.txt b/examples/docs_beta_snippets/docs_beta_snippets/guides/components/workspace/5-component-type-list.txt index 15fdc181cfe96..d856b2b7b4909 100644 --- a/examples/docs_beta_snippets/docs_beta_snippets/guides/components/workspace/5-component-type-list.txt +++ b/examples/docs_beta_snippets/docs_beta_snippets/guides/components/workspace/5-component-type-list.txt @@ -1,14 +1,12 @@ cd projects/project-1 && dg list component-type -┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━┓ -┃ Component Type ┃ Summary ┃ -┡━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━┩ -│ dagster_components.lib.DefinitionsComponent │ Wraps an arbitrary │ -│ │ set of Dagster │ -│ │ definitions. │ -│ dagster_components.lib.PipesSubprocessScriptCollectionComponent │ Assets that wrap │ -│ │ Python scripts │ -│ │ executed with │ -│ │ Dagster's │ -│ │ PipesSubprocessClie… │ -└─────────────────────────────────────────────────────────────────┴──────────────────────┘ +┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━━━┓ +┃ Component Type ┃ Summary ┃ +┡━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━━━┩ +│ dagster_components.DefinitionsComponent │ Wraps an arbitrary set │ +│ │ of Dagster definitions. │ +│ dagster_components.PipesSubprocessScriptCollectionComponent │ Assets that wrap Python │ +│ │ scripts executed with │ +│ │ Dagster's │ +│ │ PipesSubprocessClient. │ +└─────────────────────────────────────────────────────────────┴──────────────────────────┘ diff --git a/examples/docs_beta_snippets/docs_beta_snippets/guides/components/workspace/6-component-type-list.txt b/examples/docs_beta_snippets/docs_beta_snippets/guides/components/workspace/6-component-type-list.txt index 507315c5b4ad6..434eedc303674 100644 --- a/examples/docs_beta_snippets/docs_beta_snippets/guides/components/workspace/6-component-type-list.txt +++ b/examples/docs_beta_snippets/docs_beta_snippets/guides/components/workspace/6-component-type-list.txt @@ -1,19 +1,16 @@ dg list component-type Using /.../dagster-workspace/projects/project-1/.venv/bin/dagster-components -┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━┓ -┃ Component Type ┃ Summary ┃ -┡━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━┩ -│ dagster_components.lib.DefinitionsComponent │ Wraps an arbitrary │ -│ │ set of Dagster │ -│ │ definitions. │ -│ dagster_components.lib.PipesSubprocessScriptCollectionComponent │ Assets that wrap │ -│ │ Python scripts │ -│ │ executed with │ -│ │ Dagster's │ -│ │ PipesSubprocessClie… │ -│ dagster_components.lib.SlingReplicationCollectionComponent │ Expose one or more │ -│ │ Sling replications │ -│ │ to Dagster as │ -│ │ assets. │ -└─────────────────────────────────────────────────────────────────┴──────────────────────┘ +┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━━━┓ +┃ Component Type ┃ Summary ┃ +┡━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━━━┩ +│ dagster_components.DefinitionsComponent │ Wraps an arbitrary set │ +│ │ of Dagster definitions. │ +│ dagster_components.PipesSubprocessScriptCollectionComponent │ Assets that wrap Python │ +│ │ scripts executed with │ +│ │ Dagster's │ +│ │ PipesSubprocessClient. │ +│ dagster_components.SlingReplicationCollectionComponent │ Expose one or more Sling │ +│ │ replications to Dagster │ +│ │ as assets. │ +└─────────────────────────────────────────────────────────────┴──────────────────────────┘ diff --git a/examples/docs_beta_snippets/docs_beta_snippets/guides/components/workspace/9-component-type-list.txt b/examples/docs_beta_snippets/docs_beta_snippets/guides/components/workspace/9-component-type-list.txt index c42fe6b9d0f11..2bb19713b872a 100644 --- a/examples/docs_beta_snippets/docs_beta_snippets/guides/components/workspace/9-component-type-list.txt +++ b/examples/docs_beta_snippets/docs_beta_snippets/guides/components/workspace/9-component-type-list.txt @@ -1,14 +1,12 @@ cd projects/project-2 && dg list component-type -┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━┓ -┃ Component Type ┃ Summary ┃ -┡━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━┩ -│ dagster_components.lib.DefinitionsComponent │ Wraps an arbitrary │ -│ │ set of Dagster │ -│ │ definitions. │ -│ dagster_components.lib.PipesSubprocessScriptCollectionComponent │ Assets that wrap │ -│ │ Python scripts │ -│ │ executed with │ -│ │ Dagster's │ -│ │ PipesSubprocessClie… │ -└─────────────────────────────────────────────────────────────────┴──────────────────────┘ +┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━━━┓ +┃ Component Type ┃ Summary ┃ +┡━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━━━┩ +│ dagster_components.DefinitionsComponent │ Wraps an arbitrary set │ +│ │ of Dagster definitions. │ +│ dagster_components.PipesSubprocessScriptCollectionComponent │ Assets that wrap Python │ +│ │ scripts executed with │ +│ │ Dagster's │ +│ │ PipesSubprocessClient. │ +└─────────────────────────────────────────────────────────────┴──────────────────────────┘ diff --git a/examples/docs_beta_snippets/docs_beta_snippets_tests/snippet_checks/guides/components/test_components_docs.py b/examples/docs_beta_snippets/docs_beta_snippets_tests/snippet_checks/guides/components/test_components_docs.py index a06f08cb6b572..462d184c4a3ff 100644 --- a/examples/docs_beta_snippets/docs_beta_snippets_tests/snippet_checks/guides/components/test_components_docs.py +++ b/examples/docs_beta_snippets/docs_beta_snippets_tests/snippet_checks/guides/components/test_components_docs.py @@ -119,7 +119,7 @@ def next_snip_no(): # Scaffold new ingestion, validate new files run_command_and_snippet_output( - cmd="dg scaffold component 'dagster_components.lib.SlingReplicationCollectionComponent' ingest_files", + cmd="dg scaffold component 'dagster_components.SlingReplicationCollectionComponent' ingest_files", snippet_path=COMPONENTS_SNIPPETS_DIR / f"{next_snip_no()}-dg-scaffold-sling-replication.txt", update_snippets=update_snippets, @@ -218,7 +218,7 @@ def next_snip_no(): ).strip(), ) _run_command( - "dagster asset materialize --select '*' -m jaffle_platform.definitions" + "uv run dagster asset materialize --select '*' -m jaffle_platform.definitions" ) run_command_and_snippet_output( cmd='duckdb /tmp/jaffle_platform.duckdb -c "SELECT * FROM raw_customers LIMIT 5;"', @@ -251,7 +251,7 @@ def next_snip_no(): # Scaffold dbt project components run_command_and_snippet_output( - cmd="dg scaffold component dagster_components.lib.DbtProjectComponent jdbt --project-path dbt/jdbt", + cmd="dg scaffold component dagster_components.DbtProjectComponent jdbt --project-path dbt/jdbt", snippet_path=COMPONENTS_SNIPPETS_DIR / f"{next_snip_no()}-dg-scaffold-jdbt.txt", update_snippets=update_snippets, @@ -294,7 +294,7 @@ def next_snip_no(): snippet_path=COMPONENTS_SNIPPETS_DIR / f"{next_snip_no()}-project-jdbt.yaml", contents=format_multiline(""" - type: dagster_components.lib.DbtProjectComponent + type: dagster_components.DbtProjectComponent attributes: dbt: @@ -315,7 +315,7 @@ def next_snip_no(): # Run dbt, check works _run_command( - "DAGSTER_IS_DEV_CLI=1 dagster asset materialize --select '*' -m jaffle_platform.definitions" + "DAGSTER_IS_DEV_CLI=1 uv run dagster asset materialize --select '*' -m jaffle_platform.definitions" ) run_command_and_snippet_output( cmd='duckdb /tmp/jaffle_platform.duckdb -c "SELECT * FROM orders LIMIT 5;"', diff --git a/examples/docs_beta_snippets/docs_beta_snippets_tests/snippet_checks/guides/components/test_components_docs_creating_a_component.py b/examples/docs_beta_snippets/docs_beta_snippets_tests/snippet_checks/guides/components/test_components_docs_creating_a_component.py index 978cbd6bfe94f..f9e6e4737d3fb 100644 --- a/examples/docs_beta_snippets/docs_beta_snippets_tests/snippet_checks/guides/components/test_components_docs_creating_a_component.py +++ b/examples/docs_beta_snippets/docs_beta_snippets_tests/snippet_checks/guides/components/test_components_docs_creating_a_component.py @@ -156,5 +156,5 @@ def test_components_docs_index( update_snippets=update_snippets, ) _run_command( - "dagster asset materialize --select '*' -m my_component_library.definitions" + "uv run dagster asset materialize --select '*' -m my_component_library.definitions" ) diff --git a/examples/docs_beta_snippets/docs_beta_snippets_tests/snippet_checks/guides/components/test_components_docs_migrating_definitions/test_migrating_definitions.py b/examples/docs_beta_snippets/docs_beta_snippets_tests/snippet_checks/guides/components/test_components_docs_migrating_definitions/test_migrating_definitions.py index 62e07e2530240..d48c7acf2d779 100644 --- a/examples/docs_beta_snippets/docs_beta_snippets_tests/snippet_checks/guides/components/test_components_docs_migrating_definitions/test_migrating_definitions.py +++ b/examples/docs_beta_snippets/docs_beta_snippets_tests/snippet_checks/guides/components/test_components_docs_migrating_definitions/test_migrating_definitions.py @@ -61,7 +61,7 @@ def test_components_docs_migrating_definitions(update_snippets: bool) -> None: ) run_command_and_snippet_output( - cmd="dg scaffold component 'dagster_components.lib.DefinitionsComponent' elt-definitions", + cmd="dg scaffold component 'dagster_components.DefinitionsComponent' elt-definitions", snippet_path=COMPONENTS_SNIPPETS_DIR / f"{get_next_snip_number()}-scaffold.txt", update_snippets=update_snippets, @@ -102,7 +102,7 @@ def test_components_docs_migrating_definitions(update_snippets: bool) -> None: / "elt-definitions" / "component.yaml", format_multiline(""" - type: dagster_components.lib.DefinitionsComponent + type: dagster_components.DefinitionsComponent attributes: definitions_path: definitions.py @@ -156,7 +156,7 @@ def test_components_docs_migrating_definitions(update_snippets: bool) -> None: # migrate analytics _run_command( - cmd="dg scaffold component 'dagster_components.lib.DefinitionsComponent' analytics-definitions", + cmd="dg scaffold component 'dagster_components.DefinitionsComponent' analytics-definitions", ) _run_command( cmd="mv my_existing_project/analytics/* my_existing_project/components/analytics-definitions && rm -rf my_existing_project/analytics", @@ -185,7 +185,7 @@ def test_components_docs_migrating_definitions(update_snippets: bool) -> None: / "analytics-definitions" / "component.yaml", format_multiline(""" - type: dagster_components.lib.DefinitionsComponent + type: dagster_components.DefinitionsComponent attributes: definitions_path: definitions.py diff --git a/python_modules/libraries/dagster-components/dagster_components/__init__.py b/python_modules/libraries/dagster-components/dagster_components/__init__.py index 5f51487fbe99b..aad12bab76a15 100644 --- a/python_modules/libraries/dagster-components/dagster_components/__init__.py +++ b/python_modules/libraries/dagster-components/dagster_components/__init__.py @@ -1,3 +1,5 @@ +import importlib.util + from dagster_components.core.component import ( Component as Component, ComponentLoadContext as ComponentLoadContext, @@ -26,3 +28,27 @@ ) from dagster_components.scaffold import scaffold_component_yaml as scaffold_component_yaml from dagster_components.version import __version__ as __version__ + +# ######################## +# ##### COMPONENTS +# ######################## + +_has_dagster_dbt = importlib.util.find_spec("dagster_dbt") is not None +_has_dagster_sling = importlib.util.find_spec("dagster_sling") is not None + +if _has_dagster_dbt: + from dagster_components.components.dbt_project.component import ( + DbtProjectComponent as DbtProjectComponent, + ) + +if _has_dagster_sling: + from dagster_components.components.sling_replication_collection.component import ( + SlingReplicationCollectionComponent as SlingReplicationCollectionComponent, + ) + +from dagster_components.components.definitions_component.component import ( + DefinitionsComponent as DefinitionsComponent, +) +from dagster_components.components.pipes_subprocess_script_collection import ( + PipesSubprocessScriptCollectionComponent as PipesSubprocessScriptCollectionComponent, +) diff --git a/python_modules/libraries/dagster-components/dagster_components/lib/dbt_project/__init__.py b/python_modules/libraries/dagster-components/dagster_components/components/__init__.py similarity index 100% rename from python_modules/libraries/dagster-components/dagster_components/lib/dbt_project/__init__.py rename to python_modules/libraries/dagster-components/dagster_components/components/__init__.py diff --git a/python_modules/libraries/dagster-components/dagster_components/lib/definitions_component/__init__.py b/python_modules/libraries/dagster-components/dagster_components/components/dbt_project/__init__.py similarity index 100% rename from python_modules/libraries/dagster-components/dagster_components/lib/definitions_component/__init__.py rename to python_modules/libraries/dagster-components/dagster_components/components/dbt_project/__init__.py diff --git a/python_modules/libraries/dagster-components/dagster_components/lib/dbt_project/component.py b/python_modules/libraries/dagster-components/dagster_components/components/dbt_project/component.py similarity index 98% rename from python_modules/libraries/dagster-components/dagster_components/lib/dbt_project/component.py rename to python_modules/libraries/dagster-components/dagster_components/components/dbt_project/component.py index ab763cb80478e..56a407ad560fe 100644 --- a/python_modules/libraries/dagster-components/dagster_components/lib/dbt_project/component.py +++ b/python_modules/libraries/dagster-components/dagster_components/components/dbt_project/component.py @@ -14,6 +14,7 @@ from pydantic.dataclasses import dataclass from dagster_components import Component, ComponentLoadContext, FieldResolver +from dagster_components.components.dbt_project.scaffolder import DbtProjectComponentScaffolder from dagster_components.core.schema.base import ResolvableSchema from dagster_components.core.schema.metadata import ResolvableFieldInfo from dagster_components.core.schema.objects import ( @@ -24,7 +25,6 @@ PostProcessorFn, ResolutionContext, ) -from dagster_components.lib.dbt_project.scaffolder import DbtProjectComponentScaffolder from dagster_components.utils import TranslatorResolvingInfo, get_wrapped_translator_class diff --git a/python_modules/libraries/dagster-components/dagster_components/lib/dbt_project/scaffolder.py b/python_modules/libraries/dagster-components/dagster_components/components/dbt_project/scaffolder.py similarity index 100% rename from python_modules/libraries/dagster-components/dagster_components/lib/dbt_project/scaffolder.py rename to python_modules/libraries/dagster-components/dagster_components/components/dbt_project/scaffolder.py diff --git a/python_modules/libraries/dagster-components/dagster_components/lib/sling_replication_collection/__init__.py b/python_modules/libraries/dagster-components/dagster_components/components/definitions_component/__init__.py similarity index 100% rename from python_modules/libraries/dagster-components/dagster_components/lib/sling_replication_collection/__init__.py rename to python_modules/libraries/dagster-components/dagster_components/components/definitions_component/__init__.py diff --git a/python_modules/libraries/dagster-components/dagster_components/lib/definitions_component/component.py b/python_modules/libraries/dagster-components/dagster_components/components/definitions_component/component.py similarity index 91% rename from python_modules/libraries/dagster-components/dagster_components/lib/definitions_component/component.py rename to python_modules/libraries/dagster-components/dagster_components/components/definitions_component/component.py index 757d4b7bdb85b..31cae5afa5285 100644 --- a/python_modules/libraries/dagster-components/dagster_components/lib/definitions_component/component.py +++ b/python_modules/libraries/dagster-components/dagster_components/components/definitions_component/component.py @@ -9,7 +9,9 @@ from pydantic.dataclasses import dataclass from dagster_components import Component, ComponentLoadContext, ResolvableSchema -from dagster_components.lib.definitions_component.scaffolder import DefinitionsComponentScaffolder +from dagster_components.components.definitions_component.scaffolder import ( + DefinitionsComponentScaffolder, +) class DefinitionsParamSchema(ResolvableSchema): diff --git a/python_modules/libraries/dagster-components/dagster_components/lib/definitions_component/scaffolder.py b/python_modules/libraries/dagster-components/dagster_components/components/definitions_component/scaffolder.py similarity index 100% rename from python_modules/libraries/dagster-components/dagster_components/lib/definitions_component/scaffolder.py rename to python_modules/libraries/dagster-components/dagster_components/components/definitions_component/scaffolder.py diff --git a/python_modules/libraries/dagster-components/dagster_components/lib/pipes_subprocess_script_collection.py b/python_modules/libraries/dagster-components/dagster_components/components/pipes_subprocess_script_collection.py similarity index 100% rename from python_modules/libraries/dagster-components/dagster_components/lib/pipes_subprocess_script_collection.py rename to python_modules/libraries/dagster-components/dagster_components/components/pipes_subprocess_script_collection.py diff --git a/python_modules/libraries/dagster-components/dagster_components/components/sling_replication_collection/__init__.py b/python_modules/libraries/dagster-components/dagster_components/components/sling_replication_collection/__init__.py new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/python_modules/libraries/dagster-components/dagster_components/lib/sling_replication_collection/component.py b/python_modules/libraries/dagster-components/dagster_components/components/sling_replication_collection/component.py similarity index 98% rename from python_modules/libraries/dagster-components/dagster_components/lib/sling_replication_collection/component.py rename to python_modules/libraries/dagster-components/dagster_components/components/sling_replication_collection/component.py index a2be950364eee..f4f18765eb789 100644 --- a/python_modules/libraries/dagster-components/dagster_components/lib/sling_replication_collection/component.py +++ b/python_modules/libraries/dagster-components/dagster_components/components/sling_replication_collection/component.py @@ -109,7 +109,7 @@ class SlingReplicationCollectionComponent(Component): @classmethod def get_scaffolder(cls) -> ComponentScaffolder: - from dagster_components.lib.sling_replication_collection.scaffolder import ( + from dagster_components.components.sling_replication_collection.scaffolder import ( SlingReplicationComponentScaffolder, ) diff --git a/python_modules/libraries/dagster-components/dagster_components/lib/sling_replication_collection/scaffolder.py b/python_modules/libraries/dagster-components/dagster_components/components/sling_replication_collection/scaffolder.py similarity index 100% rename from python_modules/libraries/dagster-components/dagster_components/lib/sling_replication_collection/scaffolder.py rename to python_modules/libraries/dagster-components/dagster_components/components/sling_replication_collection/scaffolder.py diff --git a/python_modules/libraries/dagster-components/dagster_components/lib/__init__.py b/python_modules/libraries/dagster-components/dagster_components/lib/__init__.py deleted file mode 100644 index 75e877a133786..0000000000000 --- a/python_modules/libraries/dagster-components/dagster_components/lib/__init__.py +++ /dev/null @@ -1,21 +0,0 @@ -import importlib.util - -_has_dagster_dbt = importlib.util.find_spec("dagster_dbt") is not None -_has_dagster_sling = importlib.util.find_spec("dagster_sling") is not None - -if _has_dagster_dbt: - from dagster_components.lib.dbt_project.component import ( - DbtProjectComponent as DbtProjectComponent, - ) - -if _has_dagster_sling: - from dagster_components.lib.sling_replication_collection.component import ( - SlingReplicationCollectionComponent as SlingReplicationCollectionComponent, - ) - -from dagster_components.lib.definitions_component.component import ( - DefinitionsComponent as DefinitionsComponent, -) -from dagster_components.lib.pipes_subprocess_script_collection import ( - PipesSubprocessScriptCollectionComponent as PipesSubprocessScriptCollectionComponent, -) diff --git a/python_modules/libraries/dagster-components/dagster_components_tests/code_locations/dbt_project_location/components/jaffle_shop_dbt/component.yaml b/python_modules/libraries/dagster-components/dagster_components_tests/code_locations/dbt_project_location/components/jaffle_shop_dbt/component.yaml index e02da2d6a58a8..7fb14386c36a2 100644 --- a/python_modules/libraries/dagster-components/dagster_components_tests/code_locations/dbt_project_location/components/jaffle_shop_dbt/component.yaml +++ b/python_modules/libraries/dagster-components/dagster_components_tests/code_locations/dbt_project_location/components/jaffle_shop_dbt/component.yaml @@ -1,4 +1,4 @@ -type: dagster_components.lib.DbtProjectComponent +type: dagster_components.DbtProjectComponent attributes: dbt: diff --git a/python_modules/libraries/dagster-components/dagster_components_tests/code_locations/jaffle_shop_dbt/component.yaml b/python_modules/libraries/dagster-components/dagster_components_tests/code_locations/jaffle_shop_dbt/component.yaml index 4d5c777bad43c..bc8ada9790ba7 100644 --- a/python_modules/libraries/dagster-components/dagster_components_tests/code_locations/jaffle_shop_dbt/component.yaml +++ b/python_modules/libraries/dagster-components/dagster_components_tests/code_locations/jaffle_shop_dbt/component.yaml @@ -1,4 +1,4 @@ -type: dagster_components.lib.DbtProject +type: dagster_components.DbtProjectComponent attributes: dbt: diff --git a/python_modules/libraries/dagster-components/dagster_components_tests/code_locations/python_script_location/components/script_python_decl/component.py b/python_modules/libraries/dagster-components/dagster_components_tests/code_locations/python_script_location/components/script_python_decl/component.py index 8500165d97bcd..535df1428c8ef 100644 --- a/python_modules/libraries/dagster-components/dagster_components_tests/code_locations/python_script_location/components/script_python_decl/component.py +++ b/python_modules/libraries/dagster-components/dagster_components_tests/code_locations/python_script_location/components/script_python_decl/component.py @@ -1,11 +1,11 @@ from dagster_components import ComponentLoadContext -from dagster_components.core.component import component -from dagster_components.core.schema.objects import AssetSpecSchema -from dagster_components.lib import PipesSubprocessScriptCollectionComponent -from dagster_components.lib.pipes_subprocess_script_collection import ( +from dagster_components.components.pipes_subprocess_script_collection import ( + PipesSubprocessScriptCollectionComponent, PipesSubprocessScriptCollectionSchema, PipesSubprocessScriptSchema, ) +from dagster_components.core.component import component +from dagster_components.core.schema.objects import AssetSpecSchema @component diff --git a/python_modules/libraries/dagster-components/dagster_components_tests/code_locations/python_script_location/components/scripts/component.yaml b/python_modules/libraries/dagster-components/dagster_components_tests/code_locations/python_script_location/components/scripts/component.yaml index cf94e9e132488..6573ebc86ce6b 100644 --- a/python_modules/libraries/dagster-components/dagster_components_tests/code_locations/python_script_location/components/scripts/component.yaml +++ b/python_modules/libraries/dagster-components/dagster_components_tests/code_locations/python_script_location/components/scripts/component.yaml @@ -1,4 +1,4 @@ -type: dagster_components.lib.PipesSubprocessScriptCollectionComponent +type: dagster_components.PipesSubprocessScriptCollectionComponent attributes: scripts: diff --git a/python_modules/libraries/dagster-components/dagster_components_tests/code_locations/sling_location/components/ingest/component.yaml b/python_modules/libraries/dagster-components/dagster_components_tests/code_locations/sling_location/components/ingest/component.yaml index b73cebd48a1b7..edb48a31b0de8 100644 --- a/python_modules/libraries/dagster-components/dagster_components_tests/code_locations/sling_location/components/ingest/component.yaml +++ b/python_modules/libraries/dagster-components/dagster_components_tests/code_locations/sling_location/components/ingest/component.yaml @@ -1,4 +1,4 @@ -type: dagster_components.lib.SlingReplicationCollectionComponent +type: dagster_components.SlingReplicationCollectionComponent attributes: replications: diff --git a/python_modules/libraries/dagster-components/dagster_components_tests/code_locations/templated_custom_keys_dbt_project_location/components/jaffle_shop_dbt/component.yaml b/python_modules/libraries/dagster-components/dagster_components_tests/code_locations/templated_custom_keys_dbt_project_location/components/jaffle_shop_dbt/component.yaml index f367eb5f22136..4c60688eeea26 100644 --- a/python_modules/libraries/dagster-components/dagster_components_tests/code_locations/templated_custom_keys_dbt_project_location/components/jaffle_shop_dbt/component.yaml +++ b/python_modules/libraries/dagster-components/dagster_components_tests/code_locations/templated_custom_keys_dbt_project_location/components/jaffle_shop_dbt/component.yaml @@ -1,4 +1,4 @@ -type: dagster_components.lib.DbtProjectComponent +type: dagster_components.DbtProjectComponent attributes: dbt: diff --git a/python_modules/libraries/dagster-components/dagster_components_tests/integration_tests/components/definitions/default_file/component.yaml b/python_modules/libraries/dagster-components/dagster_components_tests/integration_tests/components/definitions/default_file/component.yaml index 51c44a9c16e2b..04b5fb62ad40b 100644 --- a/python_modules/libraries/dagster-components/dagster_components_tests/integration_tests/components/definitions/default_file/component.yaml +++ b/python_modules/libraries/dagster-components/dagster_components_tests/integration_tests/components/definitions/default_file/component.yaml @@ -1,3 +1,3 @@ -type: dagster_components.lib.DefinitionsComponent +type: dagster_components.DefinitionsComponent attributes: {} diff --git a/python_modules/libraries/dagster-components/dagster_components_tests/integration_tests/components/definitions/explicit_file/component.yaml b/python_modules/libraries/dagster-components/dagster_components_tests/integration_tests/components/definitions/explicit_file/component.yaml index 6b24e61c96c57..b206719093939 100644 --- a/python_modules/libraries/dagster-components/dagster_components_tests/integration_tests/components/definitions/explicit_file/component.yaml +++ b/python_modules/libraries/dagster-components/dagster_components_tests/integration_tests/components/definitions/explicit_file/component.yaml @@ -1,4 +1,4 @@ -type: dagster_components.lib.DefinitionsComponent +type: dagster_components.DefinitionsComponent attributes: definitions_path: some_file.py diff --git a/python_modules/libraries/dagster-components/dagster_components_tests/integration_tests/components/definitions/explicit_file_relative_imports/component.yaml b/python_modules/libraries/dagster-components/dagster_components_tests/integration_tests/components/definitions/explicit_file_relative_imports/component.yaml index 6b24e61c96c57..b206719093939 100644 --- a/python_modules/libraries/dagster-components/dagster_components_tests/integration_tests/components/definitions/explicit_file_relative_imports/component.yaml +++ b/python_modules/libraries/dagster-components/dagster_components_tests/integration_tests/components/definitions/explicit_file_relative_imports/component.yaml @@ -1,4 +1,4 @@ -type: dagster_components.lib.DefinitionsComponent +type: dagster_components.DefinitionsComponent attributes: definitions_path: some_file.py diff --git a/python_modules/libraries/dagster-components/dagster_components_tests/integration_tests/components/definitions/explicit_file_relative_imports_complex/component.yaml b/python_modules/libraries/dagster-components/dagster_components_tests/integration_tests/components/definitions/explicit_file_relative_imports_complex/component.yaml index 6b24e61c96c57..b206719093939 100644 --- a/python_modules/libraries/dagster-components/dagster_components_tests/integration_tests/components/definitions/explicit_file_relative_imports_complex/component.yaml +++ b/python_modules/libraries/dagster-components/dagster_components_tests/integration_tests/components/definitions/explicit_file_relative_imports_complex/component.yaml @@ -1,4 +1,4 @@ -type: dagster_components.lib.DefinitionsComponent +type: dagster_components.DefinitionsComponent attributes: definitions_path: some_file.py diff --git a/python_modules/libraries/dagster-components/dagster_components_tests/integration_tests/components/definitions/explicit_file_relative_imports_init/component.yaml b/python_modules/libraries/dagster-components/dagster_components_tests/integration_tests/components/definitions/explicit_file_relative_imports_init/component.yaml index 6bed73974dbbf..b070f1c2dddb5 100644 --- a/python_modules/libraries/dagster-components/dagster_components_tests/integration_tests/components/definitions/explicit_file_relative_imports_init/component.yaml +++ b/python_modules/libraries/dagster-components/dagster_components_tests/integration_tests/components/definitions/explicit_file_relative_imports_init/component.yaml @@ -1,4 +1,4 @@ -type: dagster_components.lib.DefinitionsComponent +type: dagster_components.DefinitionsComponent attributes: definitions_path: __init__.py diff --git a/python_modules/libraries/dagster-components/dagster_components_tests/integration_tests/components/definitions/validation_error_file/component.yaml b/python_modules/libraries/dagster-components/dagster_components_tests/integration_tests/components/definitions/validation_error_file/component.yaml index c517f10aedc3e..96f8ebaa1bb63 100644 --- a/python_modules/libraries/dagster-components/dagster_components_tests/integration_tests/components/definitions/validation_error_file/component.yaml +++ b/python_modules/libraries/dagster-components/dagster_components_tests/integration_tests/components/definitions/validation_error_file/component.yaml @@ -1,4 +1,4 @@ -type: dagster_components.lib.DefinitionsComponent +type: dagster_components.DefinitionsComponent attributes: definitions_path: {} diff --git a/python_modules/libraries/dagster-components/dagster_components_tests/integration_tests/test_dbt_project.py b/python_modules/libraries/dagster-components/dagster_components_tests/integration_tests/test_dbt_project.py index 86d9382a9a966..40633713bf123 100644 --- a/python_modules/libraries/dagster-components/dagster_components_tests/integration_tests/test_dbt_project.py +++ b/python_modules/libraries/dagster-components/dagster_components_tests/integration_tests/test_dbt_project.py @@ -10,13 +10,13 @@ from dagster._core.definitions.asset_spec import AssetSpec from dagster._core.definitions.assets import AssetsDefinition from dagster._core.definitions.backfill_policy import BackfillPolicy, BackfillPolicyType +from dagster_components.components.dbt_project.component import DbtProjectComponent from dagster_components.core.component_decl_builder import ComponentFileModel from dagster_components.core.component_defs_builder import ( YamlComponentDecl, build_components_from_component_folder, defs_from_components, ) -from dagster_components.lib.dbt_project.component import DbtProjectComponent from dagster_dbt import DbtProject from pydantic.dataclasses import dataclass diff --git a/python_modules/libraries/dagster-components/dagster_components_tests/integration_tests/test_sling_integration_test.py b/python_modules/libraries/dagster-components/dagster_components_tests/integration_tests/test_sling_integration_test.py index 6d95377972bde..cfd3e46f7a20a 100644 --- a/python_modules/libraries/dagster-components/dagster_components_tests/integration_tests/test_sling_integration_test.py +++ b/python_modules/libraries/dagster-components/dagster_components_tests/integration_tests/test_sling_integration_test.py @@ -17,11 +17,11 @@ from dagster._core.instance_for_test import instance_for_test from dagster._utils.env import environ from dagster_components.cli import cli -from dagster_components.core.component_decl_builder import ComponentFileModel -from dagster_components.core.component_defs_builder import YamlComponentDecl, build_component_defs -from dagster_components.lib.sling_replication_collection.component import ( +from dagster_components.components.sling_replication_collection.component import ( SlingReplicationCollectionComponent, ) +from dagster_components.core.component_decl_builder import ComponentFileModel +from dagster_components.core.component_defs_builder import YamlComponentDecl, build_component_defs from dagster_components.utils import ensure_dagster_components_tests_import from dagster_sling import SlingResource @@ -277,7 +277,7 @@ def test_scaffold_sling(): [ "scaffold", "component", - "dagster_components.lib.SlingReplicationCollectionComponent", + "dagster_components.SlingReplicationCollectionComponent", "bar/components/qux", ], ) diff --git a/python_modules/libraries/dagster-components/dagster_components_tests/integration_tests/test_templated_custom_keys_dbt_project.py b/python_modules/libraries/dagster-components/dagster_components_tests/integration_tests/test_templated_custom_keys_dbt_project.py index cdb704af883ff..16ef29f67b659 100644 --- a/python_modules/libraries/dagster-components/dagster_components_tests/integration_tests/test_templated_custom_keys_dbt_project.py +++ b/python_modules/libraries/dagster-components/dagster_components_tests/integration_tests/test_templated_custom_keys_dbt_project.py @@ -8,13 +8,13 @@ import pytest from dagster import AssetKey from dagster._utils.env import environ +from dagster_components.components.dbt_project.component import DbtProjectComponent from dagster_components.core.component_decl_builder import ComponentFileModel from dagster_components.core.component_defs_builder import ( YamlComponentDecl, build_components_from_component_folder, defs_from_components, ) -from dagster_components.lib.dbt_project.component import DbtProjectComponent from dagster_dbt import DbtProject from dagster_components_tests.utils import assert_assets, get_asset_keys, script_load_context diff --git a/python_modules/libraries/dagster-components/dagster_components_tests/registry_tests/test_registry.py b/python_modules/libraries/dagster-components/dagster_components_tests/registry_tests/test_registry.py index 777e8b39dceab..1a2b9d165f820 100644 --- a/python_modules/libraries/dagster-components/dagster_components_tests/registry_tests/test_registry.py +++ b/python_modules/libraries/dagster-components/dagster_components_tests/registry_tests/test_registry.py @@ -102,25 +102,25 @@ def test_components_from_dagster(): # No extras with _temp_venv([*common_deps, "-e", components_root]) as python_executable: component_types = _get_component_types_in_python_environment(python_executable) - assert "dagster_components.lib.PipesSubprocessScriptCollectionComponent" in component_types - assert "dagster_components.lib.DbtProjectComponent" not in component_types - assert "dagster_components.lib.SlingReplicationCollectionComponent" not in component_types + assert "dagster_components.PipesSubprocessScriptCollectionComponent" in component_types + assert "dagster_components.DbtProjectComponent" not in component_types + assert "dagster_components.SlingReplicationCollectionComponent" not in component_types with _temp_venv( [*common_deps, "-e", f"{components_root}[dbt]", "-e", dbt_root] ) as python_executable: component_types = _get_component_types_in_python_environment(python_executable) - assert "dagster_components.lib.PipesSubprocessScriptCollectionComponent" in component_types - assert "dagster_components.lib.DbtProjectComponent" in component_types - assert "dagster_components.lib.SlingReplicationCollectionComponent" not in component_types + assert "dagster_components.PipesSubprocessScriptCollectionComponent" in component_types + assert "dagster_components.DbtProjectComponent" in component_types + assert "dagster_components.SlingReplicationCollectionComponent" not in component_types with _temp_venv( [*common_deps, "-e", f"{components_root}[sling]", "-e", sling_root] ) as python_executable: component_types = _get_component_types_in_python_environment(python_executable) - assert "dagster_components.lib.PipesSubprocessScriptCollectionComponent" in component_types - assert "dagster_components.lib.DbtProjectComponent" not in component_types - assert "dagster_components.lib.SlingReplicationCollectionComponent" in component_types + assert "dagster_components.PipesSubprocessScriptCollectionComponent" in component_types + assert "dagster_components.DbtProjectComponent" not in component_types + assert "dagster_components.SlingReplicationCollectionComponent" in component_types def test_all_dagster_components_have_defined_summary(): diff --git a/python_modules/libraries/dagster-components/dagster_components_tests/test_integrity.py b/python_modules/libraries/dagster-components/dagster_components_tests/test_integrity.py index 51326267a8e41..7ab671a124009 100644 --- a/python_modules/libraries/dagster-components/dagster_components_tests/test_integrity.py +++ b/python_modules/libraries/dagster-components/dagster_components_tests/test_integrity.py @@ -3,7 +3,7 @@ from dagster_components.core.component import get_component_types_in_module _COMPONENT_LIBRARY_MODULES = [ - "dagster_components.lib", + "dagster_components", "dagster_test.components", ] diff --git a/python_modules/libraries/dagster-components/setup.py b/python_modules/libraries/dagster-components/setup.py index 828a82185c6d2..c30ab8fdc1ced 100644 --- a/python_modules/libraries/dagster-components/setup.py +++ b/python_modules/libraries/dagster-components/setup.py @@ -41,7 +41,7 @@ def get_version() -> str: "dagster-components = dagster_components.cli:main", ], "dagster.components": [ - "dagster_components = dagster_components.lib", + "dagster-components = dagster_components", ], }, extras_require={ diff --git a/python_modules/libraries/dagster-dg/dagster_dg_tests/cli_tests/test_scaffold_commands.py b/python_modules/libraries/dagster-dg/dagster_dg_tests/cli_tests/test_scaffold_commands.py index 2a55ea499487a..bc05991f2ffb5 100644 --- a/python_modules/libraries/dagster-dg/dagster_dg_tests/cli_tests/test_scaffold_commands.py +++ b/python_modules/libraries/dagster-dg/dagster_dg_tests/cli_tests/test_scaffold_commands.py @@ -504,7 +504,7 @@ def test_scaffold_dbt_project_instance(params) -> None: result = runner.invoke( "scaffold", "component", - "dagster_components.lib.DbtProjectComponent", + "dagster_components.DbtProjectComponent", "my_project", *params, ) @@ -513,7 +513,7 @@ def test_scaffold_dbt_project_instance(params) -> None: component_yaml_path = Path("foo_bar/components/my_project/component.yaml") assert component_yaml_path.exists() - assert "type: dagster_components.lib.DbtProjectComponent" in component_yaml_path.read_text() + assert "type: dagster_components.DbtProjectComponent" in component_yaml_path.read_text() assert ( cross_platfrom_string_path("stub_projects/dbt_project_location/components/jaffle_shop") in component_yaml_path.read_text()