Skip to content

Commit

Permalink
[components] Move dagster_components.lib components to top-level (#28092
Browse files Browse the repository at this point in the history 
)

## Summary & Motivation

Breaks up the public `dagster_components` entry point into three
library-specific entry points. Names have changed as follows:

- `dagster_components.PipesSubprocessScriptCollectionComponent` ->
`dagster_components.dagster.PipesSubprocessScriptCollectionComponent`
- `dagster_components.DefinitionsComponent` ->
`dagster_components.dagster.DefinitionsComponent`
- `dagster_components.DbtProjectComponent` ->
`dagster_components.dagster_dbt.DbtProjectComponent`
- `dagster_components.SlingReplicationCollectionComponent` ->
`dagster_components.dagster_sling.SlingReplicationCollectionComponent`


## How I Tested These Changes

Existing test suite.
  • Loading branch information
@smackesey
smackesey authored Feb 27, 2025
1 parent 12c4625 commit 2d7e382
Show file tree
Hide file tree
Showing 56 changed files with 231 additions and 193 deletions.
2 changes: 1 addition & 1 deletion ...docs_beta_snippets/docs_beta_snippets/guides/components/custom-subclass/basic-subclass.py
View file
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
from dagster_components.lib import SlingReplicationCollectionComponent
from dagster_components.dagster_sling import SlingReplicationCollectionComponent


class CustomSubclass(SlingReplicationCollectionComponent): ...
2 changes: 1 addition & 1 deletion ...s/docs_beta_snippets/docs_beta_snippets/guides/components/custom-subclass/custom-scope.py
View file
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
from collections.abc import Mapping
from typing import Any

from dagster_components.lib import SlingReplicationCollectionComponent
from dagster_components.dagster_sling import SlingReplicationCollectionComponent

import dagster as dg

Expand Down
2 changes: 1 addition & 1 deletion ...les/docs_beta_snippets/docs_beta_snippets/guides/components/custom-subclass/debug-mode.py
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
from collections.abc import Iterator

from dagster_components.lib import SlingReplicationCollectionComponent
from dagster_components.dagster_sling import SlingReplicationCollectionComponent
from dagster_sling import SlingResource

import dagster as dg
Expand Down
25 changes: 13 additions & 12 deletions ...ppets/docs_beta_snippets/guides/components/existing-project/8-dg-list-component-types.txt
View file
Original file line number Diff line number Diff line change
@@ -1,15 +1,16 @@
dg list component-type

Using /.../my-existing-project/.venv/bin/dagster-components
┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━┓
┃ Component Type ┃ Summary ┃
┡━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━┩
│ dagster_components.lib.DefinitionsComponent │ Wraps an arbitrary │
│ │ set of Dagster │
│ │ definitions. │
│ dagster_components.lib.PipesSubprocessScriptCollectionComponent │ Assets that wrap │
│ │ Python scripts │
│ │ executed with │
│ │ Dagster's │
│ │ PipesSubprocessClie… │
└─────────────────────────────────────────────────────────────────┴──────────────────────┘
┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━┓
┃ Component Type ┃ Summary ┃
┡━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━┩
│ dagster_components.dagster.DefinitionsComponent │ Wraps an │
│ │ arbitrary set of │
│ │ Dagster │
│ │ definitions. │
│ dagster_components.dagster.PipesSubprocessScriptCollectionComponent │ Assets that wrap │
│ │ Python scripts │
│ │ executed with │
│ │ Dagster's │
│ │ PipesSubprocess… │
└─────────────────────────────────────────────────────────────────────┴──────────────────┘
2 changes: 1 addition & 1 deletion examples/docs_beta_snippets/docs_beta_snippets/guides/components/index/11-component.yaml
View file
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
type: dagster_components.lib.SlingReplicationCollectionComponent
type: dagster_components.dagster_sling.SlingReplicationCollectionComponent

attributes:
replications:
Expand Down
42 changes: 23 additions & 19 deletions ...s_beta_snippets/docs_beta_snippets/guides/components/index/18-dg-list-component-types.txt
View file
Original file line number Diff line number Diff line change
@@ -1,22 +1,26 @@
dg list component-type

Using /.../jaffle-platform/.venv/bin/dagster-components
┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━┓
┃ Component Type ┃ Summary ┃
┡━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━┩
│ dagster_components.lib.DbtProjectComponent │ Expose a DBT project │
│ │ to Dagster as a set │
│ │ of assets. │
│ dagster_components.lib.DefinitionsComponent │ Wraps an arbitrary │
│ │ set of Dagster │
│ │ definitions. │
│ dagster_components.lib.PipesSubprocessScriptCollectionComponent │ Assets that wrap │
│ │ Python scripts │
│ │ executed with │
│ │ Dagster's │
│ │ PipesSubprocessClie… │
│ dagster_components.lib.SlingReplicationCollectionComponent │ Expose one or more │
│ │ Sling replications │
│ │ to Dagster as │
│ │ assets. │
└─────────────────────────────────────────────────────────────────┴──────────────────────┘
┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━┓
┃ Component Type ┃ Summary ┃
┡━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━┩
│ dagster_components.dagster.DefinitionsComponent │ Wraps an │
│ │ arbitrary set │
│ │ of Dagster │
│ │ definitions. │
│ dagster_components.dagster.PipesSubprocessScriptCollectionComponent │ Assets that │
│ │ wrap Python │
│ │ scripts │
│ │ executed with │
│ │ Dagster's │
│ │ PipesSubproces… │
│ dagster_components.dagster_dbt.DbtProjectComponent │ Expose a DBT │
│ │ project to │
│ │ Dagster as a │
│ │ set of assets. │
│ dagster_components.dagster_sling.SlingReplicationCollectionComponent │ Expose one or │
│ │ more Sling │
│ │ replications to │
│ │ Dagster as │
│ │ assets. │
└──────────────────────────────────────────────────────────────────────┴─────────────────┘
2 changes: 1 addition & 1 deletion ...les/docs_beta_snippets/docs_beta_snippets/guides/components/index/19-dg-scaffold-jdbt.txt
View file
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
dg scaffold component dagster_components.lib.DbtProjectComponent jdbt --project-path dbt/jdbt
dg scaffold component dagster_components.dagster_dbt.DbtProjectComponent jdbt --project-path dbt/jdbt

Creating a Dagster component instance folder at /.../jaffle-platform/jaffle_platform/defs/jdbt.
Using /.../jaffle-platform/.venv/bin/dagster-components
2 changes: 1 addition & 1 deletion ...ples/docs_beta_snippets/docs_beta_snippets/guides/components/index/20-component-jdbt.yaml
View file
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
type: dagster_components.lib.DbtProjectComponent
type: dagster_components.dagster_dbt.DbtProjectComponent

attributes:
dbt:
Expand Down
2 changes: 1 addition & 1 deletion examples/docs_beta_snippets/docs_beta_snippets/guides/components/index/23-project-jdbt.yaml
View file
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
type: dagster_components.lib.DbtProjectComponent
type: dagster_components.dagster_dbt.DbtProjectComponent

attributes:
dbt:
Expand Down
25 changes: 13 additions & 12 deletions ...cs_beta_snippets/docs_beta_snippets/guides/components/index/7-dg-list-component-types.txt
View file
Original file line number Diff line number Diff line change
@@ -1,14 +1,15 @@
dg list component-type

┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━┓
┃ Component Type ┃ Summary ┃
┡━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━┩
│ dagster_components.lib.DefinitionsComponent │ Wraps an arbitrary │
│ │ set of Dagster │
│ │ definitions. │
│ dagster_components.lib.PipesSubprocessScriptCollectionComponent │ Assets that wrap │
│ │ Python scripts │
│ │ executed with │
│ │ Dagster's │
│ │ PipesSubprocessClie… │
└─────────────────────────────────────────────────────────────────┴──────────────────────┘
┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━┓
┃ Component Type ┃ Summary ┃
┡━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━┩
│ dagster_components.dagster.DefinitionsComponent │ Wraps an │
│ │ arbitrary set of │
│ │ Dagster │
│ │ definitions. │
│ dagster_components.dagster.PipesSubprocessScriptCollectionComponent │ Assets that wrap │
│ │ Python scripts │
│ │ executed with │
│ │ Dagster's │
│ │ PipesSubprocess… │
└─────────────────────────────────────────────────────────────────────┴──────────────────┘
35 changes: 19 additions & 16 deletions ...cs_beta_snippets/docs_beta_snippets/guides/components/index/8-dg-list-component-types.txt
View file
Original file line number Diff line number Diff line change
@@ -1,19 +1,22 @@
dg list component-type

Using /.../jaffle-platform/.venv/bin/dagster-components
┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━┓
┃ Component Type ┃ Summary ┃
┡━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━┩
│ dagster_components.lib.DefinitionsComponent │ Wraps an arbitrary │
│ │ set of Dagster │
│ │ definitions. │
│ dagster_components.lib.PipesSubprocessScriptCollectionComponent │ Assets that wrap │
│ │ Python scripts │
│ │ executed with │
│ │ Dagster's │
│ │ PipesSubprocessClie… │
│ dagster_components.lib.SlingReplicationCollectionComponent │ Expose one or more │
│ │ Sling replications │
│ │ to Dagster as │
│ │ assets. │
└─────────────────────────────────────────────────────────────────┴──────────────────────┘
┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━┓
┃ Component Type ┃ Summary ┃
┡━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━┩
│ dagster_components.dagster.DefinitionsComponent │ Wraps an │
│ │ arbitrary set │
│ │ of Dagster │
│ │ definitions. │
│ dagster_components.dagster.PipesSubprocessScriptCollectionComponent │ Assets that │
│ │ wrap Python │
│ │ scripts │
│ │ executed with │
│ │ Dagster's │
│ │ PipesSubproces… │
│ dagster_components.dagster_sling.SlingReplicationCollectionComponent │ Expose one or │
│ │ more Sling │
│ │ replications to │
│ │ Dagster as │
│ │ assets. │
└──────────────────────────────────────────────────────────────────────┴─────────────────┘
2 changes: 1 addition & 1 deletion ...a_snippets/docs_beta_snippets/guides/components/index/9-dg-scaffold-sling-replication.txt
View file
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
dg scaffold component 'dagster_components.lib.SlingReplicationCollectionComponent' ingest_files
dg scaffold component 'dagster_components.dagster_sling.SlingReplicationCollectionComponent' ingest_files

Creating a Dagster component instance folder at /.../jaffle-platform/jaffle_platform/defs/ingest_files.
Using /.../jaffle-platform/.venv/bin/dagster-components
2 changes: 1 addition & 1 deletion ...s_beta_snippets/docs_beta_snippets/guides/components/migrating-definitions/3-scaffold.txt
View file
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
dg scaffold component 'dagster_components.lib.DefinitionsComponent' elt-definitions
dg scaffold component 'dagster_components.dagster.DefinitionsComponent' elt-definitions

Using /.../my-existing-project/.venv/bin/dagster-components
Creating a Dagster component instance folder at /.../my-existing-project/my_existing_project/defs/elt-definitions.
Expand Down
2 changes: 1 addition & 1 deletion ..._snippets/docs_beta_snippets/guides/components/migrating-definitions/6-component-yaml.txt
View file
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
type: dagster_components.lib.DefinitionsComponent
type: dagster_components.dagster.DefinitionsComponent

attributes:
definitions_path: definitions.py
2 changes: 1 addition & 1 deletion ...es/docs_beta_snippets/docs_beta_snippets/guides/components/python-components/component.py
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
from dagster_components import ComponentLoadContext, component
from dagster_components.lib import DbtProjectComponent
from dagster_components.dagster_dbt import DbtProjectComponent
from dagster_dbt import DagsterDbtTranslator, DbtCliResource


Expand Down
Loading

1 comment on commit 2d7e382

@github-actions
Copy link

@github-actions github-actions bot commented on 2d7e382 Feb 27, 2025 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Deploy preview for dagster-docs ready!

✅ Preview
https://dagster-docs-cdelrifmc-elementl.vercel.app

Built with commit 2d7e382.
This pull request is being automatically deployed with vercel-action

Please sign in to comment.