From 5aa34f9032366bdc0b4cd40dd7d7e02b54f842bd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Malo=20Jaffr=C3=A9?= <8469951+ghjklw@users.noreply.github.com> Date: Tue, 17 Dec 2024 21:58:27 +0000 Subject: [PATCH 1/2] Dev Container environment MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Malo Jaffré <8469951+ghjklw@users.noreply.github.com> --- .devcontainer/devcontainer.json | 56 +++++++++++++++++++++++++++++++++ .devcontainer/initialize.sh | 14 +++++++++ .gitignore | 1 + CONTRIBUTING.MD | 22 ++++++------- 4 files changed, 80 insertions(+), 13 deletions(-) create mode 100644 .devcontainer/devcontainer.json create mode 100755 .devcontainer/initialize.sh diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json new file mode 100644 index 000000000..f0f9eb479 --- /dev/null +++ b/.devcontainer/devcontainer.json @@ -0,0 +1,56 @@ +// For format details, see https://aka.ms/devcontainer.json. For config options, see the +// README at: https://github.com/devcontainers/templates/tree/main/src/python +{ + "name": "dbt-databricks", + // Or use a Dockerfile or Docker Compose file. More info: https://containers.dev/guide/dockerfile + "image": "mcr.microsoft.com/devcontainers/python:1-3.12-bullseye", + "features": { + "ghcr.io/devcontainers-extra/features/hatch:2": {} + }, + "postStartCommand": "./.devcontainer/initialize.sh", + "customizations": { + "vscode": { + "extensions": [ + "ms-python.python", + "charliermarsh.ruff", + "ms-python.mypy-type-checker" + ], + "settings": { + "editor.formatOnSave": true, + "editor.formatOnType": true, + "editor.rulers": [ + 100 + ], + "files.exclude": { + ".hatch": true, + ".mypy_cache": true, + ".pytest_cache": true, + "**/__pycache__": true + }, + "python.analysis.diagnosticMode": "workspace", + "python.analysis.enablePytestSupport": true, + "python.defaultInterpreterPath": ".hatch/bin/python", + "python.envFile": "test.env", + "python.testing.pytestEnabled": true, + "python.languageServer": "Pylance", + "ruff.fixAll": true, + "ruff.format.args": [ + "--config=pyproject.toml" + ], + "ruff.lint.args": [ + "--config=pyproject.toml" + ], + "[python]": { + "editor.defaultFormatter": "charliermarsh.ruff", + "editor.codeActionsOnSave": { + "source.organizeImports": "always", + "source.fixAll": "always" + } + }, + } + } + }, + "containerEnv": { + "WORKSPACE_FOLDER": "${containerWorkspaceFolder}" + } +} \ No newline at end of file diff --git a/.devcontainer/initialize.sh b/.devcontainer/initialize.sh new file mode 100755 index 000000000..ec38c1c4e --- /dev/null +++ b/.devcontainer/initialize.sh @@ -0,0 +1,14 @@ +#!/usr/bin/env bash +set -x + +# Create the Hatch environment +hatch env create + +# Install pre-commit +(hatch run setup-precommit && pre-commit gc) & + +# Set up test.env file if it does not already exist +if test -f ${WORKSPACE_FOLDER}/test.env; then + cp ${WORKSPACE_FOLDER}/test.env.example ${WORKSPACE_FOLDER}/test.env +fi + diff --git a/.gitignore b/.gitignore index 0725a26b2..5cf4a003d 100644 --- a/.gitignore +++ b/.gitignore @@ -20,3 +20,4 @@ logs/ *.sublime* .python-version .hatch +databricks_demo diff --git a/CONTRIBUTING.MD b/CONTRIBUTING.MD index 46c86437e..79b246c73 100644 --- a/CONTRIBUTING.MD +++ b/CONTRIBUTING.MD @@ -50,11 +50,7 @@ We follow [PEP 8](https://www.python.org/dev/peps/pep-0008/) with one exception: ### Linting -This project uses [Black](https://pypi.org/project/black/), [flake8](https://flake8.pycqa.org/en/latest/), and [mypy](https://www.mypy-lang.org/) for linting and static type checks. Run all three with the `linter` command and commit before opening your pull request. - -``` -tox -e linter -``` +This project uses [Ruff](https://docs.astral.sh/ruff/) and [mypy](https://www.mypy-lang.org/) for linting and static type checks. To simplify reviews you can commit any format changes in a separate commit. @@ -118,21 +114,21 @@ If you set your `user.name` and `user.email` git configs, you can sign your comm Unit tests do not require a Databricks account. Please confirm that your pull request passes our unit test suite before opening a pull request. ```bash -tox -e unit +hatch run unit ``` ## Functional Tests Functional tests require a Databricks account with access to a workspace containing three specific compute resources as detailed below. -The `tox` commands to run each set of these tests appear below: +The `hatch` commands to run each set of these tests appear below: -| Compute Type | Unity Catalog | Command | -| ------------------- | ------------- | ----------------------------------------------- | -| SQL Warehouse | Yes | `tox -e integration-databricks-uc-sql-endpoint` | -| All Purpose Cluster | Yes | `tox -e integration-databricks-uc-cluster` | -| All Purpose | No | `tox -e integration-databricks-cluster` | +| Compute Type | Unity Catalog | Command | +| ------------------- | ------------- | -------------------------- | +| SQL Warehouse | Yes | `hatch run sqlw-e2e` | +| All Purpose Cluster | Yes | `hatch run uc-cluster-e2e` | +| All Purpose | No | `hatch run cluster-e2e` | -These tests are configured with environment variables that `tox` reads from a file called [test.env](/test.env.example) which you can copy from the example: +These tests are configured with environment variables that `hatch` reads from a file called [test.env](/test.env.example) which you can copy from the example: ```sh cp test.env.example test.env From c684779f809bc3439022b9740773868dd04a3082 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Malo=20Jaffr=C3=A9?= <8469951+ghjklw@users.noreply.github.com> Date: Sat, 8 Mar 2025 00:17:12 +0000 Subject: [PATCH 2/2] Apply comments from PR review --- .devcontainer/devcontainer.json | 9 +-------- CHANGELOG.md | 1 + 2 files changed, 2 insertions(+), 8 deletions(-) diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index f0f9eb479..30533fd71 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -3,7 +3,7 @@ { "name": "dbt-databricks", // Or use a Dockerfile or Docker Compose file. More info: https://containers.dev/guide/dockerfile - "image": "mcr.microsoft.com/devcontainers/python:1-3.12-bullseye", + "image": "mcr.microsoft.com/devcontainers/python:1-3.9-bullseye", "features": { "ghcr.io/devcontainers-extra/features/hatch:2": {} }, @@ -27,19 +27,12 @@ ".pytest_cache": true, "**/__pycache__": true }, - "python.analysis.diagnosticMode": "workspace", "python.analysis.enablePytestSupport": true, "python.defaultInterpreterPath": ".hatch/bin/python", "python.envFile": "test.env", "python.testing.pytestEnabled": true, "python.languageServer": "Pylance", "ruff.fixAll": true, - "ruff.format.args": [ - "--config=pyproject.toml" - ], - "ruff.lint.args": [ - "--config=pyproject.toml" - ], "[python]": { "editor.defaultFormatter": "charliermarsh.ruff", "editor.codeActionsOnSave": { diff --git a/CHANGELOG.md b/CHANGELOG.md index a88c5413f..6988f689e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,7 @@ ### Under the Hood - Refactoring of some connection internals ([929](https://github.com/databricks/dbt-databricks/pull/929)) +- Adding a Dev Container configuration to simplify onboarding of new contributors ([882](https://github.com/databricks/dbt-databricks/pull/882)) ## dbt-databricks 1.9.5 (Feb 13, 2025)