Add Taskfile for testing official example dags

.gitignore

@@ -150,7 +150,3 @@ cython_debug/
 #  and can be added to the global gitignore or merged into this file.  For a more nuclear
 #  option (not recommended) you can uncomment the following to ignore the entire idea folder.
 #.idea/
-
-# Example dags for testing
-example_dags/*
-!.gitkeep

.pre-commit-config.yaml

@@ -88,9 +88,7 @@ repos:
     hooks:
       - id: pytest
         name: Run pytest
-        entry: .venv/bin/pytest tests/
-        language: script
+        entry: poetry run pytest tests/
+        language: system
         pass_filenames: false
-        # alternatively you could `types: [python]` so it only runs when python files change
-        # though tests might be invalidated if you were to say change a data file
         always_run: true

Taskfile.yml

@@ -0,0 +1,6 @@
+version: '3'
+
+tasks:
+  test:example_dags:
+    cmds:
+      - . dev/example_dags_tester.sh

dev/example_dags_tester.sh

@@ -0,0 +1,14 @@
+# Update the repo or clone it if it doesn't exist
+cd /tmp/airflow && git fetch || gh repo clone apache/airflow /tmp/airflow && cd /tmp/airflow
+example_dags_dir=/tmp/airflow/airflow/example_dags/
+# We go through all Airflow 2 tags
+for tag in $(git tag -l "2\.[0-9]*\.[0-9]*" | grep -vE "rc|b|a[0-9]*"); do
+    echo "Running for tag $tag"
+    git checkout $tag > /dev/null 2>&1
+    pip install apache-airflow==$tag > /dev/null 2>&1
+    time for file in $example_dags_dir*; do python "$file" > /dev/null 2>&1; done || true
+    airflint -a $example_dags_dir
+    autoflake --remove-all-unused-imports -r -i $example_dags_dir
+    time for file in $example_dags_dir*; do python "$file" > /dev/null 2>&1; done || true
+    git restore .
+done

pyproject.toml

@@ -17,6 +17,7 @@ refactor = "^0.4.4"
 [tool.poetry.dev-dependencies]
 pytest = "^7.1.1"
 pytest-cov = "^3.0.0"
+autoflake = "^1.4"
 
 [build-system]
 requires = ["poetry-core>=1.0.0"]