add build and release workflow (#12)

* Bump gitpython from 3.1.40 to 3.1.41 in /anomaly-detector (#4) Bumps [gitpython](https://github.com/gitpython-developers/GitPython) from 3.1.40 to 3.1.41. - [Release notes](https://github.com/gitpython-developers/GitPython/releases) - [Changelog](https://github.com/gitpython-developers/GitPython/blob/main/CHANGES) - [Commits](gitpython-developers/GitPython@3.1.40...3.1.41) --- updated-dependencies: - dependency-name: gitpython dependency-type: indirect ... Signed-off-by: dependabot[bot] <[email protected]> Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * Bump jinja2 from 3.1.2 to 3.1.3 in /anomaly-detector (#6) Bumps [jinja2](https://github.com/pallets/jinja) from 3.1.2 to 3.1.3. - [Release notes](https://github.com/pallets/jinja/releases) - [Changelog](https://github.com/pallets/jinja/blob/main/CHANGES.rst) - [Commits](pallets/jinja@3.1.2...3.1.3) --- updated-dependencies: - dependency-name: jinja2 dependency-type: indirect ... Signed-off-by: dependabot[bot] <[email protected]> Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * add build workflow * add build workflow * add build workflow * test poetry install * test poetry install * test poetry install * test poetry install * test poetry install * test poetry install * test poetry install * test poetry install * test poetry install * test poetry install * test poetry install * test poetry install * test poetry install * test poetry install * test poetry install * test poetry install * test poetry install * test poetry install * test poetry install * test poetry install * test poetry install * test poetry install * test poetry install * test poetry install * test poetry install * test poetry install * test poetry install * test poetry install * test poetry install * test poetry install * test poetry install * test poetry install * test poetry install * test poetry install * test poetry install * test poetry install * test poetry install * test poetry install * test poetry install * test poetry install * test poetry install * test poetry install * test poetry install * test poetry install * test poetry install * test poetry install * test poetry install * test poetry install * test poetry install * test poetry install * merge --------- Signed-off-by: dependabot[bot] <[email protected]> Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
microsoft · Jan 24, 2024 · 24e8554 · 24e8554
1 parent 8bb1766
commit 24e8554
Show file tree

Hide file tree

Showing 13 changed files with 523 additions and 286 deletions.
diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
@@ -0,0 +1,49 @@
+name: publish_release
+on:
+  push:
+    branches: [ "main" ]
+
+jobs:
+  release:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+#      - name: Set up Python
+#        uses: actions/setup-python@v5
+#        with:
+#          python-version: 3.11
+
+      - name: Build project
+        run: |
+          pip install Cython
+          pip install numpy 
+          python setup.py sdist bdist_wheel
+
+      - name: Create Draft Release
+        id: create_release
+        uses: actions/create-release@v1
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        with:
+          # TODO
+          tag_name: 0.1.0
+          release_name: 0.1.0
+          draft: true
+          prerelease: false
+
+      - name: Upload Release Asset
+        uses: actions/[email protected]
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        with:
+          upload_url: ${{ steps.create_release.outputs.upload_url }}
+          asset_path: ./dist/anomaly_detector-0.1.0-cp310-cp310-linux_x86_64.whl
+          asset_name: anomaly_detector-0.1.0-cp310-cp310-linux_x86_64.whl
+          asset_content_type: application/whl
+
+      - uses: eregon/publish-release@v1
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        with:
+          release_id: ${{ steps.create_release.outputs.id }}
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -6,7 +6,7 @@ name: CI
 on:
   # Triggers the workflow on push or pull request events but only for the "main" branch
   push:
-    branches: [ "main", "dev"]
+    branches: [ "main", "dev" ]
   pull_request:
     branches: [ "main", "dev" ]
 
@@ -21,7 +21,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        python-version: [ '3.8', '3.9', '3.10.8', '3.11', '3.12']
+        python-version: [ '3.9', '3.10.8', '3.11', '3.12' ]
 
     # Steps represent a sequence of tasks that will be executed as part of the job
     steps:
@@ -35,50 +35,62 @@ jobs:
         working-directory: anomaly-detector
         run: poetry install
 
+      - name: gen .pyd file
+        run: |
+          pip install Cython
+          pip install numpy
+          python setup.py build_ext --inplace
+
       - name: run the test script
         working-directory: anomaly-detector/tests
-        run: poetry run pytest -k "test_"
+        run: |
+          pip install pytest
+          poetry run pytest test_demo.py
+
 
       - name: Analyze code with pylint
         run: |
           pip install pylint
           pylint --exit-zero $(git ls-files '*.py')
 
-  test_on_poetry_whl:
+  test_on_setup_whl:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        python-version: [ '3.8', '3.9', '3.10.8', '3.11', '3.12' ]
+        python-version: [ '3.9', '3.10.8', '3.11', '3.12' ]
     steps:
       - uses: actions/checkout@v4
 
-      - name: Install poetry
-        run: pipx install poetry
-
-      - name: gen .whl file by poetry
-        working-directory: anomaly-detector
-        run: poetry build
+      - name: gen .whl file by setup
+        run: |
+          pip install Cython
+          pip install numpy 
+          python setup.py sdist bdist_wheel
 
       - name: install .whl file
-        run: pip install ./anomaly-detector/dist/*.whl
+        run: pip install ./dist/*.whl
 
       - name: run the test script
         working-directory: anomaly-detector/tests
         run: |
           pip install pytest
           python test_demo.py
-
+#          python uvad_test.py
 
   test_on_setup:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
 
       - name: install package by setup.py
-        run: pip install .
+        run: |
+          pip install Cython
+          pip install numpy
+          pip install .
 
       - name: run the test script
         working-directory: anomaly-detector/tests
         run: |
           pip install pytest
           python test_demo.py
+#          python uvad_test.py
diff --git a/MANIFEST.in b/MANIFEST.in
@@ -1,7 +1,7 @@
-recursive-include anomaly-detector/*.pyx
-recursive-include anomaly-detector/*.c
-recursive-include anomaly-detector/*.h
-recursive-include anomaly-detector/*.py
-recursive-include anomaly-detector/*.txt
-recursive-include anomaly-detector/anomaly_detector/univariate *
-include setup.py
+recursive-include ./anomaly-detector/*.pyx
+recursive-include ./anomaly-detector/*.c
+recursive-include ./anomaly-detector/*.h
+recursive-include ./anomaly-detector/*.py
+recursive-include ./anomaly-detector/*.txt
+recursive-include ./anomaly-detector/anomaly_detector/univariate *
+include ./setup.py
diff --git a/anomaly-detector/anomaly_detector/common/data_processor.py b/anomaly-detector/anomaly_detector/common/data_processor.py
@@ -12,7 +12,7 @@ def __init__(
         *,
         fill_na_method: str = FillNAMethod.Linear.name,
         fill_na_value: float = 0.0,
-        window: int | float | str,
+        window: Union[int, float, str],
         start_time: str = None,
         end_time: str = None,
     ):

diff --git a/anomaly-detector/anomaly_detector/multivariate/model.py b/anomaly-detector/anomaly_detector/multivariate/model.py
@@ -64,7 +64,7 @@ def fit(self, data: pd.DataFrame, params: Dict = None):
         train_length = int(len(data) * self.config.train_ratio)
         valid_length = int(len(data) * self.config.val_ratio)
         train_data = values[:train_length]
-        val_data = values[train_length : valid_length + train_length]
+        val_data = values[train_length: valid_length + train_length]
         max_values = np.max(train_data, axis=0)
         min_values = np.min(train_data, axis=0)
         self.config.normalize_base = NormalizeBase(
@@ -216,7 +216,7 @@ def evaluate_epoch(self, model, criterion, dataloader):
         return loss_meter.avg, vae_loss_meter.avg, pred_loss_meter.avg
 
     def predict(
-        self, context, data: pd.DataFrame, params: Optional[Dict[str, Any]] = None
+            self, context, data: pd.DataFrame, params: Optional[Dict[str, Any]] = None
     ):
         effective_timestamps, variables, values, _ = self._verify_data_and_params(
             data, params
@@ -249,16 +249,16 @@ def predict(
         thresholds = [
             get_threshold(
                 inference_scores[
-                    max(0, result_n - self.config.threshold_window - i) : len(
-                        inference_scores
-                    )
-                    - i
+                max(0, result_n - self.config.threshold_window - i): len(
+                    inference_scores
+                )
+                                                                     - i
                 ]
             )
             for i in range(result_n - 1, self.config.threshold_window - 2, -1)
         ]
-        inference_scores = inference_scores[self.config.threshold_window - 1 :]
-        contributor_scores = contributor_scores[self.config.threshold_window - 1 :]
+        inference_scores = inference_scores[self.config.threshold_window - 1:]
+        contributor_scores = contributor_scores[self.config.threshold_window - 1:]
         is_anomalies = np.array(
             [
                 (s >= t and s >= hard_th_lower) or s > hard_th_upper
@@ -281,7 +281,7 @@ def predict(
                 -1
             )
             attn_feats = (
-                attn_feats[-inference_length:] - previous_attn[-inference_length:]
+                    attn_feats[-inference_length:] - previous_attn[-inference_length:]
             )
         attn_feats = attn_feats.numpy()
         return self._pack_response(
@@ -331,26 +331,26 @@ def inference(self, values, compute_attn=True):
                     _,
                 ) = detection_results.get_loss_items()
                 batch_contributor_rmses = (
-                    torch.exp(
-                        torch.min(
-                            2 * torch.abs(x_pred - target), torch.ones_like(x_pred)
+                        torch.exp(
+                            torch.min(
+                                2 * torch.abs(x_pred - target), torch.ones_like(x_pred)
+                            )
                         )
-                    )
-                    - 1
+                        - 1
                 )
 
                 batch_contributor_probs = (
-                    torch.exp(
-                        torch.min(
-                            2 * torch.abs(x_recon - target), torch.ones_like(x_pred)
+                        torch.exp(
+                            torch.min(
+                                2 * torch.abs(x_recon - target), torch.ones_like(x_pred)
+                            )
                         )
-                    )
-                    - 1
+                        - 1
                 )
                 batch_probs = torch.mean(batch_contributor_probs, dim=1)
 
                 batch_contributor_rmses_weight = (
-                    batch_contributor_rmses.cpu().numpy() * pct_weight
+                        batch_contributor_rmses.cpu().numpy() * pct_weight
                 )
                 contributor_rmses = append_result(
                     contributor_rmses, batch_contributor_rmses_weight
@@ -438,14 +438,14 @@ def _verify_data_and_params(self, data: pd.DataFrame, params):
         return effective_timestamps, variables, values, config
 
     def _pack_response(
-        self,
-        timestamps,
-        variables,
-        is_anomalies,
-        inference_scores,
-        severity_scores,
-        contributor_scores,
-        attn_feats,
+            self,
+            timestamps,
+            variables,
+            is_anomalies,
+            inference_scores,
+            severity_scores,
+            contributor_scores,
+            attn_feats,
     ):
         contributor_scores = torch.from_numpy(contributor_scores)
         top_k_contributors_idx = torch.argsort(
@@ -499,7 +499,7 @@ def _pack_response(
                         changed_variables = []
                         for k in range(num_attentions):
                             if abs(top_attn_scores[idx, j, k]) > min(
-                                0.001, 1.0 / (1.25 * num_series_names)
+                                    0.001, 1.0 / (1.25 * num_series_names)
                             ):
                                 changed_values.append(top_attn_scores[idx, j, k])
                                 var_idx = top_attn_scores_idx[idx, j, k]

diff --git a/anomaly-detector/anomaly_detector/univariate/univariate_anomaly_detection.py b/anomaly-detector/anomaly_detector/univariate/univariate_anomaly_detection.py
@@ -1,6 +1,7 @@
 import os
 import pandas as pd
 import numpy as np
+
 from anomaly_detector.univariate.resource.error_message import *
 from anomaly_detector.univariate.util import Granularity, get_indices_from_timestamps, DEFAULT_FILL_UP_MODE, FillUpMode, BoundaryVersion
 from anomaly_detector.univariate.util.enum import default_gran_window

diff --git a/anomaly-detector/anomaly_detector/univariate/util/r_stl.py b/anomaly-detector/anomaly_detector/univariate/util/r_stl.py
@@ -26,7 +26,7 @@ def stl_core(data, np=None):
         ``data`` timeseries.
     """
     res_ts = STL(data, np, "periodic", robust=True)
-    return pandas.DataFrame({"seasonal": res_ts.seasonal,"trend": res_ts.trend, "remainder": res_ts.remainder})
+    return pandas.DataFrame({"seasonal": res_ts.seasonal, "trend": res_ts.trend, "remainder": res_ts.remainder})
 
 
 def stl_log(data, np=None):
@@ -45,7 +45,7 @@ def stl_log(data, np=None):
     base = min(data)
     if base < 1:
         data = npy.subtract(data, base)
-        data = data + 1   # add 1 along in case value scale in _data is extreme compared with 1
+        data = data + 1  # add 1 along in case value scale in _data is extreme compared with 1
 
     result = STL(npy.log(data), np, "periodic", robust=True)
     trend_log = result.trend
@@ -59,7 +59,6 @@ def stl_log(data, np=None):
         trend = trend - 1
         trend = trend + base
 
-
     try:
         res_ts = pandas.DataFrame({"seasonal": seasonal,
                                    "trend": trend,
@@ -68,7 +67,7 @@ def stl_log(data, np=None):
         raise e
 
     return res_ts
-    
+
 
 def stl(data, np=None, log_transform=False):
     if log_transform:

diff --git a/anomaly-detector/build.py b/anomaly-detector/build.py
@@ -2,6 +2,7 @@
 import os
 import numpy as np
 from distutils.errors import CCompilerError, DistutilsExecError, DistutilsPlatformError
+
 # See if Cython is installed
 try:
     from Cython.Build import cythonize
@@ -16,15 +17,18 @@ def build(setup_kwargs):
     from setuptools.dist import Distribution
     from setuptools.command.build_ext import build_ext
 # use cythonize to build the extensions
-modules = ["anomaly_detector/univariate/*.pyx"]
+modules = ["./anomaly_detector/univariate/*.pyx"]
 extensions = cythonize(modules,
                        language_level=3,
-                        compiler_directives={'linetrace': True},
+                       compiler_directives={'linetrace': True},
                        )
 
+
 # cmdclass = {'build_ext': build_ext}
 class BuildFailed(Exception):
     pass
+
+
 class ExtBuilder(build_ext):
 
     def run(self):
@@ -39,15 +43,17 @@ def build_extension(self, ext):
         except (CCompilerError, DistutilsExecError, DistutilsPlatformError, ValueError):
             raise BuildFailed('Could not compile C extension.')
 
+
 cmdclass = {"build_ext": ExtBuilder}
 
+
 def build(setup_kwargs):
     """Needed for the poetry building interface."""
-    
+
     os.environ['CFLAGS'] = '-O3'
-    
+
     setup_kwargs.update({
-        'ext_modules' : extensions,
-        'include_dirs' : [np.get_include()],
+        'ext_modules': extensions,
+        'include_dirs': [np.get_include()],
         'cmdclass': cmdclass
     })