Merge branch 'dev' into dev

Author: v-shuawei

.gitignore

@@ -401,6 +401,8 @@ mlruns/
 mlartifacts/
 model/
 venv/
+dist/
+temp/
 
 .idea/
 

anomaly-detector/anomaly_detector/__init__.py

@@ -1,2 +1,6 @@
+# ---------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# ---------------------------------------------------------
+
 from .multivariate.model import MultivariateAnomalyDetector
 from .univariate.univariate_anomaly_detection import UnivariateAnomalyDetector, EntireAnomalyDetector, LatestAnomalyDetector

anomaly-detector/anomaly_detector/base.py

@@ -1,3 +1,7 @@
+# ---------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# ---------------------------------------------------------
+
 from abc import abstractmethod
 
 import mlflow

anomaly-detector/anomaly_detector/common/__init__.py

@@ -0,0 +1,4 @@
+# ---------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# ---------------------------------------------------------
+

anomaly-detector/anomaly_detector/common/constants.py

@@ -1,3 +1,7 @@
+# ---------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# ---------------------------------------------------------
+
 import enum
 
 TIMESTAMP = "timestamp"

anomaly-detector/anomaly_detector/common/data_processor.py

@@ -1,3 +1,7 @@
+# ---------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# ---------------------------------------------------------
+
 from typing import List, Union
 
 import pandas as pd
@@ -12,9 +16,6 @@ def __init__(
         *,
         fill_na_method: str = FillNAMethod.Linear.name,
         fill_na_value: float = 0.0,
-        window: Union[int, float, str],
-        start_time: str = None,
-        end_time: str = None,
     ):
         if not hasattr(FillNAMethod, fill_na_method):
             raise InvalidParameterError(
@@ -25,32 +26,14 @@ def __init__(
 
         self.fill_na_method = FillNAMethod[fill_na_method]
         self.fill_na_value = fill_na_value
-        try:
-            self.window = int(window)
-        except TypeError:
-            raise InvalidParameterError(f"Cannot convert window to int.")
-        try:
-            self.start_time = pd.to_datetime(start_time).tz_localize(None)
-            self.end_time = pd.to_datetime(end_time).tz_localize(None)
-        except Exception as e:
-            raise InvalidParameterError(
-                f"Cannot convert start_time or end_time. {str(e)}."
-            )
-        if self.start_time > self.end_time:
-            raise InvalidParameterError(f"start_time cannot be later than end_time.")
 
     def process(self, data: pd.DataFrame):
         if not isinstance(data, pd.DataFrame):
             raise DataFormatError(f"data must be a pandas.DataFrame")
-        if TIMESTAMP not in data.columns:
-            raise DataFormatError(f"data must has a {TIMESTAMP} column.")
-        data = data.set_index(TIMESTAMP, drop=True).sort_index()  # sort indices
-        data.index = pd.to_datetime(data.index).tz_localize(None)
+        data = data.sort_index()  # sort indices
         data = data[sorted(data.columns)]  # sort columns
         data = self.fill_na(data)
-        data, effective_timestamps = self.truncate_data(data)
-        effective_timestamps = [dt_to_str(x) for x in effective_timestamps]
-        return data, effective_timestamps
+        return data
 
     def fill_na(self, data: pd.DataFrame):
         if not isinstance(data, pd.DataFrame):
@@ -72,20 +55,7 @@ def fill_na(self, data: pd.DataFrame):
                 method="linear", limit_direction="both", axis=0, limit=len(data)
             )
         elif self.fill_na_method == FillNAMethod.Fixed:
-            output_series = data.fillna(self.fill_merge_na_value)
+            output_series = data.fillna(self.fill_na_value)
         else:
             output_series = data
         return output_series.fillna(0)
-
-    def truncate_data(self, data: pd.DataFrame):
-        if not isinstance(data, pd.DataFrame):
-            raise DataFormatError(f"data must be a pandas.DataFrame")
-        effective_df = data.loc[self.start_time : self.end_time]
-        if len(effective_df) == 0:
-            raise DataFormatError(f"no effective data.")
-        first_index = effective_df.index[0]
-        start_loc = max(0, data.index.get_loc(first_index) - self.window + 1)
-        start_index = data.index[start_loc]
-        end_index = self.end_time
-        data = data.loc[start_index:end_index]
-        return data, effective_df.index.to_list()

anomaly-detector/anomaly_detector/common/error_code.py

@@ -1,3 +1,7 @@
+# ---------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# ---------------------------------------------------------
+
 class ErrorCodes:
     # invalid data format
     InvalidDataFormat = "Invalid data format. {0}"

anomaly-detector/anomaly_detector/common/exception.py

@@ -1,3 +1,7 @@
+# ---------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# ---------------------------------------------------------
+
 class AnomalyDetectorException(Exception):
     def __init__(self, code="", message=""):
         if code == "":

anomaly-detector/anomaly_detector/common/time_util.py

@@ -1,3 +1,7 @@
+# ---------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# ---------------------------------------------------------
+
 import math
 
 from dateutil import parser, tz

anomaly-detector/anomaly_detector/multivariate/__init__.py

@@ -0,0 +1,4 @@
+# ---------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# ---------------------------------------------------------
+

anomaly-detector/anomaly_detector/multivariate/contract.py

@@ -1,3 +1,7 @@
+# ---------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# ---------------------------------------------------------
+
 from dataclasses import dataclass
 from typing import List
 
@@ -9,11 +13,9 @@ class MultiADConstants:
     INFERENCE_CLIP_MAX = 1000.0
     ANOMALY_UPPER_THRESHOLD = 0.5
     ANOMALY_LOWER_THRESHOLD = 0.3
-    TOP_CONTRIBUTORS_COUNT = 30
     TOP_ATTENTION_COUNT = 10
     DEFAULT_INPUT_SIZE = 200
     DEFAULT_THRESHOLD_WINDOW = 200
-    TOP_ATTENTION_COUNT = 10
 
 
 @dataclass

anomaly-detector/anomaly_detector/multivariate/dataset.py

@@ -1,3 +1,7 @@
+# ---------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# ---------------------------------------------------------
+
 from typing import List, Union
 
 import numpy as np