add uvad，test and update setup.py

MANIFEST.in

@@ -0,0 +1,5 @@
+recursive-include anomaly-detector/anomaly_detector/univariate *.pyx
+recursive-include anomaly-detector/anomaly_detector/univariate *.c
+recursive-include anomaly-detector/anomaly_detector/univariate *.h
+recursive-include anomaly-detector/anomaly_detector/univariate *.py
+include setup.py

README.md

@@ -1,3 +1,21 @@
+# Install
+```python setup.py build_ext --inplace ```
+
+```python setup.py install```
+
+# Test
+
+```cd anomaly-detector```
+
+```python tests/uvad_test.py```
+
+# Sample
+
+```cd anomaly_detector```
+
+```python univariate/univariate_anomaly_detection.py```
+
+
 # Project
 
 > This repo has been populated by an initial template to help get you started. Please

anomaly-detector/anomaly_detector/__init__.py

@@ -1 +1,2 @@
 from .multivariate.model import MultivariateAnomalyDetector
+from .univariate.univariate_anomaly_detection import UnivariateAnomalyDetector

anomaly-detector/anomaly_detector/telemetry/__init__.py

@@ -0,0 +1,50 @@
+import os
+from telemetry.telemetry_factory import TelemetryFactory
+
+DEFAULT_TELEMETRY_TYPE = 'console'
+telemetry_type = DEFAULT_TELEMETRY_TYPE
+
+
+def validate_env(t_type, env_list):
+    for env_item in env_list:
+        if env_item not in os.environ:
+            print("Missing mandatory environment variables %s for TelemetryType %s e.g: export %s=xxxxx "
+                  % (env_item, t_type, env_item))
+            os.abort()
+
+
+def validate_telemetry_type():
+    ###
+    # verify env variables required by appinsights
+    ###
+    if telemetry_type == "appinsights":
+        appinsights_env_list = ["APP_INSIGHTS_APK"]
+        validate_env('appinsights', appinsights_env_list)
+
+    ###
+    # verify env variables required by Geneva
+    ###
+    if telemetry_type == "geneva":
+        geneva_env_list = ["STATSD_HOST", "STATSD_PORT", "STATSD_MODE",
+                           "FLUENTD_HOST", "FLUENTD_PORT"]
+        validate_env('geneva', geneva_env_list)
+
+    ###
+    # verify env variables required by Geneva
+    ###
+    if os.environ['TELEMETRY_TYPE'] == "mon3":
+        mon3_env_list = ["KENSHO2_PROFILE"]
+        validate_env('mon3', mon3_env_list)
+
+
+if 'TELEMETRY_TYPE' not in os.environ:
+    os.environ.setdefault('TELEMETRY_TYPE', DEFAULT_TELEMETRY_TYPE)
+    print("Telemetry type not provided,"
+          "will use console logger."
+          "You can set Telemetry Type to appinsights|geneva|mon3 "
+          "e.g: export TELEMETRY_TYPE=geneva")
+else:
+    telemetry_type = str(os.environ['TELEMETRY_TYPE']).strip()
+
+validate_telemetry_type()
+log = TelemetryFactory.get_telemetry(telemetry_type)

anomaly-detector/anomaly_detector/telemetry/appinsights.py

@@ -0,0 +1,42 @@
+from applicationinsights import TelemetryClient, channel
+from telemetry.telemetry_base import BaseTelemetry
+
+
+class ApplicationInsights(BaseTelemetry):
+    def __init__(self, **kwargs):
+        super().__init__()
+        self.asyncQueue = channel.AsynchronousQueue(channel.AsynchronousSender())
+        self.telemetryChannel = channel.TelemetryChannel(queue=self.asyncQueue)
+        self.telemetry = TelemetryClient(kwargs['app_insights_apk'], self.telemetryChannel)
+        self.telemetry.channel.sender.send_interval_in_milliseconds = 5 * 1000
+        self.telemetry.channel.sender.max_queue_item_count = 10000
+
+    def count(self, name, int_val, **tags):
+        self.telemetry.track_metric(name, int_val, properties=tags)
+        self.telemetry.flush()
+
+    def gauge(self, name, float_val, **tags):
+        self.telemetry.track_metric(name, float_val, properties=tags)
+        self.telemetry.flush()
+
+    def duration(self, name, time_in_seconds, **tags):
+        self.telemetry.track_request(name=name, duration=time_in_seconds * 1000, url=tags['url'],
+                                     success=tags['success'], response_code=tags['response_code'],
+                                     properties=tags)
+        self.telemetry.flush()
+
+    def info(self, msg, *args, **kwargs):
+        self.telemetry.track_trace(msg, properties=kwargs, severity='INFO')
+        self.telemetry.flush()
+
+    def warning(self, msg, *args, **kwargs):
+        self.telemetry.track_trace(msg, properties=kwargs, severity='WARNING')
+        self.telemetry.flush()
+
+    def error(self, msg, *args, **kwargs):
+        self.telemetry.track_trace(msg, properties=kwargs, severity='ERROR')
+        self.telemetry.flush()
+
+    def track_exception(self, traceback, properties):
+        self.telemetry.track_exception(tb=traceback, properties=properties)
+        self.telemetry.flush()

anomaly-detector/anomaly_detector/telemetry/console.py

@@ -0,0 +1,42 @@
+from telemetry.telemetry_base import BaseTelemetry
+import logging
+import sys
+
+
+class Console(BaseTelemetry):
+    def __init__(self):
+        super().__init__()
+        formatter = logging.Formatter(fmt='%(asctime)s %(levelname)-8s %(message)s',
+                                      datefmt='%Y-%m-%d %H:%M:%S')
+        info_handler = logging.StreamHandler(stream=sys.stdout)
+        info_handler.setFormatter(formatter)
+        self.logger = logging.getLogger('anomaly-detector-info')
+        self.logger.setLevel(logging.DEBUG)
+        self.logger.addHandler(info_handler)
+
+        err_handler = logging.StreamHandler(stream=sys.stderr)
+        err_handler.setFormatter(formatter)
+        self.err_logger = logging.getLogger('anomaly-detector-err')
+        self.err_logger.setLevel(logging.ERROR)
+        self.err_logger.addHandler(err_handler)
+
+    def count(self, name, int_val, **tags):
+        self.logger.debug('count: %s : %d ' % (name + (str(tags) if tags else ''), int_val))
+
+    def gauge(self, name, float_val, **tags):
+        self.logger.debug('gauge: %s : %d ' % (name + (str(tags) if tags else ''), float_val))
+
+    def duration(self, name, time_in_seconds, **tags):
+        self.logger.debug('duration: %s : %d ms' % (name + (str(tags) if tags else ''), time_in_seconds * 1000))
+
+    def info(self, msg, *args, **kwargs):
+        self.logger.info(msg + (str(args) if args else '') + (str(kwargs) if kwargs else ''))
+
+    def warning(self, msg, *args, **kwargs):
+        self.logger.warning(msg + (str(args) if args else '') + (str(kwargs) if kwargs else ''))
+
+    def error(self, msg, *args, **kwargs):
+        self.err_logger.error(msg + (str(args) if args else '') + (str(kwargs) if kwargs else ''))
+
+    def track_exception(self, traceback, properties):
+        self.err_logger.error(traceback + (str(properties) if properties else ''))

anomaly-detector/anomaly_detector/telemetry/fluentd_wrapper.py

@@ -0,0 +1,34 @@
+import os
+from fluent import asyncsender as sender
+
+DEFAULT_FLUENT_TAG = 'microsoft.cloudai.anomalydetector'
+fluent_tag = os.environ.get('KENSHO2_LOG_FLUENTD_PREFIX', DEFAULT_FLUENT_TAG) 
+log_app = os.environ.get("KENSHO2_LOG_APP", "AnomalyDetectorApi")
+
+class FluentdWrapper(object):
+    __logger = None
+
+    def __init__(self, host, port):
+        self.__logger = sender.FluentSender(
+            fluent_tag, host=host, port=port, queue_maxsize=100000, queue_circular=True)
+        self.addition_tag = {} if fluent_tag == DEFAULT_FLUENT_TAG else {"app": log_app}
+
+    def info(self, msg, *args, **kwargs):
+        base_log = {'level': 'INFO', 'message': msg}
+        log_content = {**base_log, **kwargs, **self.addition_tag}
+        self.__logger.emit('log', log_content)
+
+    def warning(self, msg, *args, **kwargs):
+        base_log = {'level': 'WARNING', 'message': msg}
+        log_content = {**base_log, **kwargs, **self.addition_tag}
+        self.__logger.emit('log', log_content)
+
+    def error(self, msg, *args, **kwargs):
+        base_log = {'level': 'ERROR', 'message': msg}
+        log_content = {**base_log, **kwargs, **self.addition_tag}
+        self.__logger.emit('log', log_content)
+
+    def track_exception(self,  traceback, properties):
+        base_log = {'level': 'EXCEPTION', 'traceback': traceback}
+        log_content = {**base_log, **properties, **self.addition_tag}
+        self.__logger.emit('log', log_content)

anomaly-detector/anomaly_detector/telemetry/geneva.py

@@ -0,0 +1,38 @@
+from telemetry.telemetry_base import BaseTelemetry
+from telemetry.statsd_wrapper import StatsdWrapper
+from telemetry.fluentd_wrapper import FluentdWrapper
+
+
+class Geneva(BaseTelemetry):
+    _statsd = None
+    _logger = None
+
+    def __init__(self, **kwargs):
+        super().__init__()
+        self._logger = FluentdWrapper(
+            host=kwargs['fluentd_host'], port=kwargs['fluentd_port'])
+        self._statsd = StatsdWrapper(logger=self._logger,
+                                     host=kwargs['statsd_host'], port=kwargs['statsd_port'], mode=kwargs['statsd_mode'])
+
+    def count(self, name, int_val, **tags):
+        self._statsd.gauge(name, int_val, **tags)
+
+    def gauge(self, name, float_val, **tags):
+        # TODO currently _statsd only support int type counter, should improve later
+        self._statsd.gauge(name, int(float_val), **tags)
+
+    def duration(self, name, time_in_seconds, **tags):
+        time_in_ms = int(time_in_seconds * 1000)
+        self._statsd.timing(name, time_in_ms, **tags)
+
+    def info(self, msg, *args, **kwargs):
+        self._logger.info(msg, *args, **kwargs)
+
+    def warning(self, msg, *args, **kwargs):
+        self._logger.warning(msg, *args, **kwargs)
+
+    def error(self, msg, *args, **kwargs):
+        self._logger.error(msg, *args, **kwargs)
+
+    def track_exception(self, traceback, properties):
+        self._logger.track_exception(traceback=traceback, properties=properties)

anomaly-detector/anomaly_detector/telemetry/mds/mds.configuration.xml

@@ -0,0 +1,41 @@
+<MonitoringManagement eventVersion="1" version="1.0" timestamp="2018-05-07T00:00:00Z" namespace="AicpApiAnomalyDetector">
+  <Accounts>
+    <Account moniker="aicpapianomalydetectordiag" isDefault="true" />
+    <Account moniker="aicpapianomalydetectorsecurity" alias="AzSecurityStore" />
+    <Account moniker="aicpapianomalydetectoraudit" alias="AuditStore" />
+  </Accounts>
+  <!-- Linux mdsd usage documentation: https://jarvis.dc.ad.msft.net/?page=documents&section=1363da01-b6ed-43d4-970e-f89b011d591f&id=89b7a7f4-8cf1-4a76-b541-478d37fc760f  -->
+  <Management eventVolume="Medium" defaultRetentionInDays="29">
+    <!--
+      The identity TenantRole means MONITORING_IDENTITY, MONITORING_ROLE, and MONITORING_ROLE_INSTANCE will
+      uniquely identify any of your instances
+    -->
+    <Identity type="TenantRole" />
+    <!--
+      The diskQuotaInMB is a required parameter.  For 10,000 MB there should be 15,000 MB available on the disk  The CPU, memory and network can also be limited in the
+      AgentResourceUsage element's attributes.
+    -->
+    <AgentResourceUsage diskQuotaInMB="50000" cpuPercentUsage="50" />
+  </Management>
+  <Sources>
+    <!-- personalization -->
+    <Source name="microsoft.cloudai.anomalydetector.log" dynamic_schema="true" />
+    <Source name="microsoft.cloudai.anomalydetector.exception" dynamic_schema="true" />
+  </Sources>
+  <!-- Linux events documentation: https://jarvis.dc.ad.msft.net/?page=documents&section=9c95f4eb-8689-4c9f-81bf-82d688e860fd&id=762a440d-3a1e-4ff3-86d6-03c491c163b2   -->
+  <!--
+      Events can contain one or more Event elements.  The Event elements each describe a different kind of collector.
+      Events must be defined in the order of the mds2.xsd.
+  -->
+  <Events>
+    <MdsdEvents>
+      <!-- personalization -->
+      <MdsdEventSource source="microsoft.cloudai.anomalydetector.log">
+        <RouteEvent eventName="AnomalyDetectorLog" storeType="CentralBond" priority="Normal" />
+      </MdsdEventSource>
+      <MdsdEventSource source="microsoft.cloudai.anomalydetector.exception">
+        <RouteEvent eventName="AnomalyDetectorException" storeType="CentralBond" priority="Normal" />
+      </MdsdEventSource>
+    </MdsdEvents>
+  </Events>
+</MonitoringManagement>

anomaly-detector/anomaly_detector/telemetry/mon3.py

@@ -0,0 +1,33 @@
+from telemetry.telemetry_base import BaseTelemetry
+from telemetry.mon3lib.sender import Mon3Sender
+from telemetry.mon3lib.logger import Mon3Logger
+from telemetry.mon3lib.meter import Mon3Meter
+
+
+class Mon3(BaseTelemetry):
+    def __init__(self, kensho2_profile='INT', mon3_server=None, app=None, service=None):
+        super().__init__()
+        sender = Mon3Sender(mon3_server, 500, False)
+        self.__logger = Mon3Logger(sender, app, service, kensho2_profile)
+        self.__meter = Mon3Meter(sender, app, service, kensho2_profile)
+
+    def count(self, name, int_val, **tags):
+        self.__meter.count(name, int_val, **tags)
+
+    def gauge(self, name, float_val, **tags):
+        self.__meter.gauge(name, float_val, **tags)
+
+    def duration(self, name, time_in_seconds, **tags):
+        self.__meter.duration(name, time_in_seconds, **tags)
+
+    def info(self, msg, *args, **kwargs):
+        self.__logger.log_info(msg)
+
+    def warning(self, msg, *args, **kwargs):
+        self.__logger.log_warn(msg)
+
+    def error(self, msg, *args, **kwargs):
+        self.__logger.log_error(msg=msg)
+
+    def track_exception(self, traceback, properties):
+        self.__logger.log_error(traceback)

anomaly-detector/anomaly_detector/telemetry/mon3lib/logger.py

@@ -0,0 +1,54 @@
+import os
+import sys
+import json
+import time
+import socket
+
+TAG_NODE_VAL = os.environ.get("KUBERNETES_NODE_NAME")
+internal_seq = 0
+
+class Mon3Logger(object):
+    def __init__(self, sender, app, service, profile):
+        self.sender = sender
+        self.service = service
+        self.app = app
+        self.profile = profile
+        self.hostname = socket.gethostname()
+        self.pid = os.getpid()
+
+    def log_message(self, ts, level, msg, is_error):
+        global internal_seq
+        internal_seq = internal_seq + 1
+        obj = {
+            "ts": ts,
+            "host": self.hostname,
+            "service": self.service,
+            "app": self.app,
+            "profile": self.profile,
+            "level": str(level),
+            "msg": str(msg),
+            "stack": '',
+            "category": "none",
+            "pid": self.pid,
+            "seq": internal_seq
+        }
+
+        if TAG_NODE_VAL:
+            obj["node"] = TAG_NODE_VAL
+
+        prefix = "!" if is_error else "#"
+        line = prefix + json.dumps(obj)
+
+        try:
+            self.sender.send(line.encode(encoding='utf-8'))
+        except BaseException as ex:
+            print(ex, file=sys.stderr)
+
+    def log_info(self, msg):
+        self.log_message(int(time.time() * 1000), "INFO", msg, False)
+
+    def log_error(self, msg):
+        self.log_message(int(time.time() * 1000), "ERROR", msg, True)
+
+    def log_warn(self, msg):
+        self.log_message(int(time.time() * 1000), "WARN", msg, False)