Eap/linuxhrwcert

Makefile

@@ -316,7 +316,7 @@ endif
 
 nxOMSAutomationWorker:
 	rm -rf output/staging; \
-	VERSION="1.8.0.0"; \
+	VERSION="1.9.0.0"; \
 	PROVIDERS="nxOMSAutomationWorker"; \
 	STAGINGDIR="output/staging/$@/DSCResources"; \
 	cat Providers/Modules/[email protected] | sed "s@<MODULE_VERSION>@$${VERSION}@" > intermediate/Modules/[email protected]; \

Providers/Scripts/2.4x-2.5x/Scripts/Tests/dummy_nxOMSAutomationWorker_files/configuration.py

@@ -155,11 +155,6 @@ def get_value(key):
     except KeyError:
         raise KeyError("Configuration environment variable not found. [key=" + key + "].")
 
-
-def get_jrds_get_sandbox_actions_polling_freq():
-    return get_value(JRDS_POLLING_FREQUENCY)
-
-
 def get_jrds_get_job_actions_polling_freq():
     return get_value(JRDS_POLLING_FREQUENCY)
 

Providers/Scripts/2.4x-2.5x/Scripts/nxOMSAutomationWorker.py

@@ -379,7 +379,7 @@ def is_certificate_valid(worker_conf_path, certificate_path):
         worker_conf.read(worker_conf_path)
         worker_certificate_thumbprint = worker_conf.get(SECTION_OMS_METADATA, OPTION_JRDS_CERT_THUMBPRINT)
 
-        issuer, subject, omsagent_certificate_thumbprint = linuxutil.get_cert_info(certificate_path)
+        issuer, subject, omsagent_certificate_thumbprint, not_before, not_after = linuxutil.get_cert_info(certificate_path)
 
         if worker_certificate_thumbprint == omsagent_certificate_thumbprint:
             return True

Providers/Scripts/2.6x-2.7x/Scripts/Tests/dummy_nxOMSAutomationWorker_files/configuration.py

@@ -156,10 +156,6 @@ def get_value(key):
         raise KeyError("Configuration environment variable not found. [key=" + key + "].")
 
 
-def get_jrds_get_sandbox_actions_polling_freq():
-    return get_value(JRDS_POLLING_FREQUENCY)
-
-
 def get_jrds_get_job_actions_polling_freq():
     return get_value(JRDS_POLLING_FREQUENCY)
 

Providers/Scripts/2.6x-2.7x/Scripts/nxOMSAutomationWorker.py

@@ -379,7 +379,7 @@ def is_certificate_valid(worker_conf_path, certificate_path):
         worker_conf.read(worker_conf_path)
         worker_certificate_thumbprint = worker_conf.get(SECTION_OMS_METADATA, OPTION_JRDS_CERT_THUMBPRINT)
 
-        issuer, subject, omsagent_certificate_thumbprint = linuxutil.get_cert_info(certificate_path)
+        issuer, subject, omsagent_certificate_thumbprint, not_before, not_after = linuxutil.get_cert_info(certificate_path)
 
         if worker_certificate_thumbprint == omsagent_certificate_thumbprint:
             return True

Providers/Scripts/3.x/Scripts/Tests/dummy_nxOMSAutomationWorker_files/configuration.py

@@ -156,10 +156,6 @@ def get_value(key):
         raise KeyError("Configuration environment variable not found. [key=" + key + "].")
 
 
-def get_jrds_get_sandbox_actions_polling_freq():
-    return get_value(JRDS_POLLING_FREQUENCY)
-
-
 def get_jrds_get_job_actions_polling_freq():
     return get_value(JRDS_POLLING_FREQUENCY)
 

Providers/Scripts/3.x/Scripts/nxOMSAutomationWorker.py

@@ -397,7 +397,7 @@ def is_certificate_valid(worker_conf_path, certificate_path):
         worker_conf.read(worker_conf_path)
         worker_certificate_thumbprint = worker_conf.get(SECTION_OMS_METADATA, OPTION_JRDS_CERT_THUMBPRINT)
 
-        issuer, subject, omsagent_certificate_thumbprint = linuxutil.get_cert_info(certificate_path)
+        issuer, subject, omsagent_certificate_thumbprint, not_before, not_after = linuxutil.get_cert_info(certificate_path)
 
         if worker_certificate_thumbprint == omsagent_certificate_thumbprint:
             return True

Providers/nxOMSAutomationWorker/automationworker/3.x/scripts/onboarding3.py

@@ -258,7 +258,7 @@ def register(options):
             print("Cannot create directory for certs/conf. Because of the following exception : " + str(ex))
             return
     generate_self_signed_certificate(certificate_path=certificate_path, key_path=key_path)
-    issuer, subject, thumbprint = linuxutil.get_cert_info(certificate_path)
+    issuer, subject, thumbprint, not_before, not_after = linuxutil.get_cert_info(certificate_path)
 
     # try to extract optional metadata
     unknown = "Unknown"
@@ -287,7 +287,9 @@ def register(options):
                "OperatingSystem": 2,
                "SMBIOSAssetTag": asset_tag,
                "VirtualMachineId": vm_id,
-               "Subject": subject}
+               "Subject": subject,
+               "NotBeforeUtc": not_before,
+               "NotAfterUtc": not_after}
 
     # the signature generation is based on agent service contract
     payload_hash = sha256_digest(payload)
@@ -349,7 +351,7 @@ def deregister(options):
     if os.path.exists(certificate_path) is False or os.path.exists(key_path) is False:
         raise Exception("Unable to deregister, no worker certificate/key found on disk.")
 
-    issuer, subject, thumbprint = linuxutil.get_cert_info(certificate_path)
+    issuer, subject, thumbprint, not_before, not_after = linuxutil.get_cert_info(certificate_path)
 
     if os.path.exists(worker_conf_path) is False:
         raise Exception("Missing worker configuration.")

Providers/nxOMSAutomationWorker/automationworker/3.x/scripts/register_oms.py

@@ -166,7 +166,7 @@ def get_headers_and_payload(worker_group_name, is_azure_vm, vm_id, azure_resourc
     Returns:
         A tuple containing a dictionary for the request headers and a dictionary for the payload (request body).
     """
-    issuer, subject, thumbprint = linuxutil.get_cert_info(certificate_path)
+    issuer, subject, thumbprint, not_before, not_after = linuxutil.get_cert_info(certificate_path)
     headers = {"ProtocolVersion": "2.0",
                "x-ms-date": datetime.datetime.utcnow().isoformat() + "0-00:00",
                "Content-Type": "application/json"}

Providers/nxOMSAutomationWorker/automationworker/3.x/worker/configuration3.py

@@ -58,8 +58,8 @@
 DEFAULT_VM_ID = DEFAULT_UNKNOWN
 DEFAULT_WORKER_TYPE = DEFAULT_UNKNOWN
 DEFAULT_COMPONENT = DEFAULT_UNKNOWN
-DEFAULT_WORKER_VERSION = "1.8.0.0"
-DEFAULT_JRDS_POLLING_FREQUENCY = "15"
+DEFAULT_WORKER_VERSION = "1.9.0.0"
+DEFAULT_JRDS_POLLING_FREQUENCY = "30"
 
 # state configuration keys
 STATE_PID = "pid"
@@ -186,10 +186,6 @@ def get_value(key):
         raise KeyError("Configuration environment variable not found. [key=" + key + "].")
 
 
-def get_jrds_get_sandbox_actions_polling_freq():
-    return get_value(JRDS_POLLING_FREQUENCY)
-
-
 def get_jrds_get_job_actions_polling_freq():
     return get_value(JRDS_POLLING_FREQUENCY)
 

Providers/nxOMSAutomationWorker/automationworker/3.x/worker/diydirs.py

@@ -6,7 +6,7 @@
 """Utility for DIY hybrid worker directories"""
 
 
-from worker import linuxutil
+import linuxutil
 import os
 
 NXAUTOMATION_HOME_DIR = "/home/nxautomation"

Providers/nxOMSAutomationWorker/automationworker/3.x/worker/hybridworker.py

@@ -12,6 +12,7 @@
 import threading
 import time
 import traceback
+import workerpollingfrequency
 
 # import worker module after linuxutil.daemonize() call
 
@@ -24,16 +25,17 @@ def decorated_func(*args, **kwargs):
             try:
                 # ensure required file / cert exists
                 func(*args, **kwargs)
-            except (JrdsAuthorizationException,
-                    InvalidFilePermissionException,
+            except (JrdsAuthorizationException):
+                tracer.log_worker_safe_loop_terminal_exception(traceback.format_exc())
+            except (InvalidFilePermissionException,
                     FileNotFoundException,
                     SystemExit):
                 tracer.log_worker_safe_loop_terminal_exception(traceback.format_exc())
                 time.sleep(1)  # allow the trace to make it to stdout (since traces are background threads)
                 sys.exit(-1)
             except Exception:
                 tracer.log_worker_safe_loop_non_terminal_exception(traceback.format_exc())
-            time.sleep(configuration.get_jrds_get_sandbox_actions_polling_freq())
+            time.sleep(workerpollingfrequency.get_jrds_get_sandbox_actions_polling_freq()) #polling frequency as per the value received from headers of GetSandboxActions
 
     return decorated_func
 

Providers/nxOMSAutomationWorker/automationworker/3.x/worker/jrdsclient.py

@@ -7,14 +7,19 @@
 
 from datetime import datetime
 import time
+import traceback
 
 
 import configuration3 as configuration
 import locallogger
 from workerexception import *
+import linuxutil
+import workercertificaterotation
 
 transient_status_codes = set([408, 429, 500, 502, 503, 504])
 
+DISABLE_CERT_ROTATION = 'False'
+
 class JRDSClient(object):
     def __init__(self, http_client):
         self.httpClient = http_client
@@ -46,7 +51,7 @@ def issue_request(request_function, url):
         return response
 
     def get_sandbox_actions(self):
-        """Gets any pending sandbox actions.
+        """Gets any pending sandbox actions and headers which determine polling frequency and certificate rotation of workers
 
         Returns:
             A list of sandbox actions.
@@ -69,6 +74,8 @@ def get_sandbox_actions(self):
               "&api-version=" + self.protocol_version
         response = self.issue_request(lambda u: self.httpClient.get(u), url)
 
+        import tracer
+
         if response.status_code == 200:
             try:
                 if response.deserialized_data is None or "value" not in response.deserialized_data:
@@ -77,12 +84,58 @@ def get_sandbox_actions(self):
             except TypeError:
                 locallogger.log_info("INFO: Could not deserialize get_sandbox_actions response body: %s" % str(response.deserialized_data))
                 return None
+
+            # whenever worker cert has crossed half of it's lifetime or server is initiating a forced rotation of certificate based on date, header is set on the server side
+            # based on the headers client initiates worker certificate rotation
+            try:
+                if(eval(workercertificaterotation.get_certificate_rotation_header_value())):
+                    tracer.log_debug_trace("Initiating certificate Rotation of Hybrid Worker")
+                    workercertificaterotation.set_certificate_rotation_header_value(DISABLE_CERT_ROTATION)  
+                    self.worker_certificate_rotation()
+                    tracer.log_worker_certificate_rotation_successful()
+            except Exception as ex:
+                tracer.log_debug_trace("[exception=" + str(ex) + "]")
+                tracer.log_worker_certificate_rotation_failed(ex)
 
-            # success path
             return response.deserialized_data["value"]
 
         raise Exception("Unable to get sandbox actions. [status=" + str(response.status_code) + "]")
 
+    def worker_certificate_rotation(self):
+        """ Rotate worker certificate. 
+            Steps includes creating new certificate/key and after JRDS returns 200, replace the old certificate/key with newly generated certificate/key.
+            Worker.conf is updated with the latest thumbprint.
+        """
+
+        import tracer
+
+        try:
+            temp_certificate_path, temp_key_path = workercertificaterotation.generate_cert_rotation_self_signed_certificate()
+            issuer, subject, thumbprint, not_before, not_after = linuxutil.get_cert_info(temp_certificate_path)
+
+            payload = {'Thumbprint': thumbprint,
+               'Issuer': issuer,
+               'Subject': subject,
+               'NotBefore': not_before,
+               'NotAfter': not_after}
+
+            headers = {"Content-Type": "application/json"}
+
+            url = self.base_uri + "/automationAccounts/" + self.account_id + \
+              "/hybridCertificateRotation?api-version=" + self.protocol_version
+            response = self.issue_request(lambda u: self.httpClient.post(u, headers=headers, data=payload), url)
+
+            if response.status_code == 200:
+                tracer.log_debug_trace("New worker certificate successfully added in the Database")
+                workercertificaterotation.replace_self_signed_certificate_and_key(temp_certificate_path, temp_key_path, thumbprint)
+        except Exception as ex:
+            tracer.log_debug_trace("[exception=" + str(ex) + "]" + "[stacktrace=" + str(traceback.format_exc()) + "]")
+            tracer.log_worker_certificate_rotation_failed(ex)
+        finally:
+            workercertificaterotation.clean_up_certificate_and_key(temp_certificate_path, temp_key_path)
+
+        return
+
     def get_job_actions(self, sandbox_id):
         """Gets any pending job action for the given sandbox id.
 

Providers/nxOMSAutomationWorker/automationworker/3.x/worker/linuxutil.py

@@ -9,6 +9,8 @@
 import sys
 import re
 import codecs
+import traceback
+from datetime import datetime
 
 # workaround when unexpected environment variables are present
 # sets COLUMNS wide enough so that output of ps does not get truncated
@@ -285,7 +287,7 @@ def get_cert_info(certificate_path):
     """Gets certificate information by invoking OpenSSL (OMS agent dependency).
 
     Returns:
-        A tuple containing the certificate's issuer, subject and thumbprint.
+        A tuple containing the certificate's issuer, subject, thumbprint, start date and end date.
     """
     p = subprocess.Popen(["openssl", "x509", "-noout", "-in", certificate_path, "-fingerprint", "-sha1"],
                          stdout=subprocess.PIPE,
@@ -311,9 +313,27 @@ def get_cert_info(certificate_path):
     if p.poll() != 0:
         raise Exception("Unable to get certificate subject.")
 
+    p = subprocess.Popen(["openssl", "x509", "-noout", "-in", certificate_path, "-startdate"],
+                         stdout=subprocess.PIPE,
+                         stderr=subprocess.PIPE)
+    raw_not_before, e = p.communicate()
+
+    if p.poll() != 0:
+        raise Exception("Unable to get certificate start date.")
+
+    p = subprocess.Popen(["openssl", "x509", "-noout", "-in", certificate_path, "-enddate"],
+                         stdout=subprocess.PIPE,
+                         stderr=subprocess.PIPE)
+    raw_not_after, e = p.communicate()
+
+    if p.poll() != 0:
+        raise Exception("Unable to get certificate end date.")
+
     return parse_issuer_from_openssl_output(raw_issuer.decode()), \
            parse_subject_from_openssl_output(raw_subject.decode()), \
-           parse_thumbprint_from_openssl_output(raw_fingerprint.decode())
+           parse_thumbprint_from_openssl_output(raw_fingerprint.decode()), \
+           parse_not_before_from_openssl_output(raw_not_before.decode()), \
+           parse_not_after_from_openssl_output(raw_not_after.decode())
 
 
 def parse_thumbprint_from_openssl_output(raw_fingerprint):
@@ -364,6 +384,38 @@ def parse_subject_from_openssl_output(raw_subject):
     return raw_subject.split("subject=")[1].strip()
 
 
+def parse_not_before_from_openssl_output(raw_not_before):
+    """Parses the not before value from the raw OpenSSL output.
+
+    Example output from openSSL:
+    notBefore=Jun 28 15:25:08 2022 GMT
+
+    Returns:
+        datetime : The certificate not before date.
+    """
+    not_before_date = raw_not_before.split("notBefore=")[1].replace("GMT", "").strip()
+    datetime_object = datetime.strptime(not_before_date, '%b %d %H:%M:%S %Y')
+    date_iso_format = datetime_object.isoformat()
+    return date_iso_format
+
+
+def parse_not_after_from_openssl_output(raw_not_after):
+    """Parses the not after value from the raw OpenSSL output.
+
+    Example output from openSSL:
+    notAfter=Jun 30 15:25:08 2022 GMT
+
+    Returns:
+        datetime : The certificate not after date.
+    """
+
+    not_after_date = raw_not_after.split("notAfter=")[1].replace("GMT", "").strip()
+
+    datetime_object = datetime.strptime(not_after_date, '%b %d %H:%M:%S %Y')
+    date_iso_format = datetime_object.isoformat()
+    return date_iso_format
+
+
 @posix_only
 def fork_and_exit_parent():
     """Forks and kills the parent process."""

Providers/nxOMSAutomationWorker/automationworker/3.x/worker/tracer.py

@@ -343,6 +343,17 @@ def log_worker_safe_loop_terminal_exception(exception):
     trace_generic_hybrid_worker_event_async(5109, inspect.stack()[0][3], message, 1, KEYWORD_ERROR)
 
 
+def log_worker_certificate_rotation_successful():
+    message = "Hybrid worker Certificate rotation completed."
+    trace_generic_hybrid_worker_event_async(5110, inspect.stack()[0][3], message, 1, KEYWORD_INFO)
+
+
+def log_worker_certificate_rotation_failed(exception):
+    message = "Hybrid worker Certificate rotation failed. [exception=" + \
+               str(exception) + "]"
+    trace_generic_hybrid_worker_event_async(5111, inspect.stack()[0][3], message, 1, KEYWORD_ERROR)
+
+
 # sandbox specific traces
 # traces in this section are mainly for the sandbox component
 #