diff --git a/Makefile b/Makefile index bf0fda624..b30ad42ae 100755 --- a/Makefile +++ b/Makefile @@ -316,7 +316,7 @@ endif nxOMSAutomationWorker: rm -rf output/staging; \ - VERSION="1.8.0.0"; \ + VERSION="1.9.0.0"; \ PROVIDERS="nxOMSAutomationWorker"; \ STAGINGDIR="output/staging/$@/DSCResources"; \ cat Providers/Modules/$@.psd1 | sed "s@@$${VERSION}@" > intermediate/Modules/$@.psd1; \ diff --git a/Providers/Scripts/2.4x-2.5x/Scripts/Tests/dummy_nxOMSAutomationWorker_files/configuration.py b/Providers/Scripts/2.4x-2.5x/Scripts/Tests/dummy_nxOMSAutomationWorker_files/configuration.py index 3d70284ab..12c77b5ba 100644 --- a/Providers/Scripts/2.4x-2.5x/Scripts/Tests/dummy_nxOMSAutomationWorker_files/configuration.py +++ b/Providers/Scripts/2.4x-2.5x/Scripts/Tests/dummy_nxOMSAutomationWorker_files/configuration.py @@ -155,11 +155,6 @@ def get_value(key): except KeyError: raise KeyError("Configuration environment variable not found. [key=" + key + "].") - -def get_jrds_get_sandbox_actions_polling_freq(): - return get_value(JRDS_POLLING_FREQUENCY) - - def get_jrds_get_job_actions_polling_freq(): return get_value(JRDS_POLLING_FREQUENCY) diff --git a/Providers/Scripts/2.4x-2.5x/Scripts/nxOMSAutomationWorker.py b/Providers/Scripts/2.4x-2.5x/Scripts/nxOMSAutomationWorker.py index ec4763e88..e5585ed3b 100644 --- a/Providers/Scripts/2.4x-2.5x/Scripts/nxOMSAutomationWorker.py +++ b/Providers/Scripts/2.4x-2.5x/Scripts/nxOMSAutomationWorker.py @@ -379,7 +379,7 @@ def is_certificate_valid(worker_conf_path, certificate_path): worker_conf.read(worker_conf_path) worker_certificate_thumbprint = worker_conf.get(SECTION_OMS_METADATA, OPTION_JRDS_CERT_THUMBPRINT) - issuer, subject, omsagent_certificate_thumbprint = linuxutil.get_cert_info(certificate_path) + issuer, subject, omsagent_certificate_thumbprint, not_before, not_after = linuxutil.get_cert_info(certificate_path) if worker_certificate_thumbprint == omsagent_certificate_thumbprint: return True diff --git a/Providers/Scripts/2.6x-2.7x/Scripts/Tests/dummy_nxOMSAutomationWorker_files/configuration.py b/Providers/Scripts/2.6x-2.7x/Scripts/Tests/dummy_nxOMSAutomationWorker_files/configuration.py index 3d70284ab..88f3cc5b8 100755 --- a/Providers/Scripts/2.6x-2.7x/Scripts/Tests/dummy_nxOMSAutomationWorker_files/configuration.py +++ b/Providers/Scripts/2.6x-2.7x/Scripts/Tests/dummy_nxOMSAutomationWorker_files/configuration.py @@ -156,10 +156,6 @@ def get_value(key): raise KeyError("Configuration environment variable not found. [key=" + key + "].") -def get_jrds_get_sandbox_actions_polling_freq(): - return get_value(JRDS_POLLING_FREQUENCY) - - def get_jrds_get_job_actions_polling_freq(): return get_value(JRDS_POLLING_FREQUENCY) diff --git a/Providers/Scripts/2.6x-2.7x/Scripts/nxOMSAutomationWorker.py b/Providers/Scripts/2.6x-2.7x/Scripts/nxOMSAutomationWorker.py index ec4763e88..e5585ed3b 100644 --- a/Providers/Scripts/2.6x-2.7x/Scripts/nxOMSAutomationWorker.py +++ b/Providers/Scripts/2.6x-2.7x/Scripts/nxOMSAutomationWorker.py @@ -379,7 +379,7 @@ def is_certificate_valid(worker_conf_path, certificate_path): worker_conf.read(worker_conf_path) worker_certificate_thumbprint = worker_conf.get(SECTION_OMS_METADATA, OPTION_JRDS_CERT_THUMBPRINT) - issuer, subject, omsagent_certificate_thumbprint = linuxutil.get_cert_info(certificate_path) + issuer, subject, omsagent_certificate_thumbprint, not_before, not_after = linuxutil.get_cert_info(certificate_path) if worker_certificate_thumbprint == omsagent_certificate_thumbprint: return True diff --git a/Providers/Scripts/3.x/Scripts/Tests/dummy_nxOMSAutomationWorker_files/configuration.py b/Providers/Scripts/3.x/Scripts/Tests/dummy_nxOMSAutomationWorker_files/configuration.py index 3d70284ab..88f3cc5b8 100644 --- a/Providers/Scripts/3.x/Scripts/Tests/dummy_nxOMSAutomationWorker_files/configuration.py +++ b/Providers/Scripts/3.x/Scripts/Tests/dummy_nxOMSAutomationWorker_files/configuration.py @@ -156,10 +156,6 @@ def get_value(key): raise KeyError("Configuration environment variable not found. [key=" + key + "].") -def get_jrds_get_sandbox_actions_polling_freq(): - return get_value(JRDS_POLLING_FREQUENCY) - - def get_jrds_get_job_actions_polling_freq(): return get_value(JRDS_POLLING_FREQUENCY) diff --git a/Providers/Scripts/3.x/Scripts/nxOMSAutomationWorker.py b/Providers/Scripts/3.x/Scripts/nxOMSAutomationWorker.py index 801bbfd5f..0c8ae3a15 100644 --- a/Providers/Scripts/3.x/Scripts/nxOMSAutomationWorker.py +++ b/Providers/Scripts/3.x/Scripts/nxOMSAutomationWorker.py @@ -397,7 +397,7 @@ def is_certificate_valid(worker_conf_path, certificate_path): worker_conf.read(worker_conf_path) worker_certificate_thumbprint = worker_conf.get(SECTION_OMS_METADATA, OPTION_JRDS_CERT_THUMBPRINT) - issuer, subject, omsagent_certificate_thumbprint = linuxutil.get_cert_info(certificate_path) + issuer, subject, omsagent_certificate_thumbprint, not_before, not_after = linuxutil.get_cert_info(certificate_path) if worker_certificate_thumbprint == omsagent_certificate_thumbprint: return True diff --git a/Providers/nxOMSAutomationWorker/automationworker/3.x/scripts/onboarding3.py b/Providers/nxOMSAutomationWorker/automationworker/3.x/scripts/onboarding3.py index 5557c4577..3109f2e58 100644 --- a/Providers/nxOMSAutomationWorker/automationworker/3.x/scripts/onboarding3.py +++ b/Providers/nxOMSAutomationWorker/automationworker/3.x/scripts/onboarding3.py @@ -258,7 +258,7 @@ def register(options): print("Cannot create directory for certs/conf. Because of the following exception : " + str(ex)) return generate_self_signed_certificate(certificate_path=certificate_path, key_path=key_path) - issuer, subject, thumbprint = linuxutil.get_cert_info(certificate_path) + issuer, subject, thumbprint, not_before, not_after = linuxutil.get_cert_info(certificate_path) # try to extract optional metadata unknown = "Unknown" @@ -287,7 +287,9 @@ def register(options): "OperatingSystem": 2, "SMBIOSAssetTag": asset_tag, "VirtualMachineId": vm_id, - "Subject": subject} + "Subject": subject, + "NotBeforeUtc": not_before, + "NotAfterUtc": not_after} # the signature generation is based on agent service contract payload_hash = sha256_digest(payload) @@ -349,7 +351,7 @@ def deregister(options): if os.path.exists(certificate_path) is False or os.path.exists(key_path) is False: raise Exception("Unable to deregister, no worker certificate/key found on disk.") - issuer, subject, thumbprint = linuxutil.get_cert_info(certificate_path) + issuer, subject, thumbprint, not_before, not_after = linuxutil.get_cert_info(certificate_path) if os.path.exists(worker_conf_path) is False: raise Exception("Missing worker configuration.") diff --git a/Providers/nxOMSAutomationWorker/automationworker/3.x/scripts/register_oms.py b/Providers/nxOMSAutomationWorker/automationworker/3.x/scripts/register_oms.py index 1b80c9ac5..034def66a 100644 --- a/Providers/nxOMSAutomationWorker/automationworker/3.x/scripts/register_oms.py +++ b/Providers/nxOMSAutomationWorker/automationworker/3.x/scripts/register_oms.py @@ -166,7 +166,7 @@ def get_headers_and_payload(worker_group_name, is_azure_vm, vm_id, azure_resourc Returns: A tuple containing a dictionary for the request headers and a dictionary for the payload (request body). """ - issuer, subject, thumbprint = linuxutil.get_cert_info(certificate_path) + issuer, subject, thumbprint, not_before, not_after = linuxutil.get_cert_info(certificate_path) headers = {"ProtocolVersion": "2.0", "x-ms-date": datetime.datetime.utcnow().isoformat() + "0-00:00", "Content-Type": "application/json"} diff --git a/Providers/nxOMSAutomationWorker/automationworker/3.x/worker/configuration3.py b/Providers/nxOMSAutomationWorker/automationworker/3.x/worker/configuration3.py index 60c22e14e..c075e0afa 100644 --- a/Providers/nxOMSAutomationWorker/automationworker/3.x/worker/configuration3.py +++ b/Providers/nxOMSAutomationWorker/automationworker/3.x/worker/configuration3.py @@ -58,8 +58,8 @@ DEFAULT_VM_ID = DEFAULT_UNKNOWN DEFAULT_WORKER_TYPE = DEFAULT_UNKNOWN DEFAULT_COMPONENT = DEFAULT_UNKNOWN -DEFAULT_WORKER_VERSION = "1.8.0.0" -DEFAULT_JRDS_POLLING_FREQUENCY = "15" +DEFAULT_WORKER_VERSION = "1.9.0.0" +DEFAULT_JRDS_POLLING_FREQUENCY = "30" # state configuration keys STATE_PID = "pid" @@ -186,10 +186,6 @@ def get_value(key): raise KeyError("Configuration environment variable not found. [key=" + key + "].") -def get_jrds_get_sandbox_actions_polling_freq(): - return get_value(JRDS_POLLING_FREQUENCY) - - def get_jrds_get_job_actions_polling_freq(): return get_value(JRDS_POLLING_FREQUENCY) diff --git a/Providers/nxOMSAutomationWorker/automationworker/3.x/worker/diydirs.py b/Providers/nxOMSAutomationWorker/automationworker/3.x/worker/diydirs.py index 4fc679382..2c4b328ce 100644 --- a/Providers/nxOMSAutomationWorker/automationworker/3.x/worker/diydirs.py +++ b/Providers/nxOMSAutomationWorker/automationworker/3.x/worker/diydirs.py @@ -6,7 +6,7 @@ """Utility for DIY hybrid worker directories""" -from worker import linuxutil +import linuxutil import os NXAUTOMATION_HOME_DIR = "/home/nxautomation" diff --git a/Providers/nxOMSAutomationWorker/automationworker/3.x/worker/hybridworker.py b/Providers/nxOMSAutomationWorker/automationworker/3.x/worker/hybridworker.py index b853b7b5d..498c14f2e 100644 --- a/Providers/nxOMSAutomationWorker/automationworker/3.x/worker/hybridworker.py +++ b/Providers/nxOMSAutomationWorker/automationworker/3.x/worker/hybridworker.py @@ -12,6 +12,7 @@ import threading import time import traceback +import workerpollingfrequency # import worker module after linuxutil.daemonize() call @@ -24,8 +25,9 @@ def decorated_func(*args, **kwargs): try: # ensure required file / cert exists func(*args, **kwargs) - except (JrdsAuthorizationException, - InvalidFilePermissionException, + except (JrdsAuthorizationException): + tracer.log_worker_safe_loop_terminal_exception(traceback.format_exc()) + except (InvalidFilePermissionException, FileNotFoundException, SystemExit): tracer.log_worker_safe_loop_terminal_exception(traceback.format_exc()) @@ -33,7 +35,7 @@ def decorated_func(*args, **kwargs): sys.exit(-1) except Exception: tracer.log_worker_safe_loop_non_terminal_exception(traceback.format_exc()) - time.sleep(configuration.get_jrds_get_sandbox_actions_polling_freq()) + time.sleep(workerpollingfrequency.get_jrds_get_sandbox_actions_polling_freq()) #polling frequency as per the value received from headers of GetSandboxActions return decorated_func diff --git a/Providers/nxOMSAutomationWorker/automationworker/3.x/worker/jrdsclient.py b/Providers/nxOMSAutomationWorker/automationworker/3.x/worker/jrdsclient.py index eb5c9e539..5dcc4cbfe 100644 --- a/Providers/nxOMSAutomationWorker/automationworker/3.x/worker/jrdsclient.py +++ b/Providers/nxOMSAutomationWorker/automationworker/3.x/worker/jrdsclient.py @@ -7,14 +7,19 @@ from datetime import datetime import time +import traceback import configuration3 as configuration import locallogger from workerexception import * +import linuxutil +import workercertificaterotation transient_status_codes = set([408, 429, 500, 502, 503, 504]) +DISABLE_CERT_ROTATION = 'False' + class JRDSClient(object): def __init__(self, http_client): self.httpClient = http_client @@ -46,7 +51,7 @@ def issue_request(request_function, url): return response def get_sandbox_actions(self): - """Gets any pending sandbox actions. + """Gets any pending sandbox actions and headers which determine polling frequency and certificate rotation of workers Returns: A list of sandbox actions. @@ -69,6 +74,8 @@ def get_sandbox_actions(self): "&api-version=" + self.protocol_version response = self.issue_request(lambda u: self.httpClient.get(u), url) + import tracer + if response.status_code == 200: try: if response.deserialized_data is None or "value" not in response.deserialized_data: @@ -77,12 +84,58 @@ def get_sandbox_actions(self): except TypeError: locallogger.log_info("INFO: Could not deserialize get_sandbox_actions response body: %s" % str(response.deserialized_data)) return None + + # whenever worker cert has crossed half of it's lifetime or server is initiating a forced rotation of certificate based on date, header is set on the server side + # based on the headers client initiates worker certificate rotation + try: + if(eval(workercertificaterotation.get_certificate_rotation_header_value())): + tracer.log_debug_trace("Initiating certificate Rotation of Hybrid Worker") + workercertificaterotation.set_certificate_rotation_header_value(DISABLE_CERT_ROTATION) + self.worker_certificate_rotation() + tracer.log_worker_certificate_rotation_successful() + except Exception as ex: + tracer.log_debug_trace("[exception=" + str(ex) + "]") + tracer.log_worker_certificate_rotation_failed(ex) - # success path return response.deserialized_data["value"] raise Exception("Unable to get sandbox actions. [status=" + str(response.status_code) + "]") + def worker_certificate_rotation(self): + """ Rotate worker certificate. + Steps includes creating new certificate/key and after JRDS returns 200, replace the old certificate/key with newly generated certificate/key. + Worker.conf is updated with the latest thumbprint. + """ + + import tracer + + try: + temp_certificate_path, temp_key_path = workercertificaterotation.generate_cert_rotation_self_signed_certificate() + issuer, subject, thumbprint, not_before, not_after = linuxutil.get_cert_info(temp_certificate_path) + + payload = {'Thumbprint': thumbprint, + 'Issuer': issuer, + 'Subject': subject, + 'NotBefore': not_before, + 'NotAfter': not_after} + + headers = {"Content-Type": "application/json"} + + url = self.base_uri + "/automationAccounts/" + self.account_id + \ + "/hybridCertificateRotation?api-version=" + self.protocol_version + response = self.issue_request(lambda u: self.httpClient.post(u, headers=headers, data=payload), url) + + if response.status_code == 200: + tracer.log_debug_trace("New worker certificate successfully added in the Database") + workercertificaterotation.replace_self_signed_certificate_and_key(temp_certificate_path, temp_key_path, thumbprint) + except Exception as ex: + tracer.log_debug_trace("[exception=" + str(ex) + "]" + "[stacktrace=" + str(traceback.format_exc()) + "]") + tracer.log_worker_certificate_rotation_failed(ex) + finally: + workercertificaterotation.clean_up_certificate_and_key(temp_certificate_path, temp_key_path) + + return + def get_job_actions(self, sandbox_id): """Gets any pending job action for the given sandbox id. diff --git a/Providers/nxOMSAutomationWorker/automationworker/3.x/worker/linuxutil.py b/Providers/nxOMSAutomationWorker/automationworker/3.x/worker/linuxutil.py index b42310f60..4566b0047 100644 --- a/Providers/nxOMSAutomationWorker/automationworker/3.x/worker/linuxutil.py +++ b/Providers/nxOMSAutomationWorker/automationworker/3.x/worker/linuxutil.py @@ -9,6 +9,8 @@ import sys import re import codecs +import traceback +from datetime import datetime # workaround when unexpected environment variables are present # sets COLUMNS wide enough so that output of ps does not get truncated @@ -285,7 +287,7 @@ def get_cert_info(certificate_path): """Gets certificate information by invoking OpenSSL (OMS agent dependency). Returns: - A tuple containing the certificate's issuer, subject and thumbprint. + A tuple containing the certificate's issuer, subject, thumbprint, start date and end date. """ p = subprocess.Popen(["openssl", "x509", "-noout", "-in", certificate_path, "-fingerprint", "-sha1"], stdout=subprocess.PIPE, @@ -311,9 +313,27 @@ def get_cert_info(certificate_path): if p.poll() != 0: raise Exception("Unable to get certificate subject.") + p = subprocess.Popen(["openssl", "x509", "-noout", "-in", certificate_path, "-startdate"], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + raw_not_before, e = p.communicate() + + if p.poll() != 0: + raise Exception("Unable to get certificate start date.") + + p = subprocess.Popen(["openssl", "x509", "-noout", "-in", certificate_path, "-enddate"], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + raw_not_after, e = p.communicate() + + if p.poll() != 0: + raise Exception("Unable to get certificate end date.") + return parse_issuer_from_openssl_output(raw_issuer.decode()), \ parse_subject_from_openssl_output(raw_subject.decode()), \ - parse_thumbprint_from_openssl_output(raw_fingerprint.decode()) + parse_thumbprint_from_openssl_output(raw_fingerprint.decode()), \ + parse_not_before_from_openssl_output(raw_not_before.decode()), \ + parse_not_after_from_openssl_output(raw_not_after.decode()) def parse_thumbprint_from_openssl_output(raw_fingerprint): @@ -364,6 +384,38 @@ def parse_subject_from_openssl_output(raw_subject): return raw_subject.split("subject=")[1].strip() +def parse_not_before_from_openssl_output(raw_not_before): + """Parses the not before value from the raw OpenSSL output. + + Example output from openSSL: + notBefore=Jun 28 15:25:08 2022 GMT + + Returns: + datetime : The certificate not before date. + """ + not_before_date = raw_not_before.split("notBefore=")[1].replace("GMT", "").strip() + datetime_object = datetime.strptime(not_before_date, '%b %d %H:%M:%S %Y') + date_iso_format = datetime_object.isoformat() + return date_iso_format + + +def parse_not_after_from_openssl_output(raw_not_after): + """Parses the not after value from the raw OpenSSL output. + + Example output from openSSL: + notAfter=Jun 30 15:25:08 2022 GMT + + Returns: + datetime : The certificate not after date. + """ + + not_after_date = raw_not_after.split("notAfter=")[1].replace("GMT", "").strip() + + datetime_object = datetime.strptime(not_after_date, '%b %d %H:%M:%S %Y') + date_iso_format = datetime_object.isoformat() + return date_iso_format + + @posix_only def fork_and_exit_parent(): """Forks and kills the parent process.""" diff --git a/Providers/nxOMSAutomationWorker/automationworker/3.x/worker/tracer.py b/Providers/nxOMSAutomationWorker/automationworker/3.x/worker/tracer.py index d805f8798..ef1e65403 100644 --- a/Providers/nxOMSAutomationWorker/automationworker/3.x/worker/tracer.py +++ b/Providers/nxOMSAutomationWorker/automationworker/3.x/worker/tracer.py @@ -343,6 +343,17 @@ def log_worker_safe_loop_terminal_exception(exception): trace_generic_hybrid_worker_event_async(5109, inspect.stack()[0][3], message, 1, KEYWORD_ERROR) +def log_worker_certificate_rotation_successful(): + message = "Hybrid worker Certificate rotation completed." + trace_generic_hybrid_worker_event_async(5110, inspect.stack()[0][3], message, 1, KEYWORD_INFO) + + +def log_worker_certificate_rotation_failed(exception): + message = "Hybrid worker Certificate rotation failed. [exception=" + \ + str(exception) + "]" + trace_generic_hybrid_worker_event_async(5111, inspect.stack()[0][3], message, 1, KEYWORD_ERROR) + + # sandbox specific traces # traces in this section are mainly for the sandbox component # diff --git a/Providers/nxOMSAutomationWorker/automationworker/3.x/worker/urllib2httpclient.py b/Providers/nxOMSAutomationWorker/automationworker/3.x/worker/urllib2httpclient.py index d25ddec74..d3ec30620 100644 --- a/Providers/nxOMSAutomationWorker/automationworker/3.x/worker/urllib2httpclient.py +++ b/Providers/nxOMSAutomationWorker/automationworker/3.x/worker/urllib2httpclient.py @@ -12,6 +12,8 @@ from httpclient import * from workerexception import * +import workercertificaterotation +import workerpollingfrequency PY_MAJOR_VERSION = 0 PY_MINOR_VERSION = 1 @@ -19,6 +21,11 @@ SSL_MODULE_NAME = "ssl" +GET_SANDBOX_URL = "GetSandboxActions" +POLLING_FREQUENCY_HEADER = "PollingFrequency" +ROTATE_WORKER_CERTIFICATE_HEADER = "RotateWorkerCertificate" +ENABLE_CERT_ROTATION_FOR_USER_HYBRID_WORKER = 'True' + # On some system the ssl module might be missing try: import ssl @@ -116,19 +123,64 @@ def issue_request(self, url, headers, method=None, data=None): A RequestResponse :param method: """ - https_handler = HttpsClientHandler(self.cert_path, self.key_path, self.insecure) - opener = urllib2.build_opener(https_handler) - if self.proxy_configuration is not None: - proxy_handler = urllib2.ProxyHandler({'http': self.proxy_configuration, + + import tracer + + try: + https_handler = HttpsClientHandler(self.cert_path, self.key_path, self.insecure) + opener = urllib2.build_opener(https_handler) + if self.proxy_configuration is not None: + proxy_handler = urllib2.ProxyHandler({'http': self.proxy_configuration, 'https': self.proxy_configuration}) - opener.add_handler(proxy_handler) - req = urllib2.Request(url, data=data, headers=headers) - req.get_method = lambda: method - response = opener.open(req, timeout=30) - opener.close() - https_handler.close() - - return response + opener.add_handler(proxy_handler) + req = urllib2.Request(url, data=data, headers=headers) + req.get_method = lambda: method + response = opener.open(req, timeout=30) + + if(GET_SANDBOX_URL in url): + try: + # Only Linux User Hybrid Worker certificate are rotated as they use self signed cert + if(configuration.get_worker_type()=="diy" and ROTATE_WORKER_CERTIFICATE_HEADER in response.headers): + tracer.log_debug_trace("Enabling certificate rotation for worker") + workercertificaterotation.set_certificate_rotation_header_value(ENABLE_CERT_ROTATION_FOR_USER_HYBRID_WORKER) + except Exception as ex: + tracer.log_debug_trace("[exception=" + str(ex) + "]" + "[stacktrace=" + str(traceback.format_exc()) + "]") + + try: + if(POLLING_FREQUENCY_HEADER in response.headers): + newpollingfrequency = ex.headers[POLLING_FREQUENCY_HEADER] + oldpollingfrequency = str(workerpollingfrequency.get_jrds_get_sandbox_actions_polling_freq()) + + if oldpollingfrequency != newpollingfrequency: + tracer.log_debug_trace("Changing polling frequency of worker from "+ oldpollingfrequency +" to "+ newpollingfrequency) + workerpollingfrequency.set_jrds_sandbox_actions_polling_freq(newpollingfrequency) + except Exception as ex: + tracer.log_debug_trace("[exception=" + str(ex) + "]" + "[stacktrace=" + str(traceback.format_exc()) + "]") + + opener.close() + https_handler.close() + + return response + + except Exception as ex: + if(GET_SANDBOX_URL in url): + # Cases where certificates are invalid (returns 401) or Automation Account of worker is deleted (returns 404), headers are sent as part of GetSandboxActions + # Such workers are stale and Polling frequency is set as per the values returned from the headers + try: + if((ex is not None) and (ex.headers is not None) and (ex.code is not None)) and (POLLING_FREQUENCY_HEADER in ex.headers and (ex.code==401 or ex.code==404)): + newpollingfrequency = ex.headers[POLLING_FREQUENCY_HEADER] + oldpollingfrequency = str(workerpollingfrequency.get_jrds_get_sandbox_actions_polling_freq()) + + if oldpollingfrequency != newpollingfrequency: + tracer.log_debug_trace("Changing polling frequency of worker from "+ oldpollingfrequency +" to "+ newpollingfrequency) + workerpollingfrequency.set_jrds_sandbox_actions_polling_freq(newpollingfrequency) + except Exception as e: + tracer.log_debug_trace("[exception=" + str(e) + "]" + "[stacktrace=" + str(traceback.format_exc()) + "]") + + opener.close() + https_handler.close() + raise ex + def get(self, url, headers=None): """Issues a GET request to the provided url and using the provided headers. diff --git a/Providers/nxOMSAutomationWorker/automationworker/3.x/worker/urllib3HttpClient.py b/Providers/nxOMSAutomationWorker/automationworker/3.x/worker/urllib3HttpClient.py index f23661f6c..06e8c341e 100644 --- a/Providers/nxOMSAutomationWorker/automationworker/3.x/worker/urllib3HttpClient.py +++ b/Providers/nxOMSAutomationWorker/automationworker/3.x/worker/urllib3HttpClient.py @@ -17,6 +17,8 @@ from httpclient import * from workerexception import * +import workercertificaterotation +import workerpollingfrequency PY_MAJOR_VERSION = 0 PY_MINOR_VERSION = 1 @@ -24,6 +26,11 @@ SSL_MODULE_NAME = "ssl" +GET_SANDBOX_URL = "GetSandboxActions" +POLLING_FREQUENCY_HEADER = "PollingFrequency" +ROTATE_WORKER_CERTIFICATE_HEADER = "RotateWorkerCertificate" +ENABLE_CERT_ROTATION_FOR_USER_HYBRID_WORKER = 'True' + # On some system the ssl module might be missing try: import ssl @@ -119,21 +126,65 @@ def issue_request(self, url, headers, method=None, data=None): A RequestResponse :param method: """ - https_handler = HttpsClientHandler(self.cert_path, self.key_path, self.insecure) - opener = urllib.request.build_opener(https_handler) - if self.proxy_configuration is not None: - proxy_handler = urllib.request.ProxyHandler({'http': self.proxy_configuration, + import tracer + try: + https_handler = HttpsClientHandler(self.cert_path, self.key_path, self.insecure) + opener = urllib.request.build_opener(https_handler) + if self.proxy_configuration is not None: + proxy_handler = urllib.request.ProxyHandler({'http': self.proxy_configuration, 'https': self.proxy_configuration}) - opener.add_handler(proxy_handler) - if data is not None: - data = data.encode("utf-8") - req = urllib.request.Request(url, data=data, headers=headers) - req.get_method = lambda: method - response = opener.open(req, timeout=30) - opener.close() - https_handler.close() - - return response + opener.add_handler(proxy_handler) + if data is not None: + data = data.encode("utf-8") + req = urllib.request.Request(url, data=data, headers=headers) + req.get_method = lambda: method + response = opener.open(req, timeout=30) + + if(GET_SANDBOX_URL in url): + try: + # Only Linux User Hybrid Worker certificate are rotated as they use self signed cert + if(configuration.get_worker_type()=="diy" and ROTATE_WORKER_CERTIFICATE_HEADER in response.headers): + tracer.log_debug_trace("Enabling certificate rotation for worker") + workercertificaterotation.set_certificate_rotation_header_value(ENABLE_CERT_ROTATION_FOR_USER_HYBRID_WORKER) + except Exception as ex: + tracer.log_debug_trace("[exception=" + str(ex) + "]" + "[stacktrace=" + str(traceback.format_exc()) + "]") + + try: + if(POLLING_FREQUENCY_HEADER in response.headers): + newpollingfrequency = ex.headers[POLLING_FREQUENCY_HEADER] + oldpollingfrequency = str(workerpollingfrequency.get_jrds_get_sandbox_actions_polling_freq()) + + if oldpollingfrequency != newpollingfrequency: + tracer.log_debug_trace("Changing polling frequency of worker from "+ oldpollingfrequency +" to "+ newpollingfrequency) + workerpollingfrequency.set_jrds_sandbox_actions_polling_freq(newpollingfrequency) + except Exception as ex: + tracer.log_debug_trace("[exception=" + str(ex) + "]" + "[stacktrace=" + str(traceback.format_exc()) + "]") + + opener.close() + https_handler.close() + + return response + + except Exception as ex: + if(GET_SANDBOX_URL in url): + # Cases where certificates are invalid (returns 401) or Automation Account of worker is deleted (returns 404), headers are sent as part of GetSandboxActions + # Such workers are stale and Polling frequency is set as per the values returned from the headers + try: + if((ex is not None) and (ex.headers is not None) and (ex.code is not None)) and (POLLING_FREQUENCY_HEADER in ex.headers and (ex.code==401 or ex.code==404)): + newpollingfrequency = ex.headers[POLLING_FREQUENCY_HEADER] + oldpollingfrequency = str(workerpollingfrequency.get_jrds_get_sandbox_actions_polling_freq()) + + if oldpollingfrequency != newpollingfrequency: + tracer.log_debug_trace("Changing polling frequency of worker from "+ oldpollingfrequency +" to "+ newpollingfrequency) + workerpollingfrequency.set_jrds_sandbox_actions_polling_freq(newpollingfrequency) + except Exception as e: + tracer.log_debug_trace("[exception=" + str(e) + "]" + "[stacktrace=" + str(traceback.format_exc()) + "]") + + opener.close() + https_handler.close() + + raise ex + def get(self, url, headers=None): """Issues a GET request to the provided url and using the provided headers. diff --git a/Providers/nxOMSAutomationWorker/automationworker/3.x/worker/workercertificaterotation.py b/Providers/nxOMSAutomationWorker/automationworker/3.x/worker/workercertificaterotation.py new file mode 100644 index 000000000..9b6a33fb0 --- /dev/null +++ b/Providers/nxOMSAutomationWorker/automationworker/3.x/worker/workercertificaterotation.py @@ -0,0 +1,101 @@ +#!/usr/bin/env python3 +# ==================================== +# Copyright (c) Microsoft Corporation. All rights reserved. +# ==================================== + +""" Contains functions to rotate worker certificate """ + +import configparser +import os +import subprocess +import linuxutil +import diydirs +import configuration3 + +DIY_STATE_PATH = diydirs.DIY_STATE_PATH + +SHOULD_WORKER_CERT_ROTATE = ['False'] + +def generate_cert_rotation_self_signed_certificate(): + """Creates a self-signed x509 certificate and key. + + Returns: + temp_certificate_path : string, the path of the new certificate + temp_key_path : string, the path of the new key + """ + + import tracer + + tracer.log_debug_trace("Creating Certificate/Key") + temp_certificate_path = os.path.join(DIY_STATE_PATH, "worker_diy_temp.crt") + temp_key_path = os.path.join(DIY_STATE_PATH, "worker_diy_temp.key") + cmd = ["openssl", "req", "-subj", + "/C=US/ST=Washington/L=Redmond/O=Microsoft Corporation/OU=Azure Automation/CN=Hybrid Runbook Worker", + "-new", "-newkey", "rsa:2048", "-days", "365", "-nodes", "-x509", "-keyout", temp_key_path, "-out", + temp_certificate_path] + process, certificate_creation_output, error = linuxutil.popen_communicate(cmd) + error = error.decode() if isinstance(error, bytes) else error + if process.returncode != 0: + raise Exception("Unable to create certificate/key. " + str(error)) + import tracer + tracer.log_debug_trace("Certificate/Key created for initiating certificate rotation") + + return temp_certificate_path, temp_key_path + + +def clean_up_certificate_and_key(temp_certificate_path, temp_key_path): + """ Delete the temporary certificate/key. + + Args: + temp_certificate_path : string, the path of the certificate + temp_key_path : string, the path of the key + """ + import tracer + + tracer.log_debug_trace("Cleaning up the certificate/key generated for certificate rotation") + subprocess.call(["sudo", "rm", temp_certificate_path]) + subprocess.call(["sudo", "rm", temp_key_path]) + tracer.log_debug_trace("Clean up of certificate/key generated for certificate rotation completed") + + +def replace_self_signed_certificate_and_key(temp_certificate_path, temp_key_path, thumbprint): + """ Replace the old certificate/key with new certificate/key and update worker.conf with latest thumbprint. + + Args: + temp_certificate_path : string, the path of the certificate + temp_key_path : string, the path of the key + """ + import tracer + + tracer.log_debug_trace("Replacing the old certificate/key with newly generated certificate/key") + old_certificate_path = os.path.join(DIY_STATE_PATH, "worker_diy.crt") + old_key_path = os.path.join(DIY_STATE_PATH, "worker_diy.key") + subprocess.call(["sudo", "mv", "-f", temp_certificate_path, old_certificate_path]) + subprocess.call(["sudo", "mv", "-f", temp_key_path, old_key_path]) + tracer.log_debug_trace("Worker certificate/key is updated with the latest one.") + + tracer.log_debug_trace("Updating worker.conf with latest thumbprint.") + worker_conf_path = os.path.join(DIY_STATE_PATH, "worker.conf") + + config = configparser.ConfigParser() + if os.path.isfile(worker_conf_path): + config.read(worker_conf_path) + conf_file = open(worker_conf_path, 'w') + + registration_metadata_section = "registration-metadata" + if not config.has_section(registration_metadata_section): + config.add_section(registration_metadata_section) + config.set(registration_metadata_section, configuration3.CERTIFICATE_THUMBPRINT, thumbprint) + + config.write(conf_file) + conf_file.close() + + tracer.log_debug_trace("Worker.conf is updated with newest thumbprint") + + +def get_certificate_rotation_header_value(): + return SHOULD_WORKER_CERT_ROTATE[0] + + +def set_certificate_rotation_header_value(shouldworkercertificaterotate): + SHOULD_WORKER_CERT_ROTATE[0] = shouldworkercertificaterotate \ No newline at end of file diff --git a/Providers/nxOMSAutomationWorker/automationworker/3.x/worker/workerpollingfrequency.py b/Providers/nxOMSAutomationWorker/automationworker/3.x/worker/workerpollingfrequency.py new file mode 100644 index 000000000..99d863989 --- /dev/null +++ b/Providers/nxOMSAutomationWorker/automationworker/3.x/worker/workerpollingfrequency.py @@ -0,0 +1,22 @@ +#!/usr/bin/env python3 +# ==================================== +# Copyright (c) Microsoft Corporation. All rights reserved. +# ==================================== + +"""Contains functions to get polling frequency value and set values based on the header received as part of GetSandboxActions """ + +JRDS_SANDBOX_POLLING_FREQUENCY = ['30'] +DEFAULT_JRDS_SANDBOX_POLLING_FREQUENCY = '30' + +def set_jrds_sandbox_actions_polling_freq(pollingfrequency): + if type(pollingfrequency) != str: + try: + JRDS_SANDBOX_POLLING_FREQUENCY[0] = str(pollingfrequency) + except: + JRDS_SANDBOX_POLLING_FREQUENCY[0] = DEFAULT_JRDS_SANDBOX_POLLING_FREQUENCY + else: + JRDS_SANDBOX_POLLING_FREQUENCY[0] = pollingfrequency + + +def get_jrds_get_sandbox_actions_polling_freq(): + return int(JRDS_SANDBOX_POLLING_FREQUENCY[0]) \ No newline at end of file diff --git a/Providers/nxOMSAutomationWorker/automationworker/scripts/onboarding2.py b/Providers/nxOMSAutomationWorker/automationworker/scripts/onboarding2.py index cb09ff307..af486f5a6 100644 --- a/Providers/nxOMSAutomationWorker/automationworker/scripts/onboarding2.py +++ b/Providers/nxOMSAutomationWorker/automationworker/scripts/onboarding2.py @@ -251,7 +251,7 @@ def register(options): print("Cannot create directory for certs/conf. Because of the following exception : " + str(ex)) return generate_self_signed_certificate(certificate_path=certificate_path, key_path=key_path) - issuer, subject, thumbprint = linuxutil.get_cert_info(certificate_path) + issuer, subject, thumbprint, not_before, not_after = linuxutil.get_cert_info(certificate_path) # try to extract optional metadata unknown = "Unknown" @@ -280,7 +280,9 @@ def register(options): "OperatingSystem": 2, "SMBIOSAssetTag": asset_tag, "VirtualMachineId": vm_id, - "Subject": subject} + "Subject": subject, + "NotBeforeUtc": not_before, + "NotAfterUtc": not_after} # the signature generation is based on agent service contract payload_hash = sha256_digest(payload) @@ -345,7 +347,7 @@ def deregister(options): if os.path.exists(certificate_path) is False or os.path.exists(key_path) is False: raise Exception("Unable to deregister, no worker certificate/key found on disk.") - issuer, subject, thumbprint = linuxutil.get_cert_info(certificate_path) + issuer, subject, thumbprint, not_before, not_after = linuxutil.get_cert_info(certificate_path) if os.path.exists(worker_conf_path) is False: raise Exception("Missing worker configuration.") diff --git a/Providers/nxOMSAutomationWorker/automationworker/scripts/register_oms.py b/Providers/nxOMSAutomationWorker/automationworker/scripts/register_oms.py index ef0fd5a39..25e41ac13 100644 --- a/Providers/nxOMSAutomationWorker/automationworker/scripts/register_oms.py +++ b/Providers/nxOMSAutomationWorker/automationworker/scripts/register_oms.py @@ -161,7 +161,7 @@ def get_headers_and_payload(worker_group_name, is_azure_vm, vm_id, azure_resourc Returns: A tuple containing a dictionary for the request headers and a dictionary for the payload (request body). """ - issuer, subject, thumbprint = linuxutil.get_cert_info(certificate_path) + issuer, subject, thumbprint, not_before, not_after = linuxutil.get_cert_info(certificate_path) headers = {"ProtocolVersion": "2.0", "x-ms-date": datetime.datetime.utcnow().isoformat() + "0-00:00", "Content-Type": "application/json"} diff --git a/Providers/nxOMSAutomationWorker/automationworker/worker/configuration.py b/Providers/nxOMSAutomationWorker/automationworker/worker/configuration.py index 33a84c9de..398d4f5e8 100644 --- a/Providers/nxOMSAutomationWorker/automationworker/worker/configuration.py +++ b/Providers/nxOMSAutomationWorker/automationworker/worker/configuration.py @@ -56,8 +56,8 @@ DEFAULT_VM_ID = DEFAULT_UNKNOWN DEFAULT_WORKER_TYPE = DEFAULT_UNKNOWN DEFAULT_COMPONENT = DEFAULT_UNKNOWN -DEFAULT_WORKER_VERSION = "1.8.0.0" -DEFAULT_JRDS_POLLING_FREQUENCY = "15" +DEFAULT_WORKER_VERSION = "1.9.0.0" +DEFAULT_JRDS_POLLING_FREQUENCY = "30" # state configuration keys STATE_PID = "pid" @@ -184,10 +184,6 @@ def get_value(key): raise KeyError("Configuration environment variable not found. [key=" + key + "].") -def get_jrds_get_sandbox_actions_polling_freq(): - return get_value(JRDS_POLLING_FREQUENCY) - - def get_jrds_get_job_actions_polling_freq(): return get_value(JRDS_POLLING_FREQUENCY) diff --git a/Providers/nxOMSAutomationWorker/automationworker/worker/configuration2.py b/Providers/nxOMSAutomationWorker/automationworker/worker/configuration2.py index 87400fe0c..130855304 100644 --- a/Providers/nxOMSAutomationWorker/automationworker/worker/configuration2.py +++ b/Providers/nxOMSAutomationWorker/automationworker/worker/configuration2.py @@ -55,8 +55,8 @@ DEFAULT_VM_ID = DEFAULT_UNKNOWN DEFAULT_WORKER_TYPE = DEFAULT_UNKNOWN DEFAULT_COMPONENT = DEFAULT_UNKNOWN -DEFAULT_WORKER_VERSION = "1.8.0.0" -DEFAULT_JRDS_POLLING_FREQUENCY = "15" +DEFAULT_WORKER_VERSION = "1.9.0.0" +DEFAULT_JRDS_POLLING_FREQUENCY = "30" # state configuration keys STATE_PID = "pid" @@ -183,10 +183,6 @@ def get_value(key): raise KeyError("Configuration environment variable not found. [key=" + key + "].") -def get_jrds_get_sandbox_actions_polling_freq(): - return get_value(JRDS_POLLING_FREQUENCY) - - def get_jrds_get_job_actions_polling_freq(): return get_value(JRDS_POLLING_FREQUENCY) diff --git a/Providers/nxOMSAutomationWorker/automationworker/worker/diydirs.py b/Providers/nxOMSAutomationWorker/automationworker/worker/diydirs.py index e52759c10..ccebef373 100644 --- a/Providers/nxOMSAutomationWorker/automationworker/worker/diydirs.py +++ b/Providers/nxOMSAutomationWorker/automationworker/worker/diydirs.py @@ -5,7 +5,7 @@ """Utility for DIY hybrid worker directories""" -from worker import linuxutil +import linuxutil import os NXAUTOMATION_HOME_DIR = "/home/nxautomation" diff --git a/Providers/nxOMSAutomationWorker/automationworker/worker/hybridworker.py b/Providers/nxOMSAutomationWorker/automationworker/worker/hybridworker.py index 7659a0e17..b17b63e86 100644 --- a/Providers/nxOMSAutomationWorker/automationworker/worker/hybridworker.py +++ b/Providers/nxOMSAutomationWorker/automationworker/worker/hybridworker.py @@ -11,6 +11,7 @@ import threading import time import traceback +import workerpollingfrequency # import worker module after linuxutil.daemonize() call @@ -23,8 +24,9 @@ def decorated_func(*args, **kwargs): try: # ensure required file / cert exists func(*args, **kwargs) - except (JrdsAuthorizationException, - InvalidFilePermissionException, + except (JrdsAuthorizationException): + tracer.log_worker_safe_loop_terminal_exception(traceback.format_exc()) + except (InvalidFilePermissionException, FileNotFoundException, SystemExit): tracer.log_worker_safe_loop_terminal_exception(traceback.format_exc()) @@ -32,7 +34,7 @@ def decorated_func(*args, **kwargs): sys.exit(-1) except Exception: tracer.log_worker_safe_loop_non_terminal_exception(traceback.format_exc()) - time.sleep(configuration.get_jrds_get_sandbox_actions_polling_freq()) + time.sleep(workerpollingfrequency.get_jrds_get_sandbox_actions_polling_freq()) #polling frequency as per the value received from headers of GetSandboxActions return decorated_func diff --git a/Providers/nxOMSAutomationWorker/automationworker/worker/jrdsclient.py b/Providers/nxOMSAutomationWorker/automationworker/worker/jrdsclient.py index 43eef9394..a11259a2c 100644 --- a/Providers/nxOMSAutomationWorker/automationworker/worker/jrdsclient.py +++ b/Providers/nxOMSAutomationWorker/automationworker/worker/jrdsclient.py @@ -6,14 +6,19 @@ from datetime import datetime import time +import traceback import configuration import locallogger +import linuxutil +import workercertificaterotation from workerexception import * transient_status_codes = set([408, 429, 500, 502, 503, 504]) +DISABLE_CERT_ROTATION = 'False' + class JRDSClient: def __init__(self, http_client): self.httpClient = http_client @@ -45,7 +50,7 @@ def issue_request(request_function, url): return response def get_sandbox_actions(self): - """Gets any pending sandbox actions. + """Gets any pending sandbox actions and headers which determine polling frequency and certificate rotation of workers Returns: A list of sandbox actions. @@ -68,6 +73,8 @@ def get_sandbox_actions(self): "&api-version=" + self.protocol_version response = self.issue_request(lambda u: self.httpClient.get(u), url) + import tracer + if response.status_code == 200: try: if response.deserialized_data is None or "value" not in response.deserialized_data: @@ -78,10 +85,59 @@ def get_sandbox_actions(self): return None # success path + + # whenever worker cert has crossed half of it's lifetime or server is initiating a forced rotation of certificate based on date, header is set on the server side + # based on the headers client initiates worker certificate rotation + + try: + if eval(workercertificaterotation.get_certificate_rotation_header_value()): + tracer.log_debug_trace("Initiating certificate rotation of Hybrid Worker") + workercertificaterotation.set_certificate_rotation_header_value(DISABLE_CERT_ROTATION) + self.worker_certificate_rotation() + tracer.log_worker_certificate_rotation_successful() + except Exception as ex: + tracer.log_worker_certificate_rotation_failed(ex) + tracer.log_debug_trace("[exception=" + str(ex) + "]") + return response.deserialized_data["value"] raise Exception("Unable to get sandbox actions. [status=" + str(response.status_code) + "]") + + def worker_certificate_rotation(self): + """ Rotate worker certificate. + Steps includes creating new certificate/key and after JRDS returns 200, replace the old certificate/key with newly generated certificate/key. + Worker.conf is updated with the latest thumbprint. + """ + + import tracer + + try: + temp_certificate_path, temp_key_path = workercertificaterotation.generate_cert_rotation_self_signed_certificate() + issuer, subject, thumbprint, not_before, not_after = linuxutil.get_cert_info(temp_certificate_path) + + payload = {'Thumbprint': thumbprint, + 'Issuer': issuer, + 'Subject': subject, + 'NotBefore': not_before, + 'NotAfter': not_after} + + headers = {"Content-Type": "application/json"} + + url = self.base_uri + "/automationAccounts/" + self.account_id + \ + "/hybridCertificateRotation?api-version=" + self.protocol_version + response = self.issue_request(lambda u: self.httpClient.post(u, headers=headers, data=payload), url) + + if response.status_code == 200: + tracer.log_debug_trace("New worker certificate successfully added in the Database") + workercertificaterotation.replace_self_signed_certificate_and_key(temp_certificate_path, temp_key_path, thumbprint) + except Exception as ex: + tracer.log_debug_trace("[exception=" + str(ex) + "]" + "[stacktrace=" + str(traceback.format_exc()) + "]") + tracer.log_worker_certificate_rotation_failed(ex) + finally: + workercertificaterotation.clean_up_certificate_and_key(temp_certificate_path, temp_key_path) + return + def get_job_actions(self, sandbox_id): """Gets any pending job action for the given sandbox id. diff --git a/Providers/nxOMSAutomationWorker/automationworker/worker/linuxutil.py b/Providers/nxOMSAutomationWorker/automationworker/worker/linuxutil.py index 55479380c..1db66abe8 100644 --- a/Providers/nxOMSAutomationWorker/automationworker/worker/linuxutil.py +++ b/Providers/nxOMSAutomationWorker/automationworker/worker/linuxutil.py @@ -7,6 +7,7 @@ import sys import re import codecs +from datetime import datetime # workaround when unexpected environment variables are present # sets COLUMNS wide enough so that output of ps does not get truncated @@ -272,7 +273,7 @@ def get_cert_info(certificate_path): """Gets certificate information by invoking OpenSSL (OMS agent dependency). Returns: - A tuple containing the certificate's issuer, subject and thumbprint. + A tuple containing the certificate's issuer, subject and thumbprint, start date and end date. """ p = subprocess.Popen(["openssl", "x509", "-noout", "-in", certificate_path, "-fingerprint", "-sha1"], stdout=subprocess.PIPE, @@ -298,9 +299,27 @@ def get_cert_info(certificate_path): if p.poll() != 0: raise Exception("Unable to get certificate subject.") + p = subprocess.Popen(["openssl", "x509", "-noout", "-in", certificate_path, "-startdate"], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + raw_not_before, e = p.communicate() + + if p.poll() != 0: + raise Exception("Unable to get certificate start date.") + + p = subprocess.Popen(["openssl", "x509", "-noout", "-in", certificate_path, "-enddate"], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + raw_not_after, e = p.communicate() + + if p.poll() != 0: + raise Exception("Unable to get certificate end date.") + return parse_issuer_from_openssl_output(raw_issuer), \ parse_subject_from_openssl_output(raw_subject), \ - parse_thumbprint_from_openssl_output(raw_fingerprint) + parse_thumbprint_from_openssl_output(raw_fingerprint), \ + parse_not_before_from_openssl_output(raw_not_before.decode()), \ + parse_not_after_from_openssl_output(raw_not_after.decode()) def parse_thumbprint_from_openssl_output(raw_fingerprint): @@ -351,6 +370,37 @@ def parse_subject_from_openssl_output(raw_subject): return raw_subject.split("subject=")[1].strip() +def parse_not_before_from_openssl_output(raw_not_before): + """Parses the not before value from the raw OpenSSL output. + + Example output from openSSL: + notBefore=Jun 28 15:25:08 2022 GMT + + Returns: + datetime : The certificate not before date. + """ + not_before_date = raw_not_before.split("notBefore=")[1].replace("GMT", "").strip() + datetime_object = datetime.strptime(not_before_date, '%b %d %H:%M:%S %Y') + date_iso_format = datetime_object.isoformat() + return date_iso_format + + +def parse_not_after_from_openssl_output(raw_not_after): + """Parses the not after value from the raw OpenSSL output. + + Example output from openSSL: + notAfter=Jun 30 15:25:08 2022 GMT + + Returns: + datetime : The certificate not after date. + """ + not_after_date = raw_not_after.split("notAfter=")[1].replace("GMT", "").strip() + + datetime_object = datetime.strptime(not_after_date, '%b %d %H:%M:%S %Y') + date_iso_format = datetime_object.isoformat() + return date_iso_format + + @posix_only def fork_and_exit_parent(): """Forks and kills the parent process.""" diff --git a/Providers/nxOMSAutomationWorker/automationworker/worker/tracer.py b/Providers/nxOMSAutomationWorker/automationworker/worker/tracer.py index 7f0d9b05c..4563d29e4 100644 --- a/Providers/nxOMSAutomationWorker/automationworker/worker/tracer.py +++ b/Providers/nxOMSAutomationWorker/automationworker/worker/tracer.py @@ -343,6 +343,17 @@ def log_worker_safe_loop_terminal_exception(exception): trace_generic_hybrid_worker_event_async(5109, inspect.stack()[0][3], message, 1, KEYWORD_ERROR) +def log_worker_certificate_rotation_successful(): + message = "Hybrid worker Certificate rotation completed." + trace_generic_hybrid_worker_event_async(5110, inspect.stack()[0][3], message, 1, KEYWORD_INFO) + + +def log_worker_certificate_rotation_failed(exception): + message = "Hybrid worker Certificate rotation failed. [exception=" + \ + str(exception) + "]" + trace_generic_hybrid_worker_event_async(5111, inspect.stack()[0][3], message, 1, KEYWORD_ERROR) + + # sandbox specific traces # traces in this section are mainly for the sandbox component # diff --git a/Providers/nxOMSAutomationWorker/automationworker/worker/urllib2httpclient.py b/Providers/nxOMSAutomationWorker/automationworker/worker/urllib2httpclient.py index be01d9f5c..c66e9c89b 100644 --- a/Providers/nxOMSAutomationWorker/automationworker/worker/urllib2httpclient.py +++ b/Providers/nxOMSAutomationWorker/automationworker/worker/urllib2httpclient.py @@ -12,6 +12,8 @@ from httpclient import * from workerexception import * +import workercertificaterotation +import workerpollingfrequency PY_MAJOR_VERSION = 0 PY_MINOR_VERSION = 1 @@ -19,6 +21,11 @@ SSL_MODULE_NAME = "ssl" +GET_SANDBOX_URL = "GetSandboxActions" +POLLING_FREQUENCY_HEADER = "PollingFrequency" +ROTATE_WORKER_CERTIFICATE_HEADER = "RotateWorkerCertificate" +ENABLE_CERT_ROTATION_FOR_USER_HYBRID_WORKER = 'True' + # On some system the ssl module might be missing try: import ssl @@ -116,19 +123,61 @@ def issue_request(self, url, headers, method=None, data=None): A RequestResponse :param method: """ - https_handler = HttpsClientHandler(self.cert_path, self.key_path, self.insecure) - opener = urllib2.build_opener(https_handler) - if self.proxy_configuration is not None: - proxy_handler = urllib2.ProxyHandler({'http': self.proxy_configuration, + import tracer + + try: + https_handler = HttpsClientHandler(self.cert_path, self.key_path, self.insecure) + opener = urllib2.build_opener(https_handler) + if self.proxy_configuration is not None: + proxy_handler = urllib2.ProxyHandler({'http': self.proxy_configuration, 'https': self.proxy_configuration}) - opener.add_handler(proxy_handler) - req = urllib2.Request(url, data=data, headers=headers) - req.get_method = lambda: method - response = opener.open(req, timeout=30) - opener.close() - https_handler.close() - - return response + opener.add_handler(proxy_handler) + req = urllib2.Request(url, data=data, headers=headers) + req.get_method = lambda: method + response = opener.open(req, timeout=30) + + if(GET_SANDBOX_URL in url): + try: + # Only Linux User Hybrid Worker certificate are rotated as they use self signed cert + if(configuration.get_worker_type()=="diy" and ROTATE_WORKER_CERTIFICATE_HEADER in response.headers): + tracer.log_debug_trace("Enabling certificate rotation for worker") + workercertificaterotation.set_certificate_rotation_header_value(ENABLE_CERT_ROTATION_FOR_USER_HYBRID_WORKER) + except Exception as ex: + tracer.log_debug_trace("[exception=" + str(ex) + "]" + "[stacktrace=" + str(traceback.format_exc()) + "]") + + try: + if(POLLING_FREQUENCY_HEADER in response.headers): + newpollingfrequency = ex.headers[POLLING_FREQUENCY_HEADER] + oldpollingfrequency = str(workerpollingfrequency.get_jrds_get_sandbox_actions_polling_freq()) + + if oldpollingfrequency != newpollingfrequency: + tracer.log_debug_trace("Changing polling frequency of worker from "+ oldpollingfrequency +" to "+ newpollingfrequency) + workerpollingfrequency.set_jrds_sandbox_actions_polling_freq(newpollingfrequency) + except Exception as ex: + tracer.log_debug_trace("[exception=" + str(ex) + "]" + "[stacktrace=" + str(traceback.format_exc()) + "]") + + opener.close() + https_handler.close() + return response + + except Exception as ex: + if(GET_SANDBOX_URL in url): + # Cases where certificates are invalid (returns 401) or Automation Account of worker is deleted (returns 404), headers are sent as part of GetSandboxActions + # Such workers are stale and Polling frequency is set as per the values returned from the headers + try: + if((ex is not None) and (ex.headers is not None) and (ex.code is not None)) and (POLLING_FREQUENCY_HEADER in ex.headers and (ex.code==401 or ex.code==404)): + newpollingfrequency = ex.headers[POLLING_FREQUENCY_HEADER] + oldpollingfrequency = str(workerpollingfrequency.get_jrds_get_sandbox_actions_polling_freq()) + + if oldpollingfrequency != newpollingfrequency: + tracer.log_debug_trace("Changing polling frequency of worker from "+ oldpollingfrequency +" to "+ newpollingfrequency) + workerpollingfrequency.set_jrds_sandbox_actions_polling_freq(newpollingfrequency) + except Exception as e: + tracer.log_debug_trace("[exception=" + str(e) + "]" + "[stacktrace=" + str(traceback.format_exc()) + "]") + + opener.close() + https_handler.close() + raise ex def get(self, url, headers=None): """Issues a GET request to the provided url and using the provided headers. diff --git a/Providers/nxOMSAutomationWorker/automationworker/worker/urllib3HttpClient.py b/Providers/nxOMSAutomationWorker/automationworker/worker/urllib3HttpClient.py index daa1d3d1a..4a0b59a8e 100644 --- a/Providers/nxOMSAutomationWorker/automationworker/worker/urllib3HttpClient.py +++ b/Providers/nxOMSAutomationWorker/automationworker/worker/urllib3HttpClient.py @@ -16,6 +16,8 @@ from httpclient import * from workerexception import * +import workercertificaterotation +import workerpollingfrequency PY_MAJOR_VERSION = 0 PY_MINOR_VERSION = 1 @@ -23,6 +25,12 @@ SSL_MODULE_NAME = "ssl" +GET_SANDBOX_URL = "GetSandboxActions" +POLLING_FREQUENCY_HEADER = "PollingFrequency" +ROTATE_WORKER_CERTIFICATE_HEADER = "RotateWorkerCertificate" +ENABLE_CERT_ROTATION_FOR_USER_HYBRID_WORKER = 'True' + + # On some system the ssl module might be missing try: import ssl @@ -118,21 +126,65 @@ def issue_request(self, url, headers, method=None, data=None): A RequestResponse :param method: """ - https_handler = HttpsClientHandler(self.cert_path, self.key_path, self.insecure) - opener = urllib.request.build_opener(https_handler) - if self.proxy_configuration is not None: - proxy_handler = urllib.request.ProxyHandler({'http': self.proxy_configuration, + + import tracer + + try: + + https_handler = HttpsClientHandler(self.cert_path, self.key_path, self.insecure) + opener = urllib.request.build_opener(https_handler) + if self.proxy_configuration is not None: + proxy_handler = urllib.request.ProxyHandler({'http': self.proxy_configuration, 'https': self.proxy_configuration}) - opener.add_handler(proxy_handler) - if data is not None: - data = data.encode("utf-8") - req = urllib.request.Request(url, data=data, headers=headers) - req.get_method = lambda: method - response = opener.open(req, timeout=30) - opener.close() - https_handler.close() - - return response + opener.add_handler(proxy_handler) + if data is not None: + data = data.encode("utf-8") + req = urllib.request.Request(url, data=data, headers=headers) + req.get_method = lambda: method + response = opener.open(req, timeout=30) + + if(GET_SANDBOX_URL in url): + try: + # Only Linux User Hybrid Worker certificate are rotated as they use self signed cert + if(configuration.get_worker_type()=="diy" and ROTATE_WORKER_CERTIFICATE_HEADER in response.headers): + tracer.log_debug_trace("Enabling certificate rotation for worker") + workercertificaterotation.set_certificate_rotation_header_value(ENABLE_CERT_ROTATION_FOR_USER_HYBRID_WORKER) + except Exception as ex: + tracer.log_debug_trace("[exception=" + str(ex) + "]" + "[stacktrace=" + str(traceback.format_exc()) + "]") + + try: + if(POLLING_FREQUENCY_HEADER in response.headers): + newpollingfrequency = ex.headers[POLLING_FREQUENCY_HEADER] + oldpollingfrequency = str(workerpollingfrequency.get_jrds_get_sandbox_actions_polling_freq()) + + if oldpollingfrequency != newpollingfrequency: + tracer.log_debug_trace("Changing polling frequency of worker from "+ oldpollingfrequency +" to "+ newpollingfrequency) + workerpollingfrequency.set_jrds_sandbox_actions_polling_freq(newpollingfrequency) + except Exception as ex: + tracer.log_debug_trace("[exception=" + str(ex) + "]" + "[stacktrace=" + str(traceback.format_exc()) + "]") + + opener.close() + https_handler.close() + return response + + except Exception as ex: + if(GET_SANDBOX_URL in url): + # Cases where certificates are invalid (returns 401) or Automation Account of worker is deleted (returns 404), headers are sent as part of GetSandboxActions + # Such workers are stale and Polling frequency is set as per the values returned from the headers + try: + if((ex is not None) and (ex.headers is not None) and (ex.code is not None)) and(POLLING_FREQUENCY_HEADER in ex.headers and (ex.code==401 or ex.code==404)): + newpollingfrequency = ex.headers[POLLING_FREQUENCY_HEADER] + oldpollingfrequency = str(workerpollingfrequency.get_jrds_get_sandbox_actions_polling_freq()) + + if oldpollingfrequency != newpollingfrequency: + tracer.log_debug_trace("Changing polling frequency of worker from "+ oldpollingfrequency +" to "+ newpollingfrequency) + workerpollingfrequency.set_jrds_sandbox_actions_polling_freq(newpollingfrequency) + except Exception as e: + tracer.log_debug_trace("[exception=" + str(e) + "]" + "[stacktrace=" + str(traceback.format_exc()) + "]") + + opener.close() + https_handler.close() + raise ex def get(self, url, headers=None): """Issues a GET request to the provided url and using the provided headers. diff --git a/Providers/nxOMSAutomationWorker/automationworker/worker/workercertificaterotation.py b/Providers/nxOMSAutomationWorker/automationworker/worker/workercertificaterotation.py new file mode 100644 index 000000000..3efc2b859 --- /dev/null +++ b/Providers/nxOMSAutomationWorker/automationworker/worker/workercertificaterotation.py @@ -0,0 +1,99 @@ +#!/usr/bin/env python2 +# +# Copyright (C) Microsoft Corporation, All rights reserved. + +""" Contains functions to rotate worker certificate """ + +import ConfigParser +import os +import subprocess +import linuxutil +import diydirs +import configuration2 + +DIY_STATE_PATH = diydirs.DIY_STATE_PATH + +SHOULD_WORKER_CERT_ROTATE = ['False'] + +def generate_cert_rotation_self_signed_certificate(): + """Creates a self-signed x509 certificate and key. + + Returns: + temp_certificate_path : string, the path of the new certificate + temp_key_path : string, the path of the new key + """ + import tracer + + tracer.log_debug_trace("Creating Certificate/Key") + temp_certificate_path = os.path.join(DIY_STATE_PATH, "worker_diy_temp.crt") + temp_key_path = os.path.join(DIY_STATE_PATH, "worker_diy_temp.key") + cmd = ["openssl", "req", "-subj", + "/C=US/ST=Washington/L=Redmond/O=Microsoft Corporation/OU=Azure Automation/CN=Hybrid Runbook Worker", + "-new", "-newkey", "rsa:2048", "-days", "365", "-nodes", "-x509", "-keyout", temp_key_path, "-out", + temp_certificate_path] + process, certificate_creation_output, error = linuxutil.popen_communicate(cmd) + error = error.decode() if isinstance(error, bytes) else error + if process.returncode != 0: + raise Exception("Unable to create certificate/key. " + str(error)) + import tracer + tracer.log_debug_trace("Certificate/Key created for initiating certificate rotation") + + return temp_certificate_path, temp_key_path + + +def clean_up_certificate_and_key(temp_certificate_path, temp_key_path): + """ Delete the temporary certificate/key + + Args: + temp_certificate_path : string, the path of the certificate + temp_key_path : string, the path of the key + """ + import tracer + + tracer.log_debug_trace("Cleaning up the certificate/key generated for certificate rotation") + subprocess.call(["sudo", "rm", temp_certificate_path]) + subprocess.call(["sudo", "rm", temp_key_path]) + tracer.log_debug_trace("Clean up of certificate/key generated for certificate rotation completed") + + +def replace_self_signed_certificate_and_key(temp_certificate_path, temp_key_path, thumbprint): + """ Replace the old certificate/key with new certificate/key and update worker.conf with latest thumbprint + + Args: + temp_certificate_path : string, the path of the certificate + temp_key_path : string, the path of the key + """ + import tracer + + tracer.log_debug_trace("Replacing the old certificate/key with newly generated certificate/key") + old_certificate_path = os.path.join(DIY_STATE_PATH, "worker_diy.crt") + old_key_path = os.path.join(DIY_STATE_PATH, "worker_diy.key") + subprocess.call(["sudo", "mv", "-f", temp_certificate_path, old_certificate_path]) + subprocess.call(["sudo", "mv", "-f", temp_key_path, old_key_path]) + tracer.log_debug_trace("Worker certificate/key is updated with the latest one.") + + tracer.log_debug_trace("Updating worker.conf with latest thumbprint.") + worker_conf_path = os.path.join(DIY_STATE_PATH, "worker.conf") + + config = ConfigParser.ConfigParser() + if os.path.isfile(worker_conf_path): + config.read(worker_conf_path) + conf_file = open(worker_conf_path, 'w') + + registration_metadata_section = "registration-metadata" + if not config.has_section(registration_metadata_section): + config.add_section(registration_metadata_section) + config.set(registration_metadata_section, configuration2.CERTIFICATE_THUMBPRINT, thumbprint) + + config.write(conf_file) + conf_file.close() + + tracer.log_debug_trace("Worker.conf is updated with newest thumbprint") + + +def get_certificate_rotation_header_value(): + return SHOULD_WORKER_CERT_ROTATE[0] + + +def set_certificate_rotation_header_value(shouldworkercertificaterotate): + SHOULD_WORKER_CERT_ROTATE[0] = shouldworkercertificaterotate \ No newline at end of file diff --git a/Providers/nxOMSAutomationWorker/automationworker/worker/workerpollingfrequency.py b/Providers/nxOMSAutomationWorker/automationworker/worker/workerpollingfrequency.py new file mode 100644 index 000000000..fdb5a4cdb --- /dev/null +++ b/Providers/nxOMSAutomationWorker/automationworker/worker/workerpollingfrequency.py @@ -0,0 +1,21 @@ +#!/usr/bin/env python2 +# +# Copyright (C) Microsoft Corporation, All rights reserved. + +"""Contains functions to get polling frequency value and set values based on the header received as part of GetSandboxActions """ + +JRDS_SANDBOX_POLLING_FREQUENCY = ['30'] +DEFAULT_JRDS_SANDBOX_POLLING_FREQUENCY = '30' + +def set_jrds_sandbox_actions_polling_freq(pollingfrequency): + if type(pollingfrequency) != str: + try: + JRDS_SANDBOX_POLLING_FREQUENCY[0] = str(pollingfrequency) + except: + JRDS_SANDBOX_POLLING_FREQUENCY[0] = DEFAULT_JRDS_SANDBOX_POLLING_FREQUENCY + else: + JRDS_SANDBOX_POLLING_FREQUENCY[0] = pollingfrequency + + +def get_jrds_get_sandbox_actions_polling_freq(): + return int(JRDS_SANDBOX_POLLING_FREQUENCY[0]) \ No newline at end of file