Merge pull request #627 from erwindon/zzz-orchestrate

add support for orchestrations

TODO.txt

@@ -0,0 +1,12 @@
+Branch zzz-orchestrate
+("zzz" to move it to the bottom of the list)
+PoC for supporting the orchestration mechanism
+
+PRO:
+* previously, orchestrations were not supported
+
+CON:
+* so far, no one really need this
+
+VERDICT:
+Keep until there is an actual request for it

docs/README.md

@@ -42,6 +42,7 @@ such as `yubico_client`, or execution modules such as `boto3_sns`.
 - Keyboard control for top-level navigation
 - Keyboard control to apply templates
 - Choose between live info and cached info for grains/pillar
+- View details of orchestrations and allow to start them
 
 
 ## Quick start using PAM as authentication method
@@ -347,6 +348,22 @@ saltgui_hide_saltenvs:
 Typically only one of these variables should be set.
 Jobs that were started without the `saltenv` parameter are, for this purpose only, assumed to use the value `default` for this parameter. This allows these jobs to be hidden/showed using the same mechanism. SaltGUI does not replicate the internal logic of the salt-master and/or the salt-minion to determine which saltenv would actually have been used for such jobs.
 
+## Orchestrations
+The Orchestrations page shows the available orchestrations with their steps. Name, target and function are listed in separate columns.
+All other details will be visible in the details column. The steps are listed in priority order.
+But note that additional dependencies may cause an alternative execution sequence.
+
+In the configuration files, SaltStack does not clearly distingish between state-configuration and orchestration-configuration.
+SaltGUI only shows information that has the orchestration format.
+
+An orchestration can be executed or tested. The output resembles the output of highstate commands, but now each step is a whole salt command instead of a state.
+Since the orchestration is run by the salt-master, the results are organized for only this host.
+Note that SaltStack uses a slightly different minion-name for that.
+
+Note that each stage is started as a separate job. Neither SaltStack, nor SaltGUI, has information available to somehow group the results.
+
+Unlike the highstate system, there are no events available in the SaltStack that can be used to track the progress of an orchestration.
+
 ## Issues
 The Issues page provides an overview of the system and reports any issues.
 When no issues are found, the list remains empty.

saltgui/static/scripts/Api.js

@@ -254,6 +254,15 @@ export class API {
     return this.apiRequest("POST", "/", params);
   }
 
+  getRunnerStateOrchestrateShowSls () {
+    const params = {
+      "arg": ["*"],
+      "client": "runner",
+      "fun": "state.orchestrate_show_sls"
+    };
+    return this.apiRequest("POST", "/", params);
+  }
+
   getWheelConfigValues () {
     const params = {
       "client": "wheel",

saltgui/static/scripts/Router.js

@@ -18,6 +18,7 @@ import {LogoutPage} from "./pages/Logout.js";
 import {MinionsPage} from "./pages/Minions.js";
 import {NodegroupsPage} from "./pages/Nodegroups.js";
 import {OptionsPage} from "./pages/Options.js";
+import {OrchestrationsPage} from "./pages/Orchestrations.js";
 import {Output} from "./output/Output.js";
 import {PillarsMinionPage} from "./pages/PillarsMinion.js";
 import {PillarsPage} from "./pages/Pillars.js";
@@ -57,6 +58,7 @@ export class Router {
     this._registerPage(Router.templatesPage = new TemplatesPage(this));
     this._registerPage(Router.eventsPage = new EventsPage(this));
     this._registerPage(Router.reactorsPage = new ReactorsPage(this));
+    this._registerPage(Router.orchestrationsPage = new OrchestrationsPage(this));
     this._registerPage(Router.optionsPage = new OptionsPage(this));
     this._registerPage(Router.issuesPage = new IssuesPage(this));
     this._registerPage(Router.logoutPage = new LogoutPage(this));
@@ -208,6 +210,7 @@ export class Router {
     this._registerMenuItem(null, "keys", "keys", "k");
     this._registerMenuItem(null, "jobs", "jobs", "j");
     this._registerMenuItem("jobs", "highstate", "highstate", "h");
+    this._registerMenuItem("jobs", "orchestrations", "orchestrations", "o");
     this._registerMenuItem("jobs", "templates", "templates", "t");
     this._registerMenuItem(null, "events", "events", "e");
     this._registerMenuItem("events", "reactors", "reactors", "r");
@@ -300,8 +303,9 @@ export class Router {
     Router._showMenuItem(pages, Router.beaconsPage);
     Router._showMenuItem(pages, Router.nodegroupsPage);
     Router._showMenuItem(pages, Router.keysPage);
-    Router._showMenuItem(pages, Router.jobsPage, ["highstate", "templates"]);
+    Router._showMenuItem(pages, Router.jobsPage, ["highstate", "orchestrations", "templates"]);
     Router._showMenuItem(pages, Router.highStatePage);
+    Router._showMenuItem(pages, Router.orchestrationsPage);
     Router._showMenuItem(pages, Router.templatesPage);
     Router._showMenuItem(pages, Router.eventsPage, ["reactors"]);
     Router._showMenuItem(pages, Router.reactorsPage);

saltgui/static/scripts/output/Output.js

@@ -418,7 +418,7 @@ export class Output {
 
     let txt = "";
 
-    if ("__sls__" in pTask) {
+    if ("__sls__" in pTask && pTask.__sls__) {
       txt += "\n" + pTask.__sls__.replace(/[.]/g, "/") + ".sls";
     }
 
@@ -906,6 +906,13 @@ export class Output {
 
       let minionResponse = pResponse[minionId];
 
+      if (commandCmd === "runner.state.orchestrate" && minionResponse.return && minionResponse.return.return && minionResponse.return.return.data) {
+        minionResponse = minionResponse.return.return.data[minionId];
+      }
+      if (commandCmd === "runner.state.orchestrate_single" && minionResponse.return && minionResponse.return.return && typeof minionResponse.return.return === "object") {
+        minionResponse = Object.values(minionResponse.return.return)[0];
+      }
+
       const isSuccess = Output._getIsSuccess(minionResponse);
       minionResponse = Output._getMinionResponse(pCommand, minionResponse);
       // provide the same (simplified) object for download
@@ -960,8 +967,10 @@ export class Output {
         // first put all the values in an array
         Object.keys(minionResponse).forEach(
           (taskKey) => {
-            minionResponse[taskKey].___key___ = taskKey;
-            tasks.push(minionResponse[taskKey]);
+            if (typeof minionResponse[taskKey] === "object") {
+              minionResponse[taskKey].___key___ = taskKey;
+              tasks.push(minionResponse[taskKey]);
+            }
           }
         );
         // then sort the array

saltgui/static/scripts/output/OutputHighstate.js

@@ -16,13 +16,17 @@ export class OutputHighstate {
       return false;
     }
     switch (pCommand) {
+    case "runner.state.orchestrate_single":
     case "state.apply":
     case "state.high":
     case "state.highstate":
     case "state.sls":
     case "state.sls_id":
-    case "runners.state.orchestrate":
       break;
+    case "runner.state.orchestrate":
+    case "runners.state.orchestrate":
+      // we need command-names in both variants
+      return true;
     case "state.low":
       // almost, but it is only one task
       // and we can handle only an object with tasks

saltgui/static/scripts/pages/Orchestrations.js

@@ -0,0 +1,25 @@
+/* global */
+
+import {JobsSummaryPanel} from "../panels/JobsSummary.js";
+import {OrchestrationsPanel} from "../panels/Orchestrations.js";
+import {Page} from "./Page.js";
+import {Utils} from "../Utils.js";
+
+export class OrchestrationsPage extends Page {
+
+  constructor (pRouter) {
+    super("orchestrations", "Orchestrations", "page-orchestrations", "button-orchestrations", pRouter);
+
+    this.orchestrations = new OrchestrationsPanel();
+    super.addPanel(this.orchestrations);
+    this.jobs = new JobsSummaryPanel();
+    super.addPanel(this.jobs);
+  }
+
+  /* eslint-disable class-methods-use-this */
+  isVisible () {
+  /* eslint-enable class-methods-use-this */
+    // show orchestrations menu item if orchestrations defined
+    return Utils.getStorageItemBoolean("session", "orchestrations");
+  }
+}

saltgui/static/scripts/panels/Job.js

@@ -144,7 +144,7 @@ export class JobPanel extends Panel {
     return JSON.stringify(pObj);
   }
 
-  static decodeArgumentsArray (rawArguments) {
+  static decodeArgumentsArray (rawArguments, dictsAreKwArgs = false) {
 
     if (rawArguments === undefined) {
       // no arguments
@@ -166,7 +166,9 @@ export class JobPanel extends Panel {
     let ret = "";
     for (const obj of rawArguments) {
       // all KWARGS are one entry in the parameters array
-      if (obj && typeof obj === "object" && "__kwarg__" in obj) {
+      // not all dicts have __kwarg__ to recognize them
+      // in that case, the caller must decide
+      if (obj && typeof obj === "object" && (dictsAreKwArgs || "__kwarg__" in obj)) {
         const keys = Object.keys(obj).sort();
         for (const key of keys) {
           if (key === "__kwarg__") {
@@ -217,14 +219,22 @@ export class JobPanel extends Panel {
 
     // use same formatter as direct commands
     let argumentsText = JobPanel.decodeArgumentsArray(info.Arguments);
+    if (!argumentsText && info.Function.startsWith("runner.")) {
+      // runners keep the given arguments elsewhere
+      const fakeMinion = Object.keys(info.Result)[0];
+      argumentsText = JobPanel.decodeArgumentsArray(info.Result[fakeMinion].return.fun_args, true);
+    }
 
     this.targettype = info["Target-type"];
     if (Array.isArray(info.Target)) {
       this.target = info.Target.join(",");
     } else {
       this.target = info.Target;
     }
-    this.commandtext = info.Function + argumentsText;
+    // runner commands are sometimes "runner." and sometimes "runners.".
+    // it is a big mismatch between documentation and system.
+    // we just compensate for that everywhere.
+    this.commandtext = info.Function.replace(/^runner[.]/, "runners.") + argumentsText;
     this.jobid = pJobId;
     this.minions = info.Minions;
     this.result = info.Result;
@@ -263,8 +273,12 @@ export class JobPanel extends Panel {
     } else if (info.Function.startsWith("wheel.")) {
       minions = ["WHEEL"];
       this.setWarningText("info", "WHEEL jobs are not associated with minions");
-    } else if (info.Function.startsWith("runners.")) {
-      minions = ["RUNNER"];
+    } else if (info.Function.startsWith("runner.")) {
+      if (typeof info.Result === "object") {
+        minions = Object.keys(info.Result);
+      } else {
+        minions = ["RUNNER"];
+      }
       this.setWarningText("info", "RUNNER jobs are not associated with minions");
     } else {
       minions = Object.keys(this.result);

saltgui/static/scripts/panels/Jobs.js

@@ -175,6 +175,8 @@ export class JobsPanel extends Panel {
     this._hideJobs.push("runner.jobs.list_job");
     this._hideJobs.push("runner.jobs.list_jobs");
     this._hideJobs.push("runner.manage.versions");
+    // do not hide "runner.state.orchestrate"
+    this._hideJobs.push("runner.state.orchestrate_show_sls");
     // wheel jobs
     this._hideJobs.push("wheel.config.values");
     this._hideJobs.push("wheel.key.accept");

saltgui/static/scripts/panels/Login.js

@@ -304,13 +304,21 @@ export class LoginPanel extends Panel {
 
     // We need these functions to populate the dropdown boxes
     const wheelConfigValuesPromise = this.api.getWheelConfigValues();
+    const runnerStateOrchestrateShowSlsPromise = this.api.getRunnerStateOrchestrateShowSls();
 
     // these may have been hidden on a previous logout
     Utils.hideAllMenus(false);
 
     // We need these functions to populate the dropdown boxes
+    // or determine visibility of menu items
     wheelConfigValuesPromise.then((pWheelConfigValuesData) => {
       LoginPanel._handleLoginWheelConfigValues(pWheelConfigValuesData);
+      Router.updateMainMenu();
+      return true;
+    }, () => false);
+    runnerStateOrchestrateShowSlsPromise.then((pRunnerStateOrchestrateShowSlsData) => {
+      LoginPanel._handleRunnerStateOrchestrateShowSls(pRunnerStateOrchestrateShowSlsData);
+      Router.updateMainMenu();
       return true;
     }, () => false);
 
@@ -339,6 +347,28 @@ export class LoginPanel extends Panel {
     }, 1000);
   }
 
+  static _handleRunnerStateOrchestrateShowSls (pRunnerStateOrchestrateShowSlsData) {
+    // until we prove it it available
+    Utils.setStorageItem("session", "orchestrations", "false");
+
+    const ret = pRunnerStateOrchestrateShowSlsData.return[0];
+    for (const key in ret) {
+      const obj = ret[key];
+      for (const stepkey in obj) {
+        const step = obj[stepkey].salt;
+        if (step === undefined) {
+          continue;
+        }
+        for (const item of step) {
+          if (item === "function" || item === "state" || item === "runner" || item === "wheel") {
+            Utils.setStorageItem("session", "orchestrations", "true");
+            return;
+          }
+        }
+      }
+    }
+  }
+
   static _handleLoginWheelConfigValues (pWheelConfigValuesData) {
     const wheelConfigValuesData = pWheelConfigValuesData.return[0].data.return;
 
@@ -443,8 +473,6 @@ export class LoginPanel extends Panel {
 
     const fullReturn = wheelConfigValuesData.saltgui_full_return;
     Utils.setStorageItem("session", "full_return", fullReturn);
-
-    Router.updateMainMenu();
   }
 
   _onLoginFailure (error) {