HDF2Splunk Updates

apps/frontend/src/components/global/upload_tabs/splunk/AuthStep.vue

@@ -18,20 +18,21 @@
         v-model="hostname"
         label="Hostname"
         for="hostname_field"
+        hint="https://yourdomain.com:8089"
         data-cy="splunkhostname"
       />
     </v-form>
     <v-row class="mx-1">
       <v-btn
         color="primary"
-        class="my-4 mt-0"
+        class="my-4 mt-4"
         data-cy="splunkLoginButton"
         @click="login"
       >
         Login
       </v-btn>
       <v-spacer />
-      <v-btn>
+      <v-btn @click="$emit('show-help')">
         Help
         <v-icon class="ml-2"> mdi-help-circle </v-icon>
       </v-btn>
@@ -68,15 +69,20 @@ export default class AuthStep extends Vue {
       this.username,
       this.password
     );
-    if (await splunkClient.validateCredentials()) {
-      localUsername.set(this.username);
-      localPassword.set(this.password);
-      localHostname.set(this.hostname);
-      SnackbarModule.failure('You have successfully signed in');
-      this.$emit('authenticated', splunkClient);
-    } else {
-      SnackbarModule.failure('Incorrect Username or Password');
-    }
+    splunkClient.validateCredentials().then((result) => {
+      if (result === true) {
+        localUsername.set(this.username);
+        localPassword.set(this.password);
+        localHostname.set(this.hostname);
+        SnackbarModule.notify('You have successfully signed in');
+        this.$emit('authenticated', splunkClient);
+      } else if (result === false) {
+        SnackbarModule.failure('Incorrect Username or Password');
+      } else {
+        SnackbarModule.failure(result);
+        this.$emit('error');
+      }
+    });
   }
 
   /** Init our fields */

apps/frontend/src/components/global/upload_tabs/splunk/FileList.vue

@@ -21,7 +21,7 @@
         :headers="headers"
         item-key="guid"
         :items="executions"
-        :search="search"
+        :loading="loading"
         show-select
       >
         <template #[`item.actions`]="{item}">
@@ -52,12 +52,13 @@ import {
 import _ from 'lodash';
 import Vue from 'vue';
 import Component from 'vue-class-component';
-import {Prop} from 'vue-property-decorator';
+import {Prop, Watch} from 'vue-property-decorator';
 @Component({})
 export default class FileList extends Vue {
   @Prop({type: Object, required: true}) readonly splunkClient!: SplunkClient;
 
-  search = '';
+  search = 'index="*" "meta.subtype"=header';
+  loading = false;
   executions: FileMetaData[] = [];
   selectedExecutions: FileMetaData[] = [];
 
@@ -80,19 +81,34 @@ export default class FileList extends Vue {
     }
   ];
 
+  @Watch('search')
+  async onUpdateSearch() {
+    this.mounted();
+  }
+
   async mounted() {
-    this.executions = await getAllExecutions(this.splunkClient);
+    this.loading = true;
+    this.executions = await getAllExecutions(
+      this.splunkClient,
+      this.search
+    ).then((executions) => {
+      this.loading = false;
+      return executions;
+    });
   }
 
-  loadResults() {
-    const files = this.selectedExecutions.map((execution) => {
-      return getExecution(this.splunkClient, execution.guid).then((result) =>
-        InspecIntakeModule.loadText({
-          text: JSON.stringify(result),
-          filename: _.get(result, 'meta.filename')
-        }).catch((err) => {
-          SnackbarModule.failure(String(err));
-        })
+  async loadResults() {
+    this.loading = true;
+    const files = this.selectedExecutions.map(async (execution) => {
+      return getExecution(this.splunkClient, execution.guid).then(
+        async (result) => {
+          return InspecIntakeModule.loadText({
+            text: JSON.stringify(result),
+            filename: _.get(result, 'meta.filename')
+          }).catch((err) => {
+            SnackbarModule.failure(String(err));
+          });
+        }
       );
     });
     this.$emit('got-files', files);

apps/frontend/src/components/global/upload_tabs/splunk/SplunkReader.vue

@@ -9,7 +9,11 @@
     </v-stepper-header>
     <v-stepper-items>
       <v-stepper-content step="1">
-        <AuthStep @authenticated="onAuthenticationComplete" />
+        <AuthStep
+          @authenticated="onAuthenticationComplete"
+          @error="errorCount += 1"
+          @show-help="errorCount = -1"
+        />
       </v-stepper-content>
       <v-stepper-content step="2">
         <FileList
@@ -20,6 +24,35 @@
         />
       </v-stepper-content>
     </v-stepper-items>
+    <v-overlay
+      :opacity="50"
+      absolute="absolute"
+      :value="errorCount >= 3 || errorCount < 0"
+    >
+      <div class="text-center">
+        <p>
+          <span v-if="errorCount > 0">
+            It seems you may be having trouble using the Splunk toolkit. Are you
+            sure that you have configured it properly?
+          </span>
+          <br />
+          <span>
+            For installation instructions and further information, check here:
+          </span>
+          <v-btn
+            target="_blank"
+            href="https://github.com/mitre/hdf-json-to-splunk/"
+            text
+            color="info"
+            px-0
+          >
+            <v-icon pr-2>mdi-github-circle</v-icon>
+            Splunk HDF Plugin
+          </v-btn>
+        </p>
+        <v-btn color="info" @click="errorCount = 0"> Ok </v-btn>
+      </div>
+    </v-overlay>
   </v-stepper>
 </template>
 <script lang="ts">
@@ -37,6 +70,7 @@ import FileList from './FileList.vue';
 })
 export default class SplunkReader extends Vue {
   step = 1;
+  errorCount = 0;
   splunkClient: SplunkClient | null = null;
 
   onAuthenticationComplete(splunkClient: SplunkClient) {

apps/frontend/src/types/splunk-sdk/index.d.ts

@@ -0,0 +1 @@
+declare module '@mitre/splunk-sdk-no-env';

apps/frontend/src/utilities/splunk_util.ts

@@ -1,6 +1,7 @@
 import axios from 'axios';
 import {ExecJSON} from 'inspecjs';
 import _ from 'lodash';
+import {delay} from './async_util';
 import {basic_auth, group_by, map_hash} from './helper_util';
 
 export type JobID = number;
@@ -37,20 +38,41 @@ export class SplunkClient {
 
   async validateCredentials(): Promise<boolean> {
     return apiClient
-      .get(`${this.host}/services/search/jobs`, {
+      .get(`${this.host}/services/search/jobs?output_mode=json`, {
         headers: {
           Authorization: basic_auth(this.username, this.password)
         },
+        validateStatus: () => true, // Instead of throwing a generic error we can use the response instead to create a better one
         method: 'GET'
       })
-      .then(() => {
-        apiClient.defaults.headers.common['Authorization'] = basic_auth(
-          this.username,
-          this.password
-        );
-        return true;
+      .then((response) => {
+        if (response.status === 200) {
+          apiClient.defaults.headers.common['Authorization'] = basic_auth(
+            this.username,
+            this.password
+          );
+          return true;
+        } else if (response.status === 401) {
+          return false;
+        } else {
+          // Did we get a message from Splunk?
+          if (_.get(response.data, 'messages')) {
+            return `Error Received from Splunk: ${response.data.messages
+              .map((message: {type: string; text: string}) => message.text)
+              .join(', ')}`;
+          }
+          // Something else responded, likely a corporate proxy
+          else {
+            return `Unexpected response code ${
+              response.status
+            } received, are you behind a proxy preventing you from connecting to your Splunk instance? Data received: ${_.truncate(
+              response.data,
+              {length: 256}
+            )}`;
+          }
+        }
       })
-      .catch(() => false);
+      .catch((err) => err);
   }
 }
 
@@ -63,6 +85,7 @@ async function waitForJob(
     .get(`${splunkClient.host}/services/search/jobs/${id}?output_mode=json`)
     .then(({data}) => data.entry[0].content.isDone);
   if (!completed) {
+    await delay(500);
     return waitForJob(splunkClient, id);
   } else {
     return apiClient
@@ -137,10 +160,20 @@ function consolidateFilePayloads(
     (ctrl) => ctrl.meta.profile_sha256
   );
   for (const profile of profileEvents) {
+    profile.controls = [];
     // Get the corresponding controls, and put them into the profile
     const sha = profile.meta.profile_sha256;
     const corrControls = shaGroupedControls[sha] || [];
-    profile.controls.push(...corrControls);
+    profile.controls.push(
+      ...replaceKeyValueDescriptions(
+        corrControls as unknown as (ExecJSON.Control &
+          GenericPayloadWithMetaData & {
+            descriptions?:
+              | {[key: string]: string}
+              | ExecJSON.ControlDescription[];
+          })[]
+      )
+    );
   }
 
   return exec as unknown as ExecJSON.Execution;
@@ -150,23 +183,18 @@ export async function getExecution(
   splunkClient: SplunkClient,
   guid: string
 ): Promise<ExecJSON.Execution> {
-  const jobId = await createSearch(
-    splunkClient,
-    `spath "meta.guid" | search "meta.guid"=${guid}`
-  );
+  const jobId = await createSearch(splunkClient, `"meta.guid"=${guid}`);
   const executionPayloads = waitForJob(splunkClient, jobId);
   return executionPayloads
     .then((payloads) => consolidate_payloads(payloads))
     .then((executions) => executions[0]);
 }
 
 export async function getAllExecutions(
-  splunkClient: SplunkClient
+  splunkClient: SplunkClient,
+  searchQuery: string
 ): Promise<FileMetaData[]> {
-  const jobId = await createSearch(
-    splunkClient,
-    'spath "meta.subtype" | search "meta.subtype"=header'
-  );
+  const jobId = await createSearch(splunkClient, searchQuery);
   return waitForJob(splunkClient, jobId).then(
     (executions: GenericPayloadWithMetaData[]) =>
       executions.map((execution) => execution.meta)
@@ -179,7 +207,7 @@ export async function createSearch(
   // We basically can't, and really shouldn't, do typescript here. Output is changes depending on the job called
   // eslint-disable-next-line @typescript-eslint/no-explicit-any
 ): Promise<JobID> {
-  const fullQuery = `search=search index="hdf" | ${searchString || ''}`;
+  const fullQuery = `search=search ${searchString || ''}`;
   return apiClient({
     method: 'POST',
     url: `${splunkClient.host}/services/search/jobs?output_mode=json`,

libs/hdf-converters/package.json

@@ -21,6 +21,7 @@
     "xml2json": "ts-node data/converters/xml2json.ts"
   },
   "dependencies": {
+    "@mitre/splunk-sdk-no-env": "^1.10.0",
     "@types/csv2json": "^1.4.2",
     "@types/xml2js": "^0.4.9",
     "aws-sdk": "^2.1046.0",
@@ -33,6 +34,7 @@
     "inspecjs": "^2.6.10",
     "lodash": "^4.17.21",
     "moment": "^2.29.1",
+    "winston": "^3.6.0",
     "xml2js": "^0.4.23"
   },
   "devDependencies": {
@@ -48,7 +50,7 @@
   },
   "jest": {
     "rootDir": ".",
-    "testTimeout": 50000,
+    "testTimeout": 10000000,
     "transform": {
       "^.+\\.ts$": "ts-jest"
     }