ETL Viewer

.circleci/config.yml

@@ -3,21 +3,35 @@
 # Check https://circleci.com/docs/2.0/language-php/ for more details
 #
 version: 2.1
+
+executors:
+  centos7:
+    docker:
+      - image: tools-ext-01.ccr.xdmod.org/xdmod-9.5.0:centos7.9-1.0
+  centos8: &centos8-executor
+    docker:
+      - image: tools-ext-01.ccr.xdmod.org/centos8-0.1:base
 jobs:
   build:
     parameters:
+      os:
+        type: executor
       install-type:
         type: string
-    docker:
-      # The first image entry here will be used as the image for the parent container.
-      - image: tools-ext-01.ccr.xdmod.org/xdmod-9.5.0:centos7.9-1.0
+    executor: << parameters.os >>
     environment:
       COMPOSER_ALLOW_SUPERUSER: 1
-      XDMOD_REALMS: 'jobs,storage,cloud'
+      XDMOD_REALMS: 'jobs,storage,cloud,gateways'
       XDMOD_IS_CORE: yes
       XDMOD_INSTALL_DIR: /xdmod
       XDMOD_TEST_MODE: << parameters.install-type >>
     steps:
+      - run:
+          name: Generate OpenSSL Key
+          command: openssl genrsa -rand /proc/cpuinfo:/proc/dma:/proc/filesystems:/proc/interrupts:/proc/ioports:/proc/uptime 2048 > /etc/pki/tls/private/localhost.key
+      - run:
+          name: Generate Certificate
+          command: /usr/bin/openssl req -new -key /etc/pki/tls/private/localhost.key -x509 -sha256 -days 365 -set_serial $RANDOM -extensions v3_req -out /etc/pki/tls/certs/localhost.crt -subj "/C=XX/L=Default City/O=Default Company Ltd"
       - checkout
       # We need to update our acct before we can enable docker layer caching.
       #- setup_remote_docker:
@@ -74,6 +88,19 @@ jobs:
       - run: ./tests/integration/runtests.sh --junit-output-dir ~/phpunit
       - run: ./tests/regression/post_ingest_test.sh --junit-output-dir ~/phpunit
       - run: ./tests/component/runtests.sh --junit-output-dir ~/phpunit
+      - when:
+          condition:
+            equal: [ *centos8-executor, << parameters.os >> ]
+          steps:
+            - run:
+                name: Update the chromdriver version for Centos8
+                command: sed -i 's/84.0.1/88.0.0/' /root/project/tests/ui/package.json
+            - run:
+                name: Install new chromdriver
+                command: |
+                  cd /root/project/tests/ui
+                  npm install
+                  cd /root/project
       - run: ./tests/ui/runtests.sh --headless --log-junit ~/phpunit
       - run:
           name: 'Run SSO Tests'
@@ -97,4 +124,8 @@ workflows:
       - build:
           matrix:
             parameters:
+              os: [centos7, centos8]
               install-type: ["fresh_install", "upgrade"]
+            exclude:
+              - os: centos8
+                install-type: "upgrade"

bin/xdmod-ingestor

@@ -20,6 +20,8 @@ ini_set('memory_limit', -1);
 try {
     main();
 } catch (Exception $e) {
+    global $logger;
+
     do {
         fwrite(STDERR, $e->getMessage() . "\n");
         fwrite(STDERR, $e->getTraceAsString() . "\n");

classes/CCR/Logger.php

@@ -16,7 +16,6 @@
  */
 class Logger extends \Monolog\Logger implements LoggerInterface
 {
-
     /**
      * @inheritDoc
      */

classes/Configuration/Configuration.php

@@ -73,7 +73,7 @@
 use stdClass;
 use Traversable;
 
-class Configuration extends Loggable implements iConfiguration
+class Configuration extends Loggable implements iConfiguration, \JsonSerializable
 {
     // NOTE: Any properties that need to be accessed in a transformer defined in a subclass cannot be
     // private.
@@ -1428,4 +1428,9 @@ public function __sleep()
     {
         return array_keys(get_object_vars($this));
     }
+
+    public function jsonSerialize()
+    {
+        return $this->toAssocArray();
+    }
 }  // class Configuration

classes/DataWarehouse/Data/TimeseriesDataset.php

@@ -65,7 +65,8 @@ protected function getSeriesIds($limit, $offset)
         $statement = $this->agg_query->getRawStatement($limit, $offset);
         $statement->execute();
 
-        $groupInstance = reset($this->agg_query->getGroupBys());
+        $groupBys = $this->agg_query->getGroupBys();
+        $groupInstance = reset($groupBys);
         $groupIdColumn = $groupInstance->getId() . '_id';
 
         $seriesIds = array();
@@ -114,8 +115,8 @@ public function getDatasets($limit, $offset, $summarize)
         $summaryDataset = null;
 
         list($timeGroup, $spaceGroup) = $this->getGroupByClasses();
-
-        $statObj = reset($this->query->getStats());
+        $stats = $this->query->getStats();
+        $statObj = reset($stats);
         $seriesIds = $this->getSeriesIds($limit, $offset);
 
         if (!empty($seriesIds)) {
@@ -364,7 +365,8 @@ public function export($export_title = 'title')
 
         $exportData['headers'][] = $timeGroup->getName();
 
-        $stat = reset($this->query->getStats());
+        $stats = $this->query->getStats();
+        $stat = reset($stats);
         $stat_unit  = $stat->getUnit();
 
         $seriesName = $stat->getName();

classes/ETL/DataEndpoint/File.php

@@ -8,7 +8,7 @@
 use ETL\DataEndpoint\DataEndpointOptions;
 use Psr\Log\LoggerInterface;
 
-class File extends aDataEndpoint implements iDataEndpoint
+class File extends aDataEndpoint implements iDataEndpoint, \JsonSerializable
 {
 
     /**
@@ -176,4 +176,15 @@ public function __toString()
     {
         return sprintf('%s (name=%s, path=%s)', get_class($this), $this->name, $this->path);
     }
+
+    public function jsonSerialize()
+    {
+        return array_merge(
+            parent::jsonSerialize(),
+            array(
+                'path' => $this->path,
+                'mode' => $this->mode
+            )
+        );
+    }
 }

classes/ETL/DataEndpoint/aDataEndpoint.php

@@ -11,7 +11,7 @@
 use Exception;
 use Psr\Log\LoggerInterface;
 
-abstract class aDataEndpoint extends aEtlObject
+abstract class aDataEndpoint extends aEtlObject implements \JsonSerializable
 {
     /**
      * @var string The endpoint type (e.g., mysql, pdo, file, url)
@@ -135,4 +135,16 @@ protected function generateUniqueKey()
      */
 
     abstract public function connect();
+
+    public function jsonSerialize()
+    {
+        return array_merge(
+            parent::jsonSerialize(),
+            array(
+                'type' => $this->type,
+                'key' => $this->key,
+                'key_separator' => $this->keySeparator
+            )
+        );
+    }
 }

classes/ETL/DataEndpoint/aRdbmsEndpoint.php

@@ -12,7 +12,7 @@
 use Exception;
 use PDOException;
 
-abstract class aRdbmsEndpoint extends aDataEndpoint
+abstract class aRdbmsEndpoint extends aDataEndpoint implements \JsonSerializable
 {
     /**
      * @ var string The database schema for this endpoint
@@ -357,4 +357,19 @@ abstract public function schemaExists($schemaName = null);
      */
 
     abstract public function createSchema($schemaName = null);
+
+    public function jsonSerialize()
+    {
+        return array_merge(
+            parent::jsonSerialize(),
+            array(
+                'config' => $this->config,
+                'schema' => $this->schema,
+                'host_name' => $this->hostname,
+                'port'=> $this->port,
+                'username'=> $this->username,
+                'create_schema_if_not_exists' => $this->createSchemaIfNotExists
+            )
+        );
+    }
 }

classes/ETL/DataEndpoint/aStructuredFile.php

@@ -13,7 +13,7 @@
 
 use Psr\Log\LoggerInterface;
 
-abstract class aStructuredFile extends File
+abstract class aStructuredFile extends File implements \JsonSerializable
 {
     /**
      * @const integer The default number of bytes for file read operations.
@@ -239,7 +239,7 @@ protected function attachFilters()
                         $this->logger->warning(
                             sprintf("Skipping filter '%s': %s", $config->name, implode(", ", $messages))
                         );
-                        continue;
+                        continue 2;
                     }
                     $filterName = 'xdmod.external_process';
                     $resource = @stream_filter_prepend($fd, $filterName, STREAM_FILTER_READ, $config);
@@ -613,4 +613,22 @@ abstract protected function verifyData();
      */
 
     abstract protected function discoverRecordFieldNames();
+
+    public function jsonSerialize()
+    {
+        return array_merge(
+            parent::jsonSerialize(),
+            array(
+                'record_schema_path' => $this->recordSchemaPath,
+                'filter_list' => $this->filterList,
+                'filter_definitions' => $this->filterDefinitions,
+                'record_separator' => $this->recordSeparator,
+                'field_separator' => $this->fieldSeparator,
+                'has_header' => $this->hasHeaderRecord,
+                'requested_record_field_names' => $this->requestedRecordFieldNames,
+                'discovered_record_field_names' => $this->discoveredRecordFieldNames ,
+                'record_pass_through' => $this->recordPassthrough
+            )
+        );
+    }
 }

classes/ETL/DbModel/Column.php

@@ -391,6 +391,7 @@ public function getSql($includeSchema = false)
                  ( $this->nullable && "NULL" == $this->default ) ||
                  ( "timestamp" == $this->type && is_numeric($this->default) ) ||
                  ( "timestamp" == $this->type && in_array(strtolower($this->default), $currentTimestampAliases) ) ||
+                 ( "datetime" === $this->type && in_array(strtolower($this->default), $currentTimestampAliases) ) ||
                  is_numeric($this->default) ||
                  "b'" == substr($this->default, 0, 2) ||
                  "x'" == substr(strtolower($this->default), 0, 2)

classes/ETL/DbModel/Entity.php

@@ -52,7 +52,7 @@
 use ETL\DataEndpoint;
 use ETL\DataEndpoint\DataEndpointOptions;
 
-class Entity extends Loggable
+class Entity extends Loggable implements \JsonSerializable
 {
     // The list of required properties for this model. If extending classes define their
     // own required properties they should merge them in the constructor by calling
@@ -499,4 +499,14 @@ public function __toString()
     {
         return get_class($this);
     }  // __toString()
+
+    public function jsonSerialize()
+    {
+        return array(
+            'properties' => $this->properties,
+            'required_properties' => $this->requiredProperties,
+            'default_property_values'=> $this->defaultPropertyValues,
+            'system_quote_char' => $this->systemQuoteChar
+        );
+    }
 }  // class Entity

classes/ETL/DbModel/Table.php

@@ -312,17 +312,33 @@ public function discover($source)
         // SMG: We should do a better job of detecting equivalent columns. For example "int unsigned" is
         // equivalent to "int(10) unsigned".
 
-        $sql = "SELECT
-column_name as name, column_type as type, is_nullable as nullable,
-character_set_name as charset,
-collation_name as collation,
-column_default as " . $endpoint->quoteSystemIdentifier("default") . ",
-IF('' = extra, NULL, extra) as extra,
-IF('' = column_comment, NULL, column_comment) as " . $endpoint->quoteSystemIdentifier("comment") . "
+        // NOTE: An additional `IF` statement was added to the `COLUMN_DEFAULT` clause as MariaDB 10.2+ started
+        // reporting default values differently than previous MySQL / MariaDB versions.
+        // Related links:
+        //   - https://jira.mariadb.org/browse/MDEV-15377
+        //   - https://mariadb.com/kb/en/incompatibilities-and-feature-differences-between-mariadb-102-and-mysql-57/
+        //       - "Since MariaDB supports expressions in the DEFAULT clause, in MariaDB, the INFORMATION_SCHEMA.COLUMNS
+        //         table contains extra fields, and also quotes the DEFAULT value of a string in the COLUMN_DEFAULT
+        //         field in order to distinguish it from an expression.
+        $sql = "
+SELECT column_name                                   AS name,
+       column_type                                   AS type,
+       is_nullable                                   AS nullable,
+       character_set_name                            AS charset,
+       collation_name                                AS collation,
+       IF(
+           INSTR(COLUMN_DEFAULT, '\''),
+           SUBSTR(column_default, 2, LENGTH(COLUMN_DEFAULT) - 2),
+           IF(COLUMN_DEFAULT = 'NULL',
+               NULL,
+               COLUMN_DEFAULT)
+           ) as 'default',
+       IF('' = extra, NULL, extra)                   AS extra,
+       IF('' = column_comment, NULL, column_comment) AS 'comment'
 FROM information_schema.columns
 WHERE table_schema = :schema
-AND table_name = :tablename
-ORDER BY ordinal_position ASC";
+  AND table_name = :tablename
+ORDER BY ordinal_position ASC;";
 
         try {
             $result = $endpoint->getHandle()->query($sql, $params);

classes/ETL/EtlOverseerOptions.php

@@ -15,7 +15,7 @@
 use ETL\DbModel\Query;
 use Psr\Log\LoggerInterface;
 
-class EtlOverseerOptions extends \CCR\Loggable
+class EtlOverseerOptions extends \CCR\Loggable implements \JsonSerializable
 {
     // Start of the ETL period
     private $startDate = null;
@@ -953,4 +953,29 @@ private function generateEtlChunkList()
 
         $this->etlPeriodChunkList = $chunkList;
     }  // generateEtlChunkList()
+
+    public function jsonSerialize()
+    {
+        return array(
+            'start_date' => $this->startDate,
+            'end_date' => $this->endDate,
+            'number_of_days'=> $this->numberOfDays,
+            'last_modified_start_date' => $this->lastModifiedStartDate,
+            'last_modified_end_date' => $this->lastModifiedEndDate,
+            'etl_interval_chunk_size_days' => $this->etlIntervalChunkSizeDays,
+            'etl_period_chunk_list' => $this->etlPeriodChunkList,
+            'force_operation' => $this->forceOperation,
+            'include_only_resource_codes' => $this->includeOnlyResourceCodes,
+            'exclude_resource_codes'=> $this->excludeResourceCodes,
+            'lock_dir' => $this->lockDir,
+            'lock_file_prefix' => $this->lockFilePrefix,
+            'default_module_name' => $this->defaultModuleName,
+            'resource_code_to_id_map_sql'=> $this->resourcecodeToIdMapSql,
+            'section_names'=> $this->sectionNames,
+            'action_names' => $this->actionNames,
+            'dry_run' => $this->dryrun,
+            'verbose' => $this->verbose,
+            'supported_overseer_restrictions' => $this->supportedOverseerRestrictions
+        );
+    }
 }  // class EtlOverseerOptions

classes/ETL/JsonPointer.php

@@ -84,9 +84,8 @@ public static function isValidPointer($pointer, $expectedFirstToken = null)
         }
 
         if ( null !== $expectedFirstToken ) {
-            $firstToken = array_shift(
-                array_slice(array_map('urldecode', explode('/', $pointer)), 1)
-            );
+            $tokens = array_slice(array_map('urldecode', explode('/', $pointer)), 1);
+            $firstToken = array_shift($tokens);
             return ( $firstToken == $expectedFirstToken );
         }
         return true;
@@ -106,11 +105,8 @@ public static function getFirstToken($pointer)
         if ( ! static::isValidPointer($pointer) ) {
             return false;
         }
-
-        return array_shift(
-            array_slice(array_map('urldecode', explode('/', $pointer)), 1)
-        );
-
+        $tokens = array_slice(array_map('urldecode', explode('/', $pointer)), 1);
+        return array_shift($tokens);
     }  // getFirstToken()
 
     /** -----------------------------------------------------------------------------------------