Add ability to assign an organization to a resource

classes/OpenXdmod/Migration/Version1100To1150/ConfigFilesMigration.php

@@ -18,9 +18,27 @@ public function execute()
         $this->assertPortalSettingsIsWritable();
         $this->assertModulePortalSettingsAreWritable();
         $this->assertJsonConfigIsWritable('organization');
+        $this->assertJsonConfigIsWritable('resources');
+
         $organization_config = Json::loadFile(CONFIG_DIR . '/organization.json');
+        $resources_config = Json::loadFile(CONFIG_DIR . '/resources.json');
+
+        $organization_abbrev = $organization_config['abbrev'];
         $organization_config = [$organization_config];
         $this->writeJsonConfigFile('organization', $organization_config);
+
+        $resource_config_with_organization = [];
+
+        // Add the organization field to the resource.json file and use
+        // the organization listed in organization.json.
+        foreach($resources_config as $resource)
+        {
+            $resource['organization'] = $organization_abbrev;
+            $resource_config_with_organization[] = $resource;
+        }
+
+        $this->writeJsonConfigFile('resources', $resource_config_with_organization);
+
         $this->writePortalSettingsFile();
         $this->writeModulePortalSettingsFiles();
     }

classes/OpenXdmod/Migration/Version1100To1150/DatabasesMigration.php

@@ -0,0 +1,33 @@
+<?php
+/**
+ * Update database from version 11.0.0 to 11.5.0
+ */
+
+namespace OpenXdmod\Migration\Version1100To1150;
+
+use OpenXdmod\Migration\DatabasesMigration as AbstractDatabasesMigration;
+use ETL\Utilities;
+use CCR\DB;
+use CCR\DB\MySQLHelper;
+
+/**
+ * Migrate databases from version 11.0.0 to 11.5.0
+ */
+class DatabasesMigration extends AbstractDatabasesMigration
+{
+    public function execute()
+    {
+        parent::execute();
+
+        $dbh = DB::factory('datawarehouse');
+        $mysql_helper = MySQLHelper::factory($dbh);
+
+        if ($mysql_helper->tableExists('modw.storagefact')) {
+            Utilities::runEtlPipeline(
+                ['storage-migration-11-0-0_11-5-0', 'xdw-aggregate-storage'],
+                $this->logger,
+                ['last-modified-start-date' => '2017-01-01 00:00:00']
+            );
+        }
+    }
+}

classes/OpenXdmod/Setup/AddResourceSetup.php

@@ -39,9 +39,12 @@ public function handle()
         $typeOptions = array();
         $typeDescriptionText = "";
         $availableTypes = XdmodConfiguration::assocArrayFactory('resource_types.json', CONFIG_DIR)['resource_types'];
+        $organizationOptions = array();
         $resourceAllocationTypeOptions = array();
         $resourceAllocationTypeDescriptionText = "";
         $availableResourceAllocationTypes = XdmodConfiguration::assocArrayFactory('resource_allocation_types.json', CONFIG_DIR)['resource_allocation_types'];
+        $availableOrganizations = XdmodConfiguration::assocArrayFactory('organization.json', CONFIG_DIR);
+        $organizationDescriptionText = "";
         $gpu_nodes = 0;
         $gpus = 0;
         $gpu_ppn = 0;
@@ -64,6 +67,15 @@ public function handle()
             $resourceAllocationTypeDescriptionText .= sprintf("%-10s - %s", $abbrev, $type['description']) . PHP_EOL;
         }
 
+        foreach ( $availableOrganizations as $abbrev => $type ) {
+            if ( 'UNK' == $type['abbrev'] ) {
+                continue;
+            }
+            // Note that Console::prompt() expects lowercase values for options
+            $organizationOptions[] = strtolower($type['abbrev']);
+            $organizationDescriptionText .= sprintf("%-10s - %s", $type['abbrev'], $type['name']) . PHP_EOL;
+        }
+
         $this->console->displaySectionHeader('Add A New Resource');
 
         $this->console->displayMessage(<<<"EOT"
@@ -77,6 +89,9 @@ public function handle()
 
 Available resource allocation types are:
 $resourceAllocationTypeDescriptionText
+
+Available organizations are:
+$organizationDescriptionText
 EOT
         );
         $this->console->displayBlankLine();
@@ -85,6 +100,7 @@ public function handle()
         $name     = $this->console->prompt('Formal Name:');
         $type     = $this->console->prompt('Resource Type:', 'hpc', $typeOptions);
         $resource_allocation_type     = $this->console->prompt('Resource Allocation Type:', 'cpu', $resourceAllocationTypeOptions);
+        $organization = $this->console->prompt('Organization: ', '', $organizationOptions);
         $resource_start_date = $this->getResourceStartDate();
 
         $this->console->displayBlankLine();
@@ -132,7 +148,8 @@ public function handle()
                 'gpu_processor_count' => (int)$gpus,
                 'gpu_node_count'      => (int)$gpu_nodes,
                 'gpu_ppn'             => (int)$gpu_ppn,
-                'start_date'          => $resource_start_date
+                'start_date'          => $resource_start_date,
+                'organization'        => $organization
             )
         );
     }

classes/OpenXdmod/Setup/ResourcesSetup.php

@@ -124,9 +124,11 @@ public function addResource(array $resource)
 
         $availableTypes = XdmodConfiguration::assocArrayFactory('resource_types.json', CONFIG_DIR)['resource_types'];
         $availableResourceAllocationTypes = XdmodConfiguration::assocArrayFactory('resource_allocation_types.json', CONFIG_DIR)['resource_allocation_types'];
+        $organizations = XdmodConfiguration::assocArrayFactory('organization.json', CONFIG_DIR);
 
         $typeAbbrev = 'UNK';
         $resourceAllocationTypeAbbrev = 'UNK';
+        $organizationAbbrev = 'UNK';
 
         foreach($availableTypes as $abbrev => $type) {
             if (strtolower($abbrev) === $resource['type']) {
@@ -142,11 +144,19 @@ public function addResource(array $resource)
             }
         }
 
+        foreach($organizations as $abbrev => $type) {
+            if (strtolower($type['abbrev']) === $resource['organization']) {
+                $organizationAbbrev = $type['abbrev'];
+                break;
+            }
+        }
+
         $this->resources[] = array(
             'resource'                  => $resource['resource'],
             'resource_type'             => $typeAbbrev,
             'name'                      => $resource['name'],
-            'resource_allocation_type'  => $resourceAllocationTypeAbbrev
+            'resource_allocation_type'  => $resourceAllocationTypeAbbrev,
+            'organization'              => $organizationAbbrev
         );
 
         $this->resourceSpecs[] = array(

configuration/datawarehouse.d/ref/Storage-group-bys.json

@@ -17,6 +17,9 @@
     "parentscience": {
         "$ref": "datawarehouse.d/ref/group-by-hierarchy.json#/parentscience"
     },
+    "provider": {
+        "$ref": "datawarehouse.d/ref/group-by-common.json#/provider"
+    },
     "mountpoint": {
         "attribute_table_schema": "modw",
         "attribute_to_aggregate_table_key_map": [

configuration/datawarehouse.d/ref/group-by-common.json

@@ -196,7 +196,7 @@
                 }
             ],
             "orderby": [
-                "order_id"
+                "long_name"
             ],
             "records": {
                 "id": "organization_id",

configuration/etl/etl.d/ingest_resources.json

@@ -47,7 +47,8 @@
                     "pi_column",
                     "shared_jobs",
                     "timezone",
-                    "resource_allocation_type"
+                    "resource_allocation_type",
+                    "organization"
                 ]
             },
             "destination": {

configuration/etl/etl.d/staging.json

@@ -154,7 +154,8 @@
                         "shared_jobs",
                         "timezone",
                         "resource_type",
-                        "resource_allocation_type"
+                        "resource_allocation_type",
+                        "organization"
                     ]
                 }
             }

configuration/etl/etl.d/xdmod-migration_11_0_0-11_5_0.json

@@ -19,6 +19,25 @@
                     "schema": "mod_shredder"
                 }
             }
+        },
+        "storage-migration-11-0-0_11-5-0": {
+            "namespace": "ETL\\Maintenance",
+            "options_class": "MaintenanceOptions",
+            "class": "ExecuteSql",
+            "endpoints": {
+                "source": {
+                    "type": "mysql",
+                    "name": "Datawarehouse",
+                    "config": "datawarehouse",
+                    "schema": "modw"
+                },
+                "destination": {
+                    "type": "mysql",
+                    "name": "Datawarehouse",
+                    "config": "datawarehouse",
+                    "schema": "modw"
+                }
+            }
         }
     },
     "migration-11_0_0-11_5_0": [
@@ -30,20 +49,51 @@
             "options_class": "MaintenanceOptions",
             "definition_file_list": [
                 "common/staging/organization-config.json",
-                "common/staging/organization.json"
+                "common/staging/organization.json",
+                "common/staging/resource-config.json"
             ]
         },
         {
-            "$ref": "${local_config_dir}/organizations.json#/ingest-organizations/0"
+            "$ref": "${local_config_dir}/ingest_resources.json#/ingest-resources/0"
         },
         {
-            "$ref": "${local_config_dir}/organizations.json#/ingest-organizations/1"
+            "$ref": "${local_config_dir}/ingest_resources.json#/ingest-resources/1"
         },
         {
-            "$ref": "${local_config_dir}/organizations.json#/ingest-organizations/2"
+            "$ref": "${local_config_dir}/ingest_resources.json#/ingest-resources/2"
         },
         {
-            "$ref": "${local_config_dir}/organizations.json#/ingest-organizations/3"
+            "$ref": "${local_config_dir}/ingest_resources.json#/ingest-resources/3"
+        },
+        {
+            "$ref": "${local_config_dir}/ingest_resources.json#/ingest-resources/4"
+        },
+        {
+            "$ref": "${local_config_dir}/ingest_resources.json#/ingest-resources/5"
+        },
+        {
+            "$ref": "${local_config_dir}/ingest_resources.json#/ingest-resources/6"
+        },
+        {
+            "$ref": "${local_config_dir}/ingest_resources.json#/ingest-resources/7"
+        }
+    ],
+    "storage-migration-11-0-0_11-5-0": [
+        {
+            "name": "manageStorageTables",
+            "description": "Changes to storage tables",
+            "class": "ManageTables",
+            "definition_file_list": [
+                "storage/xdw/storagefact.json",
+                "storage/xdw/storagefact.aggregation.json"
+            ]
+        },
+        {
+            "name": "updateStorageResourceOrganizationId",
+            "description": "Update resource organization for storage usage to be 1",
+            "sql_file_list": [
+                "migration/update_storage_resource_organization.sql"
+            ]
         }
     ]
 }

configuration/etl/etl_action_defs.d/common/hpcdb/resources.json

@@ -6,7 +6,7 @@
         "records": {
             "resource_id": "r.resource_id",
             "resource_type_id": "COALESCE(srt.resource_type_id, 0)",
-            "organization_id": "1",
+            "organization_id": "COALESCE(org.organization_id, -1)",
             "resource_name": "rc.name",
             "resource_code": "rc.resource",
             "resource_description": "rc.description",
@@ -40,6 +40,13 @@
                 "name": "staging_resource_allocation_type",
                 "alias": "srat",
                 "on": "rc.resource_allocation_type_abbrev = srat.resource_allocation_type_abbrev"
+            },
+            {
+                "type": "LEFT",
+                "schema": "${SOURCE_SCHEMA}",
+                "name": "staging_organization",
+                "alias": "org",
+                "on": "rc.organization = org.organization_abbrev"
             }
 
         ]

configuration/etl/etl_action_defs.d/common/staging/resource-config.json

@@ -10,7 +10,8 @@
             "shared_jobs": "shared_jobs",
             "timezone": "timezone",
             "type_abbrev": "resource_type",
-            "resource_allocation_type_abbrev": "resource_allocation_type"
+            "resource_allocation_type_abbrev": "resource_allocation_type",
+            "organization": "organization"
         }
     }
 }

configuration/etl/etl_action_defs.d/storage/xdw/storagefact.aggregation.json

@@ -19,6 +19,7 @@
             "${AGGREGATION_UNIT}": "${:PERIOD_VALUE}",
             "resource_id": "sf.resource_id",
             "resourcetype_id": "sf.resourcetype_id",
+            "resource_organization_id": "sf.resource_organization_id",
             "mountpoint_id": "sf.mountpoint_id",
             "person_id": "sf.person_id",
             "principalinvestigator_person_id": "sf.principalinvestigator_person_id",
@@ -45,6 +46,7 @@
             "${AGGREGATION_UNIT}",
             "resource_id",
             "resourcetype_id",
+            "resource_organization_id",
             "mountpoint_id",
             "person_id",
             "principalinvestigator_person_id",

configuration/etl/etl_action_defs.d/storage/xdw/storagefact.json

@@ -7,6 +7,7 @@
             "id": "su.storage_usage_id",
             "resource_id": "su.resource_id",
             "resourcetype_id": "res.resource_type_id",
+            "resource_organization_id": "res.organization_id",
             "mountpoint_id": "su.mountpoint_id",
             "person_id": "su.person_id",
             "principalinvestigator_person_id": "pi.person_id",

configuration/etl/etl_schemas.d/common/resources.schema.json

@@ -40,6 +40,11 @@
             "type": "string",
             "description": "Timezone where this resource is located",
             "maxLength": 30
+        },
+        "organization": {
+            "type": "string",
+            "description": "The organization the resource belongs to",
+            "maxLength": 100
         }
     },
     "required": [

configuration/etl/etl_sql.d/migration/update_storage_resource_organization.sql

@@ -0,0 +1 @@
+UPDATE modw.storagefact SET resource_organization_id = 1;

configuration/etl/etl_tables.d/common/staging/resource-config.json

@@ -51,6 +51,12 @@
                 "type": "varchar(30)",
                 "nullable": true,
                 "comment": "Allocation unit type such as CPU, GPU, or Node."
+            },
+            {
+                "name": "organization",
+                "type": "varchar(30)",
+                "nullable": true,
+                "comment": "Organization for the resource"
             }
         ],
         "indexes": [

configuration/etl/etl_tables.d/storage/xdw/storagefact.aggregation.json

@@ -35,6 +35,12 @@
                 "nullable": false,
                 "comment": "The ID of the resource type."
             },
+            {
+                "name": "resource_organization_id",
+                "type": "int(11)",
+                "nullable": false,
+                "comment": "The organization ID of the resource."
+            },
             {
                 "name": "mountpoint_id",
                 "type": "int(11)",

configuration/etl/etl_tables.d/storage/xdw/storagefact.json

@@ -23,6 +23,12 @@
                 "nullable": false,
                 "comment": "Resource ID"
             },
+            {
+                "name": "resource_organization_id",
+                "type": "int(11)",
+                "nullable": false,
+                "comment": "Resource Organization ID"
+            },
             {
                 "name": "resourcetype_id",
                 "type": "int(11)",