Skip to content

Commit

Permalink
Postgresql support
Browse files Browse the repository at this point in the history
Including migration changes and workarounds for auto-incremented columns and large objects.
  • Loading branch information
natechols committed Jun 25, 2019
1 parent 0f1f434 commit 9586639
Show file tree
Hide file tree
Showing 106 changed files with 946 additions and 250 deletions.
3 changes: 3 additions & 0 deletions .travis.yml
Original file line number Diff line number Diff line change
Expand Up @@ -54,6 +54,9 @@ env:
- >-
BUILD_TYPE=centaurLocal
BUILD_MYSQL=5.7
- >-
BUILD_TYPE=centaurLocal
BUILD_POSTGRESQL=11.3
- >-
BUILD_TYPE=centaurPapiV1
BUILD_MYSQL=5.7
Expand Down
7 changes: 7 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -55,6 +55,13 @@ YAML](https://cromwell.readthedocs.io/en/stable/Configuring/#yaml) for more info
* It is now possible to use `includeKey` and `excludeKey` at the same time. If so, the metadata key must match the `includeKey` **and not** match the `excludeKey` to be included.
* It is now possible to use "`calls`" as one of your `excludeKey`s, to request that only workflow metadata gets returned.

### PostgreSQL support

Cromwell now supports PostgreSQL (version 9.6 or higher, with the Large Object
extension installed) as a database backend.
See [here](https://cromwell.readthedocs.io/en/stable/Configuring/#database) for
instructions for configuring the database connection.

## 42 Release Notes

### Womtool endpoint
Expand Down
4 changes: 2 additions & 2 deletions core/src/test/resources/application.conf
Original file line number Diff line number Diff line change
Expand Up @@ -63,13 +63,13 @@ database-test-mariadb {
}
}

database-test-postgres {
database-test-postgresql {
# Run the following to (optionally) drop and (re-)create the database:
# psql postgres <<< 'drop database if exists cromwell_test; create database cromwell_test;'
profile = "slick.jdbc.PostgresProfile$"
db {
driver = "org.postgresql.Driver"
url = "jdbc:postgresql://localhost:5432/cromwell_test"
url = "jdbc:postgresql://localhost:5432/cromwell_test?reWriteBatchedInserts=true"
url = ${?CROMWELL_BUILD_POSTGRES_JDBC_URL}
user = "cromwell"
user = ${?CROMWELL_BUILD_POSTGRES_USERNAME}
Expand Down
13 changes: 13 additions & 0 deletions cromwell.example.backends/cromwell.examples.conf
Original file line number Diff line number Diff line change
Expand Up @@ -630,4 +630,17 @@ database {
# connectionTimeout = 3000
# }
#}

# Postgresql example
#database {
# profile = "slick.jdbc.PostgresProfile$"
# db {
# driver = "org.postgresql.Driver"
# url = "jdbc:postgresql://localhost:5432/cromwell"
# user = ""
# password = ""
# port = 5432
# connectionTimeout = 5000
# }
#}
}
1 change: 1 addition & 0 deletions database/migration/src/main/resources/changelog.xml
Original file line number Diff line number Diff line change
Expand Up @@ -77,6 +77,7 @@
<include file="changesets/change_max_size_workflow_url.xml" relativeToChangelogFile="true" />
<include file="changesets/docker_hash_store_add_size_column.xml" relativeToChangelogFile="true" />
<include file="changesets/enlarge_call_caching_hash_entry_id.xml" relativeToChangelogFile="true" />
<include file="changesets/postgresql_engine_schema.xml" relativeToChangelogFile="true" />
</databaseChangeLog>
<!--
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://www.liquibase.org/xml/ns/dbchangelog http://www.liquibase.org/xml/ns/dbchangelog/dbchangelog-3.3.xsd">

<changeSet id="add_attempt_in_call_caching_entry" author="tjeandet">
<changeSet id="add_attempt_in_call_caching_entry" author="tjeandet" dbms="mysql,hsqldb">
<addColumn tableName="CALL_CACHING_ENTRY">
<column name="JOB_ATTEMPT" type="INT" defaultValue="NULL">
<constraints nullable="true" />
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://www.liquibase.org/xml/ns/dbchangelog http://www.liquibase.org/xml/ns/dbchangelog/dbchangelog-3.3.xsd">

<changeSet author="tjeandet" id="add-attempt-in-execution">
<changeSet author="tjeandet" id="add-attempt-in-execution" dbms="mysql,hsqldb">
<addColumn
tableName="EXECUTION">
<column name="ATTEMPT" type="INT" />
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://www.liquibase.org/xml/ns/dbchangelog http://www.liquibase.org/xml/ns/dbchangelog/dbchangelog-3.3.xsd">

<changeSet author="tjeandet" id="add-index-in-execution">
<changeSet author="tjeandet" id="add-index-in-execution" dbms="mysql,hsqldb">
<addColumn
tableName="EXECUTION">
<column name="INDEX" type="INT" />
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://www.liquibase.org/xml/ns/dbchangelog http://www.liquibase.org/xml/ns/dbchangelog/dbchangelog-3.3.xsd">

<changeSet author="jgentry" id="add-start-end-time-in-execution">
<changeSet author="jgentry" id="add-start-end-time-in-execution" dbms="mysql,hsqldb">
<addColumn tableName="EXECUTION">
<column name="START_DT" type="datetime"/>
</addColumn>
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://www.liquibase.org/xml/ns/dbchangelog http://www.liquibase.org/xml/ns/dbchangelog/dbchangelog-3.3.xsd">

<changeSet author="kshakir" id="add_unique_constraints">
<changeSet author="kshakir" id="add_unique_constraints" dbms="mysql,hsqldb">
<comment>
Adds unique constraints to other tables besides the existing
key WORKFLOW_EXECUTION_AUX.UK_WE_AUX_WORKFLOW_EXECUTION_ID.
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@
<databaseChangeLog xmlns="http://www.liquibase.org/xml/ns/dbchangelog"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://www.liquibase.org/xml/ns/dbchangelog http://www.liquibase.org/xml/ns/dbchangelog/dbchangelog-3.3.xsd">
<changeSet author="mcovarr" id="workflow-name">
<changeSet author="mcovarr" id="workflow-name" dbms="mysql,hsqldb">
<addColumn tableName="WORKFLOW_EXECUTION">
<column name="WORKFLOW_NAME" type="VARCHAR(500)">
<constraints nullable="false" />
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://www.liquibase.org/xml/ns/dbchangelog http://www.liquibase.org/xml/ns/dbchangelog/dbchangelog-3.3.xsd">

<changeSet author="sshah" id="add_workflow_url_in_workflow_store_entry">
<changeSet author="sshah" id="add_workflow_url_in_workflow_store_entry" dbms="mysql,hsqldb">
<addColumn tableName="WORKFLOW_STORE_ENTRY">
<column name="WORKFLOW_URL" type="VARCHAR(255)" />
</addColumn>
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@
<databaseChangeLog xmlns="http://www.liquibase.org/xml/ns/dbchangelog"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://www.liquibase.org/xml/ns/dbchangelog http://www.liquibase.org/xml/ns/dbchangelog/dbchangelog-3.3.xsd">
<changeSet author="rmunshi" id="BACKEND_KV_STORE">
<changeSet author="rmunshi" id="BACKEND_KV_STORE" dbms="mysql,hsqldb">
<comment>
Storage for all Backend K/V Pairs.
</comment>
Expand Down Expand Up @@ -31,7 +31,7 @@
</createTable>
</changeSet>

<changeSet author="rmunshi" id="backend_KV_store_job_key_constraint">
<changeSet author="rmunshi" id="backend_KV_store_job_key_constraint" dbms="mysql,hsqldb">
<addUniqueConstraint
tableName="BACKEND_KV_STORE"
columnNames="WORKFLOW_EXECUTION_UUID, CALL_FQN, JOB_SCATTER_INDEX, JOB_RETRY_ATTEMPT, STORE_KEY"
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://www.liquibase.org/xml/ns/dbchangelog http://www.liquibase.org/xml/ns/dbchangelog/dbchangelog-3.3.xsd">

<changeSet author="tjeandet" id="call_caching_aggregation_entry">
<changeSet author="tjeandet" id="call_caching_aggregation_entry" dbms="mysql,hsqldb">
<comment>
Holds an aggregated version of the hashes for each cache entry.
Split into "BaseAggregation" and "InputFilesAggregation"
Expand All @@ -28,7 +28,7 @@
</createTable>
</changeSet>

<changeSet author="tjeandet" id="call_caching_aggregation_entry_keys_and_indexes">
<changeSet author="tjeandet" id="call_caching_aggregation_entry_keys_and_indexes" dbms="mysql,hsqldb">
<addForeignKeyConstraint baseColumnNames="CALL_CACHING_ENTRY_ID"
baseTableName="CALL_CACHING_AGGREGATION_ENTRY"
constraintName="FK_CALL_CACHING_AGGREGATION_ENTRY_CALL_CACHING_ENTRY_ID"
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,16 +3,16 @@
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://www.liquibase.org/xml/ns/dbchangelog http://www.liquibase.org/xml/ns/dbchangelog/dbchangelog-3.3.xsd">

<property name="boolean.type" value="BOOLEAN" dbms="!mysql"/>
<property name="boolean.type" value="BOOLEAN" dbms="hsqldb"/>
<property name="boolean.type" value="TINYINT" dbms="mysql"/>

<changeSet id="call_caching_fix_boolean" author="mcovarr">
<changeSet id="call_caching_fix_boolean" author="mcovarr" dbms="mysql,hsqldb">
<modifyDataType columnName="ALLOW_RESULT_REUSE"
newDataType="${boolean.type}"
tableName="CALL_CACHING_RESULT_METAINFO"/>
</changeSet>

<changeSet id="call_caching_reuse_default" author="mcovarr">
<changeSet id="call_caching_reuse_default" author="mcovarr" dbms="mysql,hsqldb">
<addDefaultValue columnName="ALLOW_RESULT_REUSE"
defaultValueBoolean="true"
tableName="CALL_CACHING_RESULT_METAINFO"/>
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://www.liquibase.org/xml/ns/dbchangelog http://www.liquibase.org/xml/ns/dbchangelog/dbchangelog-3.3.xsd">

<changeSet author="rmunshi" id="call_caching_job_detritus">
<changeSet author="rmunshi" id="call_caching_job_detritus" dbms="mysql,hsqldb">
<comment>
One row per job detritus file (i.e, one row for the stdout, stderr, etc)
The job detritus key contains the filename, such as hello-stdout, and the job detritus
Expand All @@ -28,13 +28,13 @@
</changeSet>

<!-- Don't want to store the same standard output key twice for each job/RESULT_METAINFO_ID) -->
<changeSet author="rmunshi" id="call_caching_job_detritus_key_uniqueness">
<changeSet author="rmunshi" id="call_caching_job_detritus_key_uniqueness" dbms="mysql,hsqldb">
<addUniqueConstraint constraintName="UK_CALL_CACHING_JOB_DETRITUS_KEY"
tableName="CALL_CACHING_JOB_DETRITUS" columnNames="JOB_DETRITUS_KEY, RESULT_METAINFO_ID" />
</changeSet>

<!-- Make RESULT_METAINFO_ID a FK into CALL_CACHING_RESULT_METAINFO -->
<changeSet author="rmunshi" id="call_caching_job_detritus_foreign_key">
<changeSet author="rmunshi" id="call_caching_job_detritus_foreign_key" dbms="mysql,hsqldb">
<addForeignKeyConstraint constraintName="CCJD_RESULT_METAINFO_ID_FK"
baseTableName="CALL_CACHING_JOB_DETRITUS"
baseColumnNames="RESULT_METAINFO_ID"
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -2,9 +2,9 @@
<databaseChangeLog xmlns="http://www.liquibase.org/xml/ns/dbchangelog"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://www.liquibase.org/xml/ns/dbchangelog http://www.liquibase.org/xml/ns/dbchangelog/dbchangelog-3.3.xsd">
<property name="boolean.type" value="BOOLEAN" dbms="!mysql"/>
<property name="boolean.type" value="BOOLEAN" dbms="hsqldb"/>
<property name="boolean.type" value="TINYINT" dbms="mysql"/>
<changeSet author="chrisl" id="call_result_caching">
<changeSet author="chrisl" id="call_result_caching" dbms="mysql,hsqldb">
<comment>Adding some tracking columns for determining eligibility for Call Result Caching.</comment>
<addColumn tableName="EXECUTION">
<column name="ALLOWS_RESULT_REUSE" type="${boolean.type}" defaultValueBoolean="true" >
Expand Down Expand Up @@ -38,4 +38,4 @@
constraintName="FK_RESULTS_CLONED_FROM"
onDelete="SET NULL" />
</changeSet>
</databaseChangeLog>
</databaseChangeLog>
16 changes: 8 additions & 8 deletions database/migration/src/main/resources/changesets/callcaching.xml
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@
<property name="clob.type" value="LONGTEXT" dbms="mysql"/>
<property name="clob.type" value="LONGVARCHAR" dbms="hsqldb"/>

<changeSet author="chrisl" id="call_caching_result_metainfo">
<changeSet author="chrisl" id="call_caching_result_metainfo" dbms="mysql,hsqldb">
<comment>
One row per cached job result.
Stores meta info about which job the result came from.
Expand Down Expand Up @@ -41,12 +41,12 @@
</changeSet>

<!-- We don't want to store results for the exact same job twice -->
<changeSet author="chrisl" id="call_caching_result_metainfo_uniqueness">
<changeSet author="chrisl" id="call_caching_result_metainfo_uniqueness" dbms="mysql,hsqldb">
<addUniqueConstraint constraintName="UK_CALL_CACHING_RESULT_METAINFO"
tableName="CALL_CACHING_RESULT_METAINFO" columnNames="WORKFLOW_UUID, CALL_FQN, JOB_SCATTER_INDEX" />
</changeSet>

<changeSet author="chrisl" id="call_caching_hashes">
<changeSet author="chrisl" id="call_caching_hashes" dbms="mysql,hsqldb">
<comment>
One row per hashkey per call cache meta info.
Allows us to link hash keys and values to any matching call cache results.
Expand All @@ -71,13 +71,13 @@
</changeSet>

<!-- We don't want to store the same hash key twice for the same RESULT_METAINFO_ID -->
<changeSet author="chrisl" id="call_caching_result_hashes_uniqueness">
<changeSet author="chrisl" id="call_caching_result_hashes_uniqueness" dbms="mysql,hsqldb">
<addUniqueConstraint constraintName="UK_CALL_CACHING_HASH"
tableName="CALL_CACHING_HASH" columnNames="HASH_KEY, RESULT_METAINFO_ID" />
</changeSet>

<!-- Make RESULT_METAINFO_ID a FK into CALL_CACHING_RESULT_METAINFO -->
<changeSet author="chrisl" id="call_caching_result_hashes_uniqueness_foreign_key">
<changeSet author="chrisl" id="call_caching_result_hashes_uniqueness_foreign_key" dbms="mysql,hsqldb">
<addForeignKeyConstraint constraintName="CCH_RESULT_METAINFO_ID_FK"
baseTableName="CALL_CACHING_HASH"
baseColumnNames="RESULT_METAINFO_ID"
Expand All @@ -86,7 +86,7 @@
/>
</changeSet>

<changeSet author="chrisl" id="call_caching_result_simpletons">
<changeSet author="chrisl" id="call_caching_result_simpletons" dbms="mysql,hsqldb">
<comment>
One row per result simpleton in the job result.
Simpleton: a single non-complex WDL value.
Expand Down Expand Up @@ -115,13 +115,13 @@
</changeSet>

<!-- Unlike the metadata journal, we never want to store the same simpleton key twice (for the same RESULT_METAINFO_ID) -->
<changeSet author="chrisl" id="call_caching_result_simpletons_uniqueness">
<changeSet author="chrisl" id="call_caching_result_simpletons_uniqueness" dbms="mysql,hsqldb">
<addUniqueConstraint constraintName="UK_CALL_CACHING_RESULT_SIMPLETON"
tableName="CALL_CACHING_RESULT_SIMPLETON" columnNames="SIMPLETON_KEY, RESULT_METAINFO_ID" />
</changeSet>

<!-- Make RESULT_METAINFO_ID a FK into CALL_CACHING_RESULT_METAINFO -->
<changeSet author="chrisl" id="call_caching_result_simpletons_foreign_key">
<changeSet author="chrisl" id="call_caching_result_simpletons_foreign_key" dbms="mysql,hsqldb">
<addForeignKeyConstraint constraintName="CCRS_RESULT_METAINFO_ID_FK"
baseTableName="CALL_CACHING_RESULT_SIMPLETON"
baseColumnNames="RESULT_METAINFO_ID"
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://www.liquibase.org/xml/ns/dbchangelog http://www.liquibase.org/xml/ns/dbchangelog/dbchangelog-3.3.xsd">

<changeSet author="tjeandet" id="change_execution_unique_constraint">
<changeSet author="tjeandet" id="change_execution_unique_constraint" dbms="mysql,hsqldb">
<comment>
Change unique constraint for Execution Table to include IDX column. For MySQL this requires
first dropping the foreign key constraint, which we then restore after adding back the enhanced
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://www.liquibase.org/xml/ns/dbchangelog http://www.liquibase.org/xml/ns/dbchangelog/dbchangelog-3.3.xsd">

<changeSet author="sshah" id="change_max_size_for_key_and_value_in_label">
<changeSet author="sshah" id="change_max_size_for_key_and_value_in_label" dbms="mysql,hsqldb">
<modifyDataType tableName="CUSTOM_LABEL_ENTRY" columnName="CUSTOM_LABEL_KEY" newDataType="VARCHAR(255)"/>
<modifyDataType tableName="CUSTOM_LABEL_ENTRY" columnName="CUSTOM_LABEL_VALUE" newDataType="VARCHAR(255)"/>
</changeSet>
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://www.liquibase.org/xml/ns/dbchangelog http://www.liquibase.org/xml/ns/dbchangelog/dbchangelog-3.3.xsd">

<changeSet author="sshah" id="change_max_size_for_workflow_url">
<changeSet author="sshah" id="change_max_size_for_workflow_url" dbms="mysql,hsqldb">
<modifyDataType tableName="WORKFLOW_STORE_ENTRY" columnName="WORKFLOW_URL" newDataType="VARCHAR(2000)"/>
</changeSet>

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://www.liquibase.org/xml/ns/dbchangelog http://www.liquibase.org/xml/ns/dbchangelog/dbchangelog-3.3.xsd">

<changeSet id="custom_LABEL_entry" author="rmunshi">
<changeSet id="custom_LABEL_entry" author="rmunshi" dbms="mysql,hsqldb">
<createTable tableName="CUSTOM_LABEL_ENTRY">
<column autoIncrement="true" name="CUSTOM_LABEL_ENTRY_ID" type="BIGINT">
<constraints primaryKey="true" primaryKeyName="PK_CUSTOM_LABEL_ENTRY"/>
Expand All @@ -20,7 +20,7 @@
</createTable>
</changeSet>

<changeSet author="rmunshi" id="WMSE_workflow_execution_uuid_foreign_key">
<changeSet author="rmunshi" id="WMSE_workflow_execution_uuid_foreign_key" dbms="mysql,hsqldb">
<addForeignKeyConstraint constraintName="FK_CUSTOM_LABEL_ENTRY_WORKFLOW_EXECUTION_UUID"
baseTableName="CUSTOM_LABEL_ENTRY"
baseColumnNames="WORKFLOW_EXECUTION_UUID"
Expand All @@ -30,13 +30,13 @@
/>
</changeSet>

<changeSet author="rmunshi" id="custom_label_entry_index">
<changeSet author="rmunshi" id="custom_label_entry_index" dbms="mysql,hsqldb">
<addUniqueConstraint constraintName="UC_CUSTOM_LABEL_ENTRY_CLK_CLV_WEU"
tableName="CUSTOM_LABEL_ENTRY" columnNames="CUSTOM_LABEL_KEY, CUSTOM_LABEL_VALUE, WORKFLOW_EXECUTION_UUID" />

</changeSet>

<changeSet author="rmunshi" id="modify label key/value data types">
<changeSet author="rmunshi" id="modify label key/value data types" dbms="mysql,hsqldb">
<modifyDataType tableName="CUSTOM_LABEL_ENTRY" columnName="CUSTOM_LABEL_KEY" newDataType="VARCHAR(63)"/>
<modifyDataType tableName="CUSTOM_LABEL_ENTRY" columnName="CUSTOM_LABEL_VALUE" newDataType="VARCHAR(63)"/>
</changeSet>
Expand Down
Loading

0 comments on commit 9586639

Please sign in to comment.