Non-Standard to Standard Concepts mapping API (#2407)

* Batch operation for fetching related standard concepts with reverse mappings
* Optimized SQL for fetching related standard concepts, eliminated vendor-specific string aggregating function
* Renaming the migration script
* Update git actions to use cache@v4.

---------

Co-authored-by: oleg-odysseus <[email protected]>
Co-authored-by: Chris Knoll <[email protected]>

Author: 3 people

.github/workflows/ci.yaml

@@ -30,7 +30,7 @@ jobs:
             java-version: 8
 
       - name: Maven cache
-        uses: actions/cache@v2
+        uses: actions/cache@v4
         with:
             # Cache gradle directories
             path: ~/.m2
@@ -57,7 +57,7 @@ jobs:
       - uses: actions/checkout@v2
 
       - name: Cache Docker layers
-        uses: actions/cache@v2
+        uses: actions/cache@v4
         with:
           path: /tmp/.buildx-cache
           key: ${{ runner.os }}-buildx-${{ github.sha }}

.github/workflows/release.yaml

@@ -26,7 +26,7 @@ jobs:
           java-version: 8
 
       - name: Maven cache
-        uses: actions/cache@v2
+        uses: actions/cache@v4
         with:
           # Cache gradle directories
           path: ~/.m2

src/main/java/org/ohdsi/webapi/service/VocabularyService.java

@@ -3,6 +3,8 @@
 import static org.ohdsi.webapi.service.cscompare.ConceptSetCompareService.CONCEPT_SET_COMPARISON_ROW_MAPPER;
 import static org.ohdsi.webapi.util.SecurityUtils.whitelist;
 
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.databind.ObjectMapper;
 import com.google.common.collect.Lists;
 import com.google.common.collect.Maps;
 
@@ -78,6 +80,7 @@
 import org.springframework.jdbc.core.RowCallbackHandler;
 import org.springframework.jdbc.core.RowMapper;
 import org.springframework.stereotype.Component;
+import org.ohdsi.webapi.vocabulary.MappedRelatedConcept;
 
  /**
   * Provides REST services for working with
@@ -138,7 +141,10 @@ public void customize(CacheManager cacheManager) {
 
   @Autowired
   private ConceptSetCompareService conceptSetCompareService;
-  
+
+  @Autowired
+  private ObjectMapper objectMapper;
+
   @Value("${datasource.driverClassName}")
   private String driver;
 
@@ -827,7 +833,75 @@ public Collection<RelatedConcept> getRelatedConcepts(@PathParam("sourceKey") Str
     return concepts.values();
   }
 
-  /**
+   @POST
+   @Path("{sourceKey}/related-standard")
+   @Produces(MediaType.APPLICATION_JSON)
+   public Collection<MappedRelatedConcept> getRelatedStandardMappedConcepts(@PathParam("sourceKey") String sourceKey, List<Long> allConceptIds) {
+     Source source = getSourceRepository().findBySourceKey(sourceKey);
+     String relatedConceptsSQLPath = "/resources/vocabulary/sql/getRelatedStandardMappedConcepts.sql";
+     String relatedMappedFromIdsSQLPath = "/resources/vocabulary/sql/getRelatedStandardMappedConcepts_getMappedFromIds.sql";
+     String tableQualifier = source.getTableQualifier(SourceDaimon.DaimonType.Vocabulary);
+
+     String[] searchStrings = {"CDM_schema"};
+     String[] replacementStrings = {tableQualifier};
+
+     String[] varNames = {"conceptIdList"};
+
+     final Map<Long, MappedRelatedConcept> resultCombinedMappedConcepts = new HashMap<>();
+     final Map<Long, RelatedConcept> relatedStandardConcepts = new HashMap<>();
+     for(final List<Long> conceptIdsBatch: Lists.partition(allConceptIds, PreparedSqlRender.getParameterLimit(source))) {
+       Object[] varValues = {conceptIdsBatch.toArray()};
+       PreparedStatementRenderer relatedConceptsRenderer = new PreparedStatementRenderer(source, relatedConceptsSQLPath, searchStrings, replacementStrings, varNames, varValues);
+       getSourceJdbcTemplate(source).query(relatedConceptsRenderer.getSql(), relatedConceptsRenderer.getSetter(), (RowMapper<Void>) (resultSet, arg1) -> {
+         addRelationships(relatedStandardConcepts, resultSet);
+         return null;
+       });
+
+       final Map<Long, Set<Long>> relatedNonStandardConceptIdsByStandardId = new HashMap<>();
+
+       PreparedStatementRenderer mappedFromConceptsRenderer = new PreparedStatementRenderer(source, relatedMappedFromIdsSQLPath, searchStrings, replacementStrings, varNames, varValues);
+       getSourceJdbcTemplate(source).query(mappedFromConceptsRenderer.getSql(), mappedFromConceptsRenderer.getSetter(), (RowMapper<Void>) (resultSet, arg1) -> {
+         populateRelatedConceptIds(relatedNonStandardConceptIdsByStandardId, resultSet);
+         return null;
+       });
+
+       enrichResultCombinedMappedConcepts(resultCombinedMappedConcepts, relatedStandardConcepts, relatedNonStandardConceptIdsByStandardId);
+      }
+     return resultCombinedMappedConcepts.values();
+   }
+
+   private void populateRelatedConceptIds(final Map<Long, Set<Long>> mappedConceptsIds, final ResultSet resultSet) throws SQLException {
+     final Long concept_id = resultSet.getLong("CONCEPT_ID");
+     if (!mappedConceptsIds.containsKey(concept_id)) {
+       Set<Long> mappedIds = new HashSet<>();
+       mappedIds.add(resultSet.getLong("MAPPED_FROM_ID"));
+       mappedConceptsIds.put(concept_id,mappedIds);
+     } else {
+       mappedConceptsIds.get(concept_id).add(resultSet.getLong("MAPPED_FROM_ID"));
+     }
+   }
+
+   void enrichResultCombinedMappedConcepts(Map<Long, MappedRelatedConcept> resultCombinedMappedConcepts,
+                                           Map<Long, RelatedConcept> relatedStandardConcepts,
+                                           Map<Long, Set<Long>> relatedNonStandardConceptIdsByStandardId) {
+    relatedNonStandardConceptIdsByStandardId.forEach((standardConceptId, mappedFromIds)->{
+      if(resultCombinedMappedConcepts.containsKey(standardConceptId)){
+        resultCombinedMappedConcepts.get(standardConceptId).mappedFromIds.addAll(mappedFromIds);
+      } else {
+        MappedRelatedConcept mappedRelatedConcept;
+         try {
+           mappedRelatedConcept = objectMapper.readValue(objectMapper.writeValueAsString(relatedStandardConcepts.get(standardConceptId)), MappedRelatedConcept.class);
+           mappedRelatedConcept.mappedFromIds=mappedFromIds;
+           resultCombinedMappedConcepts.put(standardConceptId,mappedRelatedConcept);
+         } catch (JsonProcessingException e) {
+           log.error("Could not convert RelatedConcept to MappedRelatedConcept", e);
+           throw new WebApplicationException(e);
+         }
+      }
+    });
+   }
+
+   /**
    * Get ancestor and descendant concepts for the selected concept identifier 
    * from a source. 
    * 

src/main/java/org/ohdsi/webapi/vocabulary/MappedRelatedConcept.java

@@ -0,0 +1,12 @@
+package org.ohdsi.webapi.vocabulary;
+
+import com.fasterxml.jackson.annotation.JsonInclude;
+import com.fasterxml.jackson.annotation.JsonProperty;
+
+import java.util.Set;
+
+@JsonInclude(JsonInclude.Include.NON_NULL)
+public class MappedRelatedConcept extends RelatedConcept {
+    @JsonProperty("mapped_from")
+    public Set<Long> mappedFromIds;
+}

src/main/resources/db/migration/postgresql/V2.15.0.20241009000001__add_related_standard_concepts_permission_if_not_exists.sql

@@ -0,0 +1,18 @@
+INSERT INTO  ${ohdsiSchema}.sec_permission (id, value, description)
+SELECT nextval('${ohdsiSchema}.sec_permission_id_seq'), 'vocabulary:*:related-standard:post', 'Access related mapped standard concepts resource'
+WHERE NOT EXISTS (
+        SELECT NULL FROM  ${ohdsiSchema}.sec_permission
+        WHERE value = 'vocabulary:*:related-standard:post'
+);
+
+INSERT INTO ${ohdsiSchema}.sec_role_permission(id, role_id, permission_id)
+SELECT nextval('${ohdsiSchema}.sec_role_permission_sequence'), sr.id, sp.id
+FROM ${ohdsiSchema}.sec_permission SP, ${ohdsiSchema}.sec_role sr
+WHERE sp.value IN (
+    'vocabulary:*:related-standard:post'
+    ) AND sr.name IN ('Atlas users')
+  AND NOT EXISTS (
+        SELECT NULL FROM ${ohdsiSchema}.sec_role_permission
+        WHERE permission_id = sp.id and role_id = sr.id);
+
+

src/main/resources/resources/vocabulary/sql/getRelatedStandardMappedConcepts.sql

@@ -0,0 +1,44 @@
+SELECT
+   c.CONCEPT_ID,
+   c.CONCEPT_NAME,
+   COALESCE(c.STANDARD_CONCEPT, 'N') as STANDARD_CONCEPT,
+   COALESCE(c.INVALID_REASON, 'V') as INVALID_REASON,
+   c.CONCEPT_CODE,
+   c.CONCEPT_CLASS_ID,
+   c.DOMAIN_ID,
+   c.VOCABULARY_ID,
+   c.VALID_START_DATE,
+   c.VALID_END_DATE,
+   c.RELATIONSHIP_NAME,
+   c.RELATIONSHIP_DISTANCE
+FROM (
+    SELECT
+        c.CONCEPT_ID, CONCEPT_NAME, COALESCE(c.STANDARD_CONCEPT, 'N') as STANDARD_CONCEPT, COALESCE(c.INVALID_REASON, 'V') as INVALID_REASON,
+        c.CONCEPT_CODE, c.CONCEPT_CLASS_ID, c.DOMAIN_ID, c.VOCABULARY_ID, c.VALID_START_DATE, c.VALID_END_DATE,
+        r.RELATIONSHIP_NAME, 1 as RELATIONSHIP_DISTANCE
+    FROM
+        @CDM_schema.concept_relationship cr
+    JOIN
+        @CDM_schema.concept c ON cr.CONCEPT_ID_2 = c.CONCEPT_ID
+    JOIN
+        @CDM_schema.relationship r ON cr.RELATIONSHIP_ID = r.RELATIONSHIP_ID
+    WHERE
+        cr.CONCEPT_ID_1 IN (@conceptIdList)
+        AND COALESCE(c.STANDARD_CONCEPT, 'N') IN ('S', 'C')
+        AND cr.INVALID_REASON IS NULL
+) c
+GROUP BY
+   c.CONCEPT_ID,
+   c.CONCEPT_NAME,
+   c.STANDARD_CONCEPT,
+   c.INVALID_REASON,
+   c.CONCEPT_CODE,
+   c.CONCEPT_CLASS_ID,
+   c.DOMAIN_ID,
+   c.VOCABULARY_ID,
+   c.VALID_START_DATE,
+   c.VALID_END_DATE,
+   c.RELATIONSHIP_NAME,
+   c.RELATIONSHIP_DISTANCE
+ORDER BY
+   c.RELATIONSHIP_DISTANCE ASC;

src/main/resources/resources/vocabulary/sql/getRelatedStandardMappedConcepts_getMappedFromIds.sql

@@ -0,0 +1,20 @@
+SELECT
+   c.CONCEPT_ID,
+   c.MAPPED_FROM_ID
+FROM (
+    SELECT DISTINCT
+        c.CONCEPT_ID,
+        CAST(cr.CONCEPT_ID_1 AS VARCHAR) as MAPPED_FROM_ID
+    FROM
+        @CDM_schema.concept_relationship cr
+    JOIN
+        @CDM_schema.concept c ON cr.CONCEPT_ID_2 = c.CONCEPT_ID
+    JOIN
+        @CDM_schema.relationship r ON cr.RELATIONSHIP_ID = r.RELATIONSHIP_ID
+    WHERE
+        cr.CONCEPT_ID_1 IN (@conceptIdList)
+        AND COALESCE(c.STANDARD_CONCEPT, 'N') IN ('S', 'C')
+        AND cr.INVALID_REASON IS NULL
+) c
+ORDER BY
+   c.CONCEPT_ID;