Add support for subqueries in filters (fix #602)

Author: arteymix

gemma-core/src/main/java/ubic/gemma/persistence/service/AbstractFilteringVoEnabledDao.java

@@ -12,6 +12,7 @@
 import ubic.gemma.model.common.Identifiable;
 import ubic.gemma.persistence.util.Filter;
 import ubic.gemma.persistence.util.Sort;
+import ubic.gemma.persistence.util.Subquery;
 
 import javax.annotation.Nullable;
 import javax.annotation.OverridingMethodsMustInvokeSuper;
@@ -53,6 +54,11 @@ private static class Key {
      */
     private final Set<String> filterableProperties = new HashSet<>();
 
+    /**
+     * Subset of {@link #filterableProperties} that should use a subquery for filtering.
+     */
+    private final Set<String> filterablePropertiesViaSubquery = new HashSet<>();
+
     /**
      * Aliases for filterable properties.
      */
@@ -89,15 +95,22 @@ protected class FilterablePropertiesConfigurer {
 
         private final Map<String, Class<?>> entityByPrefix = new HashMap<>();
 
+        public void registerProperty( String propertyName ) {
+            registerProperty( propertyName, false );
+        }
+
         /**
          * Register a given property.
          * @throws IllegalArgumentException if the property is already registered
          */
-        public void registerProperty( String propertyName ) {
+        public void registerProperty( String propertyName, boolean useSubquery ) {
             if ( getFilterablePropertyMeta( propertyName ) == null ) {
                 throw new IllegalArgumentException( "Property %s does not have any associated meta information." );
             }
             if ( filterableProperties.add( propertyName ) ) {
+                if ( useSubquery ) {
+                    filterablePropertiesViaSubquery.add( propertyName );
+                }
                 log.debug( String.format( "Registered property %s.", propertyName ) );
             } else {
                 throw new IllegalArgumentException( String.format( "Filterable property %s is already registered.",
@@ -150,6 +163,10 @@ public void unregisterProperties( Predicate<? super String> predicate ) throws I
             }
         }
 
+        public void registerEntity( String prefix, Class<?> entityClass, int maxDepth ) throws IllegalArgumentException {
+            registerEntity( prefix, entityClass, maxDepth, false );
+        }
+
         /**
          * Register an entity available at a given prefix.
          * <p>
@@ -159,10 +176,11 @@ public void unregisterProperties( Predicate<? super String> predicate ) throws I
          * @param maxDepth    maximum depth for visiting properties. For example, zero would expose no property but the
          *                    entity itself, 1 would expose the properties of the alias, 2 would expose the properties
          *                    of any entity directly related to the given entity, etc.
+         * @param useSubquery whether to use a subquery when filtering by this entity (and its descendant)
          * @throws IllegalArgumentException if no entity of the given type is registered under the given prefix or if
          * the prefix is invalid
          */
-        public void registerEntity( String prefix, Class<?> entityClass, int maxDepth ) throws IllegalArgumentException {
+        public void registerEntity( String prefix, Class<?> entityClass, int maxDepth, boolean useSubquery ) throws IllegalArgumentException {
             if ( !prefix.isEmpty() && !prefix.endsWith( "." ) ) {
                 throw new IllegalArgumentException( "A non-empty prefix must end with a '.' character." );
             }
@@ -190,7 +208,7 @@ public void registerEntity( String prefix, Class<?> entityClass, int maxDepth )
                 Type propertyType = propertyTypes[i];
                 if ( propertyType.isEntityType() ) {
                     if ( maxDepth > 1 ) {
-                        registerEntity( prefix + propertyName + ".", propertyType.getReturnedClass(), maxDepth - 1 );
+                        registerEntity( prefix + propertyName + ".", propertyType.getReturnedClass(), maxDepth - 1, useSubquery );
                     } else {
                         log.debug( String.format( "Max depth reached, will not recurse into %s", propertyName ) );
                     }
@@ -201,7 +219,7 @@ public void registerEntity( String prefix, Class<?> entityClass, int maxDepth )
                     log.debug( String.format( "Property %s%s of type %s was excluded in %s: BLOBs and CLOBs are not exposed by default.",
                             prefix, propertyName, propertyType.getName(), entityClass.getName() ) );
                 } else if ( Filter.getConversionService().canConvert( String.class, propertyType.getReturnedClass() ) ) {
-                    registerProperty( prefix + propertyName );
+                    registerProperty( prefix + propertyName, useSubquery );
                 } else {
                     log.warn( String.format( "Property %s%s of type %s in %s is not supported and will be skipped.",
                             prefix, propertyName, propertyType.getReturnedClass().getName(), entityClass.getName() ) );
@@ -239,16 +257,23 @@ public void unregisterEntity( String prefix, Class<?> entityClass ) {
             }
         }
 
+        /**
+         * @see #registerAlias(String, String, Class, String, int, boolean)
+         */
+        public void registerAlias( String prefix, @Nullable String objectAlias, Class<?> propertyType, @Nullable String aliasFor, int maxDepth ) {
+            registerAlias( prefix, objectAlias, propertyType, aliasFor, maxDepth, false );
+        }
+
         /**
          * Register an alias for a property.
          * <p>
          * This also registers a property under the given prefix as per {@link #registerEntity(String, Class, int)}.
          * @param objectAlias internal alias used to refer to the entity as per {@link Filter#getObjectAlias()}.
          * @see #registerEntity(String, Class, int)
          */
-        public void registerAlias( String prefix, @Nullable String objectAlias, Class<?> propertyType, @Nullable String aliasFor, int maxDepth ) {
+        public void registerAlias( String prefix, @Nullable String objectAlias, Class<?> propertyType, @Nullable String aliasFor, int maxDepth, boolean useSubquery ) {
             filterablePropertyAliases.add( new FilterablePropertyAlias( prefix, objectAlias, propertyType, aliasFor ) );
-            registerEntity( prefix, propertyType, maxDepth );
+            registerEntity( prefix, propertyType, maxDepth, useSubquery );
             log.debug( String.format( "Registered alias for %s (%s) %s.", objectAlias, propertyType.getName(), summarizePrefix( prefix ) ) );
         }
 
@@ -285,13 +310,42 @@ public List<Object> getFilterablePropertyAllowedValues( String propertyName ) th
     @Override
     public final Filter getFilter( String property, Filter.Operator operator, String value ) {
         FilterablePropertyMeta propertyMeta = getFilterablePropertyMeta( property );
-        return Filter.parse( propertyMeta.objectAlias, propertyMeta.propertyName, propertyMeta.propertyType, operator, value, property );
+        return nestIfSubquery( Filter.parse( propertyMeta.objectAlias, propertyMeta.propertyName, propertyMeta.propertyType, operator, value, property ),
+                ( property ) );
     }
 
     @Override
     public final Filter getFilter( String property, Filter.Operator operator, Collection<String> values ) {
         FilterablePropertyMeta propertyMeta = getFilterablePropertyMeta( property );
-        return Filter.parse( propertyMeta.objectAlias, propertyMeta.propertyName, propertyMeta.propertyType, operator, values, property );
+        return nestIfSubquery( Filter.parse( propertyMeta.objectAlias, propertyMeta.propertyName, propertyMeta.propertyType, operator, values, property ),
+                property );
+    }
+
+    private Filter nestIfSubquery( Filter f, String propertyName ) {
+        if ( filterablePropertiesViaSubquery.contains( propertyName ) ) {
+            String entityName = getSessionFactory().getClassMetadata( elementClass ).getEntityName();
+            List<Subquery.Alias> aliases;
+            if ( f.getObjectAlias() != null ) {
+                aliases = null;
+                for ( FilterablePropertyAlias fpa : filterablePropertyAliases ) {
+                    if ( f.getObjectAlias().equals( fpa.getObjectAlias() ) ) {
+                        // FIXME: the prefix is not always a valid path
+                        aliases = Collections.singletonList( new Subquery.Alias( fpa.prefix.substring( 0, fpa.prefix.length() - 1 ), fpa.getObjectAlias() ) );
+                        break;
+                    }
+                }
+                if ( aliases == null ) {
+                    throw new IllegalArgumentException();
+                }
+            } else {
+                // the property refers to the root entity, no need for aliases
+                aliases = Collections.emptyList();
+            }
+            return Filter.by( objectAlias, getIdentifierPropertyName(), Long.class, Filter.Operator.inSubquery,
+                    new Subquery( entityName, getIdentifierPropertyName(), aliases, f ) );
+        } else {
+            return f;
+        }
     }
 
     @Override

gemma-core/src/main/java/ubic/gemma/persistence/service/expression/arrayDesign/ArrayDesignDaoImpl.java

@@ -1084,7 +1084,7 @@ protected void configureFilterableProperties( FilterablePropertiesConfigurer con
         // see https://github.com/PavlidisLab/Gemma/issues/546
         String recursiveProperty = String.join( "|", new String[] { "subsumingArrayDesign", "mergedInto", "alternativeTo" } );
         configurer.unregisterProperties( Pattern.compile( "^(" + recursiveProperty + ")\\.(" + recursiveProperty + ")\\..+$" ).asPredicate() );
-        configurer.registerAlias( "externalReferences.", EXTERNAL_REFERENCE_ALIAS, DatabaseEntry.class, null, 2 );
+        configurer.registerAlias( "externalReferences.", EXTERNAL_REFERENCE_ALIAS, DatabaseEntry.class, null, 2, true );
         configurer.registerAlias( "taxon.", PRIMARY_TAXON_ALIAS, Taxon.class, "primaryTaxon", 2 );
     }
 

gemma-core/src/main/java/ubic/gemma/persistence/service/expression/experiment/ExpressionExperimentDaoImpl.java

@@ -1815,16 +1815,16 @@ protected void configureFilterableProperties( FilterablePropertiesConfigurer con
         configurer.unregisterProperty( "primaryPublication.pubAccession.Uri" );
 
         // attached terms
-        configurer.registerAlias( "characteristics.", CHARACTERISTIC_ALIAS, Characteristic.class, null, 1 );
+        configurer.registerAlias( "characteristics.", CHARACTERISTIC_ALIAS, Characteristic.class, null, 1, true );
         configurer.unregisterProperty( "characteristics.originalValue" );
-        configurer.registerAlias( "experimentalDesign.experimentalFactors.factorValues.characteristics.", FACTOR_VALUE_CHARACTERISTIC_ALIAS, Characteristic.class, null, 1 );
+        configurer.registerAlias( "experimentalDesign.experimentalFactors.factorValues.characteristics.", FACTOR_VALUE_CHARACTERISTIC_ALIAS, Characteristic.class, null, 1, true );
         configurer.unregisterProperty( "experimentalDesign.experimentalFactors.factorValues.characteristics.originalValue" );
-        configurer.registerAlias( "bioAssays.sampleUsed.characteristics.", BIO_MATERIAL_CHARACTERISTIC_ALIAS, Characteristic.class, null, 1 );
+        configurer.registerAlias( "bioAssays.sampleUsed.characteristics.", BIO_MATERIAL_CHARACTERISTIC_ALIAS, Characteristic.class, null, 1, true );
         configurer.unregisterProperty( "bioAssays.sampleUsed.characteristics.originalValue" );
-        configurer.registerAlias( "allCharacteristics.", ALL_CHARACTERISTIC_ALIAS, Characteristic.class, null, 1 );
+        configurer.registerAlias( "allCharacteristics.", ALL_CHARACTERISTIC_ALIAS, Characteristic.class, null, 1, true );
         configurer.unregisterProperty( "allCharacteristics.originalValue" );
 
-        configurer.registerAlias( "bioAssays.", BIO_ASSAY_ALIAS, BioAssay.class, null, 2 );
+        configurer.registerAlias( "bioAssays.", BIO_ASSAY_ALIAS, BioAssay.class, null, 2, true );
         configurer.unregisterProperty( "bioAssays.accession.Uri" );
 
         // this is not useful, unless we add an alias to the alternate names

gemma-core/src/main/java/ubic/gemma/persistence/util/Filter.java

@@ -133,6 +133,10 @@ public static <T> Filter by( @Nullable String objectAlias, String propertyName,
         return new Filter( objectAlias, propertyName, propertyType, operator, requiredValues, originalProperty );
     }
 
+    public static <T> Filter by( @Nullable String objectAlias, String propertyName, Class<T> propertyType, Operator operator, Subquery requiredValues, String originalProperty ) {
+        return new Filter( objectAlias, propertyName, propertyType, operator, requiredValues, originalProperty );
+    }
+
     /**
      * Create a new filter without an original property.
      * @see #by(String, String, Class, Operator, Object, String)
@@ -149,6 +153,10 @@ public static <T> Filter by( @Nullable String objectAlias, String propertyName,
         return new Filter( objectAlias, propertyName, propertyType, operator, requiredValues, null );
     }
 
+    public static <T> Filter by( @Nullable String objectAlias, String propertyName, Class<T> propertyType, Operator operator, Subquery requiredValues ) {
+        return new Filter( objectAlias, propertyName, propertyType, operator, requiredValues, null );
+    }
+
     /**
      * Parse filter where the right-hand side is a scalar.
      *
@@ -194,7 +202,8 @@ public enum Operator {
         greaterThan( ">", true, null ),
         lessOrEq( "<=", true, null ),
         greaterOrEq( ">=", true, null ),
-        in( "in", true, Collection.class );
+        in( "in", true, Collection.class ),
+        inSubquery( "in", true, Subquery.class );
 
         /**
          * Token used when parsing filter input.
@@ -254,7 +263,9 @@ public String toOriginalString() {
 
     private String toString( boolean withOriginalProperties ) {
         String requiredValueString;
-        if ( requiredValue instanceof Collection ) {
+        if ( requiredValue instanceof Subquery ) {
+            return ( ( Subquery ) requiredValue ).getFilter().toString( withOriginalProperties );
+        } else if ( requiredValue instanceof Collection ) {
             requiredValueString = "(" + ( ( Collection<?> ) requiredValue ).stream()
                     .map( e -> conversionService.convert( e, String.class ) )
                     .map( Filter::quoteIfNecessary )