Skip to content
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.

Commit f04ae2f

Browse files
arteymixarteymix
committedFeb 7, 2024·
Add minimal service support for adding, removing and replacing single-cell data
1 parent f1236d8 commit f04ae2f

18 files changed

+771
-238
lines changed
 

‎gemma-core/src/main/java/ubic/gemma/model/common/quantitationtype/QuantitationType.java

+5
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@
2121
import ubic.gemma.model.common.AbstractDescribable;
2222

2323
import java.io.Serializable;
24+
import java.util.Objects;
2425

2526
public class QuantitationType extends AbstractDescribable implements Serializable {
2627

@@ -214,6 +215,10 @@ public boolean equals( Object object ) {
214215
}
215216
final QuantitationType that = ( QuantitationType ) object;
216217

218+
if ( that.getId() != null && this.getId() != null ) {
219+
return Objects.equals( that.getId(), this.getId() );
220+
}
221+
217222
if ( that.getName() != null && this.getName() != null && !this.getName().equals( that.getName() ) ) {
218223
return false;
219224
}

‎gemma-core/src/main/java/ubic/gemma/model/expression/bioAssayData/BulkExpressionDataVector.java

+1-4
Original file line numberDiff line numberDiff line change
@@ -34,10 +34,7 @@ public boolean equals( Object object ) {
3434

3535
@Override
3636
public int hashCode() {
37-
if ( getId() != null ) {
38-
return Objects.hashCode( getId() );
39-
}
40-
return Objects.hash( super.hashCode(), Objects.hashCode( bioAssayDimension ) );
37+
return Objects.hash( super.hashCode(), bioAssayDimension );
4138
}
4239

4340
@Override

‎gemma-core/src/main/java/ubic/gemma/model/expression/bioAssayData/DataVector.java

+3-4
Original file line numberDiff line numberDiff line change
@@ -48,10 +48,9 @@ public abstract class DataVector implements Identifiable, Serializable {
4848
*/
4949
@Override
5050
public int hashCode() {
51-
if ( id != null ) {
52-
return Objects.hashCode( id );
53-
}
54-
return Objects.hash( expressionExperiment, quantitationType, Arrays.hashCode( data ) );
51+
// also, we cannot hash the ID because it is assigned on creation
52+
// hashing the data is wasteful because subclasses will have a design element to distinguish distinct vectors
53+
return Objects.hash( expressionExperiment, quantitationType );
5554
}
5655

5756
/**

‎gemma-core/src/main/java/ubic/gemma/model/expression/bioAssayData/DesignElementDataVector.java

-3
Original file line numberDiff line numberDiff line change
@@ -37,9 +37,6 @@ public class DesignElementDataVector extends DataVector {
3737

3838
@Override
3939
public int hashCode() {
40-
if ( getId() != null ) {
41-
return Objects.hash( getId() );
42-
}
4340
return Objects.hash( super.hashCode(), designElement );
4441
}
4542

‎gemma-core/src/main/java/ubic/gemma/model/expression/bioAssayData/SingleCellDimension.java

+51-17
Original file line numberDiff line numberDiff line change
@@ -2,19 +2,18 @@
22

33
import lombok.Getter;
44
import lombok.Setter;
5+
import org.springframework.util.Assert;
56
import ubic.gemma.core.util.ListUtils;
67
import ubic.gemma.model.common.Identifiable;
78
import ubic.gemma.model.expression.bioAssay.BioAssay;
9+
import ubic.gemma.persistence.hibernate.ByteArrayType;
810
import ubic.gemma.persistence.hibernate.CompressedStringListType;
9-
import ubic.gemma.persistence.hibernate.IntArrayType;
1011

1112
import javax.annotation.Nullable;
1213
import javax.persistence.Transient;
13-
import java.util.Arrays;
14-
import java.util.List;
15-
import java.util.Map;
16-
import java.util.Objects;
14+
import java.util.*;
1715

16+
import static java.util.Collections.unmodifiableList;
1817
import static ubic.gemma.core.util.ListUtils.getSparseRangeArrayElement;
1918

2019
@Getter
@@ -30,7 +29,7 @@ public class SingleCellDimension implements Identifiable {
3029
* <p>
3130
* This is stored as a compressed, gzipped blob in the database. See {@link CompressedStringListType} for more details.
3231
*/
33-
private List<String> cellIds;
32+
private List<String> cellIds = new ArrayList<>();
3433

3534
/**
3635
* An internal collection for mapping cell IDs to their position in {@link #cellIds}.
@@ -44,42 +43,54 @@ public class SingleCellDimension implements Identifiable {
4443
* <p>
4544
* This should always be equal to the size of {@link #cellIds}.
4645
*/
47-
private Integer numberOfCells;
46+
private int numberOfCells = 0;
4847

4948
/**
50-
* Cell types, or null if unknown.
49+
* Cell types assignment to individual cells from the {@link #cellTypeLabels} collections.
50+
* <p>
51+
* If supplied, its size must be equal to that of {@link #cellIds}.
52+
*/
53+
@Nullable
54+
private int[] cellTypes;
55+
56+
/**
57+
* Cell type labels, or null if unknown.
5158
* <p>
5259
* Those are user-supplied cell type identifiers. Its size must be equal to that of {@link #cellIds}.
5360
* <p>
5461
* This is stored as a compressed, gzipped blob in the database. See {@link CompressedStringListType} for more details.
5562
*/
5663
@Nullable
57-
private List<String> cellTypes;
64+
private List<String> cellTypeLabels;
5865

5966
/**
60-
* Number of cell types.
67+
* Number of distinct cell types.
6168
* <p>
6269
* This must always be equal to number of distinct elements of {@link #cellTypes}.
6370
*/
6471
@Nullable
65-
private Integer numberOfCellTypes;
72+
private Integer numberOfCellTypeLabels;
6673

6774
/**
6875
* List of bioassays that each cell belongs to.
6976
* <p>
7077
* The {@link BioAssay} {@code bioAssays[i]} applies to all the cells in the interval {@code [bioAssaysOffset[i], bioAssaysOffset[i+1][}.
7178
* To find the bioassay type of a given cell, use {@link #getBioAssay(int)}.
7279
*/
73-
private List<BioAssay> bioAssays;
80+
private List<BioAssay> bioAssays = new ArrayList<>();
7481

7582
/**
7683
* Offsets of the bioassays.
7784
* <p>
7885
* This always contain {@code bioAssays.size()} elements.
7986
* <p>
80-
* This is stored in the database using {@link IntArrayType}.
87+
* This is stored in the database using {@link ByteArrayType}.
8188
*/
82-
private int[] bioAssaysOffset;
89+
private int[] bioAssaysOffset = new int[0];
90+
91+
public List<String> getCellIds() {
92+
return unmodifiableList( cellIds );
93+
}
8394

8495
public void setCellIds( List<String> cellIds ) {
8596
this.cellIds = cellIds;
@@ -98,14 +109,31 @@ public BioAssay getBioAssay( int index ) {
98109
* Obtain the {@link BioAssay} for a given cell ID.
99110
*/
100111
public BioAssay getBioAssayByCellId( String cellId ) {
112+
return getBioAssay( getCellIndex( cellId ) );
113+
}
114+
115+
public String getCellTypeLabel( int index ) {
116+
Assert.notNull( cellTypes, "No cell types have been assigned." );
117+
Assert.notNull( cellTypeLabels, "No cell labels exist." );
118+
return cellTypeLabels.get( cellTypes[index] );
119+
}
120+
121+
/**
122+
* Obtain a cell type label by cell ID.
123+
*/
124+
public String getCellTypeLabelByCellId( String cellId ) {
125+
return getCellTypeLabel( getCellIndex( cellId ) );
126+
}
127+
128+
private int getCellIndex( String cellId ) {
101129
if ( cellIdToIndex == null ) {
102130
cellIdToIndex = ListUtils.indexOfElements( cellIds );
103131
}
104132
Integer index = cellIdToIndex.get( cellId );
105133
if ( index == null ) {
106134
throw new IllegalArgumentException( "Cell ID not found: " + cellId );
107135
}
108-
return getBioAssay( index );
136+
return index;
109137
}
110138

111139
@Override
@@ -114,7 +142,7 @@ public int hashCode() {
114142
return Objects.hash( id );
115143
}
116144
// no need to hash numberOfCells, it's derived from cellIds's size
117-
return Objects.hash( cellIds, cellTypes, cellTypes, bioAssays, Arrays.hashCode( bioAssaysOffset ) );
145+
return Objects.hash( cellIds, Arrays.hashCode( cellTypes ), cellTypeLabels, bioAssays, Arrays.hashCode( bioAssaysOffset ) );
118146
}
119147

120148
@Override
@@ -129,8 +157,14 @@ public boolean equals( Object obj ) {
129157
}
130158
if ( id != null && ( ( SingleCellDimension ) obj ).id != null )
131159
return id.equals( ( ( SingleCellDimension ) obj ).id );
132-
return Objects.equals( cellTypes, scd.cellTypes )
160+
return Objects.equals( cellTypeLabels, scd.cellTypeLabels )
133161
&& Objects.equals( bioAssays, scd.bioAssays )
162+
&& Arrays.equals( cellTypes, scd.cellTypes )
134163
&& Objects.equals( cellIds, scd.cellIds ); // this is the most expensive to compare
135164
}
165+
166+
@Override
167+
public String toString() {
168+
return String.format( "SingleCellDimension %s", id != null ? "Id=" + id : "" );
169+
}
136170
}

‎gemma-core/src/main/java/ubic/gemma/model/expression/bioAssayData/SingleCellExpressionDataVector.java

+19-2
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,10 @@
22

33
import lombok.Getter;
44
import lombok.Setter;
5-
import ubic.gemma.persistence.hibernate.IntArrayType;
5+
import ubic.gemma.persistence.hibernate.ByteArrayType;
6+
7+
import java.util.Arrays;
8+
import java.util.Objects;
69

710
/**
811
* An expression data vector that contains data at the resolution of a single cell.
@@ -26,7 +29,21 @@ public class SingleCellExpressionDataVector extends DesignElementDataVector {
2629
/**
2730
* Positions of the non-zero data in the {@link #getData()} vector.
2831
* <p>
29-
* This is mapped in the database using {@link IntArrayType}.
32+
* This is mapped in the database using {@link ByteArrayType}.
3033
*/
3134
private int[] dataIndices;
35+
36+
@Override
37+
public boolean equals( Object object ) {
38+
if ( this == object ) {
39+
return true;
40+
}
41+
if ( !( object instanceof SingleCellExpressionDataVector ) ) {
42+
return false;
43+
}
44+
SingleCellExpressionDataVector other = ( SingleCellExpressionDataVector ) object;
45+
return super.equals( object )
46+
&& Objects.equals( singleCellDimension, other.singleCellDimension )
47+
&& Arrays.equals( dataIndices, other.dataIndices );
48+
}
3249
}

‎gemma-core/src/main/java/ubic/gemma/persistence/hibernate/ByteArrayType.java

+166
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,166 @@
1+
package ubic.gemma.persistence.hibernate;
2+
3+
import org.hibernate.HibernateException;
4+
import org.hibernate.engine.spi.SessionImplementor;
5+
import org.hibernate.usertype.ParameterizedType;
6+
import org.hibernate.usertype.UserType;
7+
import org.springframework.jdbc.support.lob.DefaultLobHandler;
8+
import org.springframework.jdbc.support.lob.LobHandler;
9+
import org.springframework.util.Assert;
10+
import ubic.basecode.io.ByteArrayConverter;
11+
12+
import java.io.Serializable;
13+
import java.sql.PreparedStatement;
14+
import java.sql.ResultSet;
15+
import java.sql.SQLException;
16+
import java.sql.Types;
17+
import java.util.Arrays;
18+
import java.util.Properties;
19+
20+
/**
21+
* Represents a vector of scalars stored as a byte array in a single column.
22+
* <p>
23+
* The following types are supported for the {@code arrayType} parameter:
24+
* <ul>
25+
* <li>{@code int}</li>
26+
* <li>{@code double}</li>
27+
* </ul>
28+
* Other types supported by {@link ByteArrayConverter} can be added if necessary.
29+
* @author poirigui
30+
* @see ByteArrayConverter
31+
*/
32+
public class ByteArrayType implements UserType, ParameterizedType {
33+
34+
private enum ByteArrayTypes {
35+
INT( int[].class ),
36+
DOUBLE( double[].class );
37+
38+
private final Class<?> arrayClass;
39+
40+
ByteArrayTypes( Class<?> arrayClass ) {
41+
this.arrayClass = arrayClass;
42+
}
43+
}
44+
45+
private final ByteArrayConverter converter = new ByteArrayConverter();
46+
private final LobHandler lobHandler = new DefaultLobHandler();
47+
48+
private ByteArrayTypes arrayType;
49+
50+
@Override
51+
public int[] sqlTypes() {
52+
return new int[] { Types.BLOB };
53+
}
54+
55+
@Override
56+
public Class<?> returnedClass() {
57+
return arrayType.arrayClass;
58+
}
59+
60+
@Override
61+
public boolean equals( Object x, Object y ) throws HibernateException {
62+
switch ( arrayType ) {
63+
case INT:
64+
return Arrays.equals( ( int[] ) x, ( int[] ) y );
65+
case DOUBLE:
66+
return Arrays.equals( ( double[] ) x, ( double[] ) y );
67+
default:
68+
throw unsupportedArrayType( arrayType );
69+
}
70+
}
71+
72+
@Override
73+
public int hashCode( Object x ) throws HibernateException {
74+
switch ( arrayType ) {
75+
case INT:
76+
return Arrays.hashCode( ( int[] ) x );
77+
case DOUBLE:
78+
return Arrays.hashCode( ( double[] ) x );
79+
default:
80+
throw unsupportedArrayType( arrayType );
81+
}
82+
}
83+
84+
@Override
85+
public Object nullSafeGet( ResultSet rs, String[] names, SessionImplementor session, Object owner ) throws HibernateException, SQLException {
86+
byte[] data = lobHandler.getBlobAsBytes( rs, 0 );
87+
if ( data != null ) {
88+
switch ( arrayType ) {
89+
case INT:
90+
return converter.byteArrayToInts( data );
91+
case DOUBLE:
92+
return converter.byteArrayToDoubles( data );
93+
default:
94+
throw unsupportedArrayType( arrayType );
95+
}
96+
} else {
97+
return null;
98+
}
99+
}
100+
101+
@Override
102+
public void nullSafeSet( PreparedStatement st, Object value, int index, SessionImplementor session ) throws HibernateException, SQLException {
103+
byte[] blob;
104+
if ( value != null ) {
105+
switch ( arrayType ) {
106+
case INT:
107+
blob = converter.intArrayToBytes( ( int[] ) value );
108+
break;
109+
case DOUBLE:
110+
blob = converter.doubleArrayToBytes( ( double[] ) value );
111+
break;
112+
default:
113+
throw unsupportedArrayType( arrayType );
114+
}
115+
} else {
116+
blob = null;
117+
}
118+
lobHandler.getLobCreator().setBlobAsBytes( st, index, blob );
119+
}
120+
121+
@Override
122+
public Object deepCopy( Object value ) throws HibernateException {
123+
if ( value == null ) {
124+
return null;
125+
}
126+
switch ( arrayType ) {
127+
case INT:
128+
return ( ( int[] ) value ).clone();
129+
case DOUBLE:
130+
return ( ( double[] ) value ).clone();
131+
default:
132+
throw unsupportedArrayType( arrayType );
133+
}
134+
}
135+
136+
@Override
137+
public boolean isMutable() {
138+
return true;
139+
}
140+
141+
@Override
142+
public Serializable disassemble( Object value ) throws HibernateException {
143+
return ( Serializable ) deepCopy( value );
144+
}
145+
146+
@Override
147+
public Object assemble( Serializable cached, Object owner ) throws HibernateException {
148+
return deepCopy( cached );
149+
}
150+
151+
@Override
152+
public Object replace( Object original, Object target, Object owner ) throws HibernateException {
153+
return deepCopy( original );
154+
}
155+
156+
@Override
157+
public void setParameterValues( Properties parameters ) {
158+
Assert.isTrue( parameters != null && parameters.containsKey( "arrayType" ),
159+
"There must be an 'arrayType' parameter in the type declaration." );
160+
arrayType = ByteArrayTypes.valueOf( parameters.getProperty( "arrayType" ).toUpperCase() );
161+
}
162+
163+
private HibernateException unsupportedArrayType( ByteArrayTypes type ) {
164+
return new HibernateException( String.format( "Unsupported array type: %s.", type ) );
165+
}
166+
}

‎gemma-core/src/main/java/ubic/gemma/persistence/hibernate/CompressedStringListType.java

+13-11
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
package ubic.gemma.persistence.hibernate;
22

33
import org.apache.commons.io.IOUtils;
4+
import org.apache.commons.io.output.ByteArrayOutputStream;
45
import org.apache.commons.lang3.StringUtils;
56
import org.hibernate.HibernateException;
67
import org.hibernate.engine.spi.SessionImplementor;
@@ -12,17 +13,16 @@
1213

1314
import java.io.IOException;
1415
import java.io.InputStream;
16+
import java.io.OutputStream;
1517
import java.io.Serializable;
1618
import java.nio.charset.StandardCharsets;
1719
import java.sql.PreparedStatement;
1820
import java.sql.ResultSet;
1921
import java.sql.SQLException;
2022
import java.sql.Types;
21-
import java.util.Arrays;
22-
import java.util.List;
23-
import java.util.Objects;
24-
import java.util.Properties;
23+
import java.util.*;
2524
import java.util.zip.GZIPInputStream;
25+
import java.util.zip.GZIPOutputStream;
2626

2727
import static java.util.Objects.requireNonNull;
2828

@@ -82,11 +82,13 @@ public void nullSafeSet( PreparedStatement st, Object value, int index, SessionI
8282
List<String> s = ( List<String> ) value;
8383
Assert.isTrue( s.stream().noneMatch( k -> k.contains( delimiter ) ),
8484
String.format( "The list of strings may not contain the delimiter %s.", delimiter ) );
85-
try ( InputStream is = new GZIPInputStream( IOUtils.toInputStream( String.join( delimiter, s ), StandardCharsets.UTF_8 ) ) ) {
86-
blob = IOUtils.toByteArray( is );
85+
ByteArrayOutputStream baos = new ByteArrayOutputStream();
86+
try ( OutputStream out = new GZIPOutputStream( baos ) ) {
87+
IOUtils.write( String.join( delimiter, s ), out, StandardCharsets.UTF_8 );
8788
} catch ( IOException e ) {
8889
throw new HibernateException( e );
8990
}
91+
blob = baos.toByteArray();
9092
} else {
9193
blob = null;
9294
}
@@ -95,27 +97,27 @@ public void nullSafeSet( PreparedStatement st, Object value, int index, SessionI
9597

9698
@Override
9799
public Object deepCopy( Object value ) throws HibernateException {
98-
return value;
100+
return value != null ? new ArrayList<>( ( List<String> ) value ) : null;
99101
}
100102

101103
@Override
102104
public boolean isMutable() {
103-
return false;
105+
return true;
104106
}
105107

106108
@Override
107109
public Serializable disassemble( Object value ) throws HibernateException {
108-
return ( String ) value;
110+
return value != null ? String.join( delimiter, ( List<String> ) value ) : null;
109111
}
110112

111113
@Override
112114
public Object assemble( Serializable cached, Object owner ) throws HibernateException {
113-
return cached;
115+
return cached != null ? Arrays.asList( StringUtils.split( ( String ) cached, delimiter ) ) : null;
114116
}
115117

116118
@Override
117119
public Object replace( Object original, Object target, Object owner ) throws HibernateException {
118-
return original;
120+
return deepCopy( original );
119121
}
120122

121123
@Override

‎gemma-core/src/main/java/ubic/gemma/persistence/hibernate/IntArrayType.java

-94
This file was deleted.

‎gemma-core/src/main/java/ubic/gemma/persistence/service/expression/experiment/ExpressionExperimentDao.java

+5
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@
1212
import ubic.gemma.model.expression.bioAssay.BioAssay;
1313
import ubic.gemma.model.expression.bioAssayData.BioAssayDimension;
1414
import ubic.gemma.model.expression.bioAssayData.MeanVarianceRelation;
15+
import ubic.gemma.model.expression.bioAssayData.SingleCellDimension;
1516
import ubic.gemma.model.expression.biomaterial.BioMaterial;
1617
import ubic.gemma.model.expression.experiment.*;
1718
import ubic.gemma.model.genome.Gene;
@@ -304,4 +305,8 @@ Map<ExpressionExperiment, Collection<AuditEvent>> getSampleRemovalEvents(
304305
* The result is stored in the standard query cache.
305306
*/
306307
long countBioMaterials( @Nullable Filters filters );
308+
309+
void createSingleCellDimension( ExpressionExperiment ee, SingleCellDimension singleCellDimension );
310+
311+
void deleteSingleCellDimension( ExpressionExperiment ee, SingleCellDimension singleCellDimension );
307312
}

‎gemma-core/src/main/java/ubic/gemma/persistence/service/expression/experiment/ExpressionExperimentDaoImpl.java

+11
Original file line numberDiff line numberDiff line change
@@ -46,6 +46,7 @@
4646
import ubic.gemma.model.expression.bioAssay.BioAssay;
4747
import ubic.gemma.model.expression.bioAssayData.BioAssayDimension;
4848
import ubic.gemma.model.expression.bioAssayData.MeanVarianceRelation;
49+
import ubic.gemma.model.expression.bioAssayData.SingleCellDimension;
4950
import ubic.gemma.model.expression.biomaterial.BioMaterial;
5051
import ubic.gemma.model.expression.experiment.*;
5152
import ubic.gemma.model.genome.Gene;
@@ -1944,6 +1945,16 @@ public void thawForFrontEnd( final ExpressionExperiment expressionExperiment ) {
19441945
}
19451946
}
19461947

1948+
@Override
1949+
public void createSingleCellDimension( ExpressionExperiment ee, SingleCellDimension singleCellDimension ) {
1950+
getSessionFactory().getCurrentSession().persist( singleCellDimension );
1951+
}
1952+
1953+
@Override
1954+
public void deleteSingleCellDimension( ExpressionExperiment ee, SingleCellDimension singleCellDimension ) {
1955+
getSessionFactory().getCurrentSession().delete( singleCellDimension );
1956+
}
1957+
19471958
@Override
19481959
protected Query getFilteringQuery( @Nullable Filters filters, @Nullable Sort sort ) {
19491960
// the constants for aliases are messing with the inspector

‎gemma-core/src/main/java/ubic/gemma/persistence/service/expression/experiment/ExpressionExperimentServiceImpl.java

+134-31
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,7 @@
2424
import org.apache.commons.lang3.StringUtils;
2525
import org.apache.commons.math3.exception.NotStrictlyPositiveException;
2626
import org.hibernate.Hibernate;
27+
import org.hibernate.SessionFactory;
2728
import org.springframework.beans.factory.annotation.Autowired;
2829
import org.springframework.security.access.ConfigAttribute;
2930
import org.springframework.security.access.SecurityConfig;
@@ -59,6 +60,7 @@
5960
import ubic.gemma.model.expression.bioAssay.BioAssay;
6061
import ubic.gemma.model.expression.bioAssayData.*;
6162
import ubic.gemma.model.expression.biomaterial.BioMaterial;
63+
import ubic.gemma.model.expression.designElement.CompositeSequence;
6264
import ubic.gemma.model.expression.experiment.*;
6365
import ubic.gemma.model.genome.Gene;
6466
import ubic.gemma.model.genome.Taxon;
@@ -69,6 +71,7 @@
6971
import ubic.gemma.persistence.service.analysis.expression.pca.PrincipalComponentAnalysisService;
7072
import ubic.gemma.persistence.service.analysis.expression.sampleCoexpression.SampleCoexpressionAnalysisService;
7173
import ubic.gemma.persistence.service.common.auditAndSecurity.AuditEventService;
74+
import ubic.gemma.persistence.service.common.auditAndSecurity.AuditTrailService;
7275
import ubic.gemma.persistence.service.common.quantitationtype.QuantitationTypeService;
7376
import ubic.gemma.persistence.service.expression.bioAssayData.BioAssayDimensionService;
7477
import ubic.gemma.persistence.util.*;
@@ -101,6 +104,8 @@ public class ExpressionExperimentServiceImpl
101104
@Autowired
102105
private AuditEventService auditEventService;
103106
@Autowired
107+
private AuditTrailService auditTrailService;
108+
@Autowired
104109
private BioAssayDimensionService bioAssayDimensionService;
105110
@Autowired
106111
private DifferentialExpressionAnalysisService differentialExpressionAnalysisService;
@@ -1300,51 +1305,93 @@ public ExpressionExperiment replaceRawVectors( ExpressionExperiment ee,
13001305
@Override
13011306
@Transactional
13021307
public void addSingleCellDataVectors( ExpressionExperiment ee, QuantitationType quantitationType, Collection<SingleCellExpressionDataVector> vectors ) {
1308+
Assert.notNull( ee.getId() );
1309+
Assert.notNull( quantitationType.getId(), "The quantitation type must be persistent." );
13031310
Assert.isTrue( !ee.getQuantitationTypes().contains( quantitationType ),
1304-
ee + " already have vectors for the quantitation type: " + quantitationType );
1305-
Assert.isTrue( !vectors.isEmpty(), "At least one single-cell vector has to be supplied." );
1306-
Assert.isTrue( vectors.stream().allMatch( v -> v.getQuantitationType().equals( quantitationType ) ),
1307-
"All vectors must have the quantitation type: " + quantitationType );
1308-
Assert.isTrue( vectors.stream().map( SingleCellExpressionDataVector::getSingleCellDimension ).distinct().count() == 1,
1309-
"All vectors must share the same dimension." );
1310-
validateSingleCellDimension( ee, vectors.iterator().next().getSingleCellDimension() );
1311-
ExpressionExperiment finalEe = ee;
1312-
Assert.isTrue( vectors.stream().allMatch( v -> v.getExpressionExperiment() == null || v.getExpressionExperiment().equals( finalEe ) ),
1313-
"Some of the vectors belong to other expression experiments." );
1314-
ee = ensureInSession( ee );
1311+
String.format( "%s already have vectors for the quantitation type: %s; use replaceSingleCellDataVectors() to replace existing vectors.",
1312+
ee, quantitationType ) );
1313+
validateSingleCellDataVectors( ee, quantitationType, vectors );
1314+
if ( vectors.iterator().next().getSingleCellDimension().getId() == null ) {
1315+
log.info( "Creating a new single-cell dimension for " + ee );
1316+
expressionExperimentDao.createSingleCellDimension( ee, vectors.iterator().next().getSingleCellDimension() );
1317+
}
13151318
for ( SingleCellExpressionDataVector v : vectors ) {
13161319
v.setExpressionExperiment( ee );
13171320
}
1321+
int previousSize = ee.getSingleCellExpressionDataVectors().size();
1322+
log.info( String.format( "Adding %d single-cell vectors to %s for %s", vectors.size(), ee, quantitationType ) );
13181323
ee.getSingleCellExpressionDataVectors().addAll( vectors );
1324+
int numVectorsAdded = ee.getSingleCellExpressionDataVectors().size() - previousSize;
13191325
// make all other single-cell QTs non-preferred
13201326
if ( quantitationType.getIsPreferred() ) {
1321-
ee.getQuantitationTypes().forEach( q -> q.setIsPreferred( false ) );
1327+
for ( QuantitationType qt : ee.getQuantitationTypes() ) {
1328+
if ( qt.getIsPreferred() ) {
1329+
log.info( "Setting " + qt + " to non-preferred since we're adding a new set of preferred vectors to " + ee );
1330+
qt.setIsPreferred( false );
1331+
break; // there is at most 1 set of preferred vectors
1332+
}
1333+
}
13221334
}
13231335
ee.getQuantitationTypes().add( quantitationType );
13241336
update( ee ); // will take care of creating vectors
1337+
auditTrailService.addUpdateEvent( ee, DataAddedEvent.class,
1338+
String.format( "Added %d vectors for %s", numVectorsAdded, quantitationType ) );
13251339
}
13261340

13271341
@Override
13281342
@Transactional
13291343
public void replaceSingleCellDataVectors( ExpressionExperiment ee, QuantitationType quantitationType, Collection<SingleCellExpressionDataVector> vectors ) {
1344+
Assert.notNull( ee.getId() );
1345+
Assert.notNull( quantitationType.getId(), "The quantitation type must be persistent." );
13301346
Assert.isTrue( ee.getQuantitationTypes().contains( quantitationType ),
1331-
ee + " does not have the quantitation type: " + quantitationType );
1332-
Assert.isTrue( !vectors.isEmpty(), "At least one single-cell vector has to be supplied; use removeSingleCelLDataVectors() to remove vectors instead." );
1333-
Assert.isTrue( vectors.stream().allMatch( v -> v.getQuantitationType().equals( quantitationType ) ),
1334-
"All vectors must have the quantitation type: " + quantitationType );
1335-
Assert.isTrue( vectors.stream().map( SingleCellExpressionDataVector::getSingleCellDimension ).distinct().count() == 1,
1336-
"All vectors must share the same dimension." );
1337-
validateSingleCellDimension( ee, vectors.iterator().next().getSingleCellDimension() );
1338-
ExpressionExperiment finalEe = ee;
1339-
Assert.isTrue( vectors.stream().allMatch( v -> v.getExpressionExperiment() == null || v.getExpressionExperiment().equals( finalEe ) ),
1340-
"Some of the vectors belong to other expression experiments." );
1341-
ee = ensureInSession( ee );
1342-
ee.getSingleCellExpressionDataVectors().removeIf( v -> v.getQuantitationType().equals( quantitationType ) );
1347+
String.format( "%s does not have the quantitation type: %s; use addSingleCellDataVectors() to add new vectors instead.",
1348+
ee, quantitationType ) );
1349+
validateSingleCellDataVectors( ee, quantitationType, vectors );
1350+
boolean scdCreated = false;
1351+
if ( vectors.iterator().next().getSingleCellDimension().getId() == null ) {
1352+
log.info( "Creating a new single-cell dimension for " + ee );
1353+
expressionExperimentDao.createSingleCellDimension( ee, vectors.iterator().next().getSingleCellDimension() );
1354+
scdCreated = true;
1355+
}
1356+
Set<SingleCellExpressionDataVector> vectorsToBeReplaced = ee.getSingleCellExpressionDataVectors().stream()
1357+
.filter( v -> v.getQuantitationType().equals( quantitationType ) ).collect( Collectors.toSet() );
13431358
for ( SingleCellExpressionDataVector v : vectors ) {
13441359
v.setExpressionExperiment( ee );
13451360
}
1361+
int previousSize = ee.getSingleCellExpressionDataVectors().size();
1362+
if ( !vectorsToBeReplaced.isEmpty() ) {
1363+
// if the SCD was created, we do not need to check additional vectors for removing the existing one
1364+
removeSingleCellVectorsAndDimensionIfNecessary( ee, vectorsToBeReplaced, scdCreated ? null : vectors );
1365+
} else {
1366+
log.warn( "No vectors with the quantitation type: " + quantitationType );
1367+
}
1368+
int numVectorsRemoved = ee.getSingleCellExpressionDataVectors().size() - previousSize;
1369+
log.info( String.format( "Adding %d single-cell vectors to %s for %s", vectors.size(), ee, quantitationType ) );
13461370
ee.getSingleCellExpressionDataVectors().addAll( vectors );
1371+
int numVectorsAdded = ee.getSingleCellExpressionDataVectors().size() - ( previousSize - numVectorsRemoved );
13471372
update( ee );
1373+
auditTrailService.addUpdateEvent( ee, DataReplacedEvent.class,
1374+
String.format( "Replaced %d vectors with %d vectors for %s.", numVectorsRemoved, numVectorsAdded, quantitationType ) );
1375+
}
1376+
1377+
private void validateSingleCellDataVectors( ExpressionExperiment ee, QuantitationType quantitationType, Collection<SingleCellExpressionDataVector> vectors ) {
1378+
Assert.notNull( quantitationType.getId(), "The quantitation type must be persistent." );
1379+
Assert.isTrue( !vectors.isEmpty(), "At least one single-cell vector has to be supplied; use removeSingleCellDataVectors() to remove vectors instead." );
1380+
Assert.isTrue( vectors.stream().allMatch( v -> v.getExpressionExperiment() == null || v.getExpressionExperiment().equals( ee ) ),
1381+
"Some of the vectors belong to other expression experiments." );
1382+
Assert.isTrue( vectors.stream().allMatch( v -> v.getQuantitationType() == quantitationType ),
1383+
"All vectors must have the same quantitation type: " + quantitationType );
1384+
Assert.isTrue( vectors.stream().allMatch( v -> v.getDesignElement() != null && v.getDesignElement().getId() != null ),
1385+
"All vectors must have a persistent design element." );
1386+
// TODO: allow vectors from multiple platforms
1387+
CompositeSequence element = vectors.iterator().next().getDesignElement();
1388+
ArrayDesign platform = element.getArrayDesign();
1389+
Assert.isTrue( vectors.stream().allMatch( v -> v.getDesignElement().getArrayDesign().equals( platform ) ),
1390+
"All vectors must have a persistent design element from the same platform." );
1391+
SingleCellDimension singleCellDimension = vectors.iterator().next().getSingleCellDimension();
1392+
validateSingleCellDimension( ee, singleCellDimension );
1393+
Assert.isTrue( vectors.stream().allMatch( v -> v.getSingleCellDimension() == singleCellDimension ),
1394+
"All vectors must share the same dimension: " + singleCellDimension );
13481395
}
13491396

13501397
/**
@@ -1354,24 +1401,80 @@ private void validateSingleCellDimension( ExpressionExperiment ee, SingleCellDim
13541401
Assert.isTrue( scbad.getCellIds().size() == scbad.getNumberOfCells(),
13551402
"The number of cell IDs must match the number of cells." );
13561403
if ( scbad.getCellTypes() != null ) {
1357-
Assert.notNull( scbad.getNumberOfCellTypes() );
1358-
Assert.isTrue( scbad.getCellTypes().stream().distinct().count() == scbad.getNumberOfCellTypes(),
1359-
"The number of cell types must match the number of distinct values the cellTypes collection." );
1404+
Assert.notNull( scbad.getNumberOfCellTypeLabels() );
1405+
Assert.notNull( scbad.getCellTypeLabels() );
1406+
Assert.isTrue( scbad.getCellTypes().length == scbad.getCellIds().size(),
1407+
"The number of cell types must match the number of cell IDs." );
1408+
Assert.isTrue( scbad.getCellTypeLabels().size() == scbad.getNumberOfCellTypeLabels(),
1409+
"The number of cell types must match the number of values the cellTypeLabels collection." );
13601410
} else {
1361-
Assert.isNull( scbad.getNumberOfCellTypes(), "There is no cell types assigned, the number of cell types must be null." );
1411+
Assert.isNull( scbad.getCellTypeLabels() );
1412+
Assert.isNull( scbad.getNumberOfCellTypeLabels(), "There is no cell types assigned, the number of cell types must be null." );
13621413
}
13631414
Assert.isTrue( ee.getBioAssays().containsAll( scbad.getBioAssays() ), "Not all supplied BioAssays belong to " + ee );
1364-
validateSparseRangeArray( scbad.getBioAssays(), scbad.getBioAssaysOffset(), scbad.getNumberOfCells() );
1415+
validateSparseRangeArray( scbad.getBioAssays(), scbad.getBioAssaysOffset(), scbad.getNumberOfCells() );
13651416
}
13661417

13671418
@Override
13681419
@Transactional
13691420
public void removeSingleCellDataVectors( ExpressionExperiment ee, QuantitationType quantitationType ) {
1421+
Assert.notNull( ee.getId() );
1422+
Assert.notNull( quantitationType.getId() );
13701423
Assert.isTrue( ee.getQuantitationTypes().contains( quantitationType ) );
1371-
ee = ensureInSession( ee );
1372-
ee.getSingleCellExpressionDataVectors().removeIf( v -> v.getQuantitationType().equals( quantitationType ) );
1424+
Set<SingleCellExpressionDataVector> vectors = ee.getSingleCellExpressionDataVectors().stream()
1425+
.filter( v -> v.getQuantitationType().equals( quantitationType ) ).collect( Collectors.toSet() );
1426+
if ( !vectors.isEmpty() ) {
1427+
removeSingleCellVectorsAndDimensionIfNecessary( ee, vectors, null );
1428+
} else {
1429+
log.warn( "No vectors with the quantitation type: " + quantitationType );
1430+
}
13731431
ee.getQuantitationTypes().remove( quantitationType );
13741432
update( ee );
1433+
auditTrailService.addUpdateEvent( ee, DataRemovedEvent.class,
1434+
String.format( "Removed %d vectors for %s.", vectors.size(), quantitationType ) );
1435+
}
1436+
1437+
/**
1438+
* @deprecated do not use this, it's only meant as a workaround for deleting single-cell vectors
1439+
*/
1440+
@Autowired
1441+
@Deprecated
1442+
private SessionFactory sessionFactory;
1443+
1444+
/**
1445+
* Remove the given single-cell vectors and their corresponding single-cell dimension if necessary.
1446+
* @param ee the experiment to remove the vectors from.
1447+
* @param additionalVectors additional vectors to check if the single-cell dimension is still in use (i.e. vectors that are in the process of being added).
1448+
* @return true if the vectors were removed, false otherwise.
1449+
*/
1450+
private void removeSingleCellVectorsAndDimensionIfNecessary( ExpressionExperiment ee,
1451+
Collection<SingleCellExpressionDataVector> vectors,
1452+
@Nullable Collection<SingleCellExpressionDataVector> additionalVectors ) {
1453+
log.info( String.format( "Removing %d single-cell vectors for %s...", vectors.size(), ee ) );
1454+
ee.getSingleCellExpressionDataVectors().removeAll( vectors );
1455+
// FIXME: flushing shouldn't be necessary here, but Hibernate does appear to cascade vectors removal prior to removing the SCD or QT...
1456+
sessionFactory.getCurrentSession().flush();
1457+
// check if SCD is still in use else remove it
1458+
SingleCellDimension scd = vectors.iterator().next().getSingleCellDimension();
1459+
boolean scdStillUsed = false;
1460+
for ( SingleCellExpressionDataVector v : ee.getSingleCellExpressionDataVectors() ) {
1461+
if ( v.getSingleCellDimension().equals( scd ) ) {
1462+
scdStillUsed = true;
1463+
break;
1464+
}
1465+
}
1466+
if ( !scdStillUsed && additionalVectors != null ) {
1467+
for ( SingleCellExpressionDataVector v : additionalVectors ) {
1468+
if ( v.getSingleCellDimension().equals( scd ) ) {
1469+
scdStillUsed = true;
1470+
break;
1471+
}
1472+
}
1473+
}
1474+
if ( !scdStillUsed ) {
1475+
log.info( "Removing unused single-cell dimension " + scd + " for " + ee );
1476+
expressionExperimentDao.deleteSingleCellDimension( ee, scd );
1477+
}
13751478
}
13761479

13771480
/**

‎gemma-core/src/main/resources/hibernate.cfg.xml

+6-3
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
<?xml version='1.0' encoding='utf-8'?>
22
<!DOCTYPE hibernate-configuration PUBLIC
3-
"-//Hibernate/Hibernate Configuration DTD//EN"
4-
"http://www.hibernate.org/dtd/hibernate-configuration-3.0.dtd">
3+
"-//Hibernate/Hibernate Configuration DTD//EN"
4+
"http://www.hibernate.org/dtd/hibernate-configuration-3.0.dtd">
55
<hibernate-configuration>
66
<session-factory>
77
<mapping resource="gemma/gsec/model/AclEntry.hbm.xml"/>
@@ -22,7 +22,8 @@
2222
<mapping resource="ubic/gemma/model/analysis/expression/coexpression/SampleCoexpressionMatrix.hbm.xml"/>
2323
<mapping resource="ubic/gemma/model/analysis/expression/diff/ContrastResult.hbm.xml"/>
2424
<mapping resource="ubic/gemma/model/analysis/expression/diff/DifferentialExpressionAnalysisResult.hbm.xml"/>
25-
<mapping resource="ubic/gemma/model/analysis/expression/diff/GeneDifferentialExpressionMetaAnalysisResult.hbm.xml"/>
25+
<mapping
26+
resource="ubic/gemma/model/analysis/expression/diff/GeneDifferentialExpressionMetaAnalysisResult.hbm.xml"/>
2627
<mapping resource="ubic/gemma/model/analysis/expression/diff/HitListSize.hbm.xml"/>
2728
<mapping resource="ubic/gemma/model/analysis/expression/diff/PvalueDistribution.hbm.xml"/>
2829
<mapping resource="ubic/gemma/model/analysis/expression/pca/Eigenvalue.hbm.xml"/>
@@ -63,6 +64,8 @@
6364
<mapping resource="ubic/gemma/model/expression/bioAssayData/MeanVarianceRelation.hbm.xml"/>
6465
<mapping resource="ubic/gemma/model/expression/bioAssayData/ProcessedExpressionDataVector.hbm.xml"/>
6566
<mapping resource="ubic/gemma/model/expression/bioAssayData/RawExpressionDataVector.hbm.xml"/>
67+
<mapping resource="ubic/gemma/model/expression/bioAssayData/SingleCellExpressionDataVector.hbm.xml"/>
68+
<mapping resource="ubic/gemma/model/expression/bioAssayData/SingleCellDimension.hbm.xml"/>
6669
<mapping resource="ubic/gemma/model/expression/biomaterial/BioMaterial.hbm.xml"/>
6770
<mapping resource="ubic/gemma/model/expression/biomaterial/Compound.hbm.xml"/>
6871
<mapping resource="ubic/gemma/model/expression/biomaterial/Treatment.hbm.xml"/>

‎gemma-core/src/main/resources/ubic/gemma/model/analysis/Investigation.hbm.xml

+2-1
Original file line numberDiff line numberDiff line change
@@ -128,7 +128,8 @@
128128
<!-- cannot be non-null because subsets and generic experiments don't have curation details -->
129129
<column name="CURATION_DETAILS_FK" not-null="false" sql-type="BIGINT" unique="true"/>
130130
</many-to-one>
131-
<set name="singleCellExpressionDataVectors" lazy="true" fetch="select" inverse="true" cascade="all-delete-orphan">
131+
<set name="singleCellExpressionDataVectors" lazy="true" fetch="select" inverse="true"
132+
cascade="all-delete-orphan">
132133
<key foreign-key="SINGLE_CELL_DATA_VECTOR_EXPRESSION_EXPERIMENT_FKC">
133134
<column name="EXPRESSION_EXPERIMENT_FK" sql-type="BIGINT"/>
134135
</key>

‎gemma-core/src/main/resources/ubic/gemma/model/expression/bioAssayData/SingleCellDimension.hbm.xml

+17-6
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@
55

66
<hibernate-mapping>
77
<class name="ubic.gemma.model.expression.bioAssayData.SingleCellDimension"
8-
table="RAW_EXPRESSION_DATA_VECTOR" mutable="false">
8+
table="SINGLE_CELL_DIMENSION" mutable="false">
99
<id name="id" type="java.lang.Long" unsaved-value="null">
1010
<column name="ID" sql-type="BIGINT"/>
1111
<generator class="native"/>
@@ -22,11 +22,20 @@
2222
</property>
2323
<property name="cellTypes">
2424
<column name="CELL_TYPES" not-null="false" sql-type="LONGBLOB"/>
25-
<type name="ubic.gemma.persistence.hibernate.CompressedStringListType">
26-
<param name="delimiter">\n</param>
25+
<type name="ubic.gemma.persistence.hibernate.ByteArrayType">
26+
<param name="arrayType">int</param>
2727
</type>
2828
</property>
29-
<property name="numberOfCellTypes">
29+
<list name="cellTypeLabels" table="SINGLE_CELL_CELL_TYPE_LABEL" fetch="select" lazy="false">
30+
<key foreign-key="SINGLE_CELL_CELL_TYPE_LABEL_DIMENSION_FKC">
31+
<column name="SINGLE_CELL_DIMENSION_FK" sql-type="BIGINT"/>
32+
</key>
33+
<list-index column="ORDERING"/>
34+
<element type="string">
35+
<column name="CELL_TYPE_LABEL" sql-type="VARCHAR"/>
36+
</element>
37+
</list>
38+
<property name="numberOfCellTypeLabels">
3039
<column name="NUMBER_OF_CELL_TYPES" not-null="false" sql-type="INTEGER"/>
3140
</property>
3241
<list name="bioAssays" table="BIO_ASSAYS2SINGLE_CELL_DIMENSIONS" lazy="false" fetch="select">
@@ -35,13 +44,15 @@
3544
<column name="SINGLE_CELL_DIMENSIONS_FK" sql-type="BIGINT"/>
3645
</key>
3746
<list-index column="ORDERING"/>
38-
<many-to-many>
47+
<many-to-many class="ubic.gemma.model.expression.bioAssay.BioAssay">
3948
<column name="BIO_ASSAYS_FK" sql-type="BIGINT"/>
4049
</many-to-many>
4150
</list>
4251
<property name="bioAssaysOffset">
4352
<column name="BIO_ASSAYS_OFFSET" not-null="true" sql-type="LONGBLOB"/>
44-
<type name="ubic.gemma.persistence.hibernate.IntArrayType"/>
53+
<type name="ubic.gemma.persistence.hibernate.ByteArrayType">
54+
<param name="arrayType">int</param>
55+
</type>
4556
</property>
4657
</class>
4758
</hibernate-mapping>

‎gemma-core/src/main/resources/ubic/gemma/model/expression/bioAssayData/SingleCellDataVector.hbm.xml ‎gemma-core/src/main/resources/ubic/gemma/model/expression/bioAssayData/SingleCellExpressionDataVector.hbm.xml

+9-5
Original file line numberDiff line numberDiff line change
@@ -5,10 +5,11 @@
55

66
<hibernate-mapping>
77
<class name="ubic.gemma.model.expression.bioAssayData.SingleCellExpressionDataVector"
8-
table="SINGLE_CELL_DATA_VECTOR">
8+
table="SINGLE_CELL_EXPRESSION_DATA_VECTOR">
99
<id name="id" type="java.lang.Long" unsaved-value="null">
1010
<column name="ID" sql-type="BIGINT"/>
11-
<generator class="native"/>
11+
<generator class="native">
12+
</generator>
1213
</id>
1314
<many-to-one name="singleCellDimension" class="ubic.gemma.model.expression.bioAssayData.SingleCellDimension"
1415
lazy="proxy" fetch="select">
@@ -21,15 +22,18 @@
2122
<property name="data" type="org.hibernate.type.MaterializedBlobType">
2223
<column name="DATA" not-null="true" unique="false" sql-type="LONGBLOB"/>
2324
</property>
24-
<property name="dataIndices" type="ubic.gemma.persistence.hibernate.IntArrayType">
25+
<property name="dataIndices">
2526
<column name="DATA_INDICES" not-null="true" unique="false" sql-type="LONGBLOB"/>
27+
<type name="ubic.gemma.persistence.hibernate.ByteArrayType">
28+
<param name="arrayType">int</param>
29+
</type>
2630
</property>
2731
<many-to-one name="quantitationType" class="ubic.gemma.model.common.quantitationtype.QuantitationType"
2832
lazy="false" fetch="select">
29-
<column name="QUANTITATION_TYPE_FK" not-null="false" sql-type="BIGINT"/>
33+
<column name="QUANTITATION_TYPE_FK" not-null="true" sql-type="BIGINT"/>
3034
</many-to-one>
3135
<many-to-one name="expressionExperiment" class="ubic.gemma.model.expression.experiment.ExpressionExperiment"
32-
cascade="none" lazy="proxy" fetch="select">
36+
lazy="proxy" fetch="select">
3337
<column name="EXPRESSION_EXPERIMENT_FK" not-null="true" sql-type="BIGINT"/>
3438
</many-to-one>
3539
</class>

‎gemma-core/src/test/java/ubic/gemma/persistence/service/expression/bioAssayData/SingleCellExpressionDataVectorDaoTest.java

-47
This file was deleted.

‎gemma-core/src/test/java/ubic/gemma/persistence/service/expression/experiment/SingleCellExpressionExperimentServiceTest.java

+329-10
Large diffs are not rendered by default.

0 commit comments

Comments
 (0)
Please sign in to comment.