[WIP] JAVA 17 BWARE COMMIT

src/main/java/org/apache/sysds/hops/AggBinaryOp.java

@@ -439,8 +439,7 @@ private boolean isApplicableForTransitiveSparkExecType(boolean left)
 				|| (left && !isLeftTransposeRewriteApplicable(true)))
 			&& getInput(index).getParent().size()==1 //bagg is only parent
 			&& !getInput(index).areDimsBelowThreshold() 
-			&& (getInput(index).optFindExecType() == ExecType.SPARK
-				|| (getInput(index) instanceof DataOp && ((DataOp)getInput(index)).hasOnlyRDD()))
+			&& getInput(index).hasSparkOutput()
 			&& getInput(index).getOutputMemEstimate()>getOutputMemEstimate();
 	}
 

src/main/java/org/apache/sysds/hops/BinaryOp.java

@@ -747,8 +747,8 @@ protected ExecType optFindExecType(boolean transitive) {
 
 		checkAndSetForcedPlatform();
 
-		DataType dt1 = getInput().get(0).getDataType();
-		DataType dt2 = getInput().get(1).getDataType();
+		final DataType dt1 = getInput(0).getDataType();
+		final DataType dt2 = getInput(1).getDataType();
 
 		if( _etypeForced != null ) {
 			setExecType(_etypeForced);
@@ -796,18 +796,28 @@ else if ( dt1 == DataType.SCALAR && dt2 == DataType.MATRIX ) {
 			checkAndSetInvalidCPDimsAndSize();
 		}
 
-		//spark-specific decision refinement (execute unary scalar w/ spark input and
+		// spark-specific decision refinement (execute unary scalar w/ spark input and
 		// single parent also in spark because it's likely cheap and reduces intermediates)
-		if(transitive && _etype == ExecType.CP && _etypeForced != ExecType.CP && _etypeForced != ExecType.FED &&
-			getDataType().isMatrix() // output should be a matrix
-			&& (dt1.isScalar() || dt2.isScalar()) // one side should be scalar
-			&& supportsMatrixScalarOperations() // scalar operations
-			&& !(getInput().get(dt1.isScalar() ? 1 : 0) instanceof DataOp) // input is not checkpoint
-			&& getInput().get(dt1.isScalar() ? 1 : 0).getParent().size() == 1 // unary scalar is only parent
-			&& !HopRewriteUtils.isSingleBlock(getInput().get(dt1.isScalar() ? 1 : 0)) // single block triggered exec
-			&& getInput().get(dt1.isScalar() ? 1 : 0).optFindExecType() == ExecType.SPARK) {
-			// pull unary scalar operation into spark
-			_etype = ExecType.SPARK;
+		if(transitive // we allow transitive Spark operations. continue sequences of spark operations
+			&& _etype == ExecType.CP // The instruction is currently in CP
+			&& _etypeForced != ExecType.CP // not forced CP
+			&& _etypeForced != ExecType.FED // not federated
+			&& (getDataType().isMatrix() || getDataType().isFrame()) // output should be a matrix or frame
+		) {
+			final boolean v1 = getInput(0).isScalarOrVectorBellowBlockSize();
+			final boolean v2 = getInput(1).isScalarOrVectorBellowBlockSize();
+			final boolean left = v1 == true; // left side is the vector or scalar
+			final Hop sparkIn = getInput(left ? 1 : 0);
+			if((v1 ^ v2) // XOR only one side is allowed to be a vector or a scalar.
+				&& (supportsMatrixScalarOperations() || op == OpOp2.APPLY_SCHEMA)  // supported operation
+				&& sparkIn.getParent().size() == 1 // only one parent
+				&& !HopRewriteUtils.isSingleBlock(sparkIn) // single block triggered exec
+				&& sparkIn.optFindExecType() == ExecType.SPARK // input was spark op.
+				&& !(sparkIn instanceof DataOp) // input is not checkpoint
+			) {
+				// pull operation into spark
+				_etype = ExecType.SPARK;
+			}
 		}
 
 		if( OptimizerUtils.ALLOW_BINARY_UPDATE_IN_PLACE &&
@@ -837,7 +847,7 @@ else if( (op == OpOp2.CBIND && getDataType().isList())
 				|| (op == OpOp2.RBIND && getDataType().isList())) {
 			_etype = ExecType.CP;
 		}
-		
+
 		//mark for recompile (forever)
 		setRequiresRecompileIfNecessary();
 

src/main/java/org/apache/sysds/hops/Hop.java

@@ -1040,6 +1040,12 @@ public final String toString() {
 	// ========================================================================================
 
 
+	protected boolean isScalarOrVectorBellowBlockSize(){
+		return getDataType().isScalar() || (dimsKnown() && 
+			(( _dc.getRows() == 1 && _dc.getCols() < ConfigurationManager.getBlocksize())
+			|| _dc.getCols() == 1 && _dc.getRows() < ConfigurationManager.getBlocksize()));
+	}
+
 	protected boolean isVector() {
 		return (dimsKnown() && (_dc.getRows() == 1 || _dc.getCols() == 1) );
 	}
@@ -1624,6 +1630,11 @@ protected void setMemoryAndComputeEstimates(Lop lop) {
 		lop.setComputeEstimate(ComputeCost.getHOPComputeCost(this));
 	}
 
+	protected boolean hasSparkOutput(){
+		return (this.optFindExecType() == ExecType.SPARK
+			|| (this instanceof DataOp && ((DataOp)this).hasOnlyRDD()));
+	}
+
 	/**
 	 * Set parse information.
 	 *

src/main/java/org/apache/sysds/hops/UnaryOp.java

@@ -366,7 +366,11 @@ protected double computeOutputMemEstimate( long dim1, long dim2, long nnz )
 		} else {
 			sparsity = OptimizerUtils.getSparsity(dim1, dim2, nnz);
 		}
-		return OptimizerUtils.estimateSizeExactSparsity(dim1, dim2, sparsity);
+
+		if(getDataType() == DataType.FRAME)
+			return OptimizerUtils.estimateSizeExactFrame(dim1, dim2);
+		else
+			return OptimizerUtils.estimateSizeExactSparsity(dim1, dim2, sparsity);
 	}
 
 	@Override
@@ -463,6 +467,13 @@ public boolean isMetadataOperation() {
 			|| _op == OpOp1.CAST_AS_LIST;
 	}
 
+	private boolean isDisallowedSparkOps(){
+		return isCumulativeUnaryOperation() 
+			|| isCastUnaryOperation()
+			|| _op==OpOp1.MEDIAN
+			|| _op==OpOp1.IQM;
+	}
+
 	@Override
 	protected ExecType optFindExecType(boolean transitive) 
 	{
@@ -493,19 +504,22 @@ else if ( getInput().get(0).areDimsBelowThreshold() || getInput().get(0).isVecto
 			checkAndSetInvalidCPDimsAndSize();
 		}
 
+
 		//spark-specific decision refinement (execute unary w/ spark input and 
 		//single parent also in spark because it's likely cheap and reduces intermediates)
-		if( _etype == ExecType.CP && _etypeForced != ExecType.CP
-			&& getInput().get(0).optFindExecType() == ExecType.SPARK 
-			&& getDataType().isMatrix() 
-			&& !isCumulativeUnaryOperation() && !isCastUnaryOperation()
-			&& _op!=OpOp1.MEDIAN && _op!=OpOp1.IQM
-			&& !(getInput().get(0) instanceof DataOp)    //input is not checkpoint
-			&& getInput().get(0).getParent().size()==1 ) //unary is only parent
-		{
+		if(_etype == ExecType.CP // currently CP instruction
+			&& _etype != ExecType.SPARK /// currently not SP.
+			&& _etypeForced != ExecType.CP // not forced as CP instruction
+			&& getInput(0).hasSparkOutput() // input is a spark instruction
+			&& (getDataType().isMatrix() || getDataType().isFrame()) // output is a matrix or frame
+			&& !isDisallowedSparkOps() // is invalid spark instruction
+			// && !(getInput().get(0) instanceof DataOp) // input is not checkpoint
+			// && getInput(0).getParent().size() <= 1// unary is only parent
+		) {
 			//pull unary operation into spark 
 			_etype = ExecType.SPARK;
 		}
+
 
 		//mark for recompile (forever)
 		setRequiresRecompileIfNecessary();
@@ -520,7 +534,7 @@ && getInput().get(0).getParent().size()==1 ) //unary is only parent
 		} else {
 			setRequiresRecompileIfNecessary();
 		}
-		
+
 		return _etype;
 	}
 

src/main/java/org/apache/sysds/runtime/compress/CompressedMatrixBlock.java

@@ -58,8 +58,8 @@
 import org.apache.sysds.runtime.compress.lib.CLALibMMChain;
 import org.apache.sysds.runtime.compress.lib.CLALibMatrixMult;
 import org.apache.sysds.runtime.compress.lib.CLALibMerge;
-import org.apache.sysds.runtime.compress.lib.CLALibReplace;
 import org.apache.sysds.runtime.compress.lib.CLALibReorg;
+import org.apache.sysds.runtime.compress.lib.CLALibReplace;
 import org.apache.sysds.runtime.compress.lib.CLALibReshape;
 import org.apache.sysds.runtime.compress.lib.CLALibRexpand;
 import org.apache.sysds.runtime.compress.lib.CLALibScalar;
@@ -101,6 +101,7 @@
 import org.apache.sysds.runtime.util.IndexRange;
 import org.apache.sysds.utils.DMLCompressionStatistics;
 import org.apache.sysds.utils.stats.InfrastructureAnalyzer;
+import org.apache.sysds.utils.stats.Timing;
 
 public class CompressedMatrixBlock extends MatrixBlock {
 	private static final Log LOG = LogFactory.getLog(CompressedMatrixBlock.class.getName());
@@ -475,16 +476,20 @@ public void readFields(DataInput in) throws IOException {
 	}
 
 	public static CompressedMatrixBlock read(DataInput in) throws IOException {
+		Timing t = new Timing();
 		int rlen = in.readInt();
 		int clen = in.readInt();
 		long nonZeros = in.readLong();
 		boolean overlappingColGroups = in.readBoolean();
 		List<AColGroup> groups = ColGroupIO.readGroups(in, rlen);
-		return new CompressedMatrixBlock(rlen, clen, nonZeros, overlappingColGroups, groups);
+		CompressedMatrixBlock ret =  new CompressedMatrixBlock(rlen, clen, nonZeros, overlappingColGroups, groups);
+		LOG.debug("Compressed read serialization time: " + t.stop());
+		return ret;
 	}
 
 	@Override
 	public void write(DataOutput out) throws IOException {
+		Timing t = new Timing();
 		final long estimateUncompressed = nonZeros > 0 ? MatrixBlock.estimateSizeOnDisk(rlen, clen,
 			nonZeros) : Long.MAX_VALUE;
 		final long estDisk = nonZeros > 0 ? getExactSizeOnDisk() : Long.MAX_VALUE;
@@ -512,6 +517,7 @@ public void write(DataOutput out) throws IOException {
 		out.writeLong(nonZeros);
 		out.writeBoolean(overlappingColGroups);
 		ColGroupIO.writeGroups(out, _colGroups);
+		LOG.debug("Compressed write serialization time: " + t.stop());
 	}
 
 	/**
@@ -611,14 +617,6 @@ public MatrixBlock aggregateUnaryOperations(AggregateUnaryOperator op, MatrixVal
 	public MatrixBlock transposeSelfMatrixMultOperations(MatrixBlock out, MMTSJType tstype, int k) {
 		// check for transpose type
 		if(tstype == MMTSJType.LEFT) {
-			if(isEmpty())
-				return new MatrixBlock(clen, clen, true);
-			// create output matrix block
-			if(out == null)
-				out = new MatrixBlock(clen, clen, false);
-			else
-				out.reset(clen, clen, false);
-			out.allocateDenseBlock();
 			CLALibTSMM.leftMultByTransposeSelf(this, out, k);
 			return out;
 		}
@@ -1202,8 +1200,8 @@ public void examSparsity(boolean allowCSR, int k) {
 	}
 
 	@Override
-	public void sparseToDense(int k) {
-		// do nothing
+	public MatrixBlock sparseToDense(int k) {
+		return this; // do nothing
 	}
 
 	@Override

src/main/java/org/apache/sysds/runtime/compress/CompressedMatrixBlockFactory.java

@@ -63,6 +63,8 @@ public class CompressedMatrixBlockFactory {
 
 	private static final Log LOG = LogFactory.getLog(CompressedMatrixBlockFactory.class.getName());
 
+	private static final Object asyncCompressLock = new Object(); 
+
 	/** Timing object to measure the time of each phase in the compression */
 	private final Timing time = new Timing(true);
 	/** Compression statistics gathered throughout the compression */
@@ -165,21 +167,23 @@ public static Future<Void> compressAsync(ExecutionContext ec, String varName) {
 	}
 
 	public static Future<Void> compressAsync(ExecutionContext ec, String varName, InstructionTypeCounter ins) {
-		LOG.debug("Compressing Async");
 		final ExecutorService pool = CommonThreadPool.get(); // We have to guarantee that a thread pool is allocated.
 		return CompletableFuture.runAsync(() -> {
 			// method call or code to be async
 			try {
 				CacheableData<?> data = ec.getCacheableData(varName);
-				if(data instanceof MatrixObject) {
-					MatrixObject mo = (MatrixObject) data;
-					MatrixBlock mb = mo.acquireReadAndRelease();
-					MatrixBlock mbc = CompressedMatrixBlockFactory.compress(mo.acquireReadAndRelease(), ins).getLeft();
-					if(mbc instanceof CompressedMatrixBlock) {
-						ExecutionContext.createCacheableData(mb);
-						mo.acquireModify(mbc);
-						mo.release();
-						mbc.sum(); // calculate sum to forcefully materialize counts
+				synchronized(asyncCompressLock){ // synchronize on the data object to not allow multiple compressions of the same matrix.
+					if(data instanceof MatrixObject) {
+						LOG.debug("Compressing Async");
+						MatrixObject mo = (MatrixObject) data;
+						MatrixBlock mb = mo.acquireReadAndRelease();
+						MatrixBlock mbc = CompressedMatrixBlockFactory.compress(mb, ins).getLeft();
+						if(mbc instanceof CompressedMatrixBlock) {
+							ExecutionContext.createCacheableData(mb);
+							mo.acquireModify(mbc);
+							mo.release();
+							mbc.sum(); // calculate sum to forcefully materialize counts
+						}
 					}
 				}
 			}

src/main/java/org/apache/sysds/runtime/compress/colgroup/ASDCZero.java

@@ -203,6 +203,22 @@ private final void leftMultByMatrixNoPreAggRowsDense(MatrixBlock mb, double[] re
 	 */
 	protected abstract void multiplyScalar(double v, double[] resV, int offRet, AIterator it);
 
+	public void decompressToSparseBlock(SparseBlock sb, int rl, int ru, int offR, int offC, AIterator it) {
+		if(_dict instanceof MatrixBlockDictionary) {
+			final MatrixBlockDictionary md = (MatrixBlockDictionary) _dict;
+			final MatrixBlock mb = md.getMatrixBlock();
+			// The dictionary is never empty.
+			if(mb.isInSparseFormat())
+				// TODO make sparse decompression where the iterator is known in argument
+				decompressToSparseBlockSparseDictionary(sb, rl, ru, offR, offC, mb.getSparseBlock());
+			else
+				decompressToSparseBlockDenseDictionaryWithProvidedIterator(sb, rl, ru, offR, offC, mb.getDenseBlockValues(),
+					it);
+		}
+		else
+			decompressToSparseBlockDenseDictionaryWithProvidedIterator(sb, rl, ru, offR, offC, _dict.getValues(), it);
+	}
+
 	public void decompressToDenseBlock(DenseBlock db, int rl, int ru, int offR, int offC, AIterator it) {
 		if(_dict instanceof MatrixBlockDictionary) {
 			final MatrixBlockDictionary md = (MatrixBlockDictionary) _dict;
@@ -223,6 +239,9 @@ public void decompressToDenseBlockDenseDictionary(DenseBlock db, int rl, int ru,
 		decompressToDenseBlockDenseDictionaryWithProvidedIterator(db, rl, ru, offR, offC, _dict.getValues(), it);
 	}
 
+	public abstract void decompressToSparseBlockDenseDictionaryWithProvidedIterator(SparseBlock db, int rl, int ru,
+	int offR, int offC, double[] values, AIterator it);
+
 	public abstract void decompressToDenseBlockDenseDictionaryWithProvidedIterator(DenseBlock db, int rl, int ru,
 		int offR, int offC, double[] values, AIterator it);
 

src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupIO.java

@@ -94,9 +94,7 @@ public static long getExactSizeOnDisk(List<AColGroup> colGroups) {
 			}
 			ret += grp.getExactSizeOnDisk();
 		}
-		if(LOG.isWarnEnabled())
-			LOG.warn(" duplicate dicts on exact Size on Disk : " + (colGroups.size() - dicts.size()) );
-
+
 		return ret;
 	}
 

src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupSDCSingleZeros.java

@@ -109,10 +109,8 @@ protected void decompressToDenseBlockDenseDictionary(DenseBlock db, int rl, int
 			return;
 		else if(it.value() >= ru)
 			return;
-		// _indexes.cacheIterator(it, ru);
 		else {
 			decompressToDenseBlockDenseDictionaryWithProvidedIterator(db, rl, ru, offR, offC, values, it);
-			// _indexes.cacheIterator(it, ru);
 		}
 	}
 
@@ -238,8 +236,10 @@ protected void decompressToSparseBlockSparseDictionary(SparseBlock ret, int rl,
 		if(it == null)
 			return;
 		else if(it.value() >= ru)
-			_indexes.cacheIterator(it, ru);
-		else if(ru > last) {
+			return;
+			// _indexes.cacheIterator(it, ru);
+		else 
+		if(ru > last) {
 			final int apos = sb.pos(0);
 			final int alen = sb.size(0) + apos;
 			final int[] aix = sb.indexes(0);
@@ -277,8 +277,14 @@ protected void decompressToSparseBlockDenseDictionary(SparseBlock ret, int rl, i
 		if(it == null)
 			return;
 		else if(it.value() >= ru)
-			_indexes.cacheIterator(it, ru);
-		else if(ru > _indexes.getOffsetToLast()) {
+			return;
+		else
+			decompressToSparseBlockDenseDictionaryWithProvidedIterator(ret, rl, ru, offR, offC, values, it);
+	}
+
+	@Override
+	public void decompressToSparseBlockDenseDictionaryWithProvidedIterator(SparseBlock ret, int rl, int ru, int offR, int offC, double[] values, final AIterator it) {
+		if(ru > _indexes.getOffsetToLast()) {
 			final int nCol = _colIndexes.size();
 			final int lastOff = _indexes.getOffsetToLast();
 			int row = offR + it.value();