Skip to content

Commit 5e6ef38

Browse files
BaunsgaardBaunsgaard
committed
[DO NOT MERGE][skip ci] JAVA 17 BWARE COMMIT
1 parent 5f360ef commit 5e6ef38

File tree

128 files changed

+5661
-1349
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

128 files changed

+5661
-1349
lines changed

‎bin/systemds

+4
Original file line numberDiff line numberDiff line change
@@ -413,6 +413,7 @@ if [ $WORKER == 1 ]; then
413413
print_out "# starting Federated worker on port $PORT"
414414
CMD=" \
415415
java $SYSTEMDS_STANDALONE_OPTS \
416+
--add-modules=jdk.incubator.vector \
416417
$LOG4JPROPFULL \
417418
-jar $SYSTEMDS_JAR_FILE \
418419
-w $PORT \
@@ -422,6 +423,7 @@ elif [ "$FEDMONITORING" == 1 ]; then
422423
print_out "# starting Federated backend monitoring on port $PORT"
423424
CMD=" \
424425
java $SYSTEMDS_STANDALONE_OPTS \
426+
--add-modules=jdk.incubator.vector \
425427
$LOG4JPROPFULL \
426428
-jar $SYSTEMDS_JAR_FILE \
427429
-fedMonitoring $PORT \
@@ -433,6 +435,7 @@ elif [ $SYSDS_DISTRIBUTED == 0 ]; then
433435
CMD=" \
434436
java $SYSTEMDS_STANDALONE_OPTS \
435437
$LOG4JPROPFULL \
438+
--add-modules=jdk.incubator.vector \
436439
-jar $SYSTEMDS_JAR_FILE \
437440
-f $SCRIPT_FILE \
438441
-exec $SYSDS_EXEC_MODE \
@@ -442,6 +445,7 @@ else
442445
print_out "# Running script $SCRIPT_FILE distributed with opts: $*"
443446
CMD=" \
444447
spark-submit $SYSTEMDS_DISTRIBUTED_OPTS \
448+
--add-modules=jdk.incubator.vector \
445449
$SYSTEMDS_JAR_FILE \
446450
-f $SCRIPT_FILE \
447451
-exec $SYSDS_EXEC_MODE \

‎pom.xml

+9-3
Original file line numberDiff line numberDiff line change
@@ -67,7 +67,7 @@
6767
<!-- aws-java-sdk-bundle version should align with hadoop-aws version -->
6868
<!-- aws-java-sdk-bundle.version>1.12.367</aws-java-sdk-bundle.version -->
6969
<!-- Set java compile level via argument, ex: 1.8 1.9 10 11-->
70-
<java.level>11</java.level>
70+
<java.level>17</java.level>
7171
<java.version>{java.level}</java.version>
7272
<!-->Testing settings<!-->
7373
<maven.test.skip>false</maven.test.skip>
@@ -77,6 +77,7 @@
7777
<test-forkCount>1C</test-forkCount>
7878
<rerun.failing.tests.count>2</rerun.failing.tests.count>
7979
<jacoco.skip>false</jacoco.skip>
80+
<doc.skip>false</doc.skip>
8081
<jacoco.include>**</jacoco.include>
8182
<automatedtestbase.outputbuffering>false</automatedtestbase.outputbuffering>
8283
<argLine>-Xms3000m -Xmx3000m -Xmn300m</argLine>
@@ -345,6 +346,9 @@
345346
<source>${java.level}</source>
346347
<target>${java.level}</target>
347348
<release>${java.level}</release>
349+
<compilerArgs>
350+
<arg>--add-modules=jdk.incubator.vector</arg>
351+
</compilerArgs>
348352
</configuration>
349353
</plugin>
350354

@@ -367,6 +371,7 @@
367371
<systemPropertyVariables>
368372
<log4j.configurationFile>file:src/test/resources/log4j.properties</log4j.configurationFile>
369373
</systemPropertyVariables>
374+
<argLine>--add-modules=jdk.incubator.vector</argLine>
370375
</configuration>
371376
</plugin>
372377

@@ -875,9 +880,10 @@
875880
<configuration>
876881
<excludePackageNames>*.protobuf</excludePackageNames>
877882
<notimestamp>true</notimestamp>
878-
<failOnWarnings>true</failOnWarnings>
883+
<failOnWarnings>false</failOnWarnings>
879884
<quiet>true</quiet>
880-
<skip>false</skip>
885+
<additionalJOption>--add-modules=jdk.incubator.vector</additionalJOption>
886+
<skip>${doc.skip}</skip>
881887
<show>public</show>
882888
<source>${java.level}</source>
883889
</configuration>

‎src/main/java/org/apache/sysds/hops/AggBinaryOp.java

+1-2
Original file line numberDiff line numberDiff line change
@@ -439,8 +439,7 @@ private boolean isApplicableForTransitiveSparkExecType(boolean left)
439439
|| (left && !isLeftTransposeRewriteApplicable(true)))
440440
&& getInput(index).getParent().size()==1 //bagg is only parent
441441
&& !getInput(index).areDimsBelowThreshold()
442-
&& (getInput(index).optFindExecType() == ExecType.SPARK
443-
|| (getInput(index) instanceof DataOp && ((DataOp)getInput(index)).hasOnlyRDD()))
442+
&& getInput(index).hasSparkOutput()
444443
&& getInput(index).getOutputMemEstimate()>getOutputMemEstimate();
445444
}
446445

‎src/main/java/org/apache/sysds/hops/BinaryOp.java

+53-25
Original file line numberDiff line numberDiff line change
@@ -747,8 +747,8 @@ protected ExecType optFindExecType(boolean transitive) {
747747

748748
checkAndSetForcedPlatform();
749749

750-
DataType dt1 = getInput().get(0).getDataType();
751-
DataType dt2 = getInput().get(1).getDataType();
750+
final DataType dt1 = getInput(0).getDataType();
751+
final DataType dt2 = getInput(1).getDataType();
752752

753753
if( _etypeForced != null ) {
754754
setExecType(_etypeForced);
@@ -796,18 +796,28 @@ else if ( dt1 == DataType.SCALAR && dt2 == DataType.MATRIX ) {
796796
checkAndSetInvalidCPDimsAndSize();
797797
}
798798

799-
//spark-specific decision refinement (execute unary scalar w/ spark input and
799+
// spark-specific decision refinement (execute unary scalar w/ spark input and
800800
// single parent also in spark because it's likely cheap and reduces intermediates)
801-
if(transitive && _etype == ExecType.CP && _etypeForced != ExecType.CP && _etypeForced != ExecType.FED &&
802-
getDataType().isMatrix() // output should be a matrix
803-
&& (dt1.isScalar() || dt2.isScalar()) // one side should be scalar
804-
&& supportsMatrixScalarOperations() // scalar operations
805-
&& !(getInput().get(dt1.isScalar() ? 1 : 0) instanceof DataOp) // input is not checkpoint
806-
&& getInput().get(dt1.isScalar() ? 1 : 0).getParent().size() == 1 // unary scalar is only parent
807-
&& !HopRewriteUtils.isSingleBlock(getInput().get(dt1.isScalar() ? 1 : 0)) // single block triggered exec
808-
&& getInput().get(dt1.isScalar() ? 1 : 0).optFindExecType() == ExecType.SPARK) {
809-
// pull unary scalar operation into spark
810-
_etype = ExecType.SPARK;
801+
if(transitive // we allow transitive Spark operations. continue sequences of spark operations
802+
&& _etype == ExecType.CP // The instruction is currently in CP
803+
&& _etypeForced != ExecType.CP // not forced CP
804+
&& _etypeForced != ExecType.FED // not federated
805+
&& (getDataType().isMatrix() || getDataType().isFrame()) // output should be a matrix or frame
806+
) {
807+
final boolean v1 = getInput(0).isScalarOrVectorBellowBlockSize();
808+
final boolean v2 = getInput(1).isScalarOrVectorBellowBlockSize();
809+
final boolean left = v1 == true; // left side is the vector or scalar
810+
final Hop sparkIn = getInput(left ? 1 : 0);
811+
if((v1 ^ v2) // XOR only one side is allowed to be a vector or a scalar.
812+
&& (supportsMatrixScalarOperations() || op == OpOp2.APPLY_SCHEMA) // supported operation
813+
&& sparkIn.getParent().size() == 1 // only one parent
814+
&& !HopRewriteUtils.isSingleBlock(sparkIn) // single block triggered exec
815+
&& sparkIn.optFindExecType() == ExecType.SPARK // input was spark op.
816+
&& !(sparkIn instanceof DataOp) // input is not checkpoint
817+
) {
818+
// pull operation into spark
819+
_etype = ExecType.SPARK;
820+
}
811821
}
812822

813823
if( OptimizerUtils.ALLOW_BINARY_UPDATE_IN_PLACE &&
@@ -837,7 +847,7 @@ else if( (op == OpOp2.CBIND && getDataType().isList())
837847
|| (op == OpOp2.RBIND && getDataType().isList())) {
838848
_etype = ExecType.CP;
839849
}
840-
850+
841851
//mark for recompile (forever)
842852
setRequiresRecompileIfNecessary();
843853

@@ -1154,17 +1164,35 @@ && getInput().get(0) == that2.getInput().get(0)
11541164
}
11551165

11561166
public boolean supportsMatrixScalarOperations() {
1157-
return ( op==OpOp2.PLUS ||op==OpOp2.MINUS
1158-
||op==OpOp2.MULT ||op==OpOp2.DIV
1159-
||op==OpOp2.MODULUS ||op==OpOp2.INTDIV
1160-
||op==OpOp2.LESS ||op==OpOp2.LESSEQUAL
1161-
||op==OpOp2.GREATER ||op==OpOp2.GREATEREQUAL
1162-
||op==OpOp2.EQUAL ||op==OpOp2.NOTEQUAL
1163-
||op==OpOp2.MIN ||op==OpOp2.MAX
1164-
||op==OpOp2.LOG ||op==OpOp2.POW
1165-
||op==OpOp2.AND ||op==OpOp2.OR ||op==OpOp2.XOR
1166-
||op==OpOp2.BITWAND ||op==OpOp2.BITWOR ||op==OpOp2.BITWXOR
1167-
||op==OpOp2.BITWSHIFTL ||op==OpOp2.BITWSHIFTR);
1167+
switch(op) {
1168+
case PLUS:
1169+
case MINUS:
1170+
case MULT:
1171+
case DIV:
1172+
case MODULUS:
1173+
case INTDIV:
1174+
case LESS:
1175+
case LESSEQUAL:
1176+
case GREATER:
1177+
case GREATEREQUAL:
1178+
case EQUAL:
1179+
case NOTEQUAL:
1180+
case MIN:
1181+
case MAX:
1182+
case LOG:
1183+
case POW:
1184+
case AND:
1185+
case OR:
1186+
case XOR:
1187+
case BITWAND:
1188+
case BITWOR:
1189+
case BITWXOR:
1190+
case BITWSHIFTL:
1191+
case BITWSHIFTR:
1192+
return true;
1193+
default:
1194+
return false;
1195+
}
11681196
}
11691197

11701198
public boolean isPPredOperation() {

‎src/main/java/org/apache/sysds/hops/Hop.java

+11
Original file line numberDiff line numberDiff line change
@@ -1040,6 +1040,12 @@ public final String toString() {
10401040
// ========================================================================================
10411041

10421042

1043+
protected boolean isScalarOrVectorBellowBlockSize(){
1044+
return getDataType().isScalar() || (dimsKnown() &&
1045+
(( _dc.getRows() == 1 && _dc.getCols() < ConfigurationManager.getBlocksize())
1046+
|| _dc.getCols() == 1 && _dc.getRows() < ConfigurationManager.getBlocksize()));
1047+
}
1048+
10431049
protected boolean isVector() {
10441050
return (dimsKnown() && (_dc.getRows() == 1 || _dc.getCols() == 1) );
10451051
}
@@ -1624,6 +1630,11 @@ protected void setMemoryAndComputeEstimates(Lop lop) {
16241630
lop.setComputeEstimate(ComputeCost.getHOPComputeCost(this));
16251631
}
16261632

1633+
protected boolean hasSparkOutput(){
1634+
return (this.optFindExecType() == ExecType.SPARK
1635+
|| (this instanceof DataOp && ((DataOp)this).hasOnlyRDD()));
1636+
}
1637+
16271638
/**
16281639
* Set parse information.
16291640
*

‎src/main/java/org/apache/sysds/hops/TernaryOp.java

+8-2
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@
2121

2222
import org.apache.sysds.api.DMLScript;
2323
import org.apache.sysds.common.Types.DataType;
24+
import org.apache.sysds.common.Types.ExecType;
2425
import org.apache.sysds.common.Types.OpOp2;
2526
import org.apache.sysds.common.Types.OpOp3;
2627
import org.apache.sysds.common.Types.OpOpDG;
@@ -33,8 +34,8 @@
3334
import org.apache.sysds.lops.CentralMoment;
3435
import org.apache.sysds.lops.CoVariance;
3536
import org.apache.sysds.lops.Ctable;
37+
import org.apache.sysds.lops.Data;
3638
import org.apache.sysds.lops.Lop;
37-
import org.apache.sysds.common.Types.ExecType;
3839
import org.apache.sysds.lops.LopsException;
3940
import org.apache.sysds.lops.PickByCount;
4041
import org.apache.sysds.lops.SortKeys;
@@ -273,14 +274,19 @@ private void constructLopsCtable() {
273274
// F=ctable(A,B,W)
274275

275276
DataType dt1 = getInput().get(0).getDataType();
277+
278+
276279
DataType dt2 = getInput().get(1).getDataType();
277280
DataType dt3 = getInput().get(2).getDataType();
278281
Ctable.OperationTypes ternaryOpOrig = Ctable.findCtableOperationByInputDataTypes(dt1, dt2, dt3);
279282

280283
// Compute lops for all inputs
281284
Lop[] inputLops = new Lop[getInput().size()];
282285
for(int i=0; i < getInput().size(); i++) {
283-
inputLops[i] = getInput().get(i).constructLops();
286+
if(i == 0 && HopRewriteUtils.isSequenceSizeOfA(getInput(0), getInput(1)))
287+
inputLops[i] = Data.createLiteralLop(ValueType.INT64, "" +getInput(1).getDim(0));
288+
else
289+
inputLops[i] = getInput().get(i).constructLops();
284290
}
285291

286292
ExecType et = optFindExecType();

‎src/main/java/org/apache/sysds/hops/UnaryOp.java

+24-10
Original file line numberDiff line numberDiff line change
@@ -366,7 +366,11 @@ protected double computeOutputMemEstimate( long dim1, long dim2, long nnz )
366366
} else {
367367
sparsity = OptimizerUtils.getSparsity(dim1, dim2, nnz);
368368
}
369-
return OptimizerUtils.estimateSizeExactSparsity(dim1, dim2, sparsity);
369+
370+
if(getDataType() == DataType.FRAME)
371+
return OptimizerUtils.estimateSizeExactFrame(dim1, dim2);
372+
else
373+
return OptimizerUtils.estimateSizeExactSparsity(dim1, dim2, sparsity);
370374
}
371375

372376
@Override
@@ -463,6 +467,13 @@ public boolean isMetadataOperation() {
463467
|| _op == OpOp1.CAST_AS_LIST;
464468
}
465469

470+
private boolean isDisallowedSparkOps(){
471+
return isCumulativeUnaryOperation()
472+
|| isCastUnaryOperation()
473+
|| _op==OpOp1.MEDIAN
474+
|| _op==OpOp1.IQM;
475+
}
476+
466477
@Override
467478
protected ExecType optFindExecType(boolean transitive)
468479
{
@@ -493,19 +504,22 @@ else if ( getInput().get(0).areDimsBelowThreshold() || getInput().get(0).isVecto
493504
checkAndSetInvalidCPDimsAndSize();
494505
}
495506

507+
496508
//spark-specific decision refinement (execute unary w/ spark input and
497509
//single parent also in spark because it's likely cheap and reduces intermediates)
498-
if( _etype == ExecType.CP && _etypeForced != ExecType.CP
499-
&& getInput().get(0).optFindExecType() == ExecType.SPARK
500-
&& getDataType().isMatrix()
501-
&& !isCumulativeUnaryOperation() && !isCastUnaryOperation()
502-
&& _op!=OpOp1.MEDIAN && _op!=OpOp1.IQM
503-
&& !(getInput().get(0) instanceof DataOp) //input is not checkpoint
504-
&& getInput().get(0).getParent().size()==1 ) //unary is only parent
505-
{
510+
if(_etype == ExecType.CP // currently CP instruction
511+
&& _etype != ExecType.SPARK /// currently not SP.
512+
&& _etypeForced != ExecType.CP // not forced as CP instruction
513+
&& getInput(0).hasSparkOutput() // input is a spark instruction
514+
&& (getDataType().isMatrix() || getDataType().isFrame()) // output is a matrix or frame
515+
&& !isDisallowedSparkOps() // is invalid spark instruction
516+
// && !(getInput().get(0) instanceof DataOp) // input is not checkpoint
517+
// && getInput(0).getParent().size() <= 1// unary is only parent
518+
) {
506519
//pull unary operation into spark
507520
_etype = ExecType.SPARK;
508521
}
522+
509523

510524
//mark for recompile (forever)
511525
setRequiresRecompileIfNecessary();
@@ -519,7 +533,7 @@ && getInput().get(0).getParent().size()==1 ) //unary is only parent
519533
} else {
520534
setRequiresRecompileIfNecessary();
521535
}
522-
536+
523537
return _etype;
524538
}
525539

‎src/main/java/org/apache/sysds/hops/rewrite/HopRewriteUtils.java

+19
Original file line numberDiff line numberDiff line change
@@ -1392,6 +1392,25 @@ public static boolean isBasicN1Sequence(Hop hop)
13921392
return ret;
13931393
}
13941394

1395+
public static boolean isSequenceSizeOfA(Hop hop, Hop A)
1396+
{
1397+
boolean ret = false;
1398+
1399+
if( hop instanceof DataGenOp )
1400+
{
1401+
DataGenOp dgop = (DataGenOp) hop;
1402+
if( dgop.getOp() == OpOpDG.SEQ ){
1403+
Hop from = dgop.getInput().get(dgop.getParamIndex(Statement.SEQ_FROM));
1404+
Hop to = dgop.getInput().get(dgop.getParamIndex(Statement.SEQ_TO));
1405+
Hop incr = dgop.getInput().get(dgop.getParamIndex(Statement.SEQ_INCR));
1406+
ret = (from instanceof LiteralOp && getIntValueSafe((LiteralOp) from) == 1) &&
1407+
(to instanceof LiteralOp && getIntValueSafe((LiteralOp) to) == A.getDim(0)) &&
1408+
(incr instanceof LiteralOp && getIntValueSafe((LiteralOp)incr)==1);
1409+
}
1410+
}
1411+
1412+
return ret;
1413+
}
13951414

13961415
public static Hop getBasic1NSequenceMax(Hop hop) {
13971416
if( isDataGenOp(hop, OpOpDG.SEQ) ) {

‎src/main/java/org/apache/sysds/parser/BuiltinFunctionExpression.java

+2-2
Original file line numberDiff line numberDiff line change
@@ -1974,8 +1974,8 @@ else if(this.getOpCode() == Builtins.MAX_POOL || this.getOpCode() == Builtins.AV
19741974
case DECOMPRESS:
19751975
if(OptimizerUtils.ALLOW_SCRIPT_LEVEL_COMPRESS_COMMAND){
19761976
checkNumParameters(1);
1977-
checkMatrixParam(getFirstExpr());
1978-
output.setDataType(DataType.MATRIX);
1977+
checkMatrixFrameParam(getFirstExpr());
1978+
output.setDataType(getFirstExpr().getOutput().getDataType());
19791979
output.setDimensions(id.getDim1(), id.getDim2());
19801980
output.setBlocksize (id.getBlocksize());
19811981
output.setValueType(id.getValueType());

0 commit comments

Comments
 (0)