From adeebb51757b0cc5da0a525307dfea55bfedcae1 Mon Sep 17 00:00:00 2001 From: wendycwong Date: Mon, 22 Apr 2024 11:25:28 -0700 Subject: [PATCH 1/4] GH-16149: fix gam rebalance bug (#16153) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * add R test that reproduce the error. * add more to test. * GH-16149: fixed key passed to rebalance dataset to avoid collision. * Adopt adam code review comments. Co-authored-by: Veronika Maurerová --- h2o-algos/src/main/java/hex/gam/GAM.java | 4 +- .../gam/runit_GH_16149_gam_row_error.R | 53 +++++++++++++++++++ 2 files changed, 55 insertions(+), 2 deletions(-) create mode 100644 h2o-r/tests/testdir_algos/gam/runit_GH_16149_gam_row_error.R diff --git a/h2o-algos/src/main/java/hex/gam/GAM.java b/h2o-algos/src/main/java/hex/gam/GAM.java index 25f55742c28d..9f598be447e4 100644 --- a/h2o-algos/src/main/java/hex/gam/GAM.java +++ b/h2o-algos/src/main/java/hex/gam/GAM.java @@ -927,7 +927,7 @@ public void computeImpl() { if (error_count() > 0) // if something goes wrong, let's throw a fit throw H2OModelBuilderIllegalArgumentException.makeFromBuilder(GAM.this); // add gamified columns to training frame - Frame newTFrame = new Frame(rebalance(adaptTrain(), false, _result+".temporary.train")); + Frame newTFrame = new Frame(rebalance(adaptTrain(), false, Key.make()+".temporary.train")); verifyGamTransformedFrame(newTFrame); if (error_count() > 0) // if something goes wrong during gam transformation, let's throw a fit again! @@ -937,7 +937,7 @@ public void computeImpl() { int[] singleGamColsCount = new int[]{_cubicSplineNum, _iSplineNum, _mSplineNum}; _valid = rebalance(adaptValidFrame(_parms.valid(), _valid, _parms, _gamColNamesCenter, _binvD, _zTranspose, _knots, _zTransposeCS, _allPolyBasisList, _gamColMeansRaw, _oneOGamColStd, singleGamColsCount), - false, _result + ".temporary.valid"); + false, Key.make() + ".temporary.valid"); } DKV.put(newTFrame); // This one will cause deleted vectors if add to Scope.track Frame newValidFrame = _valid == null ? null : new Frame(_valid); diff --git a/h2o-r/tests/testdir_algos/gam/runit_GH_16149_gam_row_error.R b/h2o-r/tests/testdir_algos/gam/runit_GH_16149_gam_row_error.R new file mode 100644 index 000000000000..2186599a4205 --- /dev/null +++ b/h2o-r/tests/testdir_algos/gam/runit_GH_16149_gam_row_error.R @@ -0,0 +1,53 @@ +setwd(normalizePath(dirname(R.utils::commandArgs(asValues=TRUE)$"f"))) +source("../../../scripts/h2o-r-test-setup.R") + +library(data.table) + +# This test was provided by a customer. No exit condition is needed as +# the test before my fix always failed. As long as this test completes +# successfully, it should be good enough. +test.gam.dataset.error <- function(n) { + sum_insured <- seq(1, 200000, length.out = n) + d2 <- + data.table( + sum_insured = sum_insured, + sqrt = sqrt(sum_insured), + sine = sin(2 * pi * sum_insured / 40000) + ) + d2[, sine := 0.3 * sqrt * sine , ] + d2[, y := pmax(0, sqrt + sine) , ] + + d2[, x := sum_insured] + d2[, x2 := rev(x) , ] # flip axis + + # import the dataset + h2o_data2 <- as.h2o(d2) + + model2 <- + h2o.gam( + y = "y", + gam_columns = c("x2"), + bs = c(2), + spline_orders = c(3), + splines_non_negative = c(F), + training_frame = h2o_data2, + family = "tweedie", + tweedie_variance_power = 1.1, + scale = c(0), + lambda = 0, + alpha = 0, + keep_gam_cols = T, + non_negative = TRUE, + num_knots = c(10) + ) + print("model building completed.") +} + +test.model.gam.dataset.error <- function() { + # test for n=1005 + test.gam.dataset.error(1005) + # test for n=1001; + test.gam.dataset.error(1001) +} + +doTest("General Additive Model dataset size 1001 and 1005 error", test.model.gam.dataset.error) From c12b264a25d8f9742fe0f1db5279171c06ad7342 Mon Sep 17 00:00:00 2001 From: krasinski <8573352+krasinski@users.noreply.github.com> Date: Tue, 23 Apr 2024 21:37:38 +0200 Subject: [PATCH 2/4] GH-16130 Remove distutils version check (#16143) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * GH-16130 Remove distutils version check (cherry picked from commit f1c96d6e303b4447a73f3a7a4b2906bdadf2e70c) * remove matplotlib version check Co-authored-by: Tomáš Frýda (cherry picked from commit bfdb48c1e383c28ffc39ad437b3971d9169eb66a) --- h2o-py/h2o/backend/__init__.py | 1 - h2o-py/h2o/plot/_matplotlib.py | 7 +------ 2 files changed, 1 insertion(+), 7 deletions(-) diff --git a/h2o-py/h2o/backend/__init__.py b/h2o-py/h2o/backend/__init__.py index 6dd0a9df75cf..bcea98905a47 100644 --- a/h2o-py/h2o/backend/__init__.py +++ b/h2o-py/h2o/backend/__init__.py @@ -36,7 +36,6 @@ connection will be used by all subsequent calls to ``h2o.`` functions. Currently, there is no effective way to have multiple connections to separate H2O servers open at the same time. Such facility may be added in the future. """ -from distutils.version import StrictVersion import sys diff --git a/h2o-py/h2o/plot/_matplotlib.py b/h2o-py/h2o/plot/_matplotlib.py index 6658e0b74581..fa0b4212041f 100644 --- a/h2o-py/h2o/plot/_matplotlib.py +++ b/h2o-py/h2o/plot/_matplotlib.py @@ -3,12 +3,7 @@ def get_matplotlib_pyplot(server, raise_if_not_available=False): try: # noinspection PyUnresolvedReferences import matplotlib - from distutils.version import LooseVersion - if server: - if LooseVersion(matplotlib.__version__) <= LooseVersion("3.1"): - matplotlib.use("Agg", warn=False) - else: # Versions >= 3.2 don't have warn argument - matplotlib.use("Agg") + matplotlib.use("Agg") try: # noinspection PyUnresolvedReferences import matplotlib.pyplot as plt From cef5321a500e12096700beb6335914c2489b7ab1 Mon Sep 17 00:00:00 2001 From: wendycwong Date: Fri, 26 Apr 2024 09:20:03 -0700 Subject: [PATCH 3/4] GH-6722 constraint glm (#16035) * GH-6722: add ability to have user find glm coefficient names without waiting for complete model building process. User just need to set max_iterations=0 and then call the model.coef_names() in python or h2o.coef_names(model) in R. GH-6722: extract constraints from betaConstraints and from linear constraints with and without standardization. GH-6722: complete tests to make sure constraint extraction with or without standardization is correct. GH-6722: Streamline GLMConstrainedTest, build matrix representing beta and linear constraints. GH-6722: adding redundant constraint matrix check GH-6722: add QR to check for rank of constraint matrix and added test GH-6722: adding error message about redundant constraints so that user will know which one to remove. GH-6722: added contributinos from constraints for objective function. GH-6722: adding constraint contribution to gram and gradient GH-6722: added test to make sure constraints contribution to gram and gradient is correct. GH-6722: Added constraint parameters update and constraint stopping conditions. GH-6722: finished taken care of gram zero cols and added tests. GH-6722: Add exact line search. GH-6722: Complete constraints/derivative/gram contribution update from beta change. Add short circuit test. GH-6722: Allow users to set parameters so that they can control how the constraint parameters change. GH-6722 moved some linear constraint checks before expensive=true GH-6722 make objective() the function call to get training objective results with linear constraints. Incorporate Tomas F comments Incorporate Veronika Maurever comments. --- .../hex/Infogram/InfogramCVValidTest.java | 2 - .../hex/gam/MatrixFrameUtils/GamUtils.java | 7 + .../src/main/java/hex/glm/CoefIndices.java | 4 + .../main/java/hex/glm/ComputationState.java | 505 +++++++- .../java/hex/glm/ConstrainedGLMUtils.java | 859 +++++++++++++ h2o-algos/src/main/java/hex/glm/GLM.java | 748 ++++++++--- h2o-algos/src/main/java/hex/glm/GLMModel.java | 20 + h2o-algos/src/main/java/hex/glm/GLMTask.java | 13 +- h2o-algos/src/main/java/hex/glm/GLMUtils.java | 8 + h2o-algos/src/main/java/hex/gram/Gram.java | 16 + .../hex/optimization/OptimizationUtils.java | 141 ++- .../src/main/java/hex/schemas/GLMModelV3.java | 27 +- .../src/main/java/hex/schemas/GLMV3.java | 148 ++- .../test/java/hex/glm/GLMConstrainedTest.java | 1090 +++++++++++++++++ .../hex/optimization/ExactLineSearchTest.java | 149 +++ h2o-bindings/bin/custom/python/gen_glm.py | 90 ++ .../src/main/java/water/util/ArrayUtils.java | 18 + .../src/main/java/water/util/FrameUtils.java | 3 +- h2o-py/h2o/estimators/glm.py | 344 +++++- h2o-py/h2o/model/model_base.py | 9 + h2o-py/tests/pyunit_utils/__init__.py | 1 + h2o-py/tests/pyunit_utils/utilsPY.py | 6 +- .../tests/pyunit_utils/utils_for_glm_tests.py | 148 +++ ...se_lessthan_linear_constraints_binomial.py | 212 ++++ ...nstraints_binomial_objective_likelihood.py | 173 +++ ..._GH_6722_constraints_on_collinear_cols1.py | 76 ++ ..._GH_6722_constraints_on_collinear_cols2.py | 56 + ...6722_equality_constraints_only_binomial.py | 134 ++ ...se_lessthan_linear_constraints_binomial.py | 173 +++ .../glm/pyunit_GH_6722_glm_coefNames.py | 33 + ..._equality_lessthan_constraints_binomial.py | 212 ++++ ..._equality_lessthan_constraints_binomial.py | 190 +++ ..._tight_linear_constraints_only_binomial.py | 206 ++++ ...pyunit_GH_6722_linear_constraints_error.py | 31 + ..._loose_beta_linear_constraints_binomial.py | 162 +++ ..._loose_only_linear_constraints_binomial.py | 140 +++ .../pyunit_GH_6722_redundant_constraints.py | 207 ++++ ...t_GH_6722_separate_linear_beta_gaussian.py | 113 ++ ...ta_equality_linear_constraints_binomial.py | 322 +++++ ...ht_equality_linear_constraints_binomial.py | 242 ++++ ..._tight_linear_constraints_only_binomial.py | 206 ++++ .../glm/pyunit_PUBDEV_7730_plot_pr_xval.py | 4 +- h2o-r/h2o-package/R/glm.R | 105 +- h2o-r/h2o-package/R/models.R | 119 ++ h2o-r/h2o-package/pkgdown/_pkgdown.yml | 3 + .../glm/runit_GH_6722_redundant_constraints.R | 31 + .../glm/runit_gh_6722_glm_coefficient_names.R | 30 + 47 files changed, 7214 insertions(+), 322 deletions(-) create mode 100644 h2o-algos/src/main/java/hex/glm/CoefIndices.java create mode 100644 h2o-algos/src/main/java/hex/glm/ConstrainedGLMUtils.java create mode 100644 h2o-algos/src/test/java/hex/glm/GLMConstrainedTest.java create mode 100644 h2o-algos/src/test/java/hex/optimization/ExactLineSearchTest.java create mode 100644 h2o-py/tests/pyunit_utils/utils_for_glm_tests.py create mode 100644 h2o-py/tests/testdir_algos/glm/pyunit_GH_6722_beta_equality_loose_lessthan_linear_constraints_binomial.py create mode 100644 h2o-py/tests/testdir_algos/glm/pyunit_GH_6722_beta_linear_constraints_binomial_objective_likelihood.py create mode 100644 h2o-py/tests/testdir_algos/glm/pyunit_GH_6722_constraints_on_collinear_cols1.py create mode 100644 h2o-py/tests/testdir_algos/glm/pyunit_GH_6722_constraints_on_collinear_cols2.py create mode 100644 h2o-py/tests/testdir_algos/glm/pyunit_GH_6722_equality_constraints_only_binomial.py create mode 100644 h2o-py/tests/testdir_algos/glm/pyunit_GH_6722_equality_loose_lessthan_linear_constraints_binomial.py create mode 100644 h2o-py/tests/testdir_algos/glm/pyunit_GH_6722_glm_coefNames.py create mode 100644 h2o-py/tests/testdir_algos/glm/pyunit_GH_6722_light_tight_beta_equality_lessthan_constraints_binomial.py create mode 100644 h2o-py/tests/testdir_algos/glm/pyunit_GH_6722_light_tight_equality_lessthan_constraints_binomial.py create mode 100644 h2o-py/tests/testdir_algos/glm/pyunit_GH_6722_light_tight_linear_constraints_only_binomial.py create mode 100644 h2o-py/tests/testdir_algos/glm/pyunit_GH_6722_linear_constraints_error.py create mode 100644 h2o-py/tests/testdir_algos/glm/pyunit_GH_6722_loose_beta_linear_constraints_binomial.py create mode 100644 h2o-py/tests/testdir_algos/glm/pyunit_GH_6722_loose_only_linear_constraints_binomial.py create mode 100644 h2o-py/tests/testdir_algos/glm/pyunit_GH_6722_redundant_constraints.py create mode 100644 h2o-py/tests/testdir_algos/glm/pyunit_GH_6722_separate_linear_beta_gaussian.py create mode 100644 h2o-py/tests/testdir_algos/glm/pyunit_GH_6722_tight_beta_equality_linear_constraints_binomial.py create mode 100644 h2o-py/tests/testdir_algos/glm/pyunit_GH_6722_tight_equality_linear_constraints_binomial.py create mode 100644 h2o-py/tests/testdir_algos/glm/pyunit_GH_6722_tight_linear_constraints_only_binomial.py create mode 100644 h2o-r/tests/testdir_algos/glm/runit_GH_6722_redundant_constraints.R create mode 100644 h2o-r/tests/testdir_algos/glm/runit_gh_6722_glm_coefficient_names.R diff --git a/h2o-admissibleml/src/test/java/hex/Infogram/InfogramCVValidTest.java b/h2o-admissibleml/src/test/java/hex/Infogram/InfogramCVValidTest.java index 97488a82f232..f97f4e06adb8 100644 --- a/h2o-admissibleml/src/test/java/hex/Infogram/InfogramCVValidTest.java +++ b/h2o-admissibleml/src/test/java/hex/Infogram/InfogramCVValidTest.java @@ -4,9 +4,7 @@ import org.junit.Test; import org.junit.runner.RunWith; import water.*; -import water.fvec.Chunk; import water.fvec.Frame; -import water.fvec.NewChunk; import water.fvec.Vec; import water.runner.CloudSize; import water.runner.H2ORunner; diff --git a/h2o-algos/src/main/java/hex/gam/MatrixFrameUtils/GamUtils.java b/h2o-algos/src/main/java/hex/gam/MatrixFrameUtils/GamUtils.java index dc0fc4004b68..aab8dc90e379 100644 --- a/h2o-algos/src/main/java/hex/gam/MatrixFrameUtils/GamUtils.java +++ b/h2o-algos/src/main/java/hex/gam/MatrixFrameUtils/GamUtils.java @@ -146,6 +146,13 @@ public static void copy2DArray(double[][] src_array, double[][] dest_array) { src_array[colIdx].length); } } + + // copy a square array + public static double[][] copy2DArray(double[][] src_array) { + double[][] dest_array = MemoryManager.malloc8d(src_array.length, src_array[0].length); + copy2DArray(src_array, dest_array); + return dest_array; + } public static void copy2DArray(int[][] src_array, int[][] dest_array) { int numRows = src_array.length; diff --git a/h2o-algos/src/main/java/hex/glm/CoefIndices.java b/h2o-algos/src/main/java/hex/glm/CoefIndices.java new file mode 100644 index 000000000000..01743b943072 --- /dev/null +++ b/h2o-algos/src/main/java/hex/glm/CoefIndices.java @@ -0,0 +1,4 @@ +package hex.glm; + +public interface CoefIndices { +} diff --git a/h2o-algos/src/main/java/hex/glm/ComputationState.java b/h2o-algos/src/main/java/hex/glm/ComputationState.java index 38e135b7d3aa..23c343202e0d 100644 --- a/h2o-algos/src/main/java/hex/glm/ComputationState.java +++ b/h2o-algos/src/main/java/hex/glm/ComputationState.java @@ -10,25 +10,37 @@ import hex.optimization.ADMM; import hex.optimization.OptimizationUtils.GradientInfo; import hex.optimization.OptimizationUtils.GradientSolver; +import jsr166y.ForkJoinTask; +import jsr166y.RecursiveAction; +import water.H2O; +import water.H2ORuntime; import water.Job; import water.MemoryManager; import water.fvec.Frame; import water.util.ArrayUtils; +import water.util.IcedHashMap; import water.util.Log; import water.util.MathUtils; -import java.util.Arrays; -import java.util.Comparator; +import java.util.*; +import java.util.stream.Collectors; import java.util.stream.IntStream; +import static hex.glm.ComputationState.GramGrad.findZeroCols; +import static hex.glm.ConstrainedGLMUtils.*; import static hex.glm.GLMModel.GLMParameters.Family.gaussian; import static hex.glm.GLMUtils.calSmoothNess; import static hex.glm.GLMUtils.copyGInfo; +import static water.util.ArrayUtils.*; public final class ComputationState { + private static final double R2_EPS = 1e-7; + public static final double EPS_CS = 1e-6; + public static final double EPS_CS_SQUARE = EPS_CS*EPS_CS; + private static final int MIN_PAR = 1000; final boolean _intercept; final int _nbetas; - private final GLMParameters _parms; + public final GLMParameters _parms; private BetaConstraint _bc; double _alpha; double[] _ymu; @@ -46,7 +58,19 @@ public final class ComputationState { private boolean _lambdaNull; // true if lambda was not provided by user private double _gMax; // store max value of original gradient without dividing by math.max(1e-2, _parms._alpha[0]) private DataInfo _activeData; - private BetaConstraint _activeBC = null; + private BetaConstraint _activeBC; + LinearConstraints[] _equalityConstraintsLinear = null; + LinearConstraints[] _lessThanEqualToConstraintsLinear = null; + LinearConstraints[] _equalityConstraintsBeta = null; + LinearConstraints[] _lessThanEqualToConstraintsBeta = null; + LinearConstraints[] _equalityConstraints = null; + LinearConstraints[] _lessThanEqualToConstraints = null; + double[] _lambdaEqual; + double[] _lambdaLessThanEqualTo; + ConstraintsDerivatives[] _derivativeEqual = null; + ConstraintsDerivatives[] _derivativeLess = null; + ConstraintsGram[] _gramEqual = null; + ConstraintsGram[] _gramLess = null; private final GLM.BetaInfo _modelBetaInfo; private double[] _beta; // vector of coefficients corresponding to active data private double[] _ubeta; // HGLM, store coefficients of random effects; @@ -66,7 +90,10 @@ public final class ComputationState { int[][] _gamBetaIndices; int _totalBetaLength; // actual coefficient length without taking into account active columns only int _betaLengthPerClass; - + public boolean _noReg; + public boolean _hasConstraints; + public ConstrainedGLMUtils.ConstraintGLMStates _csGLMState; + public ComputationState(Job job, GLMParameters parms, DataInfo dinfo, BetaConstraint bc, GLM.BetaInfo bi){ _job = job; _parms = parms; @@ -87,6 +114,59 @@ public ComputationState(Job job, GLMParameters parms, DataInfo dinfo, BetaConstr _modelBetaInfo = bi; } + /** + * This method calculates + * 1. the contribution of constraints to the gradient; + * 2. the contribution of ||h(beta)||^2 to the gradient and the hessian. + * + * Note that this calculation is only needed once since the contributions to the derivative and hessian depends only + * on the value of linear constraint coefficients and not the actual glm model parameters. Refer to the doc, + * section VI. + */ + public void initConstraintDerivatives(LinearConstraints[] equalityConstraints, LinearConstraints[] lessThanEqualToConstraints, + List coeffNames) { + boolean hasEqualityConstraints = equalityConstraints != null; + boolean hasLessConstraints = lessThanEqualToConstraints != null; + _derivativeEqual = hasEqualityConstraints ? calDerivatives(equalityConstraints, coeffNames) : null; + _derivativeLess = hasLessConstraints ? calDerivatives(lessThanEqualToConstraints, coeffNames) : null; + // contribution to gradient and hessian from ||h(beta)||^2 without C, stays constant once calculated, active status can change + _gramEqual = hasEqualityConstraints ? calGram(_derivativeEqual) : null; + _gramLess = hasLessConstraints ? calGram(_derivativeLess) : null; + } + + /*** + * Any time when the glm coefficient changes, the constraints values will change and active constraints can be inactive + * and vice versa. In addition, the active status of the derivatives and 2nd derivatives can change as well. The + * derivative and 2nd derivatives are part of the ComputationState. It is the purpose of this method to change the + * active status of the constraint derivatives (transpose(lambda)*h(beta)) and the 2nd order derivatives of + * (ck/2*transpose(h(beta))*h(beta)). + */ + public void updateConstraintInfo(LinearConstraints[] equalityConstraints, LinearConstraints[] lessThanEqualToConstraints) { + updateDerivativeActive(_derivativeEqual, _gramEqual, equalityConstraints); + updateDerivativeActive(_derivativeLess, _gramLess, lessThanEqualToConstraints); + } + + public void updateDerivativeActive(ConstraintsDerivatives[] derivativesConst, ConstraintsGram[] gramConst, + LinearConstraints[] constraints) { + if (constraints != null) { + IntStream.range(0, derivativesConst.length).forEach(index -> { + derivativesConst[index]._active = constraints[index]._active; + gramConst[index]._active = constraints[index]._active; + }); + } + } + + public void resizeConstraintInfo(LinearConstraints[] equalityConstraints, + LinearConstraints[] lessThanEqualToConstraints) { + boolean hasEqualityConstraints = _derivativeEqual != null; + boolean hasLessConstraints = _derivativeLess != null; + List coeffNames = Arrays.stream(_activeData.coefNames()).collect(Collectors.toList()); + _derivativeEqual = hasEqualityConstraints ? calDerivatives(equalityConstraints, coeffNames) : null; + _derivativeLess = hasLessConstraints ? calDerivatives(lessThanEqualToConstraints, coeffNames) : null; + _gramEqual = hasEqualityConstraints ? calGram(_derivativeEqual) : null; + _gramLess = hasLessConstraints ? calGram(_derivativeLess) : null; + } + public ComputationState(Job job, GLMParameters parms, DataInfo dinfo, BetaConstraint bc, GLM.BetaInfo bi, double[][][] penaltyMat, int[][] gamColInd){ this (job, parms, dinfo, bc, bi); @@ -335,7 +415,8 @@ protected void applyStrongRules(double lambdaNew, double lambdaOld) { int P = _dinfo.fullN(); _activeBC = _bc; _activeData = _activeData != null?_activeData:_dinfo; - _allIn = _allIn || _alpha*lambdaNew == 0 || _activeBC.hasBounds(); + // keep all predictors for the case of beta constraints or linear constraints + _allIn = _allIn || _alpha*lambdaNew == 0 || _activeBC.hasBounds() || _parms._linear_constraints != null; if (!_allIn) { int newlySelected = 0; final double rhs = Math.max(0,_alpha * (2 * lambdaNew - lambdaOld)); @@ -513,6 +594,16 @@ public void setBC(BetaConstraint bc) { _bc = bc; _activeBC = _bc; } + + public void setLinearConstraints(LinearConstraints[] equalityC, LinearConstraints[] lessThanEqualToC, boolean forBeta) { + if (forBeta) { + _equalityConstraintsBeta = equalityC.length == 0 ? null : equalityC; + _lessThanEqualToConstraintsBeta = lessThanEqualToC.length == 0 ? null : lessThanEqualToC; + } else { + _equalityConstraintsLinear = equalityC.length == 0 ? null : equalityC; + _lessThanEqualToConstraintsLinear = lessThanEqualToC.length == 0 ? null : lessThanEqualToC; + } + } public void setActiveClass(int activeClass) {_activeClass = activeClass;} @@ -539,7 +630,7 @@ public double deviance() { * This method will grab a subset of the gradient for each multinomial class. However, if remove_collinear_columns is * on, fullInfo will only contains the gradient of active columns. */ - public static class GLMSubsetGinfo extends GLMGradientInfo { + public static class GLMSubsetGinfo extends GLM.GLMGradientInfo { public final GLMGradientInfo _fullInfo; public GLMSubsetGinfo(GLMGradientInfo fullInfo, int N, int c, int [] ids) { super(fullInfo._likelihood, fullInfo._objVal, extractSubRange(N,c,ids,fullInfo._gradient)); @@ -834,9 +925,78 @@ public double objective(double [] beta, double likelihood) { else gamVal = calSmoothNess(expandBeta(beta), _penaltyMatrix, _gamBetaIndices); // take up memory } - return likelihood * _parms._obj_reg + gamVal + penalty(beta) + (_activeBC == null?0:_activeBC.proxPen(beta)); + if (_csGLMState != null && (_equalityConstraints != null || _lessThanEqualToConstraints != null)) + return _ginfo._objVal; + else + return likelihood * _parms._obj_reg + gamVal + penalty(beta) + (_activeBC == null?0:_activeBC.proxPen(beta)); + } + + /*** + * + * This methold will calculate the first derivative of h(beta). Refer to the doc, section VI.I + * + */ + public static ConstrainedGLMUtils.ConstraintsDerivatives[] calDerivatives(LinearConstraints[] constraints, List coefNames) { + int numConstraints = constraints.length; + ConstrainedGLMUtils.ConstraintsDerivatives[] constDeriv = new ConstrainedGLMUtils.ConstraintsDerivatives[numConstraints]; + LinearConstraints oneConstraint; + for (int index=0; index coeffNames) { + ConstrainedGLMUtils.ConstraintsDerivatives constraintDerivative = new ConstrainedGLMUtils.ConstraintsDerivatives(oneConstraints._active); + IcedHashMap coeffNameValues = oneConstraints._constraints; + int index; + for (String coefName: coeffNameValues.keySet()) { + index = coeffNames.indexOf(coefName); + if (index >= 0) + constraintDerivative._constraintsDerivative.put(index, coeffNameValues.get(coefName)); + } + return constraintDerivative; + } + + /*** + * This method to calculate contribution of penalty to gram (d2H/dbidbj), refer to the doc Section VI.II + */ + public static ConstrainedGLMUtils.ConstraintsGram[] calGram(ConstrainedGLMUtils.ConstraintsDerivatives[] derivativeEqual) { + return Arrays.stream(derivativeEqual).map(x -> constructGram(x)).toArray(ConstrainedGLMUtils.ConstraintsGram[]::new); + } + + /*** + * This method is not called often. If called, it will calculate the contribution of constraints to the + * hessian. Whenever there is a predictor number change, this function should be called again as it only looks + * at the predictor index. This predictor index will change when the number of predictors change. It calculates + * the second derivative regardless of the active status because an inactive constraint may become active in the + * future. Note that here, only half of the 2nd derivatives are calculated, namely d(tranpose(h(beta))*h(beta)/dCidCj + * and not d(tranpose(h(beta))*h(beta)/dCjdCi since they are symmetric. + */ + public static ConstrainedGLMUtils.ConstraintsGram constructGram(ConstrainedGLMUtils.ConstraintsDerivatives constDeriv) { + ConstrainedGLMUtils.ConstraintsGram cGram = new ConstrainedGLMUtils.ConstraintsGram(); + List predictorIndexc = constDeriv._constraintsDerivative.keySet().stream().collect(Collectors.toList()); + Collections.sort(predictorIndexc); + while (!predictorIndexc.isEmpty()) { + Integer firstEle = predictorIndexc.get(0); + for (Integer oneCoeff : predictorIndexc) { + ConstrainedGLMUtils.CoefIndices coefPairs = new ConstrainedGLMUtils.CoefIndices(firstEle, oneCoeff); + cGram._coefIndicesValue.put(coefPairs, constDeriv._constraintsDerivative.get(firstEle)*constDeriv._constraintsDerivative.get(oneCoeff)); + } + predictorIndexc.remove(0); + } + cGram._active = constDeriv._active; // calculate for active/inactive constraints, inactive may be active in future + return cGram; } - protected double updateState(double [] beta, double likelihood) { + + + protected double updateState(double [] beta, double likelihood) { _betaDiff = ArrayUtils.linfnorm(_beta == null?beta:ArrayUtils.subtract(_beta,beta),false); double objOld = objective(); _beta = beta; @@ -860,7 +1020,7 @@ public boolean converged(){ return converged; } - protected double updateState(double [] beta,GLMGradientInfo ginfo){ + public double updateState(double [] beta,GLMGradientInfo ginfo) { double objOld; if (_beta != null && beta.length > _beta.length) { // beta is full while _beta only contains active columns double[] shortBeta = shrinkFullArray(beta); @@ -874,14 +1034,13 @@ protected double updateState(double [] beta,GLMGradientInfo ginfo){ if(_beta == null)_beta = beta.clone(); else System.arraycopy(beta,0,_beta,0,beta.length); } - _ginfo = ginfo; _likelihood = ginfo._likelihood; - return (_relImprovement = (objOld - objective())/Math.abs(objOld)); + _relImprovement = (objOld - objective()) / Math.abs(objOld); + return _relImprovement; } double getBetaDiff() {return _betaDiff;} - double getGradientErr() {return _gradientErr;} protected void setBetaDiff(double betaDiff) { _betaDiff = betaDiff; } protected void setGradientErr(double gErr) { _gradientErr = gErr; } protected void setGinfo(GLMGradientInfo ginfo) { @@ -925,6 +1084,247 @@ protected void setHGLMComputationState(double [] beta, double[] ubeta, double[] else return expandToFullArray(beta, _activeData.activeCols(), _totalBetaLength, _nbetas, _betaLengthPerClass); } + + public static class GramGrad { + public double[][] _gram; + public double[] beta; + public double[] _grad; + public double objective; + public double _sumOfRowWeights; + public double[] _xy; + + public GramGrad(double[][] gramM, double[] grad, double[] b, double obj, double sumOfRowWeights, double[] xy) { + _gram = gramM; + beta = b; + _grad = grad; + objective = obj; + _sumOfRowWeights = sumOfRowWeights; + _xy = xy; + } + + public Gram.Cholesky cholesky(Gram.Cholesky chol, double[][] xx) { + if( chol == null ) { + for( int i = 0; i < xx.length; ++i ) + xx[i] = xx[i].clone(); + chol = new Gram.Cholesky(xx, new double[0]); + } + final Gram.Cholesky fchol = chol; + final int sparseN = 0; + final int denseN = xx.length - sparseN; + // compute the cholesky of the diagonal and diagonal*dense parts + ForkJoinTask[] fjts = new ForkJoinTask[denseN]; + // compute the outer product of diagonal*dense + //Log.info("SPARSEN = " + sparseN + " DENSEN = " + denseN); + final int[][] nz = new int[denseN][]; + for( int i = 0; i < denseN; ++i ) { + final int fi = i; + fjts[i] = new RecursiveAction() { + @Override protected void compute() { + int[] tmp = new int[sparseN]; + double[] rowi = fchol._xx[fi]; + int n = 0; + for( int k = 0; k < sparseN; ++k ) + if (rowi[k] != .0) tmp[n++] = k; + nz[fi] = Arrays.copyOf(tmp, n); + } + }; + } + ForkJoinTask.invokeAll(fjts); + for( int i = 0; i < denseN; ++i ) { + final int fi = i; + fjts[i] = new RecursiveAction() { + @Override protected void compute() { + double[] rowi = fchol._xx[fi]; + int[] nzi = nz[fi]; + for( int j = 0; j <= fi; ++j ) { + double[] rowj = fchol._xx[j]; + int[] nzj = nz[j]; + double s = 0; + for (int t=0,z=0; t < nzi.length && z < nzj.length; ) { + int k1 = nzi[t]; + int k2 = nzj[z]; + if (k1 < k2) { t++; continue; } + else if (k1 > k2) { z++; continue; } + else { + s += rowi[k1] * rowj[k1]; + t++; z++; + } + } + rowi[j + sparseN] = xx[fi][j + sparseN] - s; + } + } + }; + } + ForkJoinTask.invokeAll(fjts); + // compute the cholesky of dense*dense-outer_product(diagonal*dense) + double[][] arr = new double[denseN][]; + for( int i = 0; i < arr.length; ++i ) + arr[i] = Arrays.copyOfRange(fchol._xx[i], sparseN, sparseN + denseN); + final int p = H2ORuntime.availableProcessors(); + Gram.InPlaceCholesky d = Gram.InPlaceCholesky.decompose_2(arr, 10, p); + fchol.setSPD(d.isSPD()); + arr = d.getL(); + for( int i = 0; i < arr.length; ++i ) { + // See PUBDEV-5585: we use a manual array copy instead of System.arraycopy because of behavior on Java 10 + // Used to be: System.arraycopy(arr[i], 0, fchol._xx[i], sparseN, i + 1); + for (int j = 0; j < i + 1; j++) + fchol._xx[i][sparseN + j] = arr[i][j]; + } + + return chol; + } + + public Gram.Cholesky qrCholesky(List dropped_cols, double[][] Z, boolean standardized) { + final double [][] R = new double[Z.length][]; + final double [] Zdiag = new double[Z.length]; + final double [] ZdiagInv = new double[Z.length]; + for(int i = 0; i < Z.length; ++i) + ZdiagInv[i] = 1.0/(Zdiag[i] = Z[i][i]); + for(int j = 0; j < Z.length; ++j) { + final double [] gamma = R[j] = new double[j+1]; + for(int l = 0; l <= j; ++l) // compute gamma_l_j + gamma[l] = Z[j][l]*ZdiagInv[l]; + double zjj = Z[j][j]; + for(int k = 0; k < j; ++k) // only need the diagonal, the rest is 0 (dot product of orthogonal vectors) + zjj += gamma[k] * (gamma[k] * Z[k][k] - 2*Z[j][k]); + // Check R^2 for the current column and ignore if too high (1-R^2 too low), R^2 = 1- rs_res/rs_tot + // rs_res = zjj (the squared residual) + // rs_tot = sum((yi - mean(y))^2) = mean(y^2) - mean(y)^2, + // mean(y^2) is on diagonal + // mean(y) is in the intercept (0 if standardized) + // might not be regularized with number of observations, that's why dividing by intercept diagonal + double rs_tot = standardized + ?ZdiagInv[j] + :1.0/(Zdiag[j]-Z[j][0]*ZdiagInv[0]*Z[j][0]); + if (j > 0 && zjj*rs_tot < R2_EPS) { + zjj=0; + dropped_cols.add(j-1); + ZdiagInv[j] = 0; + } else { + ZdiagInv[j] = 1. / zjj; + } + Z[j][j] = zjj; + int jchunk = Math.max(1,MIN_PAR/(Z.length-j)); + int nchunks = (Z.length - j - 1)/jchunk; + nchunks = Math.min(nchunks, H2O.NUMCPUS); + if(nchunks <= 1) { // single threaded update + updateZ(gamma,Z,j); + } else { // multi-threaded update + final int fjchunk = (Z.length - 1 - j)/nchunks; + int rem = Z.length - 1 - j - fjchunk*nchunks; + for(int i = Z.length-rem; i < Z.length; ++i) + updateZij(i,j,Z,gamma); + RecursiveAction[] ras = new RecursiveAction[nchunks]; + final int fj = j; + int k = 0; + for (int i = j + 1; i < Z.length-rem; i += fjchunk) { // update xj to zj // + final int fi = i; + ras[k++] = new RecursiveAction() { + @Override + protected final void compute() { + int max_i = Math.min(fi+fjchunk,Z.length); + for(int i = fi; i < max_i; ++i) + updateZij(i,fj,Z,gamma); + } + }; + } + ForkJoinTask.invokeAll(ras); + } + } + // update the R - we computed Rt/sqrt(diag(Z)) which we can directly use to solve the problem + if(R.length < 500) + for(int i = 0; i < R.length; ++i) + for (int j = 0; j <= i; ++j) + R[i][j] *= Math.sqrt(Z[j][j]); + else { + RecursiveAction[] ras = new RecursiveAction[R.length]; + for(int i = 0; i < ras.length; ++i) { + final int fi = i; + final double [] Rrow = R[i]; + ras[i] = new RecursiveAction() { + @Override + protected void compute() { + for (int j = 0; j <= fi; ++j) + Rrow[j] *= Math.sqrt(Z[j][j]); + } + }; + } + ForkJoinTask.invokeAll(ras); + } + // deal with dropped_cols if present + if (dropped_cols.isEmpty()) + return new Gram.Cholesky(R, new double[0], true); + else + return new Gram.Cholesky(dropIgnoredCols(R, Z, dropped_cols),new double[0], true); + } + + public static double[][] dropIgnoredCols(double[][] R, double[][] Z, List dropped_cols) { + double[][] Rnew = new double[R.length-dropped_cols.size()][]; + for(int i = 0; i < Rnew.length; ++i) + Rnew[i] = new double[i+1]; + int j = 0; + for(int i = 0; i < R.length; ++i) { + if(Z[i][i] == 0) continue; + int k = 0; + for(int l = 0; l <= i; ++l) { + if(k < dropped_cols.size() && l == (dropped_cols.get(k)+1)) { + ++k; + continue; + } + Rnew[j][l - k] = R[i][l]; + } + ++j; + } + return Rnew; + } + + private final void updateZij(int i, int j, double [][] Z, double [] gamma) { + double [] Zi = Z[i]; + double Zij = Zi[j]; + for (int k = 0; k < j; ++k) + Zij -= gamma[k] * Zi[k]; + Zi[j] = Zij; + } + private final void updateZ(final double [] gamma, final double [][] Z, int j){ + for (int i = j + 1; i < Z.length; ++i) // update xj to zj // + updateZij(i,j,Z,gamma); + } + + public static double[][] dropCols(int[] cols, double[][] xx) { + Arrays.sort(cols); + int newXXLen = xx.length-cols.length; + double [][] xxNew = new double[newXXLen][newXXLen]; + int oldXXLen = xx.length; + List newIndices = IntStream.range(0, newXXLen).boxed().collect(Collectors.toList()); + for (int index:cols) + newIndices.add(index,-1); + int newXindexX, newXindexY; + for (int rInd=0; rInd= 0 && newXindexX >= 0) { + xxNew[newXindexX][newXindexY] = xx[rInd][cInd]; + xxNew[newXindexY][newXindexX] = xx[cInd][rInd]; + } + } + } + return xxNew; + } + + public static int[] findZeroCols(double[][] xx){ + ArrayList zeros = new ArrayList<>(); + for(int i = 0; i < xx.length; ++i) { + if (sum(xx[i]) == 0) + zeros.add(i); + } + if(zeros.size() == 0) return new int[0]; + int [] ary = new int[zeros.size()]; + for(int i = 0; i < zeros.size(); ++i) + ary[i] = zeros.get(i); + return ary; + } + } /** * Cached state of COD (with covariate updates) solver. @@ -1014,7 +1414,7 @@ protected GramXY computeNewGram(DataInfo activeData, double [] beta, GLMParamete double obj_reg = _parms._obj_reg; if(_glmw == null) _glmw = new GLMModel.GLMWeightsFun(_parms); GLMTask.GLMIterationTask gt = new GLMTask.GLMIterationTask(_job._key, activeData, _glmw, beta, - _activeClass).doAll(activeData._adaptedFrame); + _activeClass, _hasConstraints).doAll(activeData._adaptedFrame); gt._gram.mul(obj_reg); if (_parms._glmType.equals(GLMParameters.GLMType.gam)) { // add contribution from GAM smoothness factor Integer[] activeCols=null; @@ -1024,7 +1424,7 @@ protected GramXY computeNewGram(DataInfo activeData, double [] beta, GLMParamete } gt._gram.addGAMPenalty(activeCols , _penaltyMatrix, _gamBetaIndices); } - ArrayUtils.mult(gt._xy,obj_reg); + mult(gt._xy,obj_reg); int [] activeCols = activeData.activeCols(); int [] zeros = gt._gram.findZeroCols(); GramXY res; @@ -1053,6 +1453,67 @@ public GramXY computeGramRCC(double[] beta, GLMParameters.Solver s) { return computeNewGram(_activeData, ArrayUtils.select(beta, _activeData.activeCols()), s); } + /*** + * This function calculates the following values: + * 1. the hessian + * 2. the xy which is basically (hessian * old_beta + gradient) + */ + protected GramGrad computeGram(double [] beta, GLMGradientInfo gradientInfo){ + DataInfo activeData = activeData(); + double obj_reg = _parms._obj_reg; + if(_glmw == null) _glmw = new GLMModel.GLMWeightsFun(_parms); + GLMTask.GLMIterationTask gt = new GLMTask.GLMIterationTask(_job._key, activeData, _glmw, beta, + _activeClass, _hasConstraints).doAll(activeData._adaptedFrame); + double[][] fullGram = gt._gram.getXX(); // only extract gram matrix + mult(fullGram, obj_reg); + if (_gramEqual != null) + elementwiseSumSymmetricArrays(fullGram, mult(sumGramConstribution(_gramEqual, fullGram.length), _csGLMState._ckCS)); + if (_gramLess != null) + elementwiseSumSymmetricArrays(fullGram, mult(sumGramConstribution(_gramLess, fullGram.length), _csGLMState._ckCS)); + if (_parms._glmType.equals(GLMParameters.GLMType.gam)) { // add contribution from GAM smoothness factor + gt._gram.addGAMPenalty(_penaltyMatrix, _gamBetaIndices, fullGram); + } + // form xy which is (Gram*beta_current + gradient) + double[] xy = formXY(fullGram, beta, gradientInfo._gradient); + // remove zeros in Gram matrix and throw an error if that coefficient is included in the constraint + int[] zeros = findZeroCols(fullGram); + if (_parms._family != Family.multinomial && zeros.length > 0 && zeros.length <= activeData.activeCols().length) { + fullGram = GramGrad.dropCols(zeros, fullGram); // shrink gram matrix + removeCols(zeros); // update activeData.activeCols(), _beta + return new GramGrad(fullGram, ArrayUtils.removeIds(gradientInfo._gradient, zeros), + ArrayUtils.removeIds(beta, zeros), gradientInfo._objVal, gt.sumOfRowWeights, ArrayUtils.removeIds(xy, zeros)); + } + return new GramGrad(fullGram, gradientInfo._gradient, beta, gradientInfo._objVal, gt.sumOfRowWeights, xy); + } + + /*** + * + * This method adds to objective function the contribution of + * transpose(lambda)*constraint vector + ck/2*transpose(constraint vector)*constraint vector + */ + public static double addConstraintObj(double[] lambda, LinearConstraints[] constraints, double ckHalf) { + int numConstraints = constraints.length; + LinearConstraints oneC; + double objValueAdd = 0; + for (int index=0; index xy[x]-grad[x]).toArray(); + } + + + // get cached gram or incrementally update or compute new one public GramXY computeGram(double [] beta, GLMParameters.Solver s){ double obj_reg = _parms._obj_reg; @@ -1092,12 +1553,22 @@ public GramXY computeGram(double [] beta, GLMParameters.Solver s){ if(!weighted || matches) { GLMTask.GLMIncrementalGramTask gt = new GLMTask.GLMIncrementalGramTask(newColsIds, activeData, _glmw, beta).doAll(activeData._adaptedFrame); // dense for (double[] d : gt._gram) - ArrayUtils.mult(d, obj_reg); - ArrayUtils.mult(gt._xy, obj_reg); + mult(d, obj_reg); + mult(gt._xy, obj_reg); // glue the update and old gram together return _currGram = GramXY.addCols(beta, activeCols, newColsIds, _currGram, gt._gram, gt._xy); } } return _currGram = computeNewGram(activeData,beta,s); } + + public void setConstraintInfo(GLMGradientInfo gradientInfo, LinearConstraints[] equalityConstraints, + LinearConstraints[] lessThanEqualToConstraints, double[] lambdaEqual, double[] lambdaLessThan) { + _ginfo = gradientInfo; + _lessThanEqualToConstraints = lessThanEqualToConstraints; + _equalityConstraints = equalityConstraints; + _lambdaEqual = lambdaEqual; + _lambdaLessThanEqualTo = lambdaLessThan; + _likelihood = gradientInfo._likelihood; + } } diff --git a/h2o-algos/src/main/java/hex/glm/ConstrainedGLMUtils.java b/h2o-algos/src/main/java/hex/glm/ConstrainedGLMUtils.java new file mode 100644 index 000000000000..ac198226e727 --- /dev/null +++ b/h2o-algos/src/main/java/hex/glm/ConstrainedGLMUtils.java @@ -0,0 +1,859 @@ +package hex.glm; + +import Jama.Matrix; +import hex.DataInfo; +import water.DKV; +import water.Iced; +import water.Key; +import water.fvec.Frame; +import water.util.ArrayUtils; +import water.util.IcedHashMap; +import water.util.TwoDimTable; + +import java.util.*; +import java.util.stream.Collectors; +import java.util.stream.IntStream; +import java.util.stream.Stream; + +import static java.util.Arrays.stream; +import static water.util.ArrayUtils.innerProduct; + +public class ConstrainedGLMUtils { + // constant setting refer to Michel Bierlaire, Optimization: Principles and Algorithms, Chapter 19, EPEL Press, + // second edition, 2018. + public static final double EPS = 1e-15; + public static final double EPS2 = 1e-12; + + public static class LinearConstraints extends Iced { // store one linear constraint + public IcedHashMap _constraints; // column names, coefficient of constraints + public double _constraintsVal; // contains evaluated constraint values + public boolean _active = true; // only applied to less than and equal to zero constraints + + public LinearConstraints() { + _constraints = new IcedHashMap<>(); + _constraintsVal = Double.NaN; // represent constraint not evaluated. + } + } + + public static class ConstraintsDerivatives extends Iced { + public IcedHashMap _constraintsDerivative; + public boolean _active; + + public ConstraintsDerivatives(boolean active) { + _constraintsDerivative = new IcedHashMap<>(); + _active = active; + } + } + + public static class ConstraintsGram extends Iced { + public IcedHashMap _coefIndicesValue; + public boolean _active; + + public ConstraintsGram() { + _coefIndicesValue = new IcedHashMap<>(); + } + } + + public static class CoefIndices implements hex.glm.CoefIndices { + final int _firstCoefIndex; + final int _secondCoefIndex; + + public CoefIndices(int firstInd, int secondInd) { + _firstCoefIndex = firstInd; + _secondCoefIndex = secondInd; + } + + @Override + public boolean equals(Object o) { + if (this == o) + return true; + else if (o == null) + return false; + else if (this._firstCoefIndex == ((CoefIndices) o)._firstCoefIndex && + this._secondCoefIndex == ((CoefIndices) o)._secondCoefIndex) + return true; + return false; + } + + public String toString() { + return "first coefficient index: " + _firstCoefIndex + ", second coefficient index " + _secondCoefIndex; + } + } + + public static class ConstraintGLMStates { + double _ckCS; + double _ckCSHalf; // = ck/2 + double _epsilonkCS; + double _epsilonkCSSquare; + double _etakCS; + double _etakCSSquare; + double _epsilon0; + String[] _constraintNames; + double[][] _initCSMatrix; + double _gradientMagSquare; + double _constraintMagSquare; + + public ConstraintGLMStates(String[] constrainNames, double[][] initMatrix, GLMModel.GLMParameters parms) { + _constraintNames = constrainNames; + _initCSMatrix = initMatrix; + _ckCS = parms._constraint_c0; + _ckCSHalf = parms._constraint_c0*0.5; + _epsilonkCS = 1.0/parms._constraint_c0; + _epsilonkCSSquare =_epsilonkCS*_epsilonkCS; + _etakCS = parms._constraint_eta0/Math.pow(parms._constraint_c0, parms._constraint_alpha); + _etakCSSquare = _etakCS*_etakCS; + _epsilon0 = 1.0/parms._constraint_c0; + } + } + + public static LinearConstraints[] combineConstraints(LinearConstraints[] const1, LinearConstraints[] const2) { + List allList = new ArrayList<>(); + if (const1 != null) + allList.addAll(stream(const1).collect(Collectors.toList())); + if (const2 != null) + allList.addAll(stream(const2).collect(Collectors.toList())); + return allList.size()==0 ? null : allList.stream().toArray(LinearConstraints[]::new); + } + + /*** + * + * This method will extract the constraints specified in beta constraint and combine it with the linear constraints + * later. Note that the linear constraints are only accepted in standard form, meaning we only accept the following + * constraint forms: 2*beta_1-3*beta_4-3 == 0 or 2*beta_1-3*beta_4-3 <= 0. + * + * The beta constraints on the other hand is specified in several forms: + * 1): -Infinity <= beta <= Infinity: ignored, no constrain here; + * 2): -Infinity <= beta <= high_val: transformed to beta - high_val <= 0, add to lessThanEqualTo constraint; + * 3): low_val <= beta <= Infinity: transformed to low_val - beta <= 0, add to lessThanEqualTo constraint; + * 4): low_val <= beta <= high_val: transformed to two constraints, low_val-beta <= 0, beta-high_val <= 0, add to + * lessThanEqualTo constraint. + * 5): val <= beta <= val: transformed to beta-val == 0, add to equalTo constraint. + * + * The newly extracted constraints will be added to fields in state. + * + */ + public static int[] extractBetaConstraints(ComputationState state, String[] coefNames) { + GLM.BetaConstraint betaC = state.activeBC(); + List equalityC = new ArrayList<>(); + List lessThanEqualToC = new ArrayList<>(); + List betaIndexOnOff = new ArrayList<>(); + if (betaC._betaLB != null) { + int numCons = betaC._betaLB.length-1; + for (int index=0; indexx).toArray(); + } + + /*** + * This method will extract the equality constraint and add to equalityC from beta constraint by doing the following + * transformation: val <= beta <= val: transformed to beta-val == 0, add to equalTo constraint. + */ + public static void addBCEqualityConstraint(List equalityC, GLM.BetaConstraint betaC, + String[] coefNames, int index) { + LinearConstraints oneEqualityConstraint = new LinearConstraints(); + oneEqualityConstraint._constraints.put(coefNames[index], 1.0); + oneEqualityConstraint._constraints.put("constant", -betaC._betaLB[index]); + equalityC.add(oneEqualityConstraint); + } + + /*** + * This method will extract the greater than constraint and add to lessThanC from beta constraint by doing the following + * transformation: low_val <= beta <= Infinity: transformed to low_val - beta <= 0. + */ + public static void addBCGreaterThanConstraint(List lessThanC, GLM.BetaConstraint betaC, + String[] coefNames, int index) { + LinearConstraints lessThanEqualToConstraint = new LinearConstraints(); + lessThanEqualToConstraint._constraints.put(coefNames[index], -1.0); + lessThanEqualToConstraint._constraints.put("constant", betaC._betaLB[index]); + lessThanC.add(lessThanEqualToConstraint); + } + + /*** + * This method will extract the less than constraint and add to lessThanC from beta constraint by doing the following + * transformation: -Infinity <= beta <= high_val: transformed to beta - high_val <= 0. + */ + public static void addBCLessThanConstraint(List lessThanC, GLM.BetaConstraint betaC, + String[] coefNames, int index) { + LinearConstraints greaterThanConstraint = new LinearConstraints(); + greaterThanConstraint._constraints.put(coefNames[index], 1.0); + greaterThanConstraint._constraints.put("constant", -betaC._betaUB[index]); + lessThanC.add(greaterThanConstraint); + } + + /*** + * This method will extract the constraints specified in the Frame with key linearConstraintFrameKey. For example, + * the following constraints a*beta_1+b*beta_2-c*beta_5 == 0, d*beta_2+e*beta_6-f <= 0 can be specified as the + * following rows: + * names values Type constraint_numbers + * beta_1 a Equal 0 + * beta_2 b Equal 0 + * beta_5 -c Equal 0 + * beta_2 d LessThanEqual 1 + * beta_6 e LessThanEqual 1 + * constant -f LessThanEqual 1 + */ + public static void extractLinearConstraints(ComputationState state, Key linearConstraintFrameKey, DataInfo dinfo) { + List equalityC = new ArrayList<>(); + List lessThanEqualToC = new ArrayList<>(); + Frame linearConstraintF = DKV.getGet(linearConstraintFrameKey); + List colNamesList = Stream.of(dinfo._adaptedFrame.names()).collect(Collectors.toList()); + List coefNamesList = Stream.of(dinfo.coefNames()).collect(Collectors.toList()); + int numberOfConstraints = linearConstraintF.vec("constraint_numbers").toCategoricalVec().domain().length; + int numRow = (int) linearConstraintF.numRows(); + List rowIndices = IntStream.range(0,numRow).boxed().collect(Collectors.toList()); + String constraintType; + int rowIndex; + for (int conInd = 0; conInd < numberOfConstraints; conInd++) { + if (!rowIndices.isEmpty()) { + rowIndex = rowIndices.get(0); + constraintType = linearConstraintF.vec("types").stringAt(rowIndex).toLowerCase(); + if ("equal".equals(constraintType)) { + extractConstraint(linearConstraintF, rowIndices, equalityC, dinfo, coefNamesList, colNamesList); + } else if ("lessthanequal".equals(constraintType)) { + extractConstraint(linearConstraintF, rowIndices, lessThanEqualToC, dinfo, coefNamesList, + colNamesList); + } else { + throw new IllegalArgumentException("Type of linear constraints can only be Equal to LessThanEqualTo."); + } + } + } + state.setLinearConstraints(equalityC.toArray(new LinearConstraints[0]), + lessThanEqualToC.toArray(new LinearConstraints[0]), false); + } + + public static void extractConstraint(Frame constraintF, List rowIndices, List equalC, + DataInfo dinfo, List coefNames, List colNames) { + List processedRowIndices = new ArrayList<>(); + int constraintNumberFrame = (int) constraintF.vec("constraint_numbers").at(rowIndices.get(0)); + LinearConstraints currentConstraint = new LinearConstraints(); + String constraintType = constraintF.vec("types").stringAt(rowIndices.get(0)).toLowerCase(); + boolean standardize = dinfo._normMul != null; + boolean constantFound = false; + for (Integer rowIndex : rowIndices) { + String coefName = constraintF.vec("names").stringAt(rowIndex); + String currType = constraintF.vec("types").stringAt(rowIndex).toLowerCase(); + if (!coefNames.contains(coefName) && !"constant".equals(coefName)) + throw new IllegalArgumentException("Coefficient name " + coefName + " is not a valid coefficient name. It " + + "be a valid coefficient name or it can be constant"); + if ((int) constraintF.vec("constraint_numbers").at(rowIndex) == constraintNumberFrame) { + if (!constraintType.equals(currType)) + throw new IllegalArgumentException("Constraint type "+" of the same constraint must be the same but is not." + + " Expected type: "+constraintType+". Actual type: "+currType); + if ("constant".equals(coefName)) + constantFound = true; + processedRowIndices.add(rowIndex); + // coefNames is valid + int colInd = colNames.indexOf(coefName)-dinfo._cats; + if (standardize && colNames.contains(coefName) && colInd >= 0) { // numerical column with standardization + currentConstraint._constraints.put(coefName, constraintF.vec("values").at(rowIndex)*dinfo._normMul[colInd]); + } else { // categorical column, constant or numerical column without standardization + currentConstraint._constraints.put(coefName, constraintF.vec("values").at(rowIndex)); + } + } + } + if (!constantFound) + currentConstraint._constraints.put("constant", 0.0); // put constant of 0.0 + if (currentConstraint._constraints.size() < 3) + throw new IllegalArgumentException("Linear constraint must have at least two coefficients. For constraints on" + + " just one coefficient: "+ constraintF.vec("names").stringAt(0)+", use betaConstraints instead."); + equalC.add(currentConstraint); + rowIndices.removeAll(processedRowIndices); + } + + public static double[][] formConstraintMatrix(ComputationState state, List constraintNamesList, int[] betaEqualLessThanInd) { + // extract coefficient names from constraints + constraintNamesList.addAll(extractConstraintCoeffs(state)); + // form double matrix + int numRow = (betaEqualLessThanInd == null ? 0 : ArrayUtils.sum(betaEqualLessThanInd)) + + (state._equalityConstraintsLinear == null ? 0 : state._equalityConstraintsLinear.length) + + (state._lessThanEqualToConstraintsLinear == null ? 0 : state._lessThanEqualToConstraintsLinear.length); + double[][] initConstraintMatrix = new double[numRow][constraintNamesList.size()]; + fillConstraintValues(state, constraintNamesList, initConstraintMatrix, betaEqualLessThanInd); + return initConstraintMatrix; + } + + public static void fillConstraintValues(ComputationState state, List constraintNamesList, + double[][] initCMatrix, int[] betaLessThan) { + int rowIndex = 0; + if (state._equalityConstraintsBeta != null) + rowIndex = extractConstraintValues(state._equalityConstraintsBeta, constraintNamesList, initCMatrix, rowIndex, + null); + if (state._lessThanEqualToConstraintsBeta != null) + rowIndex= extractConstraintValues(state._lessThanEqualToConstraintsBeta, constraintNamesList, initCMatrix, + rowIndex, betaLessThan); + if (state._equalityConstraintsLinear != null) + rowIndex = extractConstraintValues(state._equalityConstraintsLinear, constraintNamesList, initCMatrix, rowIndex, null); + if (state._lessThanEqualToConstraintsLinear != null) + extractConstraintValues(state._lessThanEqualToConstraintsLinear, constraintNamesList, initCMatrix, rowIndex, null); + } + + public static int extractConstraintValues(LinearConstraints[] constraints, List constraintNamesList, + double[][] initCMatrix, int rowIndex, int[] betaLessThan) { + int numConstr = constraints.length; + for (int index=0; index coeffKeys = constraints[index]._constraints.keySet(); + for (String oneKey : coeffKeys) { + if (constraintNamesList.contains(oneKey)) + initCMatrix[rowIndex][constraintNamesList.indexOf(oneKey)] = constraints[index]._constraints.get(oneKey); + } + rowIndex++; + } + } + return rowIndex; + } + + public static void printConstraintSummary(GLMModel model, ComputationState state, String[] coefNames) { + LinearConstraintConditions cCond = printConstraintSummary(state, coefNames); + model._output._linear_constraint_states = cCond._constraintDescriptions; + model._output._all_constraints_satisfied = cCond._allConstraintsSatisfied; + makeConstraintSummaryTable(model, cCond); + } + + public static void makeConstraintSummaryTable(GLMModel model, LinearConstraintConditions cCond) { + int numRow = cCond._constraintBounds.length; + String[] colHeaders = new String[]{"constraint", "values", "condition", "condition_satisfied"}; + String[] colTypes = new String[]{"string", "double", "string", "string"}; + String[] colFormats = new String[]{"%s", "%5.2f", "%s", "%s"}; + TwoDimTable cTable = new TwoDimTable("Beta (if exists) and Linear Constraints Table", null, + new String[numRow], colHeaders, colTypes, colFormats, "constraint"); + for (int index=0; index coefNameList = Arrays.stream(coefNames).collect(Collectors.toList()); + List constraintConditions = new ArrayList<>(); + List cSatisfied = new ArrayList<>(); + List cValues = new ArrayList<>(); + List cConditions = new ArrayList<>(); + List constraintStrings = new ArrayList<>(); + + if (state._equalityConstraintsBeta != null) + constraintsSatisfied = evaluateConstraint(state, state._equalityConstraintsBeta, true, beta, + coefNameList, "Beta equality constraint: ", constraintConditions, cSatisfied, cValues, + cConditions, constraintStrings) && constraintsSatisfied; + + if (state._lessThanEqualToConstraintsBeta != null) + constraintsSatisfied = evaluateConstraint(state, state._lessThanEqualToConstraintsBeta, false, + beta, coefNameList, "Beta inequality constraint: ", constraintConditions, cSatisfied, cValues, + cConditions, constraintStrings) && constraintsSatisfied; + + if (state._equalityConstraintsLinear != null) + constraintsSatisfied = evaluateConstraint(state, state._equalityConstraintsLinear, true, beta, + coefNameList, "Linear equality constraint: ", constraintConditions, cSatisfied, cValues, + cConditions, constraintStrings) && constraintsSatisfied; + + if (state._lessThanEqualToConstraints != null) + constraintsSatisfied = evaluateConstraint(state, state._lessThanEqualToConstraints, false, beta, + coefNameList, "Linear inequality constraint: ", constraintConditions, cSatisfied, cValues, + cConditions, constraintStrings) && constraintsSatisfied; + + return new LinearConstraintConditions(constraintConditions.stream().toArray(String[]::new), + cSatisfied.stream().toArray(String[]::new), cValues.stream().mapToDouble(x->x).toArray(), + cConditions.stream().toArray(String[]::new), constraintStrings.stream().toArray(String[]::new), + constraintsSatisfied); + } + + /** + * Print constraints without any standardization applied so that people can see the setting in their original + * form without standardization. The beta coefficients are non-standardized. However, if standardized, the + * constraint values are changed to accommodate the standardized coefficients. + */ + public static boolean evaluateConstraint(ComputationState state, LinearConstraints[] constraints, boolean equalityConstr, + double[] beta, List coefNames, String startStr, + List constraintCond, List cSatisfied, List cValues, + List cConditions, List constraintsStrings) { + int constLen = constraints.length; + LinearConstraints oneC; + String constrainStr; + boolean allSatisfied = true; + for (int index=0; index trainNames = stream(dinfo.coefNames()).collect(Collectors.toList()); + double constantVal = 0; + int colInd = -1; + int coefOffset = (dinfo._catOffsets == null || dinfo._catOffsets.length == 0) ? 0 : dinfo._catOffsets[dinfo._catOffsets.length - 1]; + for (String coefName : oneConst._constraints.keySet()) { + double constrVal = oneConst._constraints.get(coefName); + if (constrVal != 0) { + if ("constant".equals(coefName)) { + constantVal = constrVal; + } else if (trainNames.contains(coefName)) { + colInd = trainNames.indexOf(coefName) - coefOffset; + if (standardize && colInd >= 0 && !isBetaConstraint) { + if (constrVal > 0) + sb.append('+'); + sb.append(constrVal / dinfo._normMul[colInd]); + } else { + sb.append(constrVal); + } + sb.append('*'); + sb.append(coefName); + } + } + } + // add constant value here + // add constant value to the end + if (constantVal != 0) { + if (constantVal > 0) + sb.append("+"); + if (isBetaConstraint && colInd >= 0 && standardize) + sb.append(constantVal * dinfo._normMul[colInd]); + else + sb.append(constantVal); + } + return sb.toString(); + } + + public static List extractConstraintCoeffs(ComputationState state) { + List tConstraintCoeffName = new ArrayList<>(); + boolean nonZeroConstant = false; + if (state._equalityConstraintsBeta != null) + nonZeroConstant = extractCoeffNames(tConstraintCoeffName, state._equalityConstraintsBeta); + + if (state._lessThanEqualToConstraintsBeta != null) + nonZeroConstant = extractCoeffNames(tConstraintCoeffName, state._lessThanEqualToConstraintsBeta) || nonZeroConstant; + + if (state._equalityConstraintsLinear != null) + nonZeroConstant = extractCoeffNames(tConstraintCoeffName, state._equalityConstraintsLinear) || nonZeroConstant; + + if (state._lessThanEqualToConstraintsLinear != null) + nonZeroConstant = extractCoeffNames(tConstraintCoeffName, state._lessThanEqualToConstraintsLinear) || nonZeroConstant; + + // remove duplicates in the constraints names + Set noDuplicateNames = new HashSet<>(tConstraintCoeffName); + if (!nonZeroConstant) // no non-Zero constant present + noDuplicateNames.remove("constant"); + return new ArrayList<>(noDuplicateNames); + } + + public static boolean extractCoeffNames(List coeffList, LinearConstraints[] constraints) { + int numConst = constraints.length; + boolean nonZeroConstant = false; + for (int index=0; index keys = constraints[index]._constraints.keySet(); + coeffList.addAll(keys); + if (keys.contains("constant")) + nonZeroConstant = constraints[index]._constraints.get("constant") != 0.0; + } + return nonZeroConstant; + } + + public static List foundRedundantConstraints(ComputationState state, final double[][] initConstraintMatrix) { + Matrix constMatrix = new Matrix(initConstraintMatrix); + Matrix constMatrixLessConstant = constMatrix.getMatrix(0, constMatrix.getRowDimension() -1, 1, constMatrix.getColumnDimension()-1); + Matrix constMatrixTConstMatrix = constMatrixLessConstant.times(constMatrixLessConstant.transpose()); + int rank = constMatrixLessConstant.rank(); + if (rank < constMatrix.getRowDimension()) { // redundant constraints are specified + double[][] rMatVal = constMatrixTConstMatrix.qr().getR().getArray(); + List diag = IntStream.range(0, rMatVal.length).mapToDouble(x->Math.abs(rMatVal[x][x])).boxed().collect(Collectors.toList()); + int[] sortedIndices = IntStream.range(0, diag.size()).boxed().sorted((i, j) -> diag.get(i).compareTo(diag.get(j))).mapToInt(ele->ele).toArray(); + List duplicatedEleIndice = IntStream.range(0, diag.size()-rank).map(x -> sortedIndices[x]).boxed().collect(Collectors.toList()); + return genRedundantConstraint(state, duplicatedEleIndice); + } + return null; + } + + public static List genRedundantConstraint(ComputationState state, List duplicatedEleIndics) { + List redundantConstraint = new ArrayList<>(); + for (Integer redIndex : duplicatedEleIndics) + redundantConstraint.add(grabRedundantConstraintMessage(state, redIndex)); + + return redundantConstraint; + } + + public static String grabRedundantConstraintMessage(ComputationState state, Integer constraintIndex) { + // figure out which constraint among state._fromBetaConstraints, state._equalityConstraints, + // state._lessThanEqualToConstraints is actually redundant + LinearConstraints redundantConst = getConstraintFromIndex(state, constraintIndex); + if (redundantConst != null) { + boolean standardize = state.activeData()._normMul != null ? true : false; + boolean isBetaConstraint = redundantConst._constraints.size() < 3; + StringBuilder sb = new StringBuilder(); + DataInfo dinfo = state.activeData(); + List trainNames = stream(dinfo.coefNames()).collect(Collectors.toList()); + sb.append("This constraint is redundant "); + double constantVal = 0; + int colInd = -1; + int coefOffset = (dinfo._catOffsets == null || dinfo._catOffsets.length == 0) ? 0 : dinfo._catOffsets[dinfo._catOffsets.length - 1]; + for (String coefName : redundantConst._constraints.keySet()) { + double constrVal = redundantConst._constraints.get(coefName); + if (constrVal != 0) { + if ("constant".equals(coefName)) { + constantVal = constrVal; + } else if (trainNames.contains(coefName)) { + colInd = trainNames.indexOf(coefName) - coefOffset; + if (standardize && colInd >= 0 && !isBetaConstraint) { + if (constrVal > 0) + sb.append('+'); + sb.append(constrVal * dinfo._normMul[colInd]); + } else { + sb.append(constrVal); + } + sb.append('*'); + sb.append(coefName); + } + } + } + // add constant value here + // add constant value to the end + if (constantVal != 0) { + if (constantVal > 0) + sb.append("+"); + if (isBetaConstraint && colInd >= 0) + sb.append(constantVal * dinfo._normMul[colInd]); + else + sb.append(constantVal); + } + sb.append(" <= or == 0."); + sb.append(" Please remove it from your beta/linear constraints."); + return sb.toString(); + } else { + return null; + } + } + + public static LinearConstraints getConstraintFromIndex(ComputationState state, Integer constraintIndex) { + int constIndexWOffset = constraintIndex; + if (state._equalityConstraintsBeta != null) { + if (constIndexWOffset < state._equalityConstraintsBeta.length) { + return state._equalityConstraintsBeta[constIndexWOffset]; + } else { + constIndexWOffset -= state._equalityConstraintsBeta.length; + } + } + + if (state._lessThanEqualToConstraintsBeta != null) { + if (constIndexWOffset < state._lessThanEqualToConstraintsBeta.length) { + return state._lessThanEqualToConstraintsBeta[constIndexWOffset]; + } else { + constIndexWOffset -= state._lessThanEqualToConstraintsBeta.length; + } + } + + if (state._equalityConstraintsLinear != null) { + if (constIndexWOffset < state._equalityConstraintsLinear.length) { + return state._equalityConstraintsLinear[constIndexWOffset]; + } else { + constIndexWOffset -= state._equalityConstraintsLinear.length; + } + } + + if (state._lessThanEqualToConstraints != null && constIndexWOffset < state._lessThanEqualToConstraints.length) { + return state._lessThanEqualToConstraints[constIndexWOffset]; + } + return null; + } + + /*** + * + * This method will evaluate the value of a constraint given the GLM coefficients and the coefficicent name list. + * Note that the beta should be the normalized beta if standardize = true and the coefficients to the coefficients + * are set correctly for the standardized coefficients. + */ + public static void evalOneConstraint(LinearConstraints constraint, double[] beta, List coefNames) { + double sumV = 0.0; + Map constraints = constraint._constraints; + for (String coef : constraints.keySet()) { + if ("constant".equals(coef)) + sumV += constraints.get(coef); + else + sumV += constraints.get(coef)*beta[coefNames.indexOf(coef)]; + } + constraint._constraintsVal = sumV; + } + + /*** + * + * The initial value of lambda values really do not matter that much. The lambda update will take care of making + * sure it is the right sign at the end of the iteration. + * + */ + public static void genInitialLambda(Random randObj, LinearConstraints[] constraints, double[] lambda) { + int numC = constraints.length; + LinearConstraints oneC; + for (int index=0; index x._constraintsVal*x._constraintsVal).sum(); + if (lessThanConst != null) // only counts magnitude when the constraint is active + sumSquare += stream(lessThanConst).filter(x -> x._active).mapToDouble(x -> x._constraintsVal*x._constraintsVal).sum(); + state._csGLMState._constraintMagSquare = sumSquare; + } + + public static void updateLambda(double[] lambda, double ckCS, LinearConstraints[] constraints) { + int numC = constraints.length; + LinearConstraints oneC; + for (int index=0; index x._active).count() > 0; + } + + /*** + * This method calls getGradient to calculate the gradient, likelhood and objective function values. In addition, + * it will add to the gradient and objective function the contribution from the linear constraints. + */ + public static GLM.GLMGradientInfo calGradient(double[] betaCnd, ComputationState state, GLM.GLMGradientSolver ginfo, + double[] lambdaE, double[] lambdaL, LinearConstraints[] constraintE, + LinearConstraints[] constraintL) { + // todo: need to add support for predictors removed for whatever reason + // calculate gradients + GLM.GLMGradientInfo gradientInfo = ginfo.getGradient(betaCnd, state); // gradient without constraints + boolean hasEqualConstraints = constraintE != null; + boolean hasLessConstraints = constraintL != null; + // add gradient, objective and likelihood contribution from constraints + if (hasEqualConstraints) { + addConstraintGradient(lambdaE, state._derivativeEqual, gradientInfo); + addPenaltyGradient(state._derivativeEqual, constraintE, gradientInfo, state._csGLMState._ckCS); + gradientInfo._objVal += state.addConstraintObj(lambdaE, constraintE, state._csGLMState._ckCSHalf); + } + if (hasLessConstraints) { + addConstraintGradient(lambdaL, state._derivativeLess, gradientInfo); + addPenaltyGradient(state._derivativeLess, constraintL, gradientInfo, state._csGLMState._ckCS); + gradientInfo._objVal += state.addConstraintObj(lambdaL, constraintL, state._csGLMState._ckCSHalf); + } + return gradientInfo; + } + + /** + * Simple method to all linear constraints given the coefficient values. In addition, it will determine if a + * linear constraint is active. Only active constraints are included in the objective function calculations. + * + * For an equality constraint, any constraint value not equal to zero is active. + * For a less than or equality constraint, any constraint value that exceed zero is active. + */ + public static void updateConstraintValues(double[] betaCnd, List coefNames, + LinearConstraints[] equalityConstraints, + LinearConstraints[] lessThanEqualToConstraints) { + if (equalityConstraints != null) // equality constraints + Arrays.stream(equalityConstraints).forEach(constraint -> { + evalOneConstraint(constraint, betaCnd, coefNames); + constraint._active = (Math.abs(constraint._constraintsVal) > EPS2); + }); + + if (lessThanEqualToConstraints != null) // less than or equal to constraints + Arrays.stream(lessThanEqualToConstraints).forEach(constraint -> { + evalOneConstraint(constraint, betaCnd, coefNames); + constraint._active = constraint._constraintsVal > 0; + }); + } + + public static String[] collinearInConstraints(String[] collinear_cols, String[] constraintNames) { + List cNames = Arrays.stream(constraintNames).collect(Collectors.toList()); + return Arrays.stream(collinear_cols).filter(x -> (cNames.contains(x))).toArray(String[]::new); + } + + public static int countNumConst(ComputationState state) { + int numConst = 0; + // check constraints from beta constrains + numConst += state._equalityConstraintsBeta == null ? 0 : state._equalityConstraintsBeta.length; + numConst += state._lessThanEqualToConstraintsBeta == null ? 0 : state._lessThanEqualToConstraintsBeta.length/2; + numConst += state._equalityConstraintsLinear == null ? 0 : state._equalityConstraintsLinear.length; + numConst += state._lessThanEqualToConstraints == null ? 0 : state._lessThanEqualToConstraints.length; + return numConst; + } + + public static class LinearConstraintConditions { + final String[] _constraintDescriptions; // 0.5C2 + 1.3C2+3 + final String[] _constraintSatisfied; + final double[] _constraintValues; + final String[] _constraintBounds; // == 0 for equality constraint, <= 0 for lessThanEqual to constraint + final String[] _constraintNValues; // 0.5C2+1.4C2-0.5 = 2.0 + final boolean _allConstraintsSatisfied; + + public LinearConstraintConditions(String[] constraintC, String[] cSatisfied, double[] cValues, String[] cBounds, + String[] cNV, boolean conditionS) { + _constraintDescriptions = constraintC; + _constraintSatisfied = cSatisfied; + _constraintValues = cValues; + _constraintBounds = cBounds; + _constraintNValues = cNV; + _allConstraintsSatisfied = conditionS; + } + } +} diff --git a/h2o-algos/src/main/java/hex/glm/GLM.java b/h2o-algos/src/main/java/hex/glm/GLM.java index 433f8c058dc5..8e13acb10215 100644 --- a/h2o-algos/src/main/java/hex/glm/GLM.java +++ b/h2o-algos/src/main/java/hex/glm/GLM.java @@ -1,6 +1,8 @@ package hex.glm; +import Jama.Matrix; import hex.*; +import hex.gam.MatrixFrameUtils.GamUtils; import hex.glm.GLMModel.GLMOutput; import hex.glm.GLMModel.GLMParameters.Family; import hex.glm.GLMModel.GLMParameters.Link; @@ -48,10 +50,9 @@ import java.util.stream.IntStream; import static hex.ModelMetrics.calcVarImp; -import static hex.gam.MatrixFrameUtils.GamUtils.copy2DArray; import static hex.gam.MatrixFrameUtils.GamUtils.keepFrameKeys; -import static hex.glm.ComputationState.extractSubRange; -import static hex.glm.ComputationState.fillSubRange; +import static hex.glm.ComputationState.*; +import static hex.glm.ConstrainedGLMUtils.*; import static hex.glm.DispersionUtils.*; import static hex.glm.GLMModel.GLMParameters; import static hex.glm.GLMModel.GLMParameters.CHECKPOINT_NON_MODIFIABLE_FIELDS; @@ -69,10 +70,10 @@ * Generalized linear model implementation. */ public class GLM extends ModelBuilder { + static double BAD_CONDITION_NUMBER = 20000; static NumberFormat lambdaFormatter = new DecimalFormat(".##E0"); static NumberFormat devFormatter = new DecimalFormat(".##"); private static final DateTimeFormatter DATE_TIME_FORMATTER = DateTimeFormat.forPattern("yyyy-MM-dd HH:mm:ss"); - public static final int SCORING_INTERVAL_MSEC = 15000; // scoreAndUpdateModel every minute unless score every iteration is set public int[] _randC; // contains categorical column levels for random columns for HGLM public String _generatedWeights = null; @@ -86,6 +87,7 @@ public class GLM extends ModelBuilder { private boolean _checkPointFirstIter = false; // indicate first iteration for checkpoint model private boolean _betaConstraintsOn = false; private boolean _tweedieDispersionOnly = false; + private boolean _linearConstraintsOn = false; public GLM(boolean startup_once){super(new GLMParameters(),startup_once);} public GLM(GLMModel.GLMParameters parms) { @@ -814,6 +816,7 @@ void restoreFromCheckpoint(TwoDimTable sHist, int[] colIndices) { private GLMGradientInfo _ginfoStart; private double _betaDiffStart; private double[] _betaStart; + private int _initIter = 0; @Override public int nclasses() { @@ -966,6 +969,15 @@ public void init(boolean expensive) { } } } + if (_parms._max_iterations == 0) { + warn("max_iterations", "for GLM, must be >= 1 (or -1 for unlimited or default setting) " + + "to obtain proper model. Setting it to be 0 will only return the correct coefficient names and an empty" + + " model."); + warn("_max_iterations", H2O.technote(2 , "for GLM, if specified, must be >= 1 or == -1.")); + } + if (_parms._linear_constraints != null) { + checkInitLinearConstraints(); + } if (expensive) { if (_parms._build_null_model) { if (!(tweedie.equals(_parms._family) || gamma.equals(_parms._family) || negativebinomial.equals(_parms._family))) @@ -973,8 +985,6 @@ public void init(boolean expensive) { else removePredictors(_parms, _train); } - if (_parms._max_iterations == 0) - error("_max_iterations", H2O.technote(2, "if specified, must be >= 1.")); if (error_count() > 0) return; if (_parms._lambda_search && (_parms._stopping_rounds > 0)) { error("early stop", " cannot run when lambda_search=True. Lambda_search has its own " + @@ -1173,6 +1183,7 @@ public void init(boolean expensive) { } } boolean betaContsOn = _parms._beta_constraints != null || _parms._non_negative; + _linearConstraintsOn = _parms._linear_constraints != null; _betaConstraintsOn = (betaContsOn && (Solver.AUTO.equals(_parms._solver) || Solver.COORDINATE_DESCENT.equals(_parms._solver) || IRLSM.equals(_parms._solver )|| Solver.L_BFGS.equals(_parms._solver))); @@ -1185,17 +1196,20 @@ public void init(boolean expensive) { _enumInCS = true; // make sure we only do this once } } + if (_parms._expose_constraints && _parms._linear_constraints == null) + error("_expose_constraints", "can only be enabled when there are linear constraints."); BetaConstraint bc = _parms._beta_constraints != null ? new BetaConstraint(_parms._beta_constraints.get()) : new BetaConstraint(); if (betaContsOn && !_betaConstraintsOn) { - warn("Beta Constraints", " will be disabled except for solver AUTO, COORDINATE_DESCENT, " + + warn("Beta Constraints", "will be disabled except for solver AUTO, COORDINATE_DESCENT, " + "IRLSM or L_BFGS. It is not available for ordinal or multinomial families."); } - if((bc.hasBounds() || bc.hasProximalPenalty()) && _parms._compute_p_values) - error("_compute_p_values","P-values can not be computed for constrained problems"); + if (bc.hasProximalPenalty() && _parms._compute_p_values) + error("_compute_p_values","P-values can not be computed for constrained problems with proximal penalty"); if (bc.hasBounds() && _parms._early_stopping) warn("beta constraint and early_stopping", "if both are enabled may degrade model performance."); _state.setBC(bc); + if(hasOffsetCol() && _parms._intercept && !ordinal.equals(_parms._family)) { // fit intercept GLMGradientSolver gslvr = gam.equals(_parms._glmType) ? new GLMGradientSolver(_job,_parms, _dinfo.filterExpandedColumns(new int[0]), 0, _state.activeBC(), _betaInfo, _penaltyMatrix, _gamColIndices) @@ -1223,12 +1237,16 @@ public void init(boolean expensive) { " be the following:\n"+String.join("\n", _dinfo._adaptedFrame._names)+"\n Intercept.\n " + "Run your model without specifying startval to find out the actual coefficients names and " + "lengths."); - } else + } else { System.arraycopy(_parms._startval, 0, beta, 0, beta.length); + } + } else if (_parms._linear_constraints != null && _parms._init_optimal_glm) { // start value is not assigned + beta = genInitBeta(); } + GLMGradientInfo ginfo = gam.equals(_parms._glmType) ? new GLMGradientSolver(_job, _parms, _dinfo, 0, _state.activeBC(), _betaInfo, _penaltyMatrix, _gamColIndices).getGradient(beta) : new GLMGradientSolver(_job, - _parms, _dinfo, 0, _state.activeBC(), _betaInfo).getGradient(beta); // gradient obtained with zero penalty + _parms, _dinfo, 0, _state.activeBC(), _betaInfo).getGradient(beta); // gradient with L2 penalty, no constraints _lmax = lmax(ginfo._gradient); _gmax = _lmax*Math.max(1e-2, _parms._alpha[0]); // each alpha should have its own best lambda _state.setLambdaMax(_lmax); @@ -1339,10 +1357,101 @@ else if (!Solver.IRLSM.equals(_parms._solver)) _parms._compute_p_values = true; // automatically turned these on _parms._remove_collinear_columns = true; } + if (_parms._linear_constraints != null) { + checkAssignLinearConstraints(); + } buildModel(); } } - + + public double[] genInitBeta() { + if (_checkPointFirstIter) + return _model._betaCndCheckpoint; + Key linear_constr = _parms._linear_constraints; + Key beta_constr = _parms._beta_constraints; + _parms._linear_constraints = _parms._beta_constraints = null; + GLMModel model = new GLM(_parms).trainModel().get(); + Scope.track_generic(model); + _parms._linear_constraints= linear_constr; + _parms._beta_constraints = beta_constr; + ScoringInfo[] scInfo = model.getScoringInfo(); + _initIter = ((GLMScoringInfo) scInfo[scInfo.length-1]).iterations; + return _parms._standardize ? model._output.getNormBeta() : model._output.beta(); // constraint values evaluation will take care of normalization + } + + void checkInitLinearConstraints() { + if (!IRLSM.equals(_parms._solver)) { // only solver irlsm is allowed + error("solver", "constrained GLM is only available for IRLSM. PLease set solver to" + + " IRLSM/irlsm explicitly."); + return; + } + if (_parms._constraint_eta0 <= 0) { + error("constraint_eta0", "must be > 0."); + return; + } + if (_parms._constraint_tau <= 0) { + error("constraint_tau", "must be > 0."); + return; + } + if (_parms._constraint_c0 <= 0) { + error("constraint_c0", "must be > 0."); + return; + } + if (!_parms._intercept) { + error("intercept", "constrained GLM is only supported with when intercept=true."); + return; + } + // no regularization for constrainted GLM except during testing + if ((notZeroLambdas(_parms._lambda) || _parms._lambda_search) && !_parms._testCSZeroGram) { + error("lambda or lambda_search", "Regularization is not allowed for constrained GLM."); + return; + } + if ("multinomial".equals(_parms._solver) || "ordinal".equals(_parms._solver)) { + error("solver", "Constrained GLM is not supported for multinomial and ordinal families"); + return; + } + if ("ml".equals(_parms._dispersion_parameter_method)) { + error("dispersion_parameter_method", "Only pearson and deviance is supported for dipsersion" + + " parameter calculation."); + return; + } + } + + /** + * This method will extract the constraints from beta_constraints followed by the constraints specified in the + * linear_constraints. The constraints are extracted into equality and lessthanequalto constraints from + * beta_constraints and linear constraints. + * + * In addition, we extract all the constraints into a matrix and if the matrix is not full rank, constraints + * are redundant and an error will be thrown and the redundant constraints will be included in the error message + * so users can know which constraints to remove. + */ + void checkAssignLinearConstraints() { + String[] coefNames = _dinfo.coefNames(); + int[] betaEqualLessThanArr = null; + if (_parms._beta_constraints != null) + betaEqualLessThanArr = extractBetaConstraints(_state, coefNames); + // extract constraints from linear_constraints into equality of lessthanequalto constraints + extractLinearConstraints(_state, _parms._linear_constraints, _dinfo); + // make sure constraints have full rank. If not, generate messages stating what constraints are redundant, error out + List constraintNames = new ArrayList<>(); + double[][] initConstraintMatrix = formConstraintMatrix(_state, constraintNames, betaEqualLessThanArr); + String[] constraintCoefficientNames = constraintNames.toArray(new String[0]); + if (countNumConst(_state) > coefNames.length) + warn("number of constraints", " exceeds the number of coefficients. The system is" + + " over-constraints, and probably may not yield a valid solution due to possible conflicting " + + "constraints. Consider reducing the number of constraints."); + List redundantConstraints = foundRedundantConstraints(_state, initConstraintMatrix); + if (redundantConstraints != null) { + int numRedundant = redundantConstraints.size(); + for (int index = 0; index < numRedundant; index++) + error("redundant and possibly conflicting linear constraints", redundantConstraints.get(index)); + } else { + _state._csGLMState = new ConstraintGLMStates(constraintCoefficientNames, initConstraintMatrix, _parms); + _state._hasConstraints = true; + } + } + /** * initialize the following parameters for HGLM from either user initial inputs or from a GLM model if user did not * provide any starting values. @@ -1596,34 +1705,72 @@ private void doCleanup() { private transient Cholesky _chol; private transient L1Solver _lslvr; + /*** + * Use cholesky decomposition to solve for GLM Coefficients from the augmented Langrangian objective. In addition, + * it will check for collinear columns and removed them when found. + */ + private double[] constraintGLM_solve(GramGrad gram) { + if (!_parms._intercept) throw H2O.unimpl(); + ArrayList ignoredCols = new ArrayList<>(); + double[] xy = gram._xy.clone(); + Cholesky chol = ((_state._iter == 0) ? gram.qrCholesky(ignoredCols, GamUtils.copy2DArray(gram._gram), _parms._standardize) : gram.cholesky(null, gram._gram)); + if (!chol.isSPD()) throw new NonSPDMatrixException(); + if (!ignoredCols.isEmpty()) { + int[] collinearCols = ignoredCols.stream().mapToInt(x -> x).toArray(); + String[] ignoredConstraints = collinearInConstraints(ArrayUtils.select(_dinfo.coefNames(), collinearCols), + _state._csGLMState._constraintNames); + String collinearColNames = Arrays.toString(ArrayUtils.select(_dinfo.coefNames(), collinearCols)); + if (ignoredConstraints != null && ignoredConstraints.length > 0) + throw new IllegalArgumentException("Found constraints " + Arrays.toString(ignoredConstraints) + + " included on collinear columns that are going to be removed. Please remove any constraints " + + "involving collinear columns."); + if (!_parms._remove_collinear_columns) + throw new Gram.CollinearColumnsException("Found collinear columns in the dataset. Set " + + "remove_collinear_columns flag to true to remove collinear columns automatically. " + + "Found collinear columns " + collinearColNames); + _model.addWarning("Removed collinear columns "+collinearColNames); + Log.warn("Removed collinear columns "+collinearColNames); + _state.removeCols(collinearCols); + gram._gram = GramGrad.dropCols(collinearCols, gram._gram); + gram._grad = ArrayUtils.removeIds(gram._grad, collinearCols); + xy = ArrayUtils.removeIds(xy, collinearCols); + } + _chol = chol; + chol.solve(xy); + return xy; + } + private double[] ADMM_solve(Gram gram, double[] xy) { if (_parms._remove_collinear_columns || _parms._compute_p_values) { if (!_parms._intercept) throw H2O.unimpl(); ArrayList ignoredCols = new ArrayList<>(); Cholesky chol = ((_state._iter == 0) ? gram.qrCholesky(ignoredCols, _parms._standardize) : gram.cholesky(null)); if (!ignoredCols.isEmpty() && !_parms._remove_collinear_columns) { - int[] collinear_cols = new int[ignoredCols.size()]; - for (int i = 0; i < collinear_cols.length; ++i) - collinear_cols[i] = ignoredCols.get(i); - throw new Gram.CollinearColumnsException("Found collinear columns in the dataset. P-values can not be computed with collinear columns in the dataset. Set remove_collinear_columns flag to true to remove collinear columns automatically. Found collinear columns " + Arrays.toString(ArrayUtils.select(_dinfo.coefNames(), collinear_cols))); + int[] collinearCols = new int[ignoredCols.size()]; + for (int i = 0; i < collinearCols.length; ++i) + collinearCols[i] = ignoredCols.get(i); + throw new Gram.CollinearColumnsException("Found collinear columns in the dataset. P-values can not be " + + "computed with collinear columns in the dataset. Set remove_collinear_columns flag to true to remove " + + "collinear columns automatically. Found collinear columns " + + Arrays.toString(ArrayUtils.select(_dinfo.coefNames(), collinearCols))); } if (!chol.isSPD()) throw new NonSPDMatrixException(); _chol = chol; if (!ignoredCols.isEmpty()) { // got some redundant cols - int[] collinear_cols = new int[ignoredCols.size()]; - for (int i = 0; i < collinear_cols.length; ++i) - collinear_cols[i] = ignoredCols.get(i); - String[] collinear_col_names = ArrayUtils.select(_state.activeData().coefNames(), collinear_cols); + int[] collinearCols = new int[ignoredCols.size()]; + for (int i = 0; i < collinearCols.length; ++i) + collinearCols[i] = ignoredCols.get(i); + String[] collinearColNames = ArrayUtils.select(_state.activeData().coefNames(), collinearCols); // need to drop the cols from everywhere - _model.addWarning("Removed collinear columns " + Arrays.toString(collinear_col_names)); - Log.warn("Removed collinear columns " + Arrays.toString(collinear_col_names)); - _state.removeCols(collinear_cols); - gram.dropCols(collinear_cols); - xy = ArrayUtils.removeIds(xy, collinear_cols); + _model.addWarning("Removed collinear columns " + Arrays.toString(collinearColNames)); + Log.warn("Removed collinear columns " + Arrays.toString(collinearColNames)); + _state.removeCols(collinearCols); + gram.dropCols(collinearCols); + xy = ArrayUtils.removeIds(xy, collinearCols); } xy = xy.clone(); chol.solve(xy); - } else { + } else { // ADMM solve is only used when there is l1 regularization (lasso). gram = gram.deep_clone(); xy = xy.clone(); GramSolver slvr = new GramSolver(gram.clone(), xy.clone(), _parms._intercept, _state.l2pen(), _state.l1pen(), _state.activeBC()._betaGiven, _state.activeBC()._rho, _state.activeBC()._betaLB, _state.activeBC()._betaUB); @@ -2200,6 +2347,162 @@ private void fitIRLSM(Solver s) { Log.warn(LogMsg("Got Non SPD matrix, stopped.")); } } + + /*** + * This method fits the constraint GLM for IRLSM. We implemented the algorithm depicted in the document (H2O + * Constrained GLM Implementation.pdf) attached to this issue: https://github.com/h2oai/h2o-3/issues/6722. We will + * hereby use the word the doc to refere to this document. In particular, we following the algorithm described in + * Section VII (and table titled Algorithm 19.1) of the doc. + */ + private void fitIRLSMCS() { + double[] betaCnd = _checkPointFirstIter ? _model._betaCndCheckpoint : _state.beta(); + double[] tempBeta = _parms._separate_linear_beta ? new double[betaCnd.length] : null; + List coefNames = Arrays.stream(_state.activeData()._coefNames).collect(Collectors.toList()); + LinearConstraints[] equalityConstraints; + LinearConstraints[] lessThanEqualToConstraints; + final BetaConstraint bc = _state.activeBC(); + if (_parms._separate_linear_beta) { // keeping linear and beta constraints separate in this case + equalityConstraints = _state._equalityConstraintsLinear; + lessThanEqualToConstraints = _state._lessThanEqualToConstraintsLinear; + } else { // combine beta and linear constraints together + equalityConstraints = combineConstraints(_state._equalityConstraintsBeta, _state._equalityConstraintsLinear); + lessThanEqualToConstraints = combineConstraints(_state._lessThanEqualToConstraintsBeta, + _state._lessThanEqualToConstraintsLinear); + } + boolean hasEqualityConstraints = equalityConstraints != null; + boolean hasLessConstraints = lessThanEqualToConstraints != null; + double[] lambdaEqual = hasEqualityConstraints ? new double[equalityConstraints.length] : null; + double[] lambdaLessThan = hasLessConstraints ? new double[lessThanEqualToConstraints.length] : null; + Long startSeed = _parms._seed == -1 ? new Random().nextLong() : _parms._seed; + Random randObj = new Random(startSeed); + updateConstraintValues(betaCnd, coefNames, equalityConstraints, lessThanEqualToConstraints); + if (hasEqualityConstraints) // set lambda values for constraints + genInitialLambda(randObj, equalityConstraints, lambdaEqual); + if (hasLessConstraints) + genInitialLambda(randObj, lessThanEqualToConstraints, lambdaLessThan); + ExactLineSearch ls = null; + int iterCnt = (_checkPointFirstIter ? _state._iter : 0)+_initIter; + // contribution to gradient and hessian from constraints + _state.initConstraintDerivatives(equalityConstraints, lessThanEqualToConstraints, coefNames); + + GLMGradientSolver ginfo = gam.equals(_parms._glmType) ? new GLMGradientSolver(_job, _parms, _dinfo, 0, + _state.activeBC(), _betaInfo, _penaltyMatrix, _gamColIndices) : new GLMGradientSolver(_job, _parms, + _dinfo, 0, _state.activeBC(), _betaInfo); + GLMGradientInfo gradientInfo = calGradient(betaCnd, _state, ginfo, lambdaEqual, lambdaLessThan, + equalityConstraints, lessThanEqualToConstraints); + _state.setConstraintInfo(gradientInfo, equalityConstraints, lessThanEqualToConstraints, lambdaEqual, lambdaLessThan); // update state ginfo with contributions from GLMGradientInfo + boolean predictorSizeChange; + boolean applyBetaConstraints = _parms._separate_linear_beta && _betaConstraintsOn; + // short circuit check here: if gradient magnitude is small and all constraints are satisfied, quit right away + if (constraintsStop(gradientInfo, _state)) { + Log.info(LogMsg("GLM with constraints model building completed successfully!!")); + return; + } + double gradMagSquare; + int origIter = iterCnt+1; + try { + while (true) { + do { // implement Algorithm 11.8 of the doc to find coefficients with epsilon k as the precision + iterCnt++; + long t1 = System.currentTimeMillis(); + ComputationState.GramGrad gram = _state.computeGram(betaCnd, gradientInfo); // calculate gram (hessian), xy, objective values + if (iterCnt == origIter) { + Matrix gramMatrix = new Matrix(gram._gram); + if (gramMatrix.cond() >= BAD_CONDITION_NUMBER) + if (_parms._init_optimal_glm) { + warn("init_optimal_glm", " should be disabled. This lead to gram matrix being close to" + + " singular. Please re-run with init_optimal_glm set to false."); + } + } + predictorSizeChange = !coefNames.equals(Arrays.asList(_state.activeData().coefNames())); + if (predictorSizeChange) { // reset if predictors changed + coefNames = changeCoeffBetainfo(_state.activeData()._coefNames); + _state.resizeConstraintInfo(equalityConstraints, lessThanEqualToConstraints); + ginfo = gam.equals(_parms._glmType) ? new GLMGradientSolver(_job, _parms, _state.activeData(), 0, + _state.activeBC(), _betaInfo, _penaltyMatrix, _gamColIndices) : new GLMGradientSolver(_job, _parms, + _state.activeData(), 0, _state.activeBC(), _betaInfo); + tempBeta = new double[coefNames.size()]; + } + // solve for GLM coefficients + betaCnd = constraintGLM_solve(gram); // beta_k+1 = beta_k+dk where dk = beta_k+1-beta_k + predictorSizeChange = !coefNames.equals(Arrays.asList(_state.activeData().coefNames())); + if (predictorSizeChange) { // reset if predictors changed + coefNames = changeCoeffBetainfo(_state.activeData()._coefNames); + _state.resizeConstraintInfo(equalityConstraints, lessThanEqualToConstraints); + ginfo = gam.equals(_parms._glmType) ? new GLMGradientSolver(_job, _parms, _state.activeData(), 0, + _state.activeBC(), _betaInfo, _penaltyMatrix, _gamColIndices) : new GLMGradientSolver(_job, _parms, + _state.activeData(), 0, _state.activeBC(), _betaInfo); + tempBeta = new double[betaCnd.length]; + } + // add exact line search for GLM coefficients. Refer to the doc, Algorithm 11.5 + if (ls == null) + ls = new ExactLineSearch(betaCnd, _state, coefNames); + else + ls.reset(betaCnd, _state, coefNames); + + if (ls.findAlpha(lambdaEqual, lambdaLessThan, _state, equalityConstraints, lessThanEqualToConstraints, ginfo)) { + gradMagSquare = ArrayUtils.innerProduct(gradientInfo._gradient, gradientInfo._gradient); + betaCnd = ls._newBeta; + gradientInfo = ls._ginfoOriginal; + } else { // ls failed, reset to + if (applyBetaConstraints) // separate beta and linear constraints + bc.applyAllBounds(_state.beta()); + ls.setBetaConstraintsDeriv(lambdaEqual, lambdaLessThan, _state, equalityConstraints, lessThanEqualToConstraints, + ginfo, _state.beta()); + Log.info(LogMsg("Line search failed " + ls)); + return; + } + + if (applyBetaConstraints) { // if beta constraints are applied, may need to update constraints, derivatives, gradientInfo + System.arraycopy(betaCnd, 0, tempBeta, 0, betaCnd.length); + bc.applyAllBounds(betaCnd); + ArrayUtils.subtract(betaCnd, tempBeta, tempBeta); + ls.setBetaConstraintsDeriv(lambdaEqual, lambdaLessThan, _state, equalityConstraints, + lessThanEqualToConstraints, ginfo, betaCnd); + gradientInfo = ls._ginfoOriginal; + } + + // check for stopping conditions + if (checkIterationDone(betaCnd, gradientInfo, iterCnt)) // ratio of objective drops. + return; + Log.info(LogMsg("computed in " + (System.currentTimeMillis() - t1) + "ms, step = " + iterCnt + + ((_lslvr != null) ? ", l1solver " + _lslvr : ""))); + } while (gradMagSquare > _state._csGLMState._epsilonkCSSquare); + // update constraint parameters, ck, lambdas and others + updateConstraintParameters(_state, lambdaEqual, lambdaLessThan, equalityConstraints, lessThanEqualToConstraints, _parms); + } + } catch (NonSPDMatrixException e) { + Log.warn(LogMsg("Got Non SPD matrix, stopped.")); + } + } + + /*** + * This method will first check if enough progress has been made with progress method. + * If no more progress is made, we will check it the constraint stopping conditions are met. + * The model building process will stop if no more progress is made regardless of whether the constraint stopping + * conditions are met or not. + */ + public boolean checkIterationDone(double[] betaCnd, GLMGradientInfo gradientInfo, int iterCnt) { + // check for stopping conditions + boolean done = !progress(betaCnd, gradientInfo); // no good change in coeff, time-out or max_iteration reached + if (done) { + _model._betaCndCheckpoint = betaCnd; + boolean kktAchieved = constraintsStop(gradientInfo, _state); + if (kktAchieved) + Log.info("KKT Conditions achieved after " + iterCnt + " iterations "); + else + Log.warn("KKT Conditions not achieved but no further progress made due to time out or no changes" + + " to coefficients after " + iterCnt + " iterations"); + return true; + } + return false; + } + + public List changeCoeffBetainfo(String[] coefNames) { + _betaInfo = new BetaInfo(fractionalbinomial.equals(_parms._family) ? 2 : + (multinomial.equals(_parms._family) || ordinal.equals(_parms._family)) ? nclasses() : 1, coefNames.length); + return Arrays.stream(coefNames).collect(Collectors.toList()); + } private void fitIRLSMML(Solver s) { double[] betaCnd = _checkPointFirstIter ? _model._betaCndCheckpoint : _state.beta(); @@ -2998,13 +3301,15 @@ private void fitModel() { fitIRLSM_multinomial(solver); else if (ordinal.equals(_parms._family)) fitIRLSM_ordinal_default(solver); - else if (gaussian.equals(_parms._family) && Link.identity.equals(_parms._link)) - fitLSM(solver); + else if (gaussian.equals(_parms._family) && Link.identity.equals(_parms._link) && _parms._linear_constraints == null) + fitLSM(solver); // not constrained GLM else { if (_parms._dispersion_parameter_method.equals(ml)) fitIRLSMML(solver); - else + else if (_parms._linear_constraints == null) fitIRLSM(solver); + else + fitIRLSMCS(); // constrained GLM IRLSM } break; case GRADIENT_DESCENT_LH: @@ -3082,7 +3387,7 @@ else if (gaussian.equals(_parms._family) && Link.identity.equals(_parms._link)) double[][] inv = chol.getInv(); if (_parms._influence != null) { _cholInvInfluence = new double[inv.length][inv.length]; - copy2DArray(inv, _cholInvInfluence); + GamUtils.copy2DArray(inv, _cholInvInfluence); ArrayUtils.mult(_cholInvInfluence, _parms._obj_reg); g.mul(1.0/_parms._obj_reg); } @@ -3537,13 +3842,16 @@ public void computeImpl() { try { doCompute(); } finally { + final List keep = new ArrayList<>(); if ((!_doInit || !_cvRuns) && _betaConstraints != null) { DKV.remove(_betaConstraints._key); _betaConstraints.delete(); } + if ((!_doInit || !_cvRuns) && _parms._linear_constraints != null) { + keepFrameKeys(keep, _parms._linear_constraints); + } if (_model != null) { if (_parms._influence != null) { - final List keep = new ArrayList<>(); keepFrameKeys(keep, _model._output._regression_influence_diagnostics); if (_parms._keepBetaDiffVar) keepFrameKeys(keep, _model._output._betadiff_var); @@ -3579,177 +3887,189 @@ private void doCompute() { if (error_count() > 0) throw H2OModelBuilderIllegalArgumentException.makeFromBuilder(GLM.this); _model._output._start_time = System.currentTimeMillis(); //quickfix to align output duration with other models - if (_parms._lambda_search) { - if (ordinal.equals(_parms._family)) - nullDevTrain = new GLMResDevTaskOrdinal(_job._key, _state._dinfo, getNullBeta(), _nclass).doAll(_state._dinfo._adaptedFrame).avgDev(); - else - nullDevTrain = multinomial.equals(_parms._family) - ? new GLMResDevTaskMultinomial(_job._key, _state._dinfo, getNullBeta(), _nclass).doAll(_state._dinfo._adaptedFrame).avgDev() - : new GLMResDevTask(_job._key, _state._dinfo, _parms, getNullBeta()).doAll(_state._dinfo._adaptedFrame).avgDev(); - if (_validDinfo != null) { + if (_parms._expose_constraints && _parms._linear_constraints != null) { + _model._output._equalityConstraintsBeta = _state._equalityConstraintsBeta; + _model._output._lessThanEqualToConstraintsBeta = _state._lessThanEqualToConstraintsBeta; + _model._output._equalityConstraintsLinear = _state._equalityConstraintsLinear; + _model._output._lessThanEqualToConstraintsLinear = _state._lessThanEqualToConstraintsLinear; + _model._output._constraintCoefficientNames = _state._csGLMState._constraintNames; + _model._output._initConstraintMatrix = _state._csGLMState._initCSMatrix; + } + if (_parms._max_iterations == 0) { + return; + } else { + if (_parms._lambda_search) { if (ordinal.equals(_parms._family)) - nullDevValid = new GLMResDevTaskOrdinal(_job._key, _validDinfo, getNullBeta(), _nclass).doAll(_validDinfo._adaptedFrame).avgDev(); + nullDevTrain = new GLMResDevTaskOrdinal(_job._key, _state._dinfo, getNullBeta(), _nclass).doAll(_state._dinfo._adaptedFrame).avgDev(); else - nullDevValid = multinomial.equals(_parms._family) - ? new GLMResDevTaskMultinomial(_job._key, _validDinfo, getNullBeta(), _nclass).doAll(_validDinfo._adaptedFrame).avgDev() - : new GLMResDevTask(_job._key, _validDinfo, _parms, getNullBeta()).doAll(_validDinfo._adaptedFrame).avgDev(); + nullDevTrain = multinomial.equals(_parms._family) + ? new GLMResDevTaskMultinomial(_job._key, _state._dinfo, getNullBeta(), _nclass).doAll(_state._dinfo._adaptedFrame).avgDev() + : new GLMResDevTask(_job._key, _state._dinfo, _parms, getNullBeta()).doAll(_state._dinfo._adaptedFrame).avgDev(); + if (_validDinfo != null) { + if (ordinal.equals(_parms._family)) + nullDevValid = new GLMResDevTaskOrdinal(_job._key, _validDinfo, getNullBeta(), _nclass).doAll(_validDinfo._adaptedFrame).avgDev(); + else + nullDevValid = multinomial.equals(_parms._family) + ? new GLMResDevTaskMultinomial(_job._key, _validDinfo, getNullBeta(), _nclass).doAll(_validDinfo._adaptedFrame).avgDev() + : new GLMResDevTask(_job._key, _validDinfo, _parms, getNullBeta()).doAll(_validDinfo._adaptedFrame).avgDev(); + } + _workPerIteration = WORK_TOTAL / _parms._nlambdas; + } else + _workPerIteration = 1 + (WORK_TOTAL / _parms._max_iterations); + + if (!Solver.L_BFGS.equals(_parms._solver) && (multinomial.equals(_parms._family) || + ordinal.equals(_parms._family))) { + Vec[] vecs = genGLMVectors(_dinfo, getNullBeta()); + addGLMVec(vecs, false, _dinfo); } - _workPerIteration = WORK_TOTAL / _parms._nlambdas; - } else - _workPerIteration = 1 + (WORK_TOTAL / _parms._max_iterations); - - if (!Solver.L_BFGS.equals(_parms._solver) && (multinomial.equals(_parms._family) || - ordinal.equals(_parms._family))) { - Vec[] vecs = genGLMVectors(_dinfo, getNullBeta()); - addGLMVec(vecs, false, _dinfo); - } - - if (_parms._HGLM) { // add w, augZ, etaOld and random columns to response for easy access inside _dinfo - addWdataZiEtaOld2Response(); - } - - double oldDevTrain = nullDevTrain; - double oldDevTest = nullDevValid; - double[] devHistoryTrain = new double[5]; - double[] devHistoryTest = new double[5]; - - if (!_parms._HGLM) { // only need these for non HGLM - _ginfoStart = GLMUtils.copyGInfo(_state.ginfo()); - _betaDiffStart = _state.getBetaDiff(); - } - - if (_parms.hasCheckpoint()) { // restore _state parameters - _state.copyCheckModel2State(_model, _gamColIndices); - if (_model._output._submodels.length == 1) - _model._output._submodels = null; // null out submodel only for single alpha/lambda values - } - - if (!_parms._lambda_search & !_parms._HGLM) - updateProgress(false); - - // alpha, lambda search loop - int alphaStart = 0; - int lambdaStart = 0; - int submodelCount = 0; - if (_parms.hasCheckpoint() && _model._output._submodels != null) { // multiple alpha/lambdas or lambda search - submodelCount = Family.gaussian.equals(_parms._family) ? _model._output._submodels.length - : _model._output._submodels.length - 1; - alphaStart = submodelCount / _parms._lambda.length; - lambdaStart = submodelCount % _parms._lambda.length; - } - _model._output._lambda_array_size = _parms._lambda.length; - for (int alphaInd = alphaStart; alphaInd < _parms._alpha.length; alphaInd++) { - _state.setAlpha(_parms._alpha[alphaInd]); // loop through the alphas - if ((!_parms._HGLM) && (alphaInd > 0) && !_checkPointFirstIter) // no need for cold start during the first iteration - coldStart(devHistoryTrain, devHistoryTest); // reset beta, lambda, currGram - for (int i = lambdaStart; i < _parms._lambda.length; ++i) { // for lambda search, can quit before it is done - if (_job.stop_requested() || (timeout() && _model._output._submodels.length > 0)) - break; //need at least one submodel on timeout to avoid issues. - if (_parms._max_iterations != -1 && _state._iter >= _parms._max_iterations) - break; // iterations accumulate across all lambda/alpha values when coldstart = false - if ((!_parms._HGLM && (_parms._cold_start || (!_parms._lambda_search && _parms._cold_start))) && (i > 0) - && !_checkPointFirstIter) // default: cold_start for non lambda_search - coldStart(devHistoryTrain, devHistoryTest); - Submodel sm = computeSubmodel(submodelCount, _parms._lambda[i], nullDevTrain, nullDevValid); - if (_checkPointFirstIter) - _checkPointFirstIter = false; - double trainDev = sm.devianceTrain; // this is stupid, they are always -1 except for lambda_search=True - double testDev = sm.devianceValid; - devHistoryTest[submodelCount % devHistoryTest.length] = - (oldDevTest - testDev) / oldDevTest; // only remembers 5 - oldDevTest = testDev; - devHistoryTrain[submodelCount % devHistoryTrain.length] = - (oldDevTrain - trainDev) / oldDevTrain; - oldDevTrain = trainDev; - if (_parms._lambda[i] < _lmax && Double.isNaN(_lambdaCVEstimate) /** if we have cv lambda estimate we should use it, can not stop before reaching it */) { - if (_parms._early_stopping && _state._iter >= devHistoryTrain.length) { - double s = ArrayUtils.maxValue(devHistoryTrain); - if (s < 1e-4) { - Log.info(LogMsg("converged at lambda[" + i + "] = " + _parms._lambda[i] + "alpha[" + alphaInd + "] = " - + _parms._alpha[alphaInd] + ", improvement on train = " + s)); - break; // started overfitting - } - if (_validDinfo != null && _parms._nfolds <= 1) { // check for early stopping on test with no xval - s = ArrayUtils.maxValue(devHistoryTest); - if (s < 0) { - Log.info(LogMsg("converged at lambda[" + i + "] = " + _parms._lambda[i] + "alpha[" + alphaInd + - "] = " + _parms._alpha[alphaInd] + ", improvement on test = " + s)); + + if (_parms._HGLM) { // add w, augZ, etaOld and random columns to response for easy access inside _dinfo + addWdataZiEtaOld2Response(); + } else { // only need these for non HGLM + _ginfoStart = GLMUtils.copyGInfo(_state.ginfo()); + _betaDiffStart = _state.getBetaDiff(); + } + + double oldDevTrain = nullDevTrain; + double oldDevTest = nullDevValid; + double[] devHistoryTrain = new double[5]; + double[] devHistoryTest = new double[5]; + + if (_parms.hasCheckpoint()) { // restore _state parameters + _state.copyCheckModel2State(_model, _gamColIndices); + if (_model._output._submodels.length == 1) + _model._output._submodels = null; // null out submodel only for single alpha/lambda values + } + + if (!_parms._lambda_search & !_parms._HGLM) + updateProgress(false); + + // alpha, lambda search loop + int alphaStart = 0; + int lambdaStart = 0; + int submodelCount = 0; + if (_parms.hasCheckpoint() && _model._output._submodels != null) { // multiple alpha/lambdas or lambda search + submodelCount = Family.gaussian.equals(_parms._family) ? _model._output._submodels.length + : _model._output._submodels.length - 1; + alphaStart = submodelCount / _parms._lambda.length; + lambdaStart = submodelCount % _parms._lambda.length; + } + _model._output._lambda_array_size = _parms._lambda.length; + for (int alphaInd = alphaStart; alphaInd < _parms._alpha.length; alphaInd++) { + _state.setAlpha(_parms._alpha[alphaInd]); // loop through the alphas + if ((!_parms._HGLM) && (alphaInd > 0) && !_checkPointFirstIter) // no need for cold start during the first iteration + coldStart(devHistoryTrain, devHistoryTest); // reset beta, lambda, currGram + for (int i = lambdaStart; i < _parms._lambda.length; ++i) { // for lambda search, can quit before it is done + if (_job.stop_requested() || (timeout() && _model._output._submodels.length > 0)) + break; //need at least one submodel on timeout to avoid issues. + if (_parms._max_iterations != -1 && _state._iter >= _parms._max_iterations) + break; // iterations accumulate across all lambda/alpha values when coldstart = false + if ((!_parms._HGLM && (_parms._cold_start || (!_parms._lambda_search && _parms._cold_start))) && (i > 0) + && !_checkPointFirstIter) // default: cold_start for non lambda_search + coldStart(devHistoryTrain, devHistoryTest); + Submodel sm = computeSubmodel(submodelCount, _parms._lambda[i], nullDevTrain, nullDevValid); + if (_checkPointFirstIter) + _checkPointFirstIter = false; + double trainDev = sm.devianceTrain; // this is stupid, they are always -1 except for lambda_search=True + double testDev = sm.devianceValid; + devHistoryTest[submodelCount % devHistoryTest.length] = + (oldDevTest - testDev) / oldDevTest; // only remembers 5 + oldDevTest = testDev; + devHistoryTrain[submodelCount % devHistoryTrain.length] = + (oldDevTrain - trainDev) / oldDevTrain; + oldDevTrain = trainDev; + if (_parms._lambda[i] < _lmax && Double.isNaN(_lambdaCVEstimate) /** if we have cv lambda estimate we should use it, can not stop before reaching it */) { + if (_parms._early_stopping && _state._iter >= devHistoryTrain.length) { // implement early stopping for lambda search + double s = ArrayUtils.maxValue(devHistoryTrain); + if (s < 1e-4) { + Log.info(LogMsg("converged at lambda[" + i + "] = " + _parms._lambda[i] + "alpha[" + alphaInd + "] = " + + _parms._alpha[alphaInd] + ", improvement on train = " + s)); break; // started overfitting } + if (_validDinfo != null && _parms._nfolds <= 1) { // check for early stopping on test with no xval + s = ArrayUtils.maxValue(devHistoryTest); + if (s < 0) { + Log.info(LogMsg("converged at lambda[" + i + "] = " + _parms._lambda[i] + "alpha[" + alphaInd + + "] = " + _parms._alpha[alphaInd] + ", improvement on test = " + s)); + break; // started overfitting + } + } } } - } - if ((_parms._lambda_search || _parms._generate_scoring_history) && (_parms._score_each_iteration || - timeSinceLastScoring() > _scoringInterval || ((_parms._score_iteration_interval > 0) && - ((_state._iter % _parms._score_iteration_interval) == 0)))) { - _model._output.setSubmodelIdx(_model._output._best_submodel_idx = submodelCount, _parms); // quick and easy way to set submodel parameters - scoreAndUpdateModel(); // update partial results + if ((_parms._lambda_search || _parms._generate_scoring_history) && (_parms._score_each_iteration || + timeSinceLastScoring() > _scoringInterval || ((_parms._score_iteration_interval > 0) && + ((_state._iter % _parms._score_iteration_interval) == 0)))) { + _model._output.setSubmodelIdx(_model._output._best_submodel_idx = submodelCount, _parms); // quick and easy way to set submodel parameters + scoreAndUpdateModel(); // update partial results + } + _job.update(_workPerIteration, "iter=" + _state._iter + " lmb=" + + lambdaFormatter.format(_state.lambda()) + " alpha=" + lambdaFormatter.format(_state.alpha()) + + "deviance trn/tst= " + devFormatter.format(trainDev) + "/" + devFormatter.format(testDev) + + " P=" + ArrayUtils.countNonzeros(_state.beta())); + submodelCount++; // updata submodel index count here } - _job.update(_workPerIteration, "iter=" + _state._iter + " lmb=" + - lambdaFormatter.format(_state.lambda()) + " alpha=" + lambdaFormatter.format(_state.alpha()) + - "deviance trn/tst= " + devFormatter.format(trainDev) + "/" + devFormatter.format(testDev) + - " P=" + ArrayUtils.countNonzeros(_state.beta())); - submodelCount++; // updata submodel index count here } - } - // if beta constraint is enabled, check and make sure coefficients are within bounds - if (_betaConstraintsOn && betaConstraintsCheckEnabled()) - checkCoeffsBounds(); + // if beta constraint is enabled, check and make sure coefficients are within bounds + if (_betaConstraintsOn && betaConstraintsCheckEnabled() && (!_linearConstraintsOn || _parms._separate_linear_beta)) + checkCoeffsBounds(); - if (stop_requested() || _earlyStop) { - if (timeout()) { - Log.info("Stopping GLM training because of timeout"); - } else if (_earlyStop) { - Log.info("Stopping GLM training due to hitting earlyStopping criteria."); - } else { - throw new Job.JobCancelledException(); + if (stop_requested() || _earlyStop) { + if (timeout()) { + Log.info("Stopping GLM training because of timeout"); + } else if (_earlyStop) { + Log.info("Stopping GLM training due to hitting early stopping criteria."); + } else { + throw new Job.JobCancelledException(); + } } - } - if (_state._iter >= _parms._max_iterations) - _job.warn("Reached maximum number of iterations " + _parms._max_iterations + "!"); - if (_parms._nfolds > 1 && !Double.isNaN(_lambdaCVEstimate) && _bestCVSubmodel < _model._output._submodels.length) - _model._output.setSubmodelIdx(_model._output._best_submodel_idx = _bestCVSubmodel, _model._parms); // reset best_submodel_idx to what xval has found - else - _model._output.pickBestModel(_model._parms); - if (_vcov != null) { // should move this up, otherwise, scoring will never use info in _vcov - _model.setVcov(_vcov); + if (_state._iter >= _parms._max_iterations) + _job.warn("Reached maximum number of iterations " + _parms._max_iterations + "!"); + if (_parms._nfolds > 1 && !Double.isNaN(_lambdaCVEstimate) && _bestCVSubmodel < _model._output._submodels.length) + _model._output.setSubmodelIdx(_model._output._best_submodel_idx = _bestCVSubmodel, _model._parms); // reset best_submodel_idx to what xval has found + else + _model._output.pickBestModel(_model._parms); + if (_vcov != null) { // should move this up, otherwise, scoring will never use info in _vcov + _model.setVcov(_vcov); + _model.update(_job._key); + } + if (!_parms._HGLM) { // no need to do for HGLM + _model._finalScoring = true; // enables likelihood calculation while scoring + scoreAndUpdateModel(); + _model._finalScoring = false; // avoid calculating likelihood in case of further updates + } + + if (dfbetas.equals(_parms._influence)) + genRID(); + + if (_parms._generate_variable_inflation_factors) { + _model._output._vif_predictor_names = _model.buildVariableInflationFactors(_train, _dinfo); + }// build variable inflation factors for numerical predictors + TwoDimTable scoring_history_early_stop = ScoringInfo.createScoringHistoryTable(_model.getScoringInfo(), + (null != _parms._valid), false, _model._output.getModelCategory(), false, _parms.hasCustomMetricFunc()); + _model._output._scoring_history = combineScoringHistory(_model._output._scoring_history, + scoring_history_early_stop); + _model._output._varimp = _model._output.calculateVarimp(); + _model._output._variable_importances = calcVarImp(_model._output._varimp); + if (_linearConstraintsOn) + printConstraintSummary(_model, _state, _dinfo.coefNames()); + _model.update(_job._key); - } - if (!_parms._HGLM) { // no need to do for HGLM - _model._finalScoring = true; // enables likelihood calculation while scoring - scoreAndUpdateModel(); - _model._finalScoring = false; // avoid calculating likelihood in case of further updates - } - - if (dfbetas.equals(_parms._influence)) - genRID(); - - if (_parms._generate_variable_inflation_factors) { - _model._output._vif_predictor_names = _model.buildVariableInflationFactors(_train, _dinfo); - }// build variable inflation factors for numerical predictors - TwoDimTable scoring_history_early_stop = ScoringInfo.createScoringHistoryTable(_model.getScoringInfo(), - (null != _parms._valid), false, _model._output.getModelCategory(), false, _parms.hasCustomMetricFunc()); - _model._output._scoring_history = combineScoringHistory(_model._output._scoring_history, - scoring_history_early_stop); - _model._output._varimp = _model._output.calculateVarimp(); - _model._output._variable_importances = calcVarImp(_model._output._varimp); - - _model.update(_job._key); /* if (_vcov != null) { _model.setVcov(_vcov); _model.update(_job._key); }*/ - if (!(_parms)._lambda_search && _state._iter < _parms._max_iterations) { - _job.update(_workPerIteration * (_parms._max_iterations - _state._iter)); - } - if (_iceptAdjust != 0) { // apply the intercept adjust according to prior probability - assert _parms._intercept; - double[] b = _model._output._global_beta; - b[b.length - 1] += _iceptAdjust; - for (Submodel sm : _model._output._submodels) - sm.beta[sm.beta.length - 1] += _iceptAdjust; - _model.update(_job._key); + if (!(_parms)._lambda_search && _state._iter < _parms._max_iterations) { + _job.update(_workPerIteration * (_parms._max_iterations - _state._iter)); + } + if (_iceptAdjust != 0) { // apply the intercept adjust according to prior probability + assert _parms._intercept; + double[] b = _model._output._global_beta; + b[b.length - 1] += _iceptAdjust; + for (Submodel sm : _model._output._submodels) + sm.beta[sm.beta.length - 1] += _iceptAdjust; + _model.update(_job._key); + } } } @@ -3834,11 +4154,14 @@ private void checkCoeffsBounds() { if (bc._betaLB == null || bc._betaUB == null || coeffs == null) return; int coeffsLen = bc._betaLB.length; + StringBuffer errorMessage = new StringBuffer(); for (int index=0; index < coeffsLen; index++) { if (!(coeffs[index] == 0 || (coeffs[index] >= bc._betaLB[index] && coeffs[index] <= bc._betaUB[index]))) - throw new H2OFailException("GLM model coefficient" + coeffs[index]+" exceeds beta constraint bounds. Lower: " - +bc._betaLB[index]+", upper: "+bc._betaUB[index]); + errorMessage.append("GLM model coefficient " + coeffs[index]+" exceeds beta constraint bounds. Lower: " + +bc._betaLB[index]+", upper: "+bc._betaUB[index]+"\n"); } + if (errorMessage.length() > 0) + throw new H2OFailException("\n"+errorMessage.toString()); } /*** @@ -4474,6 +4797,47 @@ public GLMGradientInfo getMultinomialLikelihood(double[] beta) { smoothval, null); } + /*** + * + * This method calculates the gradient for constrained GLM without taking into account the contribution of the + * constraints in this case. The likelihood, objective are calculated without the contribution of the constraints + * either. + */ + public GLMGradientInfo getGradient(double[] beta, ComputationState state) { + DataInfo dinfo = state.activeData()._activeCols == null ? _dinfo : state.activeData(); + assert beta.length == dinfo.fullN() + 1; + assert _parms._intercept || (beta[beta.length-1] == 0); + GLMGradientTask gt; + if((_parms._family == binomial && _parms._link == Link.logit) || + (_parms._family == Family.fractionalbinomial && _parms._link == Link.logit)) + gt = new GLMBinomialGradientTask(_job == null?null:_job._key,dinfo,_parms,_l2pen, beta, _penaltyMatrix, + _gamColIndices).doAll(dinfo._adaptedFrame); + else if(_parms._family == Family.gaussian && _parms._link == Link.identity) + gt = new GLMGaussianGradientTask(_job == null?null:_job._key,dinfo,_parms,_l2pen, beta, _penaltyMatrix, + _gamColIndices).doAll(dinfo._adaptedFrame); + else if (Family.negativebinomial.equals(_parms._family)) + gt = new GLMNegativeBinomialGradientTask(_job == null?null:_job._key,dinfo, + _parms,_l2pen, beta, _penaltyMatrix, _gamColIndices).doAll(dinfo._adaptedFrame); + else if(_parms._family == Family.poisson && _parms._link == Link.log) + gt = new GLMPoissonGradientTask(_job == null?null:_job._key,dinfo,_parms,_l2pen, beta, _penaltyMatrix, + _gamColIndices).doAll(dinfo._adaptedFrame); + else if(_parms._family == Family.quasibinomial) + gt = new GLMQuasiBinomialGradientTask(_job == null?null:_job._key,dinfo,_parms,_l2pen, beta, + _penaltyMatrix, _gamColIndices).doAll(dinfo._adaptedFrame); + else + gt = new GLMGenericGradientTask(_job == null?null:_job._key, dinfo, _parms, _l2pen, beta, _penaltyMatrix, + _gamColIndices).doAll(dinfo._adaptedFrame); + double [] gradient = gt._gradient; + double likelihood = gt._likelihood; + if (!_parms._intercept) // no intercept, null the ginfo + gradient[gradient.length - 1] = 0; + + double gamSmooth = gam.equals(_parms._glmType)? + calSmoothNess(expandVec(beta, dinfo._activeCols, _betaInfo.totalBetaLength()), _penaltyMatrix, _gamColIndices):0; + double obj = likelihood * _parms._obj_reg + .5 * _l2pen * ArrayUtils.l2norm2(beta, true)+gamSmooth; + return new GLMGradientInfo(likelihood, obj, gradient); + } + @Override public GLMGradientInfo getGradient(double[] beta) { if (multinomial.equals(_parms._family) || ordinal.equals(_parms._family)) { @@ -4607,10 +4971,10 @@ public void setNonNegative() { public void setNonNegative(Frame otherConst) { List constraintNames = extractVec2List(otherConst); // maximum size = number of predictors, an integer - List coeffNames = Arrays.stream(_dinfo.coefNames()).collect(Collectors.toList()); - int numCoef = coeffNames.size(); + List coefNames = Arrays.stream(_dinfo.coefNames()).collect(Collectors.toList()); + int numCoef = coefNames.size(); for (int index=0; index _beta_constraints = null; + public Key _linear_constraints = null; + public boolean _expose_constraints = false; // internal parameter for testing only. public Key _plug_values = null; // internal parameter, handle with care. GLM will stop when there is more than this number of active predictors (after strong rule screening) public int _max_active_predictors = -1; @@ -534,6 +537,14 @@ public enum MissingValuesHandling { public double _dispersion_learning_rate = 0.5; public Influence _influence; // if set to dfbetas will calculate the difference of betas obtained from including and excluding a data row public boolean _keepBetaDiffVar = false; // if true, will keep the frame generating the beta without from i and the variance estimation + boolean _testCSZeroGram = false; // internal parameter, to test zero gram dropped column is correctly implemented + public boolean _separate_linear_beta = false; // if true, will perform the beta and linear constraint separately + public boolean _init_optimal_glm = false; // only used when there is linear constraints + public double _constraint_eta0 = 0.1258925; // eta_k = constraint_eta0/pow(constraint_c0, constraint_alpha) + public double _constraint_tau = 10; // ck+1 = tau*ck + public double _constraint_alpha = 0.1; // eta_k = constraint_eta_0/pow(constraint_c0, constraint_alpha) + public double _constraint_beta = 0.9; // eta_k+1 = eta_k/pow(c_k, beta) + public double _constraint_c0 = 10; // set initial epsilon k as 1/c0 public void validate(GLM glm) { if (_remove_collinear_columns) { @@ -1552,6 +1563,9 @@ public static class GLMOutput extends Model.Output { public int _selected_submodel_idx; // submodel index with best deviance public int _best_submodel_idx; // submodel index with best deviance public int _best_lambda_idx; // the same as best_submodel_idx, kept to ensure backward compatibility + public String[] _linear_constraint_states; + public boolean _all_constraints_satisfied; + public TwoDimTable _linear_constraints_table; public Key _regression_influence_diagnostics; public Key _betadiff_var; // for debugging only, no need to serialize public double lambda_best(){return _submodels.length == 0 ? -1 : _submodels[_best_submodel_idx].lambda_value;} @@ -1577,6 +1591,12 @@ public double lambda_selected(){ private double _dispersion; private boolean _dispersionEstimated; public int[] _activeColsPerClass; + public ConstrainedGLMUtils.LinearConstraints[] _equalityConstraintsLinear = null; + public ConstrainedGLMUtils.LinearConstraints[] _lessThanEqualToConstraintsLinear = null; + public ConstrainedGLMUtils.LinearConstraints[] _equalityConstraintsBeta = null; + public ConstrainedGLMUtils.LinearConstraints[] _lessThanEqualToConstraintsBeta = null; + public String[] _constraintCoefficientNames = null; + public double[][] _initConstraintMatrix = null; public boolean hasPValues(){return _zvalues != null;} public boolean hasVIF() { return _vif_predictor_names != null; } diff --git a/h2o-algos/src/main/java/hex/glm/GLMTask.java b/h2o-algos/src/main/java/hex/glm/GLMTask.java index bfbafc98ccd6..75fac2d5c4c2 100644 --- a/h2o-algos/src/main/java/hex/glm/GLMTask.java +++ b/h2o-algos/src/main/java/hex/glm/GLMTask.java @@ -575,7 +575,7 @@ public final void reduce(GLMGradientTask gmgt){ _likelihood += gmgt._likelihood; } @Override public final void postGlobal(){ - ArrayUtils.mult(_gradient,_reg); + ArrayUtils.mult(_gradient,_reg); // reg is obj_reg for(int j = 0; j < _beta.length - 1; ++j) _gradient[j] += _currentLambda * _beta[j]; // add L2 constraint for gradient if ((_penalty_mat != null) && (_gamBetaIndices != null)) // update contribution from gam smoothness constraint @@ -1522,6 +1522,7 @@ public static class GLMIterationTask extends FrameTask2 { double wsum, sumOfRowWeights; double _sumsqe; int _c = -1; + boolean _hasConstraints = false; public double[] getXY() { return _xy; @@ -1546,6 +1547,15 @@ public GLMIterationTask(Key jobKey, DataInfo dinfo, GLMWeightsFun glmw, double _c = c; } + public GLMIterationTask(Key jobKey, DataInfo dinfo, GLMWeightsFun glmw, double [] beta, int c, boolean hasConst) { + super(null,dinfo,jobKey); + _beta = beta; + _ymu = null; + _glmf = glmw; + _c = c; + _hasConstraints = hasConst; + } + @Override public boolean handlesSparseData(){return true;} transient private double _sparseOffset; @@ -1659,7 +1669,6 @@ public boolean hasNaNsOrInf() { return ArrayUtils.hasNaNsOrInfs(_xy) || _gram.hasNaNsOrInfs(); } } - /* public static class GLMCoordinateDescentTask extends FrameTask2 { final GLMParameters _params; diff --git a/h2o-algos/src/main/java/hex/glm/GLMUtils.java b/h2o-algos/src/main/java/hex/glm/GLMUtils.java index b1b825ce7d11..f1086084c12a 100644 --- a/h2o-algos/src/main/java/hex/glm/GLMUtils.java +++ b/h2o-algos/src/main/java/hex/glm/GLMUtils.java @@ -365,4 +365,12 @@ public static Frame buildRIDFrame(GLMModel.GLMParameters parms, Frame train, Fra train.add(parms._response_column, responseVec); return train; } + + public static boolean notZeroLambdas(double[] lambdas) { + if (lambdas == null) { + return false; + } else { + return ((int) Arrays.stream(lambdas).filter(x -> x != 0.0).boxed().count()) > 0; + } + } } diff --git a/h2o-algos/src/main/java/hex/gram/Gram.java b/h2o-algos/src/main/java/hex/gram/Gram.java index 784d54054de6..7d20bc430200 100644 --- a/h2o-algos/src/main/java/hex/gram/Gram.java +++ b/h2o-algos/src/main/java/hex/gram/Gram.java @@ -122,6 +122,22 @@ public void addGAMPenalty(Integer[] activeColumns, double[][][] ds, int[][] gamI } } } + + public void addGAMPenalty(double[][][] ds, int[][] gamIndices, double[][] xmatrix) { + int numGamCols = gamIndices.length; + int numKnots; + int betaIndex, betaIndexJ; + for (int gamInd=0; gamInd i) { diff --git a/h2o-algos/src/main/java/hex/optimization/OptimizationUtils.java b/h2o-algos/src/main/java/hex/optimization/OptimizationUtils.java index f75d9c567a19..221afa9779cc 100644 --- a/h2o-algos/src/main/java/hex/optimization/OptimizationUtils.java +++ b/h2o-algos/src/main/java/hex/optimization/OptimizationUtils.java @@ -1,17 +1,23 @@ package hex.optimization; +import hex.glm.ComputationState; +import hex.glm.ConstrainedGLMUtils; +import hex.glm.GLM; import water.Iced; import water.util.ArrayUtils; import water.util.Log; import java.util.Arrays; +import java.util.List; + +import static hex.glm.ComputationState.EPS_CS_SQUARE; +import static hex.glm.ConstrainedGLMUtils.*; /** * Created by tomasnykodym on 9/29/15. */ public class OptimizationUtils { - public static class GradientInfo extends Iced { public double _objVal; public double [] _gradient; @@ -456,4 +462,137 @@ public GradientInfo ginfo() { } } + /*** + * This class implements the exact line search described in the doc, Algorithm 11.5 + */ + public static final class ExactLineSearch { + public final double _betaLS1 = 1e-4; + public final double _betaLS2 = 0.99; + public final double _lambdaLS = 2; + public double _alphal; + public double _alphar; + public double _alphai; + public double[] _direction; // beta_k+1 - beta_k + public int _maxIteration = 50; // 40 too low for tight constraints + public double[] _originalBeta; + public double[] _newBeta; + public GLM.GLMGradientInfo _ginfoOriginal; + public double _currGradDirIP; // current gradient and direction inner product + public String[] _coeffNames; + + public ExactLineSearch(double[] betaCnd, ComputationState state, List coeffNames) { + reset(betaCnd, state, coeffNames); + } + + public void reset(double[] betaCnd, ComputationState state, List coeffNames) { + _direction = new double[betaCnd.length]; + ArrayUtils.subtract(betaCnd, state.beta(), _direction); + _ginfoOriginal = state.ginfo(); + _originalBeta = state.beta(); + _alphai = 1; + _alphal = 0; + _alphar = Double.POSITIVE_INFINITY; + _coeffNames = coeffNames.toArray(new String[0]); + _currGradDirIP = ArrayUtils.innerProduct(_ginfoOriginal._gradient, _direction); + } + + /*** + * Evaluate and make sure that step size alphi is not too big so that objective function is still decreasing. Refer + * to the doc, Definition 11.6 + */ + public boolean evaluateFirstWolfe(GLM.GLMGradientInfo ginfoNew) { + double newObj = ginfoNew._objVal; + double rhs = _ginfoOriginal._objVal+_alphai*_betaLS1* _currGradDirIP; + return (newObj <= rhs); + } + + /*** + * Evaluate and make sure that step size alphi is not too small so that good progress is made in reducing the + * loss function. Refer to the doc, Definition 11.8 + */ + public boolean evaluateSecondWolfe(GLM.GLMGradientInfo ginfo) { + double lhs = ArrayUtils.innerProduct(ginfo._gradient, _direction); + return lhs >= _betaLS2* _currGradDirIP; + } + + public boolean setAlphai(boolean firstWolfe, boolean secondWolfe) { + if (!firstWolfe && secondWolfe) { // step is too long + _alphar = _alphai; + _alphai = 0.5*(_alphal+_alphar); + return true; + } else if (firstWolfe && !secondWolfe) { // step is too short + _alphal = _alphai; + if (_alphar < Double.POSITIVE_INFINITY) + _alphai = 0.5*(_alphal+_alphar); + else + _alphai = _lambdaLS * _alphai; + return true; + } + return false; + } + + public void setBetaConstraintsDeriv(double[] lambdaEqual, double[] lambdaLessThan, ComputationState state, + ConstrainedGLMUtils.LinearConstraints[] equalityConstraints, + ConstrainedGLMUtils.LinearConstraints[] lessThanEqualToConstraints, + GLM.GLMGradientSolver gradientSolver, double[] betaCnd) { + _newBeta = betaCnd; + updateConstraintValues(betaCnd, Arrays.asList(_coeffNames), equalityConstraints, lessThanEqualToConstraints); + calculateConstraintSquare(state, equalityConstraints, lessThanEqualToConstraints); + // update gradient from constraints transpose(lambda)*h(beta)(value, not changed, active status may change) + // and gram contribution from ck/2*transpose(h(beta))*h(beta)), value not changed but active status may change + state.updateConstraintInfo(equalityConstraints, lessThanEqualToConstraints); + // calculate new gradient and objective function; + _ginfoOriginal = calGradient(betaCnd, state, gradientSolver, lambdaEqual, lambdaLessThan, + equalityConstraints, lessThanEqualToConstraints); + } + + /*** + * Implements the Line Search algorithm in the doc, Algorithm 11.5. + */ + public boolean findAlpha(double[] lambdaEqual, double[] lambdaLessThan, ComputationState state, + ConstrainedGLMUtils.LinearConstraints[] equalityConstraints, + ConstrainedGLMUtils.LinearConstraints[] lessThanEqualToConstraints, + GLM.GLMGradientSolver gradientSolver) { + if (_currGradDirIP > 0) { + return false; + } + GLM.GLMGradientInfo newGrad; + double[] newCoef; + int betaLen = _originalBeta.length; + double[] tempDirection = new double[betaLen]; + boolean firstWolfe; + boolean secondWolfe; + boolean alphaiChange; + for (int index=0; index<_maxIteration; index++) { + ArrayUtils.mult(_direction, tempDirection, _alphai); // tempCoef=alpha_i*direction + newCoef = ArrayUtils.add(tempDirection, _originalBeta); // newCoef = coef + alpha_i*direction + // calculate constraint values with new coefficients, constraints magnitude square + updateConstraintValues(newCoef, Arrays.asList(_coeffNames), equalityConstraints, lessThanEqualToConstraints); + calculateConstraintSquare(state, equalityConstraints, lessThanEqualToConstraints); + // update gradient from constraints transpose(lambda)*h(beta)(value, not changed, active status may change) + // and gram contribution from ck/2*transpose(h(beta))*h(beta)), value not changed but active status may change + state.updateConstraintInfo(equalityConstraints, lessThanEqualToConstraints); + // calculate new gradient and objective function for new coefficients newCoef + newGrad = calGradient(newCoef, state, gradientSolver, lambdaEqual, lambdaLessThan, + equalityConstraints, lessThanEqualToConstraints); + // evaluate if first Wolfe condition is satisfied; + firstWolfe = evaluateFirstWolfe(newGrad); + // evaluate if second Wolfe condition is satisfied; + secondWolfe = evaluateSecondWolfe(newGrad); + // return if both conditions are satisfied; + if (firstWolfe && secondWolfe) { + _newBeta = newCoef; + _ginfoOriginal = newGrad; + return true; + } + + // set alphai if first Wolfe condition is not satisfied, set alpha i if second Wolfe condition is not satisfied; + alphaiChange = setAlphai(firstWolfe, secondWolfe); + if (!alphaiChange || _alphar < EPS_CS_SQUARE) { // if alphai, alphar value are not changed and alphar is too small, quit + return false; + } + } + return false; + } + } } diff --git a/h2o-algos/src/main/java/hex/schemas/GLMModelV3.java b/h2o-algos/src/main/java/hex/schemas/GLMModelV3.java index 927c679cfbf3..4d8d79663980 100644 --- a/h2o-algos/src/main/java/hex/schemas/GLMModelV3.java +++ b/h2o-algos/src/main/java/hex/schemas/GLMModelV3.java @@ -63,12 +63,24 @@ public static final class GLMModelOutputV3 extends ModelOutputSchemaV3 validVIFNames = impl.hasVIF() ? Stream.of(vif_predictor_names).collect(Collectors.toList()) : null; @@ -215,11 +230,13 @@ public GLMModelOutputV3 fillFromImpl(GLMModel.GLMOutput impl) { random_coefficients_table.fillFromImpl(buildRandomCoefficients2DTable(impl.ubeta(), impl.randomcoefficientNames())); } double [] beta = impl.beta(); - final double [] magnitudes = new double[beta.length]; - int len = magnitudes.length - 1; - int[] indices = new int[len]; - for (int i = 0; i < indices.length; ++i) - indices[i] = i; + final double [] magnitudes = beta==null?null:new double[beta.length]; + int len = beta==null?0:magnitudes.length - 1; + int[] indices = beta==null?null:new int[len]; + if (beta != null) { + for (int i = 0; i < indices.length; ++i) + indices[i] = i; + } if(beta == null) beta = MemoryManager.malloc8d(names.length); String [] colTypes = new String[]{"double"}; diff --git a/h2o-algos/src/main/java/hex/schemas/GLMV3.java b/h2o-algos/src/main/java/hex/schemas/GLMV3.java index 96a5693b642a..58ef841fbbf8 100644 --- a/h2o-algos/src/main/java/hex/schemas/GLMV3.java +++ b/h2o-algos/src/main/java/hex/schemas/GLMV3.java @@ -96,19 +96,28 @@ public static final class GLMParametersV3 extends ModelParametersSchemaV3 _coeffNames1; + String[][] _betaConstraintNames1Equal; + double[][] _betaConstraintValStandard1Equal; + double[][] _betaConstraintVal1Equal; + String[][] _betaConstraintNames1Less; + double[][] _betaConstraintValStandard1Less; + double[][] _betaConstraintVal1Less; + String[][] _equalityNames1; + double[][] _equalityValuesStandard1; + double[][] _equalityValues1; + String[][] _lessThanNames1; + double[][] _lessThanValuesStandard1; + double[][] _lessThanValues1; + String[][] _equalityNames2; + double[][] _equalityValues2; + double[][] _equalityValuesStandard2; + String[][] _lessThanNames2; + double[][] _lessThanValues2; + double[][] _lessThanValuesStandard2; + int[] _catCol = new int[]{0, 1, 2, 3, 4, 5, 6, 7, 8, 9}; + ConstrainedGLMUtils.LinearConstraints[] _equalityConstr; + ConstrainedGLMUtils.LinearConstraints[] _lessThanConstr; + ConstrainedGLMUtils.ConstraintsDerivatives[] _cdEqual; + ConstrainedGLMUtils.ConstraintsDerivatives[] _cdLess; + ConstrainedGLMUtils.ConstraintsGram[] _cGEqual; + ConstrainedGLMUtils.ConstraintsGram[] _cGLess; + List _coeffsDG; + double[] _lambdaE; + double[] _lambdaL; + double[][] _equalGramContr; + double[][] _lessGramContr; + double[] _equalGradContr; + double[] _lessGradContr; + double _ck = 10; + double[] _beta; + double[] _equalGradPenalty; + double[] _lessGradPenalty; + Random _obj = new Random(123); + + + @Before + public void setup() { + Scope.enter(); + Frame train = parseAndTrackTestFile("smalldata/glm_test/gaussian_20cols_10000Rows.csv"); + _coeffNames1 = transformFrameCreateCoefNames(train); + generateConstraint1FrameNAnswer(train); + generateConstraint2FrameNAnswer(train); + generateConstraint3FrameNAnswer(); + generateConstraint4FrameNAnswer(); + generateConstraints(); + generateDerivativeAnswer(); + generateGramAnswer(); + } + + /** + * We have fake predictors C1, C2, C3, C4, C5, C6, C7, C8, C9, C10, C11 and constraints of + * a). 0.4*C1 + 0.2*C2+10 <= 0, b) C1-0.3*C3 == 0, c). 10*C3-4*C4 -100 <= 0, d) 11*C4 - 2.4*C5 == 0, + * e) -1.5*C6-2.8*C7 <= 0; f) -9.3*C10+1.3*C11+0.4 == 0. + * + * The coefficient list is C1, C2, C3, C4, C5, C6, C7, C8, C9, C10, C11 with C1 at index 0, C2 at index 1 and ... + */ + public void generateConstraints() { + _beta = IntStream.range(0, _coeffNames1.size()).mapToDouble(x->_obj.nextGaussian()).toArray(); + _coeffsDG = new ArrayList<>(Arrays.asList("C1", "C2", "C3", "C4", "C5", "C6", "C7", "C8", "C9", "C10", "C11", "C12")); + _equalityConstr = new ConstrainedGLMUtils.LinearConstraints[3]; + _lessThanConstr = new ConstrainedGLMUtils.LinearConstraints[3]; + _equalityConstr[0] = makeOneConstraint(new String[]{"C1", "C3"}, new double[]{1, -0.3}, true); + _equalityConstr[1] = makeOneConstraint(new String[]{"C4", "C5"}, new double[]{11, -2.4}, true); + _equalityConstr[2] = makeOneConstraint(new String[]{"C10", "C11", "Constant"}, new double[]{-9.3, 1.3, 0.4}, true); + + _lessThanConstr[0] = makeOneConstraint(new String[]{"C1", "C2", "Constant"}, new double[]{0.4, 0.2, 10}, false); + _lessThanConstr[1] = makeOneConstraint(new String[]{"C3", "C4", "Constant"}, new double[]{10, -4, -100}, false); + _lessThanConstr[2] = makeOneConstraint(new String[]{"C6", "C7"}, new double[]{-1.5, -2.8}, false); + } + + public ConstrainedGLMUtils.LinearConstraints makeOneConstraint(String[] coeffNames, double[] constrVal, boolean equalConstraints) { + ConstrainedGLMUtils.LinearConstraints constraint = new ConstrainedGLMUtils.LinearConstraints(); + int val = 0; + int numCoeffs = coeffNames.length; + for (int index=0; index= 0) + val += _beta[coefInd]*constrVal[index]; + } + } + constraint._constraintsVal = val; + if (equalConstraints) + constraint._active = constraint._constraintsVal != 0; + else + constraint._active = val > 0; + return constraint; + } + + /*** + * We are testing the correct implementation of derivatives generation from constrainted GLM. We have fake + * predictors C1, C2, C3, C4, C5, C6, C7, C8, C9, C10, C11 and constraints of + * a). 0.4*C1 + 0.2*C2+10 <= 0, + * b) C1-0.3*C3 == 0, + * c). 10*C3-4*C4 -100 <= 0, + * d) 11*C4 - 2.4*C5 == 0, + * e) -1.5*C6-2.8*C7 <= 0; + * f) -9.3*C10+1.3*C11 == 0. + * + * The coefficient list is C1, C2, C3, C4, C5, C6, C7, C8, C9, C10, C11 with C1 at index 0, C2 at index 1 and ... + * + * In this case, the correct answer should be: + * Equality constraints derivatives: + * (0, 1), (2, -0.3), + * (3, 11), (4, -2.4), + * (9, -9,3), (10, 1.3) + * + * less than equal to constraints derivatives: + * (0, 0.4), (1, 0.2) + * (2, 10), (3, -4) + * (5, -1.5), (6, -2.8) + */ + public void generateDerivativeAnswer() { + int coefSize = _coeffNames1.size()+1; // +1 for intercept + _equalGradContr = new double[coefSize]; + _lessGradContr = new double[coefSize]; + // generate gradient contributions from tranpose(lambda)*h(beta) + _lambdaE = IntStream.range(0,3).mapToDouble(x-> _obj.nextGaussian()).toArray(); + _cdEqual = new ConstrainedGLMUtils.ConstraintsDerivatives[3]; + _cdEqual[0] = genOneCDerivative(new int[]{0, 2}, new double[]{1, -0.3}, true); + _equalGradContr[0] += 1*_lambdaE[0]; + _equalGradContr[2] += -0.3*_lambdaE[0]; + _cdEqual[1] = genOneCDerivative(new int[]{3, 4}, new double[]{11, -2.4}, true); + _equalGradContr[3] += 11*_lambdaE[1]; + _equalGradContr[4] += -2.4*_lambdaE[1]; + _cdEqual[2] = genOneCDerivative(new int[]{9,10}, new double[]{-9.3, 1.3}, true); + _equalGradContr[9] += -9.3*_lambdaE[2]; + _equalGradContr[10] += 1.3*_lambdaE[2]; + + _lambdaL = IntStream.range(0,3).mapToDouble(x->_obj.nextGaussian()).toArray(); + _cdLess = new ConstrainedGLMUtils.ConstraintsDerivatives[3]; + _cdLess[0] = genOneCDerivative(new int[]{0,1}, new double[]{0.4, 0.2}, _lessThanConstr[0]._active); + if (_lessThanConstr[0]._active) { + _lessGradContr[0] += 0.4 * _lambdaL[0]; + _lessGradContr[1] += 0.2 * _lambdaL[0]; + } + _cdLess[1] = genOneCDerivative(new int[]{2,3}, new double[]{10, -4}, _lessThanConstr[1]._active); + if (_lessThanConstr[1]._active) { + _lessGradContr[2] += 10 * _lambdaL[1]; + _lessGradContr[3] += -4 * _lambdaL[1]; + } + _cdLess[2] = genOneCDerivative(new int[]{5,6}, new double[]{-1.5, -2.8}, _lessThanConstr[2]._active); + if (_lessThanConstr[2]._active) { + _lessGradContr[5] += -1.5 * _lambdaL[2]; + _lessGradContr[6] += -2.8 * _lambdaL[2]; + } + + // generate gradient contributions from penalty: Ck/2*transpose(h(beta))*h(beta) + _equalGradContr[0] += _ck*_cdEqual[0]._constraintsDerivative.get(0)*_equalityConstr[0]._constraintsVal; + _equalGradContr[2] += _ck*_cdEqual[0]._constraintsDerivative.get(2)*_equalityConstr[0]._constraintsVal; + _equalGradContr[3] += _ck*_cdEqual[1]._constraintsDerivative.get(3)*_equalityConstr[1]._constraintsVal; + _equalGradContr[4] += _ck*_cdEqual[1]._constraintsDerivative.get(4)*_equalityConstr[1]._constraintsVal; + _equalGradContr[9] += _ck*_cdEqual[2]._constraintsDerivative.get(9)*_equalityConstr[2]._constraintsVal; + _equalGradContr[10] += _ck*_cdEqual[2]._constraintsDerivative.get(10)*_equalityConstr[2]._constraintsVal; + + if (_lessThanConstr[0]._active) { + _lessGradContr[0] += _ck*_cdLess[0]._constraintsDerivative.get(0)*_lessThanConstr[0]._constraintsVal; + _lessGradContr[1] += _ck*_cdLess[0]._constraintsDerivative.get(1)*_lessThanConstr[0]._constraintsVal; + } + if (_lessThanConstr[1]._active) { + _lessGradContr[2] += _ck*_cdLess[1]._constraintsDerivative.get(2)*_lessThanConstr[1]._constraintsVal; + _lessGradContr[3] += _ck*_cdLess[1]._constraintsDerivative.get(3)*_lessThanConstr[1]._constraintsVal; + } + if (_lessThanConstr[2]._active) { + _lessGradContr[5] += _ck*_cdLess[2]._constraintsDerivative.get(5)*_lessThanConstr[2]._constraintsVal; + _lessGradContr[6] += _ck*_cdLess[2]._constraintsDerivative.get(6)*_lessThanConstr[2]._constraintsVal; + } + } + + + /*** + * This generates the contribution to the gram from Linear constraints. + * Equality constraints Gram contribution: + * ((0,0), 1), ((0,2), -0.3), ((2,2), 0.09) + * ((3,3), 121), ((3,4), -26.4), ((4,4), 5.76) + * ((9,9), 86.49), ((9,10), 12.09), ((10,10), 1.69) + * + * Less than: + * ((0,0), 0.16), ((0,1), 0.08), ((1,1), 0.04) + * ((2,2),100), ((2,3), -40), ((3,3), 16) + * ((5,5), 2.25), ((5,6), 4.2), ((6,6), 7.84) + */ + public void generateGramAnswer() { + int coefSize = _coeffNames1.size()+1; + _equalGramContr = new double[coefSize][coefSize]; + _lessGramContr = new double[coefSize][coefSize]; + _cGEqual = new ConstrainedGLMUtils.ConstraintsGram[3]; + _cGEqual[0] = genOneGram(new ConstrainedGLMUtils.CoefIndices[]{new ConstrainedGLMUtils.CoefIndices(0,0), + new ConstrainedGLMUtils.CoefIndices(0,2), new ConstrainedGLMUtils.CoefIndices(2,2)}, + new double[]{1, -0.3, 0.09}, true); + _equalGramContr[0][0] += 1; + _equalGramContr[0][2] += -0.3; + _equalGramContr[2][0] += -0.3; + _equalGramContr[2][2] += 0.09; + _cGEqual[1] = genOneGram(new ConstrainedGLMUtils.CoefIndices[]{new ConstrainedGLMUtils.CoefIndices(3,3), + new ConstrainedGLMUtils.CoefIndices(3,4), new ConstrainedGLMUtils.CoefIndices(4,4)}, + new double[]{121, -26.4, 5.76}, true); + _equalGramContr[3][3] += 121; + _equalGramContr[3][4] += -26.4; + _equalGramContr[4][3] += -26.4; + _equalGramContr[4][4] += 5.76; + _cGEqual[2] = genOneGram(new ConstrainedGLMUtils.CoefIndices[]{new ConstrainedGLMUtils.CoefIndices(9,9), + new ConstrainedGLMUtils.CoefIndices(9,10), new ConstrainedGLMUtils.CoefIndices(10,10)}, + new double[]{86.49, -12.09, 1.69}, true); + _equalGramContr[9][9] += 86.49; + _equalGramContr[9][10] += -12.09; + _equalGramContr[10][9] += -12.09; + _equalGramContr[10][10] += 1.69; + _cGLess = new ConstrainedGLMUtils.ConstraintsGram[3]; + _cGLess[0] = genOneGram(new ConstrainedGLMUtils.CoefIndices[]{new ConstrainedGLMUtils.CoefIndices(0,0), + new ConstrainedGLMUtils.CoefIndices(0,1), new ConstrainedGLMUtils.CoefIndices(1,1)}, + new double[]{0.16,0.08,0.04}, _lessThanConstr[0]._active); + if (_lessThanConstr[0]._active) { + _lessGramContr[0][0] += 0.16; + _lessGramContr[0][1] += 0.08; + _lessGramContr[1][0] += 0.08; + _lessGramContr[1][1] += 0.04; + } + _cGLess[1] = genOneGram(new ConstrainedGLMUtils.CoefIndices[]{new ConstrainedGLMUtils.CoefIndices(2,2), + new ConstrainedGLMUtils.CoefIndices(2,3), new ConstrainedGLMUtils.CoefIndices(3,3)}, + new double[]{100,-40,16}, _lessThanConstr[1]._active); + if (_lessThanConstr[1]._active) { + _lessGramContr[2][2] += 100; + _lessGramContr[2][3] += -40; + _lessGramContr[3][2] += -40; + _lessGramContr[3][3] += 16; + } + _cGLess[2] = genOneGram(new ConstrainedGLMUtils.CoefIndices[]{new ConstrainedGLMUtils.CoefIndices(5,5), + new ConstrainedGLMUtils.CoefIndices(5,6), new ConstrainedGLMUtils.CoefIndices(6,6)}, + new double[]{2.25,4.2,7.84}, _lessThanConstr[2]._active); + if (_lessThanConstr[2]._active) { + _lessGramContr[5][5] += 2.25; + _lessGramContr[5][6] += 4.2; + _lessGramContr[6][5] += 4.2; + _lessGramContr[6][6] += 7.84; + } + } + + public ConstrainedGLMUtils.ConstraintsGram genOneGram(ConstrainedGLMUtils.CoefIndices[] coefIndices, double[] vals, boolean active) { + ConstrainedGLMUtils.ConstraintsGram oneG = new ConstrainedGLMUtils.ConstraintsGram(); + int numV = vals.length; + for (int index=0; index transformFrameCreateCoefNames(Frame train) { + for (int colInd : _catCol) + train.replace((colInd), train.vec(colInd).toCategoricalVec()).remove(); + DKV.put(train); + GLMModel.GLMParameters params = new GLMModel.GLMParameters(gaussian); + params._standardize=true; + params._response_column = "C21"; + params._max_iterations = 0; + params._lambda = new double[]{0.0}; + params._train = train._key; + GLMModel glm = new GLM(params).trainModel().get(); + Scope.track_generic(glm); + return Stream.of(glm._output._coefficient_names).collect(Collectors.toList()); + } + + @After + public void teardown() { + Scope.exit(); + } + + + + public void assertCorrectDerivatives(ConstrainedGLMUtils.ConstraintsDerivatives[] actual, + ConstrainedGLMUtils.ConstraintsDerivatives[] expected) { + int numV = actual.length; + assertTrue("expected array length: " + expected.length + " and actual array length: " + actual.length + + ". They are different", actual.length == numV); + for (int index=0; index Math.abs(e.getValue()-cd2._constraintsDerivative.get(e.getKey()))<1e-6)); + } + } + + // test and make sure the contributions from constraints to the gradient calculations are correct. This one does not + // take into account the contribution from the penalty terms. + @Test + public void testConstraintsDerivativesSum() { + GLM.GLMGradientInfo ginfo = new GLM.GLMGradientInfo(0,0, new double[_equalGradContr.length]); + addConstraintGradient(_lambdaE, _cdEqual, ginfo); + addPenaltyGradient(_cdEqual, _equalityConstr, ginfo, _ck); + checkArrays(_equalGradContr, ginfo._gradient, 1e-6); + ginfo = new GLM.GLMGradientInfo(0,0, new double[_lessGradContr.length]); + addConstraintGradient(_lambdaL, _cdLess, ginfo); + addPenaltyGradient(_cdLess, _lessThanConstr, ginfo, _ck); + checkArrays(_lessGradContr, ginfo._gradient, 1e-6); + } + + // This test will form a double[][] matrix which should capture the contributions from all the constraints. + @Test + public void testGramConstraintsSum() { + double[][] equalGramC = sumGramConstribution(_cGEqual, _coeffNames1.size()+1); + double[][] lessGramC = sumGramConstribution(_cGLess, _coeffNames1.size()+1); + checkDoubleArrays(equalGramC, _equalGramContr, 1e-6); + checkDoubleArrays(lessGramC, _lessGramContr, 1e-6); + } + + // make sure constraints (the penalty part) contributions to the Gram matrix is calculated correctly. The final + // result should be an array of ConstraintsGram. + @Test + public void testConstraintsGram() { + ConstrainedGLMUtils.ConstraintsGram[] gramEqual = calGram(_cdEqual); + ConstrainedGLMUtils.ConstraintsGram[] gramLess = calGram( _cdLess); + assertCorrectGrams(gramEqual, _cGEqual); + assertCorrectGrams(gramLess, _cGLess); + } + + public void assertCorrectGrams(ConstrainedGLMUtils.ConstraintsGram[] actual, ConstrainedGLMUtils.ConstraintsGram[] expected) { + int numV = actual.length; + assertTrue("expected array length: " + expected.length + " and actual array length: " + actual.length + + ". They are different", actual.length == numV); + for (int index=0; index actual, + IcedHashMap expected) { + List actualKey = actual.keySet().stream().collect(Collectors.toList()); + List expectedKey = expected.keySet().stream().collect(Collectors.toList()); + int numK = actualKey.size(); + assertTrue(numK == expectedKey.size()); + + for (CoefIndices oneKey : actualKey) { + int index = expectedKey.indexOf(oneKey); + assertTrue(index >= 0); + double expectedValue = expected.get(expectedKey.get(index)); + assertTrue("Expected value: " + actual.get(oneKey) + ", Actual value: " + expectedValue, + Math.abs(actual.get(oneKey) - expectedValue) < 1e-6); + } + } + + // beta and linear constraints conflict and we should catch it + @Test + public void testConflictConstraints() { + Scope.enter(); + try { + // beta constraints: beta0 >= 2, beta1 >= 2 + Frame betaConstraint = + new TestFrameBuilder() + .withColNames("names", "lower_bounds", "upper_bounds") + .withVecTypes(T_STR, T_NUM, T_NUM) + .withDataForCol(0, new String[] {_coeffNames1.get(30), _coeffNames1.get(31)}) + .withDataForCol(1, new double [] {2, 2}) + .withDataForCol(2, new double[] {Double.POSITIVE_INFINITY, Double.POSITIVE_INFINITY}).build(); + Scope.track(betaConstraint); + + // linear constraint: beta0 + beta1 <= 2, contradicts with beta0 >= 2 and beta1 >= 2 + Frame linearConstraint = new TestFrameBuilder() + .withColNames("names", "values", "types", "constraint_numbers") + .withVecTypes(T_STR, T_NUM, T_STR, T_NUM) + .withDataForCol(0, new String[] {_coeffNames1.get(30), _coeffNames1.get(31), "constant"}) + .withDataForCol(1, new double [] {1,1,-2}) + .withDataForCol(2, new String[] {"lessthanequal", "lessthanequal", "lessthanequal"}) + .withDataForCol(3, new int[]{0,0,0}).build(); + Scope.track(linearConstraint); + + Frame train = parseAndTrackTestFile("smalldata/glm_test/gaussian_20cols_10000Rows.csv"); + transformFrameCreateCoefNames(train); + GLMModel.GLMParameters params = new GLMModel.GLMParameters(gaussian); + params._standardize = false; + params._response_column = "C21"; + params._solver = IRLSM; + params._train = train._key; + params._beta_constraints = betaConstraint._key; + params._max_iterations = 1; + params._expose_constraints = true; + params._linear_constraints = linearConstraint._key; + params._lambda = new double[]{0}; + GLMModel glm2 = new GLM(params).trainModel().get(); + Scope.track_generic(glm2); + assertTrue("Should have thrown an error due to duplicated constraints.", 1==2); + } catch(IllegalArgumentException ex) { + assert ex.getMessage().contains("redundant and possibly conflicting linear constraints") : "Wrong error message. Error should be about" + + " redundant linear constraints"; + } finally { + Scope.exit(); + } + } + + // linear constraints with two duplicated constraints + @Test + public void testDuplicateLinearConstraints() { + Scope.enter(); + try { + Frame train = parseAndTrackTestFile("smalldata/glm_test/gaussian_20cols_10000Rows.csv"); + transformFrameCreateCoefNames(train); + GLMModel.GLMParameters params = new GLMModel.GLMParameters(gaussian); + params._standardize = false; + params._response_column = "C21"; + params._solver = IRLSM; + params._train = train._key; + params._max_iterations = 1; + params._expose_constraints = true; + params._linear_constraints = _linearConstraint3._key; + params._lambda = new double[]{0}; + GLMModel glm2 = new GLM(params).trainModel().get(); + Scope.track_generic(glm2); + assert 1==2 : "Should have thrown an error due to duplicated constraints."; + } catch(IllegalArgumentException ex) { + assert ex.getMessage().contains("redundant and possibly conflicting linear constraints") : "Wrong error message. Error should be about" + + " redundant linear constraints"; + } finally { + Scope.exit(); + } + } + + // make sure duplicated bete or linear constraints specified are caught. + @Test + public void testDuplicateBetaLinearConstraints() { + Scope.enter(); + try { + Frame train = parseAndTrackTestFile("smalldata/glm_test/gaussian_20cols_10000Rows.csv"); + transformFrameCreateCoefNames(train); + GLMModel.GLMParameters params = new GLMModel.GLMParameters(gaussian); + params._standardize = true; + params._response_column = "C21"; + params._solver = IRLSM; + params._train = train._key; + params._max_iterations = 1; + params._expose_constraints = true; + params._linear_constraints = _linearConstraint4._key; + params._beta_constraints = _betaConstraint2._key; + params._lambda = new double[]{0}; + GLMModel glm2 = new GLM(params).trainModel().get(); + Scope.track_generic(glm2); + assert 1==2 : "Should have thrown an error due to duplicated constraints."; + } catch(IllegalArgumentException ex) { + assert ex.getMessage().contains("redundant and possibly conflicting linear constraints") : "Wrong error message. Error should be about" + + " redundant linear constraints"; + } finally { + Scope.exit(); + } + } + + + // make sure correct constraint matrix is formed after extracting constraints from linear constraints + @Test + public void testLinearConstraintMatrix() { + Scope.enter(); + try { + Frame train = parseAndTrackTestFile("smalldata/glm_test/gaussian_20cols_10000Rows.csv"); + transformFrameCreateCoefNames(train); + GLMModel.GLMParameters params = new GLMModel.GLMParameters(gaussian); + params._standardize = false; + params._response_column = "C21"; + params._solver = IRLSM; + params._train = train._key; + params._max_iterations = 1; + params._expose_constraints = true; + params._linear_constraints = _linearConstraint2._key; + params._lambda = new double[]{0}; + GLMModel glm2 = new GLM(params).trainModel().get(); + Scope.track_generic(glm2); + // check that constraint matrix is extracted correctly + List constraintNames = Arrays.stream(glm2._output._constraintCoefficientNames).collect(Collectors.toList()); + double[][] initConstraintMatrix = glm2._output._initConstraintMatrix; + // check rows from beta constraints + int rowIndex = 0; + int[] compare = new int[]{1, 1, 1, 1}; + assertCorrectConstraintMatrix(constraintNames, initConstraintMatrix, _equalityNames2, _equalityValues2, + rowIndex, compare); + // check row from linear constraints with lessThanEqualTo + rowIndex += ArrayUtils.sum(compare); + compare = new int[]{1, 1, 1}; + assertCorrectConstraintMatrix(constraintNames, initConstraintMatrix, _lessThanNames2, _lessThanValues2, + rowIndex, compare); + } finally { + Scope.exit(); + } + } + + // make sure correct constraint matrix is formed after extracting constraints from beta constraints and linear + // constraints + @Test + public void testBetaLinearConstraintMatrix() { + Scope.enter(); + try { + Frame train = parseAndTrackTestFile("smalldata/glm_test/gaussian_20cols_10000Rows.csv"); + transformFrameCreateCoefNames(train); + GLMModel.GLMParameters params = new GLMModel.GLMParameters(gaussian); + params._standardize = true; + params._response_column = "C21"; + params._solver = IRLSM; + params._train = train._key; + // build the beta_constraints + Frame beta_constraints = _betaConstraint1; + Frame linear_constraints = _linearConstraint1; + params._max_iterations = 1; + params._expose_constraints = true; + params._beta_constraints = beta_constraints._key; + params._linear_constraints = linear_constraints._key; + params._lambda = new double[]{0}; + GLMModel glm2 = new GLM(params).trainModel().get(); + Scope.track_generic(glm2); + // check that constraint matrix is extracted correctly + List constraintNames = Arrays.stream(glm2._output._constraintCoefficientNames).collect(Collectors.toList()); + double[][] initConstraintMatrix = glm2._output._initConstraintMatrix; + // check rows from beta constraints + int rowIndex = 0; + int[] compare = new int[]{1}; + assertCorrectConstraintMatrix(constraintNames, initConstraintMatrix, _betaConstraintNames1Equal, + _betaConstraintValStandard1Equal, rowIndex, compare); + // check rows from linear constraints with equality + rowIndex += ArrayUtils.sum(compare); + compare = new int[]{1, 0, 1, 1}; + assertCorrectConstraintMatrix(constraintNames, initConstraintMatrix, _betaConstraintNames1Less, + _betaConstraintValStandard1Less, rowIndex, compare); + // check rows from linear constraints with equality + rowIndex += ArrayUtils.sum(compare); + compare = new int[]{1}; + assertCorrectConstraintMatrix(constraintNames, initConstraintMatrix, _equalityNames1, _equalityValuesStandard1, + rowIndex, compare); + // check row from linear contraints with lessThanEqualTo + rowIndex += ArrayUtils.sum(compare); + compare = new int[]{1}; + assertCorrectConstraintMatrix(constraintNames, initConstraintMatrix, _lessThanNames1, _lessThanValuesStandard1, + rowIndex, compare); + } finally { + Scope.exit(); + } + } + + public void assertCorrectConstraintMatrix(List constraintNames, double[][] constraintMatrix, + String[][] origNames, double[][] originalValues, int rowStart, + int[] compare) { + int numConstraints = origNames.length; + int count = 0; + for (int index=0; index 0) { + int rowIndex = count + rowStart; + String[] constNames = origNames[index]; + double[] constValues = originalValues[index]; + int numNames = constNames.length; + for (int index2 = 0; index2 < numNames; index2++) { + int cNamesIndex = constraintNames.indexOf(constNames[index2]); + assertTrue("Expected value: " + constValues[index2] + " for constraint " + constNames[index2] + " but actual: " + + constraintMatrix[rowIndex][cNamesIndex], Math.abs(constraintMatrix[rowIndex][cNamesIndex] - constValues[index2]) < EPS); + } + count++; + } + } + } + + // make sure we can get coefficient names without building a GLM model. We compare the coefficient names + // obtained without building a model and with building a model. They should be the same. + @Test + public void testCoefficientNames() { + Scope.enter(); + try { + Frame train = parseAndTrackTestFile("smalldata/glm_test/gaussian_20cols_10000Rows.csv"); + transformFrameCreateCoefNames(train); + GLMModel.GLMParameters params = new GLMModel.GLMParameters(gaussian); + params._standardize=true; + params._response_column = "C21"; + params._max_iterations = 0; + params._train = train._key; + params._lambda = new double[]{0}; + GLMModel glm = new GLM(params).trainModel().get(); + Scope.track_generic(glm); + List coefNames = Stream.of(glm._output._coefficient_names).collect(Collectors.toList()); ; + + params._max_iterations = 1; + GLMModel glm2 = new GLM(params).trainModel().get(); + Scope.track_generic(glm2); + String[] coeffNames2 = glm2.coefficients().keySet().toArray(new String[0]); + + for (String oneName : coeffNames2) + assertTrue(coefNames.contains(oneName)); + } finally { + Scope.exit(); + } + } + + // test constraints specified in beta_constraint and linear constraints and extracted correctly with + // standardization. + @Test + public void testConstraintsInBetaLinearStandard() { + Scope.enter(); + try { + Frame train = parseAndTrackTestFile("smalldata/glm_test/gaussian_20cols_10000Rows.csv"); + transformFrameCreateCoefNames(train); + GLMModel.GLMParameters params = new GLMModel.GLMParameters(gaussian); + params._standardize = true; + params._response_column = "C21"; + params._solver = IRLSM; + params._train = train._key; + // build the beta_constraints + Frame beta_constraints = _betaConstraint1; + Frame linear_constraints = _linearConstraint1; + params._max_iterations = 1; + params._expose_constraints = true; + params._beta_constraints = beta_constraints._key; + params._linear_constraints = linear_constraints._key; + params._lambda = new double[]{0}; + GLMModel glm2 = new GLM(params).trainModel().get(); + Scope.track_generic(glm2); + // check constraints from betaConstraints are extracted properly + ConstrainedGLMUtils.LinearConstraints[] equalConstraintstBeta = glm2._output._equalityConstraintsBeta; + ConstrainedGLMUtils.LinearConstraints[] lessThanEqualToConstraintstBeta = glm2._output._lessThanEqualToConstraintsBeta; + + assertTrue("Expected constraint length: "+ _betaConstraintValStandard1Equal.length+" but" + + " actual is "+equalConstraintstBeta.length,_betaConstraintNames1Equal.length == equalConstraintstBeta.length); + assertTrue("Expected constraint length: "+ _betaConstraintNames1Less.length+" but" + + " actual is "+lessThanEqualToConstraintstBeta.length,_betaConstraintNames1Less.length == lessThanEqualToConstraintstBeta.length); + assertCorrectConstraintContent(_betaConstraintNames1Equal, _betaConstraintValStandard1Equal, equalConstraintstBeta); + assertCorrectConstraintContent(_betaConstraintNames1Less, _betaConstraintValStandard1Less, lessThanEqualToConstraintstBeta); + + // check constraints from linear constraints are extracted properly + // check equality constraint + assertCorrectConstraintContent(_equalityNames1, _equalityValuesStandard1, glm2._output._equalityConstraintsLinear); + // check lessThanEqual to constraint + assertCorrectConstraintContent(_lessThanNames1, _lessThanValuesStandard1, glm2._output._lessThanEqualToConstraintsLinear); + } finally { + Scope.exit(); + } + } + + // test constraints specified in beta_constraint and linear constraints and extracted correctly without + // standardization. + @Test + public void testConstraintsInBetaLinear() { + Scope.enter(); + try { + Frame train = parseAndTrackTestFile("smalldata/glm_test/gaussian_20cols_10000Rows.csv"); + transformFrameCreateCoefNames(train); + List coeffNames = _coeffNames1; + // build the beta_constraints + int coefLen = coeffNames.size()-1; + Frame beta_constraints = _betaConstraint1; + // there are two constraints in the linear_constraints, the first one is 2*beta_0+0.5*beta_5 -1<= 0, the second + // one is 0.5*beta_36-1.5*beta_38 == 0 + Frame linear_constraints = _linearConstraint1; + GLMModel.GLMParameters params = new GLMModel.GLMParameters(gaussian); + params._standardize=false; + params._response_column = "C21"; + params._solver = IRLSM; + params._train = train._key; + params._max_iterations = 1; + params._expose_constraints = true; + params._beta_constraints = beta_constraints._key; + params._linear_constraints = linear_constraints._key; + params._lambda = new double[]{0}; + GLMModel glm2 = new GLM(params).trainModel().get(); + Scope.track_generic(glm2); + ConstrainedGLMUtils.LinearConstraints[] fromBetaConstraintE = glm2._output._equalityConstraintsBeta; + assertTrue("Expected constraint length: "+ + _betaConstraintNames1Equal.length+" but actual is "+fromBetaConstraintE.length, + _betaConstraintNames1Equal.length == fromBetaConstraintE.length); + assertCorrectConstraintContent(_betaConstraintNames1Equal, _betaConstraintVal1Equal, fromBetaConstraintE); + + ConstrainedGLMUtils.LinearConstraints[] fromBetaConstraintL = glm2._output._lessThanEqualToConstraintsBeta; + assertTrue("Expected constraint length: "+ _betaConstraintNames1Less.length + " but actual is " + + fromBetaConstraintL.length, _betaConstraintNames1Less.length == fromBetaConstraintL.length); + assertCorrectConstraintContent(_betaConstraintNames1Less, _betaConstraintVal1Less, fromBetaConstraintL); + + // check constraints from linear constraints are extracted properly + // check equality constraint + assertCorrectConstraintContent(_equalityNames1, _equalityValues1, glm2._output._equalityConstraintsLinear); + // check lessThanEqual to constraint + assertCorrectConstraintContent(_lessThanNames1, _lessThanValues1, glm2._output._lessThanEqualToConstraintsLinear); + } finally { + Scope.exit(); + } + } + + + // test constraints specified only in linear_constraint and extracted correctly without standardization + @Test + public void testConstraintsInLinear() { + Scope.enter(); + try { + Frame train = parseAndTrackTestFile("smalldata/glm_test/gaussian_20cols_10000Rows.csv"); + transformFrameCreateCoefNames(train); + Frame linear_constraints = _linearConstraint2; + + GLMModel.GLMParameters params = new GLMModel.GLMParameters(gaussian); + params._standardize=false; + params._response_column = "C21"; + params._solver = IRLSM; + params._train = train._key; + params._max_iterations = 1; + params._expose_constraints = true; + params._linear_constraints = linear_constraints._key; + params._lambda = new double[]{0}; + GLMModel glm2 = new GLM(params).trainModel().get(); + Scope.track_generic(glm2); + // check constraints from linear constraints are extracted properly + // check equality constraint + assertCorrectConstraintContent(_equalityNames2, _equalityValues2, glm2._output._equalityConstraintsLinear); + // check lessThanEqual to constraint + assertCorrectConstraintContent(_lessThanNames2, _lessThanValues2, glm2._output._lessThanEqualToConstraintsLinear); + } finally { + Scope.exit(); + } + } + + // this test will make sure that the find zero columns function is working + @Test + public void testFindDropZeroColumns() { + Matrix initMat = Matrix.random(11, 11); + double[][] doubleValsOrig = (initMat.plus(initMat.transpose())).getArray(); + double[][] doubleVals = new double[doubleValsOrig.length][doubleValsOrig.length]; + GamUtils.copy2DArray(doubleValsOrig, doubleVals); + // no zero columns + int[] numZeroCol = findZeroCols(doubleVals); + assertTrue("number of zero columns is zero in this case but is not.", numZeroCol.length==0); + // introduce one zero column + testDropCols(doubleValsOrig, doubleVals, new int[]{8}, 8); + // introduce two zero columns + testDropCols(doubleValsOrig, doubleVals, new int[]{4, 8}, 4); + // introduce three zero columns + testDropCols(doubleValsOrig, doubleVals, new int[]{3, 4, 8}, 3); + // introduce four zero columns + testDropCols(doubleValsOrig, doubleVals, new int[]{3, 4, 6, 8}, 6); + // introduce five zero columns + testDropCols(doubleValsOrig, doubleVals, new int[]{0, 3, 4, 6, 8}, 0); + // introduce six zero columns + testDropCols(doubleValsOrig, doubleVals, new int[]{0, 3, 4, 6, 8, 9}, 9); + // introduce seven zero columns + testDropCols(doubleValsOrig, doubleVals, new int[]{0, 1, 3, 4, 6, 8, 9}, 1); + // introduce eight zero columns + testDropCols(doubleValsOrig, doubleVals, new int[]{0, 1, 3, 4, 6, 7, 8, 9}, 7); + // introduce nine zero columns + testDropCols(doubleValsOrig, doubleVals, new int[]{0, 1, 3, 4, 5, 6, 7, 8, 9}, 5); + // introduce 10 zero columns + testDropCols(doubleValsOrig, doubleVals, new int[]{0, 1, 2, 3, 4, 5, 6, 7, 8, 9}, 2); + } + + public void testDropCols(double[][] valsOrig, double[][] doubleVals, int[] zeroIndices, int newZeroIndex) { + addOneZeroCol(doubleVals, newZeroIndex); + int[] numZeroCol = findZeroCols(doubleVals); + assertArrayEquals(numZeroCol, zeroIndices); + // drop zero column + double[][] noZeroCols = dropCols(zeroIndices, valsOrig); + assertCorrectColDrops(noZeroCols, doubleVals, zeroIndices); + } + + public void assertCorrectColDrops(double[][] actual, double[][] arrayWithZeros, int[] zeroIndices) { + assertTrue("Incorrect dropped column matrix size", + (actual.length+zeroIndices.length)==arrayWithZeros.length); + Arrays.sort(zeroIndices); + int matLen = arrayWithZeros.length; + List indiceList = IntStream.range(0, actual.length).boxed().collect(Collectors.toList()); + for (int val:zeroIndices) + indiceList.add(val, -1); + + for (int rIndex=0; rIndex= 0 && actCInd >= 0) { // rows/cols not involve in dropped cols/rows + assertTrue("Non-zero elements differ.", + arrayWithZeros[rIndex][cIndex] == actual[actRInd][actCInd]); + assertTrue("Non-zero elements differ.", + arrayWithZeros[cIndex][rIndex] == actual[actCInd][actRInd]); + } else { // we are at rows/cols that are zero and should be dropped + assertTrue("Non-zero elements differ.", + arrayWithZeros[rIndex][cIndex] == 0); + assertTrue("Non-zero elements differ.", + arrayWithZeros[cIndex][rIndex] == 0); + } + } + } + } + + public void addOneZeroCol(double[][] vals, int zeroIndex) { + int len = vals.length; + for (int index = 0; index < len; index++) { + vals[zeroIndex][index] = 0; + vals[index][zeroIndex] = 0; + } + } + + // test constraints specified only in linear_constraint and extracted correctly with standardization + @Test + public void testConstraintsInLinearStandard() { + Scope.enter(); + try { + Frame train = parseAndTrackTestFile("smalldata/glm_test/gaussian_20cols_10000Rows.csv"); + transformFrameCreateCoefNames(train); + GLMModel.GLMParameters params = new GLMModel.GLMParameters(gaussian); + params._standardize = true; + params._response_column = "C21"; + params._max_iterations = 0; + params._solver = IRLSM; + params._train = train._key; + params._lambda = new double[]{0}; + GLMModel glm = new GLM(params).trainModel().get(); + Scope.track_generic(glm); + // build the beta_constraints + Frame linear_constraints = _linearConstraint2; + params._max_iterations = 1; + params._expose_constraints = true; + params._linear_constraints = linear_constraints._key; + GLMModel glm2 = new GLM(params).trainModel().get(); + Scope.track_generic(glm2); + // check constraints from linear constraints are extracted properly + // check equality constraint + assertCorrectConstraintContent(_equalityNames2, _equalityValuesStandard2, glm2._output._equalityConstraintsLinear); + // check lessThanEqual to constraint + assertCorrectConstraintContent(_lessThanNames2, _lessThanValuesStandard2, glm2._output._lessThanEqualToConstraintsLinear); + } finally { + Scope.exit(); + } + } + + public void assertCorrectConstraintContent(String[][] coefNames, double[][] value, + ConstrainedGLMUtils.LinearConstraints[] consts) { + assertTrue("array length does not match", coefNames.length == consts.length); + int constLen = consts.length; + for (int index=0; index coefKeys = oneConstraint._constraints.keySet(); + String[] coefName = coefNames[index]; + int entryLen = coefName.length; + for (int ind = 0; ind < entryLen; ind++) { + assertTrue(coefKeys.contains(coefName[ind])); + assertTrue("Expected: "+value[index][ind]+ + ". Actual: "+oneConstraint._constraints.get(coefName[ind])+" for coefficient: "+coefName[ind]+".", + Math.abs(value[index][ind] - oneConstraint._constraints.get(coefName[ind])) < EPS); + } + } + } +} diff --git a/h2o-algos/src/test/java/hex/optimization/ExactLineSearchTest.java b/h2o-algos/src/test/java/hex/optimization/ExactLineSearchTest.java new file mode 100644 index 000000000000..efd809a9eeed --- /dev/null +++ b/h2o-algos/src/test/java/hex/optimization/ExactLineSearchTest.java @@ -0,0 +1,149 @@ +package hex.optimization; + +import hex.DataInfo; +import hex.glm.ComputationState; +import hex.glm.ConstrainedGLMUtils; +import hex.glm.GLM; +import hex.glm.GLMModel; +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import water.*; +import water.fvec.Frame; +import water.runner.CloudSize; +import water.runner.H2ORunner; +import water.util.ArrayUtils; + +import java.util.Arrays; +import java.util.Random; + +import static hex.glm.ConstrainedGLMUtils.calGradient; +import static hex.glm.GLMModel.GLMParameters.Family.gaussian; + +@RunWith(H2ORunner.class) +@CloudSize(1) +public class ExactLineSearchTest extends TestUtil { + DataInfo _dinfo; + ComputationState _state; + GLMModel.GLMParameters _params; + String _responseCol; + Frame _train; + GLM.BetaInfo _betaInfo; + GLM.GLMGradientSolver _ginfo; + Random _generator; + double[] _beta; + String[] _coefNames; + public Job _job; + + @Before + public void setup() { + Scope.enter(); + prepareTrainFrame("smalldata/glm_test/gaussian_20cols_10000Rows.csv"); + prepareParms(124); + } + + public void genRandomArrays(long seed) { + _generator = new Random(seed); + _beta = _generator.doubles(_coefNames.length+1, -1, 1).toArray(); + } + public void prepareParms(long seed) { + _params = new GLMModel.GLMParameters(gaussian); + _params._response_column = "C21"; + _params._standardize = true; + _params._train = _train._key; + _params._alpha = new double[]{0.0}; + _dinfo = new DataInfo(_train.clone(), null, 1, _params._use_all_factor_levels || _params._lambda_search, + _params._standardize ? DataInfo.TransformType.STANDARDIZE : DataInfo.TransformType.NONE, + DataInfo.TransformType.NONE, _params.missingValuesHandling()==GLMModel.GLMParameters.MissingValuesHandling.Skip, + _params.imputeMissing(), _params.makeImputer(), false, false, false, false, _params.interactionSpec()); + _coefNames = _dinfo.coefNames(); + genRandomArrays(seed); + _betaInfo = new GLM.BetaInfo(1, _dinfo.fullN() + 1); + Key jobKey = Key.make(); + _job = new Job<>(jobKey, _params.javaName(), _params.algoName()); + _state = new ComputationState(null, _params, _dinfo, null, _betaInfo, null, null); + _state._csGLMState = new ConstrainedGLMUtils.ConstraintGLMStates(_coefNames, null, _params); + _ginfo = new GLM.GLMGradientSolver(_job, _params, _dinfo, 0,null, _betaInfo); + GLM.GLMGradientInfo gradientInfo = calGradient(_beta, _state, _ginfo, null, null, + null, null); + _state.updateState(_beta, gradientInfo); + } + + public void prepareTrainFrame(String fileWPath) { + _train = parseAndTrackTestFile(fileWPath); + for (int colInd=0; colInd < 10; colInd++) + _train.replace((colInd), _train.vec(colInd).toCategoricalVec()).remove(); + _responseCol = "C21"; + DKV.put(_train); + } + + // in this test, call ExactLinesearch with inner product of direction and gradient to be +ve + @Test + public void testExactLineSearchBadDir() { + try { + double[] dir = _generator.doubles(_coefNames.length + 1, -1, 1).toArray(); + if (ArrayUtils.innerProduct(dir, _state.ginfo()._gradient) <= 0) + ArrayUtils.mult(dir, -1); // make sure inner product of direction and gradient > 0 + double[] betaCnd = _beta.clone(); + ArrayUtils.add(betaCnd, dir); + OptimizationUtils.ExactLineSearch ls = new OptimizationUtils.ExactLineSearch(betaCnd, _state, Arrays.asList(_coefNames)); + ls.findAlpha(null, null, _state, null, null, _ginfo); + } catch(AssertionError ex) { + Assert.assertTrue(ex.getMessage().contains("direction is not an descent direction!")); + } + } + + @After + public void teardown() { + Scope.exit(); + } + + @Test + public void testWolfeConditions() { + double[] dir = _generator.doubles(_coefNames.length + 1, -1, 1).toArray(); + if (ArrayUtils.innerProduct(dir, _state.ginfo()._gradient) > 0) + ArrayUtils.mult(dir, -1); // make sure inner product of direction and gradient <= 0 + double[] betaCnd = _beta.clone(); + ArrayUtils.add(betaCnd, dir); + OptimizationUtils.ExactLineSearch ls = new OptimizationUtils.ExactLineSearch(betaCnd, _state, Arrays.asList(_coefNames)); + GLM.GLMGradientInfo newGrad = calGradient(betaCnd, _state, _ginfo, null, null, null, null); + boolean firstWolfe = ls.evaluateFirstWolfe(newGrad); + Assert.assertTrue("First Wolfe should equal to objective condition but is not.", + firstWolfe==(newGrad._objVal <= _state.ginfo()._objVal)); + boolean secondWolfe = ls.evaluateSecondWolfe(newGrad); + Assert.assertFalse("Second Wolfe should equal condition on gradient info but is not!", + secondWolfe == (ArrayUtils.innerProduct(newGrad._gradient, ls._direction) < ls._currGradDirIP)); + } + + @Test + public void testFindAlphai() { + // direction is exactly the same as the gradient + assertAlphai(_state.ginfo()._gradient.clone(), 5000); + // random directions with different magnitude distributions + assertAlphai(_generator.doubles(_coefNames.length + 1, -1, 1).toArray(), 20); + assertAlphai(_generator.doubles(_coefNames.length + 1, -2, 2).toArray(), 200); + assertAlphai(_generator.doubles(_coefNames.length + 1, -5, 5).toArray(), 2000); + } + + public void assertAlphai(double[] dir, int maxiter) { + if (ArrayUtils.innerProduct(dir, _state.ginfo()._gradient) > 0) + ArrayUtils.mult(dir, -1); // make sure inner product of direction and gradient <= 0 + double[] betaCnd = _beta.clone(); + ArrayUtils.add(betaCnd, dir); + OptimizationUtils.ExactLineSearch ls = new OptimizationUtils.ExactLineSearch(betaCnd, _state, Arrays.asList(_coefNames)); + ls._maxIteration = maxiter; + boolean foundAlpha = ls.findAlpha(null, null, _state, null, + null, _ginfo); + // new ExactLineSearch with newly found beta. If foundAlpha = true, WolfeConditions on ls2 should be both true. + // If foundAlpha is false, then at least one of the WolfeConditions is false + if (foundAlpha) + betaCnd = ls._newBeta; + OptimizationUtils.ExactLineSearch ls2 = new OptimizationUtils.ExactLineSearch(betaCnd, _state, Arrays.asList(_coefNames)); + GLM.GLMGradientInfo newGrad = calGradient(betaCnd, _state, _ginfo, null, null, null, + null); + Assert.assertTrue("Wolfe conditions and foundAlphai disagree!", + foundAlpha == (ls2.evaluateFirstWolfe(newGrad) && ls2.evaluateSecondWolfe(newGrad))); + } +} diff --git a/h2o-bindings/bin/custom/python/gen_glm.py b/h2o-bindings/bin/custom/python/gen_glm.py index e99d89f6cf2c..a7e25f0c8108 100644 --- a/h2o-bindings/bin/custom/python/gen_glm.py +++ b/h2o-bindings/bin/custom/python/gen_glm.py @@ -217,6 +217,96 @@ def makeGLMModel(model, coefs, threshold=.5): m._resolve_model(model_json["model_id"]["name"], model_json) return m + @staticmethod + def getConstraintsInfo(model): + """ + + Given a constrained GLM model, the constraints descriptions, constraints values, constraints conditions and + whether the constraints are satisfied (true) or not (false) are returned. + + :param model: GLM model with linear and beta (if applicable) constraints + :return: H2OTwoDimTable containing the above constraints information. + + :example: + >>> train = h2o.import_file("https://s3.amazonaws.com/h2o-public-test-data/smalldata/glm_test/binomial_20_cols_10KRows.csv") + >>> response = "C21" + >>> predictors = list(range(0,20)) + >>> loose_init_const = [] # this constraint is satisfied by default coefficient initialization + >>> # add loose constraints + >>> name = "C19" + >>> values = 0.5 + >>> types = "LessThanEqual" + >>> contraint_numbers = 0 + >>> loose_init_const.append([name, values, types, contraint_numbers]) + >>> name = "C20" + >>> values = -0.8 + >>> types = "LessThanEqual" + >>> contraint_numbers = 0 + >>> loose_init_const.append([name, values, types, contraint_numbers]) + >>> name = "constant" + >>> values = -1000 + >>> types = "LessThanEqual" + >>> contraint_numbers = 0 + >>> loose_init_const.append([name, values, types, contraint_numbers]) + >>> linear_constraints2 = h2o.H2OFrame(loose_init_const) + >>> linear_constraints2.set_names(["names", "values", "types", "constraint_numbers"]) + >>> # GLM model with GLM coefficients with default initialization + >>> h2o_glm = H2OGeneralizedLinearEstimator(family="binomial", compute_p_values=True, remove_collinear_columns=True, + ... lambda_=0.0, solver="irlsm", linear_constraints=linear_constraints2, + ... init_optimal_glm = False, seed=12345) + >>> h2o_glm.train(x=predictors, y=response, training_frame=train) + >>> print(H2OGeneralizedLinearEstimator.getConstraintsInfo(h2o_glm)) + """ + if model.actual_params["linear_constraints"] is not None: + return model._model_json["output"]["linear_constraints_table"] + else: + raise H2OValueError("getConstraintsInfo can only be called when there are linear constraints.") + + @staticmethod + def allConstraintsPassed(model): + """ + + Given a constrainted GLM model, this will return true if all beta (if exists) and linear constraints are + satified. It will return false even if one constraint is not satisfied. To see which ones failed, use + getConstraintsInfo function. + + :param model: GLM model with linear and beta (if applicable) constraints + :return: boolean True or False + + :example: + >>> train = h2o.import_file("https://s3.amazonaws.com/h2o-public-test-data/smalldata/glm_test/binomial_20_cols_10KRows.csv") + >>> response = "C21" + >>> predictors = list(range(0,20)) + >>> loose_init_const = [] # this constraint is satisfied by default coefficient initialization + >>> # add loose constraints + >>> name = "C19" + >>> values = 0.5 + >>> types = "LessThanEqual" + >>> contraint_numbers = 0 + >>> loose_init_const.append([name, values, types, contraint_numbers]) + >>> name = "C20" + >>> values = -0.8 + >>> types = "LessThanEqual" + >>> contraint_numbers = 0 + >>> loose_init_const.append([name, values, types, contraint_numbers]) + >>> name = "constant" + >>> values = -1000 + >>> types = "LessThanEqual" + >>> contraint_numbers = 0 + >>> loose_init_const.append([name, values, types, contraint_numbers]) + >>> linear_constraints2 = h2o.H2OFrame(loose_init_const) + >>> linear_constraints2.set_names(["names", "values", "types", "constraint_numbers"]) + >>> # GLM model with GLM coefficients with default initialization + >>> h2o_glm = H2OGeneralizedLinearEstimator(family="binomial", compute_p_values=True, remove_collinear_columns=True, + ... lambda_=0.0, solver="irlsm", linear_constraints=linear_constraints2, + ... init_optimal_glm = False, seed=12345) + >>> h2o_glm.train(x=predictors, y=response, training_frame=train) + >>> print(H2OGeneralizedLinearEstimator.allConstraintsPassed(h2o_glm)) + """ + if model.actual_params["linear_constraints"] is not None: + return model._model_json["output"]["all_constraints_satisfied"] + else: + raise H2OValueError("allConstraintsPassed can only be called when there are linear constraints.") extensions = dict( __imports__="""import h2o""", diff --git a/h2o-core/src/main/java/water/util/ArrayUtils.java b/h2o-core/src/main/java/water/util/ArrayUtils.java index a9ec123afd2b..cc603198cf5e 100644 --- a/h2o-core/src/main/java/water/util/ArrayUtils.java +++ b/h2o-core/src/main/java/water/util/ArrayUtils.java @@ -45,6 +45,17 @@ public static long suml(final int[] from) { for( int d : from ) result += d; return result; } + + public static void elementwiseSumSymmetricArrays(double[][] a, double[][] b) { + int arrSize = a.length; + for (int index=0; index=1. A value of 0 is only set when only the + model coefficient names and model coefficient dimensions are needed. Defaults to ``-1``. :type max_iterations: int :param objective_epsilon: Converge if objective value changes less than this. Default (of -1.0) indicates: If @@ -272,8 +280,8 @@ def __init__(self, value of lambda the default value of objective_epsilon is set to .0001. Defaults to ``-1.0``. :type objective_epsilon: float - :param beta_epsilon: Converge if beta changes less (using L-infinity norm) than beta esilon, ONLY applies to - IRLSM solver + :param beta_epsilon: Converge if beta changes less (using L-infinity norm) than beta esilon. ONLY applies to + IRLSM solver. Defaults to ``0.0001``. :type beta_epsilon: float :param gradient_epsilon: Converge if objective changes less (using L-infinity norm) than this, ONLY applies to @@ -288,13 +296,15 @@ def __init__(self, :param rand_link: Link function array for random component in HGLM. Defaults to ``None``. :type rand_link: List[Literal["[identity]", "[family_default]"]], optional - :param startval: double array to initialize fixed and random coefficients for HGLM, coefficients for GLM. + :param startval: double array to initialize fixed and random coefficients for HGLM, coefficients for GLM. If + standardize is true, the standardized coefficients should be used. Otherwise, use the regular + coefficients. Defaults to ``None``. :type startval: List[float], optional :param calc_like: if true, will return likelihood function value. Defaults to ``False``. :type calc_like: bool - :param HGLM: If set to true, will return HGLM model. Otherwise, normal GLM model will be returned + :param HGLM: If set to true, will return HGLM model. Otherwise, normal GLM model will be returned. Defaults to ``False``. :type HGLM: bool :param prior: Prior probability for y==1. To be used only for logistic regression iff the data has been sampled @@ -307,9 +317,9 @@ def __init__(self, Defaults to ``False``. :type cold_start: bool :param lambda_min_ratio: Minimum lambda used in lambda search, specified as a ratio of lambda_max (the smallest - lambda that drives all coefficients to zero). Default indicates: if the number of observations is greater - than the number of variables, then lambda_min_ratio is set to 0.0001; if the number of observations is - less than the number of variables, then lambda_min_ratio is set to 0.01. + lambda that drives all coefficients to zero). Default indicates: if the number of observations is + greater than the number of variables, then lambda_min_ratio is set to 0.0001; if the number of + observations is less than the number of variables, then lambda_min_ratio is set to 0.01. Defaults to ``-1.0``. :type lambda_min_ratio: float :param beta_constraints: Beta constraints @@ -327,7 +337,7 @@ def __init__(self, :param interaction_pairs: A list of pairwise (first order) column interactions. Defaults to ``None``. :type interaction_pairs: List[tuple], optional - :param obj_reg: Likelihood divider in objective value computation, default (of -1.0) will set it to 1/nobs + :param obj_reg: Likelihood divider in objective value computation, default (of -1.0) will set it to 1/nobs. Defaults to ``-1.0``. :type obj_reg: float :param stopping_rounds: Early stopping based on convergence of stopping_metric. Stop if simple moving average of @@ -357,7 +367,7 @@ def __init__(self, Defaults to ``5.0``. :type max_after_balance_size: float :param max_confusion_matrix_size: [Deprecated] Maximum size (# classes) for confusion matrices to be printed in - the Logs + the Logs. Defaults to ``20``. :type max_confusion_matrix_size: int :param max_runtime_secs: Maximum allowed runtime in seconds for model training. Use 0 to disable. @@ -416,6 +426,37 @@ def __init__(self, binning. Defaults to ``-1``. :type gainslift_bins: int + :param linear_constraints: Linear constraints: used to specify linear constraints involving more than one + coefficients in standard form. It is only supported for solver IRLSM. It contains four columns: names + (strings for coefficient names or constant), values, types ( strings of 'Equal' or 'LessThanEqual'), + constraint_numbers (0 for first linear constraint, 1 for second linear constraint, ...). + Defaults to ``None``. + :type linear_constraints: Union[None, str, H2OFrame], optional + :param init_optimal_glm: If true, will initialize coefficients with values derived from GLM runs without linear + constraints. Only available for linear constraints. + Defaults to ``False``. + :type init_optimal_glm: bool + :param separate_linear_beta: If true, will keep the beta constraints and linear constraints separate. After new + coefficients are found, first beta constraints will be applied followed by the application of linear + constraints. Note that the beta constraints in this case will not be part of the objective function. If + false, will combine the beta and linear constraints. + Defaults to ``False``. + :type separate_linear_beta: bool + :param constraint_eta0: For constrained GLM only. It affects the setting of eta_k+1=eta_0/power(ck+1, alpha). + Defaults to ``0.1258925``. + :type constraint_eta0: float + :param constraint_tau: For constrained GLM only. It affects the setting of c_k+1=tau*c_k. + Defaults to ``10.0``. + :type constraint_tau: float + :param constraint_alpha: For constrained GLM only. It affects the setting of eta_k = eta_0/pow(c_0, alpha). + Defaults to ``0.1``. + :type constraint_alpha: float + :param constraint_beta: For constrained GLM only. It affects the setting of eta_k+1 = eta_k/pow(c_k, beta). + Defaults to ``0.9``. + :type constraint_beta: float + :param constraint_c0: For constrained GLM only. It affects the initial setting of epsilon_k = 1/c_0. + Defaults to ``10.0``. + :type constraint_c0: float """ super(H2OGeneralizedLinearEstimator, self).__init__() self._parms = {} @@ -497,6 +538,14 @@ def __init__(self, self.dispersion_learning_rate = dispersion_learning_rate self.influence = influence self.gainslift_bins = gainslift_bins + self.linear_constraints = linear_constraints + self.init_optimal_glm = init_optimal_glm + self.separate_linear_beta = separate_linear_beta + self.constraint_eta0 = constraint_eta0 + self.constraint_tau = constraint_tau + self.constraint_alpha = constraint_alpha + self.constraint_beta = constraint_beta + self.constraint_c0 = constraint_c0 @property def training_frame(self): @@ -633,7 +682,7 @@ def export_checkpoints_dir(self, export_checkpoints_dir): @property def seed(self): """ - Seed for pseudo random number generator (if applicable) + Seed for pseudo random number generator (if applicable). Type: ``int``, defaults to ``-1``. @@ -925,7 +974,7 @@ def score_each_iteration(self, score_each_iteration): @property def score_iteration_interval(self): """ - Perform scoring for every score_iteration_interval iterations + Perform scoring for every score_iteration_interval iterations. Type: ``int``, defaults to ``-1``. """ @@ -1077,7 +1126,7 @@ def tweedie_variance_power(self, tweedie_variance_power): @property def tweedie_link_power(self): """ - Tweedie link power + Tweedie link power. Type: ``float``, defaults to ``1.0``. @@ -1228,7 +1277,7 @@ def lambda_(self, lambda_): @property def lambda_search(self): """ - Use lambda search starting at lambda max, given lambda is then interpreted as lambda min + Use lambda search starting at lambda max, given lambda is then interpreted as lambda min. Type: ``bool``, defaults to ``False``. @@ -1256,7 +1305,7 @@ def lambda_search(self, lambda_search): @property def early_stopping(self): """ - Stop early when there is no more relative improvement on train or validation (if provided) + Stop early when there is no more relative improvement on train or validation (if provided). Type: ``bool``, defaults to ``True``. @@ -1315,7 +1364,7 @@ def nlambdas(self, nlambdas): @property def standardize(self): """ - Standardize numeric columns to have zero mean and unit variance + Standardize numeric columns to have zero mean and unit variance. Type: ``bool``, defaults to ``True``. @@ -1373,7 +1422,7 @@ def missing_values_handling(self, missing_values_handling): def plug_values(self): """ Plug Values (a single row frame containing values that will be used to impute missing values of the - training/validation frame, use with conjunction missing_values_handling = PlugValues) + training/validation frame, use with conjunction missing_values_handling = PlugValues). Type: ``Union[None, str, H2OFrame]``. @@ -1406,7 +1455,7 @@ def plug_values(self, plug_values): @property def compute_p_values(self): """ - Request p-values computation, p-values work only with IRLSM solver and no regularization + Request p-values computation, p-values work only with IRLSM solver. Type: ``bool``, defaults to ``False``. @@ -1457,7 +1506,7 @@ def dispersion_parameter_method(self, dispersion_parameter_method): def init_dispersion_parameter(self): """ Only used for Tweedie, Gamma and Negative Binomial GLM. Store the initial value of dispersion parameter. If - fix_dispersion_parameter is set, this value will be used in the calculation of p-values.Default to 1.0. + fix_dispersion_parameter is set, this value will be used in the calculation of p-values. Type: ``float``, defaults to ``1.0``. """ @@ -1471,7 +1520,7 @@ def init_dispersion_parameter(self, init_dispersion_parameter): @property def remove_collinear_columns(self): """ - In case of linearly dependent columns, remove some of the dependent columns + In case of linearly dependent columns, remove the dependent columns. Type: ``bool``, defaults to ``False``. @@ -1535,7 +1584,7 @@ def intercept(self, intercept): @property def non_negative(self): """ - Restrict coefficients (not intercept) to be non-negative + Restrict coefficients (not intercept) to be non-negative. Type: ``bool``, defaults to ``False``. @@ -1569,7 +1618,8 @@ def non_negative(self, non_negative): @property def max_iterations(self): """ - Maximum number of iterations + Maximum number of iterations. Value should >=1. A value of 0 is only set when only the model coefficient names + and model coefficient dimensions are needed. Type: ``int``, defaults to ``-1``. @@ -1629,7 +1679,7 @@ def objective_epsilon(self, objective_epsilon): @property def beta_epsilon(self): """ - Converge if beta changes less (using L-infinity norm) than beta esilon, ONLY applies to IRLSM solver + Converge if beta changes less (using L-infinity norm) than beta esilon. ONLY applies to IRLSM solver. Type: ``float``, defaults to ``0.0001``. @@ -1731,7 +1781,8 @@ def rand_link(self, rand_link): @property def startval(self): """ - double array to initialize fixed and random coefficients for HGLM, coefficients for GLM. + double array to initialize fixed and random coefficients for HGLM, coefficients for GLM. If standardize is + true, the standardized coefficients should be used. Otherwise, use the regular coefficients. Type: ``List[float]``. """ @@ -1759,7 +1810,7 @@ def calc_like(self, calc_like): @property def HGLM(self): """ - If set to true, will return HGLM model. Otherwise, normal GLM model will be returned + If set to true, will return HGLM model. Otherwise, normal GLM model will be returned. Type: ``bool``, defaults to ``False``. """ @@ -1818,9 +1869,9 @@ def cold_start(self, cold_start): def lambda_min_ratio(self): """ Minimum lambda used in lambda search, specified as a ratio of lambda_max (the smallest lambda that drives all - coefficients to zero). Default indicates: if the number of observations is greater than the number of variables, - then lambda_min_ratio is set to 0.0001; if the number of observations is less than the number of variables, then - lambda_min_ratio is set to 0.01. + coefficients to zero). Default indicates: if the number of observations is greater than the number of + variables, then lambda_min_ratio is set to 0.0001; if the number of observations is less than the number of + variables, then lambda_min_ratio is set to 0.01. Type: ``float``, defaults to ``-1.0``. @@ -1990,7 +2041,7 @@ def interaction_pairs(self, interaction_pairs): @property def obj_reg(self): """ - Likelihood divider in objective value computation, default (of -1.0) will set it to 1/nobs + Likelihood divider in objective value computation, default (of -1.0) will set it to 1/nobs. Type: ``float``, defaults to ``-1.0``. @@ -2156,7 +2207,7 @@ def max_after_balance_size(self, max_after_balance_size): @property def max_confusion_matrix_size(self): """ - [Deprecated] Maximum size (# classes) for confusion matrices to be printed in the Logs + [Deprecated] Maximum size (# classes) for confusion matrices to be printed in the Logs. Type: ``int``, defaults to ``20``. """ @@ -2399,6 +2450,124 @@ def gainslift_bins(self, gainslift_bins): assert_is_type(gainslift_bins, None, int) self._parms["gainslift_bins"] = gainslift_bins + @property + def linear_constraints(self): + """ + Linear constraints: used to specify linear constraints involving more than one coefficients in standard form. + It is only supported for solver IRLSM. It contains four columns: names (strings for coefficient names or + constant), values, types ( strings of 'Equal' or 'LessThanEqual'), constraint_numbers (0 for first linear + constraint, 1 for second linear constraint, ...). + + Type: ``Union[None, str, H2OFrame]``. + """ + return self._parms.get("linear_constraints") + + @linear_constraints.setter + def linear_constraints(self, linear_constraints): + self._parms["linear_constraints"] = H2OFrame._validate(linear_constraints, 'linear_constraints') + + @property + def init_optimal_glm(self): + """ + If true, will initialize coefficients with values derived from GLM runs without linear constraints. Only + available for linear constraints. + + Type: ``bool``, defaults to ``False``. + """ + return self._parms.get("init_optimal_glm") + + @init_optimal_glm.setter + def init_optimal_glm(self, init_optimal_glm): + assert_is_type(init_optimal_glm, None, bool) + self._parms["init_optimal_glm"] = init_optimal_glm + + @property + def separate_linear_beta(self): + """ + If true, will keep the beta constraints and linear constraints separate. After new coefficients are found, + first beta constraints will be applied followed by the application of linear constraints. Note that the beta + constraints in this case will not be part of the objective function. If false, will combine the beta and linear + constraints. + + Type: ``bool``, defaults to ``False``. + """ + return self._parms.get("separate_linear_beta") + + @separate_linear_beta.setter + def separate_linear_beta(self, separate_linear_beta): + assert_is_type(separate_linear_beta, None, bool) + self._parms["separate_linear_beta"] = separate_linear_beta + + @property + def constraint_eta0(self): + """ + For constrained GLM only. It affects the setting of eta_k+1=eta_0/power(ck+1, alpha). + + Type: ``float``, defaults to ``0.1258925``. + """ + return self._parms.get("constraint_eta0") + + @constraint_eta0.setter + def constraint_eta0(self, constraint_eta0): + assert_is_type(constraint_eta0, None, numeric) + self._parms["constraint_eta0"] = constraint_eta0 + + @property + def constraint_tau(self): + """ + For constrained GLM only. It affects the setting of c_k+1=tau*c_k. + + Type: ``float``, defaults to ``10.0``. + """ + return self._parms.get("constraint_tau") + + @constraint_tau.setter + def constraint_tau(self, constraint_tau): + assert_is_type(constraint_tau, None, numeric) + self._parms["constraint_tau"] = constraint_tau + + @property + def constraint_alpha(self): + """ + For constrained GLM only. It affects the setting of eta_k = eta_0/pow(c_0, alpha). + + Type: ``float``, defaults to ``0.1``. + """ + return self._parms.get("constraint_alpha") + + @constraint_alpha.setter + def constraint_alpha(self, constraint_alpha): + assert_is_type(constraint_alpha, None, numeric) + self._parms["constraint_alpha"] = constraint_alpha + + @property + def constraint_beta(self): + """ + For constrained GLM only. It affects the setting of eta_k+1 = eta_k/pow(c_k, beta). + + Type: ``float``, defaults to ``0.9``. + """ + return self._parms.get("constraint_beta") + + @constraint_beta.setter + def constraint_beta(self, constraint_beta): + assert_is_type(constraint_beta, None, numeric) + self._parms["constraint_beta"] = constraint_beta + + @property + def constraint_c0(self): + """ + For constrained GLM only. It affects the initial setting of epsilon_k = 1/c_0. + + Type: ``float``, defaults to ``10.0``. + """ + return self._parms.get("constraint_c0") + + @constraint_c0.setter + def constraint_c0(self, constraint_c0): + assert_is_type(constraint_c0, None, numeric) + self._parms["constraint_c0"] = constraint_c0 + Lambda = deprecated_property('Lambda', lambda_) def get_regression_influence_diagnostics(self): @@ -2599,3 +2768,94 @@ def makeGLMModel(model, coefs, threshold=.5): m = H2OGeneralizedLinearEstimator() m._resolve_model(model_json["model_id"]["name"], model_json) return m + + @staticmethod + def getConstraintsInfo(model): + """ + + Given a constrained GLM model, the constraints descriptions, constraints values, constraints conditions and + whether the constraints are satisfied (true) or not (false) are returned. + + :param model: GLM model with linear and beta (if applicable) constraints + :return: H2OTwoDimTable containing the above constraints information. + + :example: + >>> train = h2o.import_file("https://s3.amazonaws.com/h2o-public-test-data/smalldata/glm_test/binomial_20_cols_10KRows.csv") + >>> response = "C21" + >>> predictors = list(range(0,20)) + >>> loose_init_const = [] # this constraint is satisfied by default coefficient initialization + >>> # add loose constraints + >>> name = "C19" + >>> values = 0.5 + >>> types = "LessThanEqual" + >>> contraint_numbers = 0 + >>> loose_init_const.append([name, values, types, contraint_numbers]) + >>> name = "C20" + >>> values = -0.8 + >>> types = "LessThanEqual" + >>> contraint_numbers = 0 + >>> loose_init_const.append([name, values, types, contraint_numbers]) + >>> name = "constant" + >>> values = -1000 + >>> types = "LessThanEqual" + >>> contraint_numbers = 0 + >>> loose_init_const.append([name, values, types, contraint_numbers]) + >>> linear_constraints2 = h2o.H2OFrame(loose_init_const) + >>> linear_constraints2.set_names(["names", "values", "types", "constraint_numbers"]) + >>> # GLM model with GLM coefficients with default initialization + >>> h2o_glm = H2OGeneralizedLinearEstimator(family="binomial", compute_p_values=True, remove_collinear_columns=True, + ... lambda_=0.0, solver="irlsm", linear_constraints=linear_constraints2, + ... init_optimal_glm = False, seed=12345) + >>> h2o_glm.train(x=predictors, y=response, training_frame=train) + >>> print(H2OGeneralizedLinearEstimator.getConstraintsInfo(h2o_glm)) + """ + if model.actual_params["linear_constraints"] is not None: + return model._model_json["output"]["linear_constraints_table"] + else: + raise H2OValueError("getConstraintsInfo can only be called when there are linear constraints.") + + @staticmethod + def allConstraintsPassed(model): + """ + + Given a constrainted GLM model, this will return true if all beta (if exists) and linear constraints are + satified. It will return false even if one constraint is not satisfied. To see which ones failed, use + getConstraintsInfo function. + + :param model: GLM model with linear and beta (if applicable) constraints + :return: boolean True or False + + :example: + >>> train = h2o.import_file("https://s3.amazonaws.com/h2o-public-test-data/smalldata/glm_test/binomial_20_cols_10KRows.csv") + >>> response = "C21" + >>> predictors = list(range(0,20)) + >>> loose_init_const = [] # this constraint is satisfied by default coefficient initialization + >>> # add loose constraints + >>> name = "C19" + >>> values = 0.5 + >>> types = "LessThanEqual" + >>> contraint_numbers = 0 + >>> loose_init_const.append([name, values, types, contraint_numbers]) + >>> name = "C20" + >>> values = -0.8 + >>> types = "LessThanEqual" + >>> contraint_numbers = 0 + >>> loose_init_const.append([name, values, types, contraint_numbers]) + >>> name = "constant" + >>> values = -1000 + >>> types = "LessThanEqual" + >>> contraint_numbers = 0 + >>> loose_init_const.append([name, values, types, contraint_numbers]) + >>> linear_constraints2 = h2o.H2OFrame(loose_init_const) + >>> linear_constraints2.set_names(["names", "values", "types", "constraint_numbers"]) + >>> # GLM model with GLM coefficients with default initialization + >>> h2o_glm = H2OGeneralizedLinearEstimator(family="binomial", compute_p_values=True, remove_collinear_columns=True, + ... lambda_=0.0, solver="irlsm", linear_constraints=linear_constraints2, + ... init_optimal_glm = False, seed=12345) + >>> h2o_glm.train(x=predictors, y=response, training_frame=train) + >>> print(H2OGeneralizedLinearEstimator.allConstraintsPassed(h2o_glm)) + """ + if model.actual_params["linear_constraints"] is not None: + return model._model_json["output"]["all_constraints_satisfied"] + else: + raise H2OValueError("allConstraintsPassed can only be called when there are linear constraints.") diff --git a/h2o-py/h2o/model/model_base.py b/h2o-py/h2o/model/model_base.py index 8146136a10af..239c8817ed69 100644 --- a/h2o-py/h2o/model/model_base.py +++ b/h2o-py/h2o/model/model_base.py @@ -833,6 +833,15 @@ def get_variable_inflation_factors(self): "generate_variable_inflation_factors is enabled.") else: raise ValueError("variable inflation factors are only found in GLM models for numerical predictors.") + + def coef_names(self): + """ + Return the coefficient names of glm model + """ + if self.algo == 'glm': + coefs = self._model_json['output']['coefficient_names'] + coefs.remove('Intercept') + return coefs def coef(self): """ diff --git a/h2o-py/tests/pyunit_utils/__init__.py b/h2o-py/tests/pyunit_utils/__init__.py index c05f173f6611..b2987ec0859d 100644 --- a/h2o-py/tests/pyunit_utils/__init__.py +++ b/h2o-py/tests/pyunit_utils/__init__.py @@ -1,4 +1,5 @@ from .utilsPY import * from .utils_model_metrics import * from .utils_model_custom_distribution import * +from .utils_for_glm_tests import * from .sklearn_multinomial_auc_method import roc_auc_score diff --git a/h2o-py/tests/pyunit_utils/utilsPY.py b/h2o-py/tests/pyunit_utils/utilsPY.py index a4e238ec5e6d..b690cad7d452 100644 --- a/h2o-py/tests/pyunit_utils/utilsPY.py +++ b/h2o-py/tests/pyunit_utils/utilsPY.py @@ -1871,16 +1871,20 @@ def generate_sign_vec(table1, table2): break # found what we need. Goto next column return sign_vec + def equal_two_dicts(dict1, dict2, tolerance=1e-6, throwError=True): size1 = len(dict1) if (size1 == len(dict2)): # only proceed if lengths are the same for key1 in dict1.keys(): diff = abs(dict1[key1]-dict2[key1]) - if (diff > tolerance): + if diff > tolerance: + print("values for key {0} are {1}, {2} and they differ by {3} and is more than " + "{4}".format(key1, dict1[key1], dict2[key1], diff, tolerance)) if throwError: assert False, "Dict 1 value {0} and Dict 2 value {1} do not agree.".format(dict1[key1], dict2[key1]) else: return False + return True def equal_two_dicts_string(dict1, dict2, throwError=True): size1 = len(dict1) diff --git a/h2o-py/tests/pyunit_utils/utils_for_glm_tests.py b/h2o-py/tests/pyunit_utils/utils_for_glm_tests.py new file mode 100644 index 000000000000..010845eb56e0 --- /dev/null +++ b/h2o-py/tests/pyunit_utils/utils_for_glm_tests.py @@ -0,0 +1,148 @@ +import h2o +from h2o.estimators import H2OGeneralizedLinearEstimator as glm +from h2o.exceptions import H2OValueError +from h2o.grid.grid_search import H2OGridSearch + + +def gen_constraint_glm_model(training_dataset, x, y, solver="AUTO", family="gaussian", linear_constraints=None, + beta_constraints=None, separate_linear_beta=False, init_optimal_glm=False, startval=None, + constraint_eta0=0.1258925, constraint_tau=10, constraint_alpha=0.1, + constraint_beta=0.9, constraint_c0=10): + """ + This function given the parameters will return a constraint GLM model. + """ + if linear_constraints is None: + raise H2OValueError("linear_constraints cannot be None") + + params = {"family":family, "lambda_":0.0, "seed":12345, "remove_collinear_columns":True, "solver":solver, + "linear_constraints":linear_constraints, "init_optimal_glm":init_optimal_glm, + "constraint_eta0":constraint_eta0, "constraint_tau":constraint_tau, "constraint_alpha":constraint_alpha, + "constraint_beta":constraint_beta, "constraint_c0":constraint_c0} + if beta_constraints is not None: + params['beta_constraints']=beta_constraints + params["separate_linear_beta"]=separate_linear_beta + if startval is not None: + params["startval"]=startval + + constraint_glm = glm(**params) + constraint_glm.train(x=x, y=y, training_frame=training_dataset) + return constraint_glm + + +def constraint_glm_gridsearch(training_dataset, x, y, solver="AUTO", family="gaussia", linear_constraints=None, + beta_constraints=None, metric="logloss", return_best=True, startval=None, + init_optimal_glm=False, constraint_eta0=[0.1258925], constraint_tau=[10], + constraint_alpha=[0.1], constraint_beta=[0.9], constraint_c0=[10], epsilon=1e-3): + """ + This function given the obj_eps_hyper and inner_loop_hyper will build and run a gridsearch model and return the one + with the best metric. + """ + if linear_constraints is None: + raise H2OValueError("linear_constraints cannot be None") + + params = {"family":family, "lambda_":0.0, "seed":12345, "remove_collinear_columns":True, "solver":solver, + "linear_constraints":linear_constraints} + hyper_parameters = {"constraint_eta0":constraint_eta0, "constraint_tau":constraint_tau, "constraint_alpha":constraint_alpha, + "constraint_beta":constraint_beta, "constraint_c0":constraint_c0} + if beta_constraints is not None: + params['beta_constraints']=beta_constraints + hyper_parameters["separate_linear_beta"] = [False, True] + if startval is not None: + params["startval"]=startval + if init_optimal_glm: + params["init_optimal_glm"]=True + + glmGrid = H2OGridSearch(glm(**params), hyper_params=hyper_parameters) + glmGrid.train(x=x, y=y, training_frame=training_dataset) + sortedGrid = glmGrid.get_grid() + print(sortedGrid) + if return_best: + print_model_hyperparameters(sortedGrid.models[0], hyper_parameters) + return sortedGrid.models[0] + else: + return grid_models_analysis(sortedGrid.models, hyper_parameters, metric=metric, epsilon=epsilon) + + +def grid_models_analysis(grid_models, hyper_parameters, metric="logloss", epsilon=1e-3): + """ + This method will search within the grid search models that have metrics within epsilon calculated as + abs(metric1-metric2)/abs(metric1) as the best model. We are wanting to send the best model that has the best + constraint values meaning either they have the smallest magnitude or if less than constraints, it has the smallest + magnitude and the correct sign. Else, the original top model will be returned. + """ + base_metric = grid_models[0].model_performance()._metric_json[metric] + base_constraints_table = grid_models[0]._model_json["output"]["linear_constraints_table"] + cond_index = base_constraints_table.col_header.index("condition") + [best_equality_constraints, best_lessthan_constraints] = grab_constraint_values( + base_constraints_table, cond_index, len(base_constraints_table.cell_values)) + + base_iteration = find_glm_iterations(grid_models[0]) + num_models = len(grid_models) + best_model_ind = 0 + model_indices = [] + model_equality_constraints_values = [] + model_lessthan_constraints_values = [] + iterations = [] + for ind in range(1, num_models): + curr_model = grid_models[ind] + curr_metric = grid_models[ind].model_performance()._metric_json[metric] + metric_diff = abs(base_metric-curr_metric)/abs(base_metric) + if metric_diff < epsilon: + curr_constraint_table = curr_model._model_json["output"]["linear_constraints_table"] + [equality_constraints_values, lessthan_constraints_values] = grab_constraint_values( + curr_constraint_table, cond_index, len(curr_constraint_table.cell_values)) + # conditions used to choose the best model + if (sum(equality_constraints_values) < sum(best_equality_constraints)) and (sum(lessthan_constraints_values) < sum(best_lessthan_constraints)): + best_model_ind = ind + base_iteration = find_glm_iterations(curr_model) + best_equality_constraints = equality_constraints_values + best_lessthan_constraints = lessthan_constraints_values + model_equality_constraints_values.append(equality_constraints_values) + model_lessthan_constraints_values.append(lessthan_constraints_values) + model_indices.append(ind) + iterations.append(find_glm_iterations(curr_model)) + print("Maximum iterations: {0} and it is from model index: {1}".format(base_iteration, best_model_ind)) + print_model_hyperparameters(grid_models[best_model_ind], hyper_parameters) + return grid_models[best_model_ind] + + +def print_model_hyperparameters(model, hyper_parameters): + print("***** Hyper parameter values for best model chosen are:") + params = model.actual_params + param_keys = hyper_parameters.keys() + actual_keys = params.keys() + for param in actual_keys: + if param in param_keys: + print("{0} value {1}.".format(param, params[param])) + + +def grab_constraint_values(curr_constraint_table, cond_index, num_constraints): + equality_constraints_values = [] + lessthan_constraints_values = [] + for ind2 in range(0, num_constraints): # collect all equality constraint info + if curr_constraint_table.cell_values[ind2][cond_index]=="== 0": + equality_constraints_values.append(curr_constraint_table.cell_values[ind2][cond_index-1]) + else: + if curr_constraint_table.cell_values[ind2][cond_index-1] < 0: + lessthan_constraints_values.append(0) + else: + lessthan_constraints_values.append(curr_constraint_table.cell_values[ind2][cond_index-1]) + return [equality_constraints_values, lessthan_constraints_values] + + +def is_always_lower_than(original_tuple, new_tuple): + """ + This function will return True if new_tuple has smaller magnitude elements than what is in original_tuple. + """ + assert len(original_tuple) == len(new_tuple), "expected tuple length: {0}, actual length: {1} and they are " \ + "different".format(len(original_tuple), len(new_tuple)) + return all(abs(orig) > abs(new) for orig, new in zip(original_tuple, new_tuple)) + +def find_glm_iterations(glm_model): + """ + Given a glm constrainted model, this method will obtain the number of iterations from the model summary. + """ + cell_values = glm_model._model_json["output"]["model_summary"].cell_values + lengths = len(cell_values) + iteration_index = glm_model._model_json["output"]["model_summary"].col_header.index("number_of_iterations") + return cell_values[lengths-1][iteration_index] diff --git a/h2o-py/tests/testdir_algos/glm/pyunit_GH_6722_beta_equality_loose_lessthan_linear_constraints_binomial.py b/h2o-py/tests/testdir_algos/glm/pyunit_GH_6722_beta_equality_loose_lessthan_linear_constraints_binomial.py new file mode 100644 index 000000000000..4c01ed1f67d7 --- /dev/null +++ b/h2o-py/tests/testdir_algos/glm/pyunit_GH_6722_beta_equality_loose_lessthan_linear_constraints_binomial.py @@ -0,0 +1,212 @@ +import h2o +from h2o.estimators.glm import H2OGeneralizedLinearEstimator as glm +from tests import pyunit_utils +from tests.pyunit_utils import utils_for_glm_tests + +def test_constraints_binomial(): + ''' + check and make sure coefficients close to GLM without constraints are generated with loose constraints + that are satisfied with coefficients from GLM without constraints. All constraints are present, beta, equal, + and less than and equal to. + ''' + train = h2o.import_file(path=pyunit_utils.locate("smalldata/glm_test/binomial_20_cols_10KRows.csv")) + for ind in range(10): + train[ind] = train[ind].asfactor() + train["C21"] = train["C21"].asfactor() + response = "C21" + predictors = list(range(0,20)) + + loose_init_const = [] # this constraint is satisfied by default coefficient initialization + + h2o_glm = glm(family="binomial", remove_collinear_columns=True, lambda_=0.0, solver="irlsm", seed=12345, + standardize=True) + h2o_glm.train(x=predictors, y=response, training_frame=train) + logloss = h2o_glm.model_performance()._metric_json['logloss'] + print("logloss with no constraints: {0}".format(logloss)) + # add beta constraints + bc = [] + name = "C11" + lower_bound = -8 + upper_bound = 0 + bc.append([name, lower_bound, upper_bound]) + + name = "C18" + lower_bound = 6 + upper_bound = 8 + bc.append([name, lower_bound, upper_bound]) + + name = "C15" + lower_bound = -9 + upper_bound = 0 + bc.append([name, lower_bound, upper_bound]) + + name = "C16" + lower_bound = -20 + upper_bound = 0.3 + bc.append([name, lower_bound, upper_bound]) + + beta_constraints = h2o.H2OFrame(bc) + beta_constraints.set_names(["names", "lower_bounds", "upper_bounds"]) + + # add loose constraints + name = "C19" + values = 0.5 + types = "Equal" + contraint_numbers = 0 + loose_init_const.append([name, values, types, contraint_numbers]) + + name = "C10.1" + values = -0.3 + types = "Equal" + contraint_numbers = 0 + loose_init_const.append([name, values, types, contraint_numbers]) + + name = "constant" + values = -1.0044392439227687 + types = "Equal" + contraint_numbers = 0 + loose_init_const.append([name, values, types, contraint_numbers]) + + name = "C18" + values = 0.75 + types = "Equal" + contraint_numbers = 1 + loose_init_const.append([name, values, types, contraint_numbers]) + + name = "C20" + values = -0.13 + types = "Equal" + contraint_numbers = 1 + loose_init_const.append([name, values, types, contraint_numbers]) + + name = "constant" + values = -6.941462080282776 + types = "Equal" + contraint_numbers = 1 + loose_init_const.append([name, values, types, contraint_numbers]) + + # add loose constraints + name = "C19" + values = 0.5 + types = "LessThanEqual" + contraint_numbers = 2 + loose_init_const.append([name, values, types, contraint_numbers]) + + name = "C20" + values = -0.8 + types = "LessThanEqual" + contraint_numbers = 2 + loose_init_const.append([name, values, types, contraint_numbers]) + + name = "constant" + values = -10 + types = "LessThanEqual" + contraint_numbers = 2 + loose_init_const.append([name, values, types, contraint_numbers]) + + name = "C12" + values = 2 + types = "LessThanEqual" + contraint_numbers = 3 + loose_init_const.append([name, values, types, contraint_numbers]) + + name = "C13" + values = -3 + types = "LessThanEqual" + contraint_numbers = 3 + loose_init_const.append([name, values, types, contraint_numbers]) + + name = "constant" + values = -36 + types = "LessThanEqual" + contraint_numbers = 3 + loose_init_const.append([name, values, types, contraint_numbers]) + + linear_constraints2 = h2o.H2OFrame(loose_init_const) + linear_constraints2.set_names(["names", "values", "types", "constraint_numbers"]) + # GLM model with with GLM coefficients set to GLM model coefficients built without constraints + h2o_glm_optimal_init = glm(family="binomial", lambda_=0.0, seed=12345, remove_collinear_columns=True,solver="irlsm", + linear_constraints=linear_constraints2, init_optimal_glm=True, beta_constraints=beta_constraints) + h2o_glm_optimal_init.train(x=predictors, y=response, training_frame=train) + init_logloss = h2o_glm_optimal_init.model_performance()._metric_json['logloss'] + print("logloss with constraints and coefficients initialized with glm model built without constraints: {0}".format(init_logloss)) + print(glm.getConstraintsInfo(h2o_glm_optimal_init)) + + h2o_glm_optimal_init_sep = glm(family="binomial", lambda_=0.0, seed=12345, remove_collinear_columns=True, + solver="irlsm", linear_constraints=linear_constraints2, init_optimal_glm=True, + beta_constraints=beta_constraints, separate_linear_beta=True) + h2o_glm_optimal_init_sep.train(x=predictors, y=response, training_frame=train) + init_logloss_sep = h2o_glm_optimal_init_sep.model_performance()._metric_json['logloss'] + print("logloss with constraints and coefficients initialized with glm model built without constraints and separate " + "linear and beta constraints: {0}".format(init_logloss_sep)) + print(glm.getConstraintsInfo(h2o_glm_optimal_init_sep)) + + # GLM model with with GLM coefficients set to random GLM model coefficients + random_coef = [0.9740393731418461, 0.9021970400494406, 0.8337282995102272, 0.20588758679724872, 0.12522385214612453, + 0.6390730524643073, 0.7055779213989253, 0.9004255614099713, 0.4075431157767999, 0.161093231584713, + 0.15250197544465616, 0.7172682822215489, 0.60836236371404, 0.07086628306822396, 0.263719138602719, + 0.16102036359390437, 0.0065987448849305075, 0.5881312311814277, 0.7836567678399617, 0.9104401158881326, + 0.8432891635016235, 0.033440093086177236, 0.8514611306363931, 0.2855332934628241, 0.36525972112514427, + 0.7526593301495519, 0.9963694184200753, 0.5614168317678196, 0.7950126291921057, 0.6212978800904426, + 0.176936615687169, 0.8817788599562331, 0.13699370230879637, 0.5754950980437555, 0.1507294463182668, + 0.23409699287029495, 0.6949148063429461, 0.47140569181488556, 0.1470896240551064, 0.8475557222612405, + 0.05957485472498203, 0.07490903723892406, 0.8412381196460251, 0.26874846387453943, 0.13669341206289243, + 0.8525684329438777, 0.46716360402752777, 0.8522055745422484, 0.3129394551398561, 0.908966336417204, + 0.26259461196353984, 0.07245314277889847, 0.41429401839807156, 0.22772860293274222, 0.26662443208488784, + 0.9875655504027848, 0.5832266083052889, 0.24205847206862052, 0.9843760682096272, 0.16269008279311103, + 0.4941250734508458, 0.5446841276322587, 0.19222703209695946, 0.9232239752817498, 0.8824688635063289, + 0.224690851359456, 0.5809304720756304, 0.36863807988348585] + constraint_eta0 = [0.05] + constraint_tau = [1.2] + constraint_alpha = [0.01] + constraint_beta = [0.5, 0.9] + constraint_c0 = [40] + h2o_glm_random_init = utils_for_glm_tests.constraint_glm_gridsearch(train, predictors, response, solver="IRLSM", + family="binomial", + linear_constraints=linear_constraints2, + beta_constraints=beta_constraints, + startval=random_coef, + constraint_eta0=constraint_eta0, + constraint_tau=constraint_tau, + constraint_alpha=constraint_alpha, + constraint_beta=constraint_beta, + constraint_c0=constraint_c0, + return_best=False) + init_random_logloss = h2o_glm_random_init.model_performance()._metric_json['logloss'] + print("logloss with constraints and coefficients initialized random initial values: {0}, number of iterations" + " taken to build the model: {1}".format(init_random_logloss, utils_for_glm_tests.find_glm_iterations(h2o_glm_random_init))) + print(glm.getConstraintsInfo(h2o_glm_random_init)) + + + # GLM model with GLM coefficients with default initialization + h2o_glm_default_init = utils_for_glm_tests.constraint_glm_gridsearch(train, predictors, response, solver="IRLSM", + family="binomial", + linear_constraints=linear_constraints2, + beta_constraints=beta_constraints, + constraint_eta0=constraint_eta0, + constraint_tau=constraint_tau, + constraint_alpha=constraint_alpha, + constraint_beta=constraint_beta, + constraint_c0=constraint_c0, + return_best=False) + default_init_logloss = h2o_glm_default_init.model_performance()._metric_json['logloss'] + print("logloss with constraints and default coefficients initialization: {0}, number of iterations" + " taken to build the model: {1}".format(default_init_logloss, utils_for_glm_tests.find_glm_iterations(h2o_glm_default_init))) + print(glm.getConstraintsInfo(h2o_glm_default_init)) + + + assert abs(logloss-init_logloss)<2e-6, "logloss from optimal GLM {0} and logloss from GLM with loose constraints " \ + "and initialized with optimal GLM {1} should equal but is not." \ + "".format(logloss, init_logloss) + assert logloss <= init_random_logloss, "logloss from optimal GLM {0} should be less than GLM with constraints " \ + "and with random initial coefficients {1} but is" \ + " not.".format(logloss, init_random_logloss) + assert logloss <= default_init_logloss, "logloss from optimal GLM {0} should be less than GLM with constraints " \ + "and with default initial coefficients {1} but is" \ + " not.".format(logloss, default_init_logloss) + + +if __name__ == "__main__": + pyunit_utils.standalone_test(test_constraints_binomial) +else: + test_constraints_binomial() diff --git a/h2o-py/tests/testdir_algos/glm/pyunit_GH_6722_beta_linear_constraints_binomial_objective_likelihood.py b/h2o-py/tests/testdir_algos/glm/pyunit_GH_6722_beta_linear_constraints_binomial_objective_likelihood.py new file mode 100644 index 000000000000..add3f09430e5 --- /dev/null +++ b/h2o-py/tests/testdir_algos/glm/pyunit_GH_6722_beta_linear_constraints_binomial_objective_likelihood.py @@ -0,0 +1,173 @@ +import h2o +from h2o.estimators.glm import H2OGeneralizedLinearEstimator as glm +from tests import pyunit_utils +from tests.pyunit_utils import utils_for_glm_tests + +def test_constraints_objective_likelihood(): + ''' + In this test, I want to make sure that the correct loglikelihood and objective functions are calculated when + a user specified constraints and want the likelihood and objective functions. + ''' + train = h2o.import_file(path=pyunit_utils.locate("smalldata/glm_test/binomial_20_cols_10KRows.csv")) + for ind in range(10): + train[ind] = train[ind].asfactor() + train["C21"] = train["C21"].asfactor() + response = "C21" + predictors = list(range(0,20)) + + loose_init_const = [] # this constraint is satisfied by default coefficient initialization + # add beta constraints + bc = [] + name = "C11" + lower_bound = -8 + upper_bound = 0 + bc.append([name, lower_bound, upper_bound]) + + name = "C18" + lower_bound = 6 + upper_bound = 8 + bc.append([name, lower_bound, upper_bound]) + + beta_constraints = h2o.H2OFrame(bc) + beta_constraints.set_names(["names", "lower_bounds", "upper_bounds"]) + + # add loose constraints + name = "C19" + values = 0.5 + types = "Equal" + contraint_numbers = 0 + loose_init_const.append([name, values, types, contraint_numbers]) + + name = "C10.1" + values = -0.3 + types = "Equal" + contraint_numbers = 0 + loose_init_const.append([name, values, types, contraint_numbers]) + + name = "constant" + values = -1.0044392439227687 + types = "Equal" + contraint_numbers = 0 + loose_init_const.append([name, values, types, contraint_numbers]) + + # add loose constraints + name = "C19" + values = 0.5 + types = "LessThanEqual" + contraint_numbers = 1 + loose_init_const.append([name, values, types, contraint_numbers]) + + name = "C20" + values = -0.8 + types = "LessThanEqual" + contraint_numbers = 1 + loose_init_const.append([name, values, types, contraint_numbers]) + + name = "constant" + values = -10 + types = "LessThanEqual" + contraint_numbers = 1 + loose_init_const.append([name, values, types, contraint_numbers]) + + linear_constraints2 = h2o.H2OFrame(loose_init_const) + linear_constraints2.set_names(["names", "values", "types", "constraint_numbers"]) + # glm without constraints + h2o_glm = glm(family="binomial", lambda_=0.0, seed=12345, remove_collinear_columns=True,solver="irlsm", + calc_like=True, generate_scoring_history=True) + h2o_glm.train(x=predictors, y=response, training_frame=train) + ll = h2o_glm.loglikelihood() + aic = h2o_glm.aic() + coef = h2o_glm.coef() + obj = h2o_glm.average_objective() + logloss = h2o_glm.model_performance()._metric_json['logloss'] + print("GLM losloss: {0}, aic: {1}, llh: {2}, average_objective: {3}.".format(logloss, aic, ll, obj)) + + # GLM model with with GLM coefficients set to GLM model coefficients built without constraints + h2o_glm_optimal_init = glm(family="binomial", lambda_=0.0, seed=12345, remove_collinear_columns=True,solver="irlsm", + linear_constraints=linear_constraints2, init_optimal_glm=True, + beta_constraints=beta_constraints, calc_like=True, generate_scoring_history=True) + h2o_glm_optimal_init.train(x=predictors, y=response, training_frame=train) + ll_optimal = h2o_glm_optimal_init.loglikelihood(train=True) + aic_optimal = h2o_glm_optimal_init.aic(train=True) + coef_optimal = h2o_glm_optimal_init.coef() + init_logloss = h2o_glm_optimal_init.model_performance()._metric_json['logloss'] + obj_optimal = h2o_glm_optimal_init.average_objective() + print("logloss with constraints and coefficients initialized with glm model built without constraints: {0}, aic: " + "{2}, llh: {3}, average_objective: {4}, number of iterations taken to build the model: " + "{1}".format(init_logloss, utils_for_glm_tests.find_glm_iterations(h2o_glm_optimal_init), aic_optimal, + ll_optimal, obj_optimal)) + print(glm.getConstraintsInfo(h2o_glm_optimal_init)) + + # GLM model with with GLM coefficients set to random GLM model coefficients + random_coef = [0.9740393731418461, 0.9021970400494406, 0.8337282995102272, 0.20588758679724872, 0.12522385214612453, + 0.6390730524643073, 0.7055779213989253, 0.9004255614099713, 0.4075431157767999, 0.161093231584713, + 0.15250197544465616, 0.7172682822215489, 0.60836236371404, 0.07086628306822396, 0.263719138602719, + 0.16102036359390437, 0.0065987448849305075, 0.5881312311814277, 0.7836567678399617, 0.9104401158881326, + 0.8432891635016235, 0.033440093086177236, 0.8514611306363931, 0.2855332934628241, 0.36525972112514427, + 0.7526593301495519, 0.9963694184200753, 0.5614168317678196, 0.7950126291921057, 0.6212978800904426, + 0.176936615687169, 0.8817788599562331, 0.13699370230879637, 0.5754950980437555, 0.1507294463182668, + 0.23409699287029495, 0.6949148063429461, 0.47140569181488556, 0.1470896240551064, 0.8475557222612405, + 0.05957485472498203, 0.07490903723892406, 0.8412381196460251, 0.26874846387453943, 0.13669341206289243, + 0.8525684329438777, 0.46716360402752777, 0.8522055745422484, 0.3129394551398561, 0.908966336417204, + 0.26259461196353984, 0.07245314277889847, 0.41429401839807156, 0.22772860293274222, 0.26662443208488784, + 0.9875655504027848, 0.5832266083052889, 0.24205847206862052, 0.9843760682096272, 0.16269008279311103, + 0.4941250734508458, 0.5446841276322587, 0.19222703209695946, 0.9232239752817498, 0.8824688635063289, + 0.224690851359456, 0.5809304720756304, 0.36863807988348585] + h2o_glm_random_init = glm(family="binomial", lambda_=0.0, seed=12345, remove_collinear_columns=True,solver="irlsm", + linear_constraints=linear_constraints2, startval=random_coef, + beta_constraints=beta_constraints, calc_like=True, generate_scoring_history=True) + h2o_glm_random_init.train(x=predictors, y=response, training_frame=train) + ll_random = h2o_glm_random_init.loglikelihood(train=True) + aic_random = h2o_glm_random_init.aic(train=True) + coef_random = h2o_glm_random_init.coef() + obj_random = h2o_glm_random_init.average_objective() + init_random_logloss = h2o_glm_random_init.model_performance()._metric_json['logloss'] + print("logloss with constraints and coefficients initialized random initial values: {0}, aic: {2}, llh: {3}, " + "average objective: {4}, number of iterations taken to build the model: " + "{1}".format(init_random_logloss, utils_for_glm_tests.find_glm_iterations(h2o_glm_random_init), aic_random, + ll_random, obj_random)) + print(glm.getConstraintsInfo(h2o_glm_random_init)) + + + # GLM model with GLM coefficients with default initialization + h2o_glm_default_init = glm(family="binomial", lambda_=0.0, seed=12345, remove_collinear_columns=True,solver="irlsm", + linear_constraints=linear_constraints2, beta_constraints=beta_constraints, + calc_like=True, generate_scoring_history=True) + h2o_glm_default_init.train(x=predictors, y=response, training_frame=train) + ll_default = h2o_glm_default_init.loglikelihood(train=True) + aic_default = h2o_glm_default_init.aic(train=True) + coef_default = h2o_glm_default_init.coef() + obj_default = h2o_glm_default_init.average_objective() + default_init_logloss = h2o_glm_default_init.model_performance()._metric_json['logloss'] + print("logloss with constraints and default coefficients initialization: {0}, aic: {2}, llh: {3}, average objective:" + " {4}, number of iterations taken to build the model: " + "{1}".format(default_init_logloss, utils_for_glm_tests.find_glm_iterations(h2o_glm_default_init), aic_default, + ll_default, obj_default)) + print(glm.getConstraintsInfo(h2o_glm_default_init)) + + # if coefficients are close enough, we will compare the objective and aic + if pyunit_utils.equal_two_dicts(coef, coef_optimal, throwError=False): + assert abs(ll-ll_optimal)/abs(ll) < 1e-6, "loglikelihood of glm: {0}, should be close to constrained GLM with" \ + " optimal coefficient init: {1} but is not.".format(ll, ll_optimal) + assert abs(aic-aic_optimal)/abs(aic) < 1e-6, "AIC of glm: {0}, should be close to constrained GLM with" \ + " optimal coefficient init: {1} but is not.".format(aic, aic_optimal) + assert abs(obj-obj_optimal)/abs(obj) < 1e-6, "average objective of glm: {0}, should be close to constrained GLM with" \ + " optimal coefficient init: {1} but is not.".format(obj, obj_optimal) + if pyunit_utils.equal_two_dicts(coef, coef_random, tolerance=2.1e-3, throwError=False): + assert abs(ll-ll_random)/abs(ll) < 1e-3, "loglikelihood of glm: {0}, should be close to constrained GLM with" \ + " random coefficient init: {1} but is not.".format(ll, ll_random) + assert abs(aic-aic_random)/abs(aic) < 1e-3, "AIC of glm: {0}, should be close to constrained GLM with" \ + " random coefficient init: {1} but is not.".format(aic, aic_random) + assert abs(obj-obj_random)/abs(obj) < 1e-3, "average objective of glm: {0}, should be close to constrained GLM with" \ + " random coefficient init: {1} but is not.".format(obj, obj_random) + if pyunit_utils.equal_two_dicts(coef, coef_default, tolerance=2e-3, throwError=False): + assert abs(ll-ll_default)/abs(ll) < 1e-3, "loglikelihood of glm: {0}, should be close to constrained GLM with" \ + " default coefficient init: {1} but is not.".format(ll, ll_default) + assert abs(aic-aic_default)/abs(aic) < 1e-3, "AIC of glm: {0}, should be close to constrained GLM with" \ + " default coefficient init: {1} but is not.".format(aic, aic_default) + assert abs(obj-obj_default)/abs(obj) < 1e-3, "average objective of glm: {0}, should be close to constrained GLM with" \ + " default coefficient init: {1} but is not.".format(obj, obj_default) +if __name__ == "__main__": + pyunit_utils.standalone_test(test_constraints_objective_likelihood) +else: + test_constraints_objective_likelihood() diff --git a/h2o-py/tests/testdir_algos/glm/pyunit_GH_6722_constraints_on_collinear_cols1.py b/h2o-py/tests/testdir_algos/glm/pyunit_GH_6722_constraints_on_collinear_cols1.py new file mode 100644 index 000000000000..39c5d10eb317 --- /dev/null +++ b/h2o-py/tests/testdir_algos/glm/pyunit_GH_6722_constraints_on_collinear_cols1.py @@ -0,0 +1,76 @@ +import h2o +from h2o.estimators.glm import H2OGeneralizedLinearEstimator +from tests import pyunit_utils + +# The purpose of this test to make sure that constrainted GLM works with collinear column removal. In this case, +# the collinear columns are added to the front of the frame. There are two collinear columns and they should be +# removed. +def test_constraints_collinear_columns(): + # first two columns are enums, the last 4 are real columns + h2o_data = pyunit_utils.genTrainFrame(10000, 6, enumCols=2, enumFactors=2, responseLevel=2, miscfrac=0, randseed=12345) + # create extra collinear columns + num1 = h2o_data[2]*0.2-0.5*h2o_data[3] + num2 = -0.8*h2o_data[4]+0.1*h2o_data[5] + h2o_collinear = num1.cbind(num2) + h2o_collinear.set_names(["corr1", "corr2"]) + train_data = h2o_collinear.cbind(h2o_data) + + y = "response" + x = train_data.names + x.remove(y) + lc2 = [] + + name = "C4" + values = 1 + types = "LessThanEqual" + contraint_numbers = 0 + lc2.append([name, values, types, contraint_numbers]) + + name = "corr2" + values = 1 + types = "LessThanEqual" + contraint_numbers = 0 + lc2.append([name, values, types, contraint_numbers]) + + name = "constant" + values = -2 + types = "LessThanEqual" + contraint_numbers = 0 + lc2.append([name, values, types, contraint_numbers]) + + name = "corr1" + values = 1 + types = "LessThanEqual" + contraint_numbers = 1 + lc2.append([name, values, types, contraint_numbers]) + + name = "C6" + values = 1 + types = "LessThanEqual" + contraint_numbers = 1 + lc2.append([name, values, types, contraint_numbers]) + + name = "constant" + values = -2 + types = "LessThanEqual" + contraint_numbers = 1 + lc2.append([name, values, types, contraint_numbers]) + + linear_constraints2 = h2o.H2OFrame(lc2) + linear_constraints2.set_names(["names", "values", "types", "constraint_numbers"]) + + h2o_glm = H2OGeneralizedLinearEstimator(family="binomial", compute_p_values=True, remove_collinear_columns=True, + lambda_=0.0, solver="irlsm", linear_constraints=linear_constraints2, + seed = 1234) + h2o_glm.train(x=x, y=y, training_frame=train_data ) + # there should be two coefficients with zero + coefs = h2o_glm.coef().values() + numZero = [x for x in coefs if x == 0] + assert len(numZero) == 2, "Length of non-zero coefficients should be 2 but is not." + + + +if __name__ == "__main__": + pyunit_utils.standalone_test(test_constraints_collinear_columns) +else: + test_constraints_collinear_columns() diff --git a/h2o-py/tests/testdir_algos/glm/pyunit_GH_6722_constraints_on_collinear_cols2.py b/h2o-py/tests/testdir_algos/glm/pyunit_GH_6722_constraints_on_collinear_cols2.py new file mode 100644 index 000000000000..e8356ba3f0fe --- /dev/null +++ b/h2o-py/tests/testdir_algos/glm/pyunit_GH_6722_constraints_on_collinear_cols2.py @@ -0,0 +1,56 @@ +import h2o +from h2o.estimators.glm import H2OGeneralizedLinearEstimator +from tests import pyunit_utils + +# The purpose of this test to make sure that constrainted GLM works with collinear column removal. In this case, +# the collinear columns are added to the back of the frame. +def test_constraints_collinear_columns(): + # first two columns are enums, the last 4 are real columns + h2o_data = pyunit_utils.genTrainFrame(10000, 6, enumCols=2, enumFactors=2, responseLevel=2, miscfrac=0, randseed=12345) + # create extra collinear columns + num1 = h2o_data[2]*0.2-0.5*h2o_data[3] + num2 = -0.8*h2o_data[4]+0.1*h2o_data[5] + h2o_collinear = num1.cbind(num2) + h2o_collinear.set_names(["corr1", "corr2"]) + train_data = h2o_data.cbind(h2o_collinear) + + y = "response" + x = train_data.names + x.remove(y) + lc2 = [] + + name = "C10" + values = 1 + types = "LessThanEqual" + contraint_numbers = 0 + lc2.append([name, values, types, contraint_numbers]) + + name = "corr2" + values = 1 + types = "LessThanEqual" + contraint_numbers = 0 + lc2.append([name, values, types, contraint_numbers]) + + name = "constant" + values = -2 + types = "LessThanEqual" + contraint_numbers = 0 + lc2.append([name, values, types, contraint_numbers]) + + linear_constraints2 = h2o.H2OFrame(lc2) + linear_constraints2.set_names(["names", "values", "types", "constraint_numbers"]) + + h2o_glm = H2OGeneralizedLinearEstimator(family="binomial", compute_p_values=True, remove_collinear_columns=True, + lambda_=0.0, solver="irlsm", linear_constraints=linear_constraints2, + seed = 1234) + h2o_glm.train(x=x, y=y, training_frame=train_data ) + # there should be two coefficients with zero + coefs = h2o_glm.coef().values() + numZero = [x for x in coefs if x == 0] + assert len(numZero) == 2, "Length of non-zero coefficients should be 2 but is not." + + +if __name__ == "__main__": + pyunit_utils.standalone_test(test_constraints_collinear_columns) +else: + test_constraints_collinear_columns() diff --git a/h2o-py/tests/testdir_algos/glm/pyunit_GH_6722_equality_constraints_only_binomial.py b/h2o-py/tests/testdir_algos/glm/pyunit_GH_6722_equality_constraints_only_binomial.py new file mode 100644 index 000000000000..8a47d3773279 --- /dev/null +++ b/h2o-py/tests/testdir_algos/glm/pyunit_GH_6722_equality_constraints_only_binomial.py @@ -0,0 +1,134 @@ +import h2o +from h2o.estimators.glm import H2OGeneralizedLinearEstimator as glm +from tests import pyunit_utils +from tests.pyunit_utils import utils_for_glm_tests + +def test_constraints_binomial(): + ''' + This test checks and make sure the equality constraints work with binomial family. Coefficients are initialized + with glm coefficients built without contraints, default coefficients and random coefficients. + ''' + train = h2o.import_file(path=pyunit_utils.locate("smalldata/glm_test/binomial_20_cols_10KRows.csv")) + for ind in range(10): + train[ind] = train[ind].asfactor() + train["C21"] = train["C21"].asfactor() + response = "C21" + predictors = list(range(0,20)) + + loose_init_const = [] # this constraint is satisfied by default coefficient initialization + + h2o_glm = glm(family="binomial", remove_collinear_columns=True, lambda_=0.0, solver="irlsm", seed=12345, + standardize=True) + h2o_glm.train(x=predictors, y=response, training_frame=train) + logloss = h2o_glm.model_performance()._metric_json['logloss'] + print("*** logloss with no constraints: {0}".format(logloss)) + + # add loose constraints + name = "C19" + values = 0.5 + types = "Equal" + contraint_numbers = 0 + loose_init_const.append([name, values, types, contraint_numbers]) + + name = "C10.1" + values = -0.3 + types = "Equal" + contraint_numbers = 0 + loose_init_const.append([name, values, types, contraint_numbers]) + + name = "constant" + values = -1.0044392439227687 + types = "Equal" + contraint_numbers = 0 + loose_init_const.append([name, values, types, contraint_numbers]) + + name = "C18" + values = 0.75 + types = "Equal" + contraint_numbers = 1 + loose_init_const.append([name, values, types, contraint_numbers]) + + name = "C20" + values = -0.13 + types = "Equal" + contraint_numbers = 1 + loose_init_const.append([name, values, types, contraint_numbers]) + + name = "constant" + values = -6.941462080282776 + types = "Equal" + contraint_numbers = 1 + loose_init_const.append([name, values, types, contraint_numbers]) + linear_constraints2 = h2o.H2OFrame(loose_init_const) + linear_constraints2.set_names(["names", "values", "types", "constraint_numbers"]) + + # GLM model with with GLM coefficients set to GLM model coefficients built without constraints + h2o_glm_optimal_init = glm(family="binomial", lambda_=0.0, seed=12345, remove_collinear_columns=True,solver="irlsm", + linear_constraints=linear_constraints2, init_optimal_glm=True) + h2o_glm_optimal_init.train(x=predictors, y=response, training_frame=train) + init_logloss = h2o_glm_optimal_init.model_performance()._metric_json['logloss'] + print("**** logloss with constraints and coefficients initialized with glm model built without constraints:" + " {0}".format(init_logloss)) + print(glm.getConstraintsInfo(h2o_glm_optimal_init)) + assert abs(logloss-init_logloss)<2e-6, "logloss from optimal GLM {0} and logloss from GLM with equal constraints " \ + "and initialized with optimal GLM {1} should equal but is not." \ + "".format(logloss, init_logloss) + + # GLM model with with GLM coefficients set to random GLM model coefficients + random_coef = [0.9740393731418461, 0.9021970400494406, 0.8337282995102272, 0.20588758679724872, 0.12522385214612453, + 0.6390730524643073, 0.7055779213989253, 0.9004255614099713, 0.4075431157767999, 0.161093231584713, + 0.15250197544465616, 0.7172682822215489, 0.60836236371404, 0.07086628306822396, 0.263719138602719, + 0.16102036359390437, 0.0065987448849305075, 0.5881312311814277, 0.7836567678399617, 0.9104401158881326, + 0.8432891635016235, 0.033440093086177236, 0.8514611306363931, 0.2855332934628241, 0.36525972112514427, + 0.7526593301495519, 0.9963694184200753, 0.5614168317678196, 0.7950126291921057, 0.6212978800904426, + 0.176936615687169, 0.8817788599562331, 0.13699370230879637, 0.5754950980437555, 0.1507294463182668, + 0.23409699287029495, 0.6949148063429461, 0.47140569181488556, 0.1470896240551064, 0.8475557222612405, + 0.05957485472498203, 0.07490903723892406, 0.8412381196460251, 0.26874846387453943, 0.13669341206289243, + 0.8525684329438777, 0.46716360402752777, 0.8522055745422484, 0.3129394551398561, 0.908966336417204, + 0.26259461196353984, 0.07245314277889847, 0.41429401839807156, 0.22772860293274222, 0.26662443208488784, + 0.9875655504027848, 0.5832266083052889, 0.24205847206862052, 0.9843760682096272, 0.16269008279311103, + 0.4941250734508458, 0.5446841276322587, 0.19222703209695946, 0.9232239752817498, 0.8824688635063289, + 0.224690851359456, 0.5809304720756304, 0.36863807988348585] + constraint_eta0 = [0.1258925] + constraint_tau = [15] + constraint_alpha = [0.01] + constraint_beta = [0.1] + constraint_c0 = [15, 20] + h2o_glm_random_init = utils_for_glm_tests.constraint_glm_gridsearch(train, predictors, response, solver="IRLSM", + family="binomial", + linear_constraints=linear_constraints2, + startval=random_coef, + constraint_eta0=constraint_eta0, + constraint_tau=constraint_tau, + constraint_alpha=constraint_alpha, + constraint_beta=constraint_beta, + constraint_c0=constraint_c0, + return_best=False) + init_random_logloss = h2o_glm_random_init.model_performance()._metric_json['logloss'] + print("**** logloss with random initial values: {0}, iterations: {1}." + "".format(init_random_logloss, h2o_glm_random_init._model_json["output"]["model_summary"].cell_values[0][6])) + print(glm.getConstraintsInfo(h2o_glm_random_init)) + + # GLM model with GLM coefficients with default initialization + h2o_glm_default_init = utils_for_glm_tests.constraint_glm_gridsearch(train, predictors, response, solver="IRLSM", + family="binomial", + linear_constraints=linear_constraints2, + constraint_eta0=constraint_eta0, + constraint_tau=constraint_tau, + constraint_alpha=constraint_alpha, + constraint_beta=constraint_beta, + constraint_c0=constraint_c0, + return_best=False) + default_init_logloss = h2o_glm_default_init.model_performance()._metric_json['logloss'] + print("logloss with default coefficients initialization: {0}, iterations: {1}." + "".format(default_init_logloss, h2o_glm_default_init._model_json["output"]["model_summary"].cell_values[0][6])) + print(glm.getConstraintsInfo(h2o_glm_default_init)) + + assert init_random_logloss >= logloss, "Random initialization logloss with constraints should be worst than GLM " \ + "without constraints but is not." + + +if __name__ == "__main__": + pyunit_utils.standalone_test(test_constraints_binomial) +else: + test_constraints_binomial() diff --git a/h2o-py/tests/testdir_algos/glm/pyunit_GH_6722_equality_loose_lessthan_linear_constraints_binomial.py b/h2o-py/tests/testdir_algos/glm/pyunit_GH_6722_equality_loose_lessthan_linear_constraints_binomial.py new file mode 100644 index 000000000000..acf74d648dea --- /dev/null +++ b/h2o-py/tests/testdir_algos/glm/pyunit_GH_6722_equality_loose_lessthan_linear_constraints_binomial.py @@ -0,0 +1,173 @@ +import h2o +from h2o.estimators.glm import H2OGeneralizedLinearEstimator as glm +from tests import pyunit_utils +from tests.pyunit_utils import utils_for_glm_tests + +def test_constraints_binomial(): + ''' + This test checks and make sure the equality constraints work with binomial family. Coefficients are initialized + with glm coefficients built without constraints, default coefficients and random coefficients. Note in this case, + coefficients from glm built without constraints will satisfy the equality constraints. + ''' + train = h2o.import_file(path=pyunit_utils.locate("smalldata/glm_test/binomial_20_cols_10KRows.csv")) + for ind in range(10): + train[ind] = train[ind].asfactor() + train["C21"] = train["C21"].asfactor() + response = "C21" + predictors = list(range(0,20)) + + loose_init_const = [] # this constraint is satisfied by default coefficient initialization + + h2o_glm = glm(family="binomial", remove_collinear_columns=True, lambda_=0.0, solver="irlsm", seed=12345, + standardize=True) + h2o_glm.train(x=predictors, y=response, training_frame=train) + logloss = h2o_glm.model_performance()._metric_json['logloss'] + print("logloss with no constraints: {0}".format(logloss)) + + # add loose constraints + name = "C19" + values = 0.5 + types = "Equal" + contraint_numbers = 0 + loose_init_const.append([name, values, types, contraint_numbers]) + + name = "C10.1" + values = -0.3 + types = "Equal" + contraint_numbers = 0 + loose_init_const.append([name, values, types, contraint_numbers]) + + name = "constant" + values = -1.0044392439227687 + types = "Equal" + contraint_numbers = 0 + loose_init_const.append([name, values, types, contraint_numbers]) + + name = "C18" + values = 0.75 + types = "Equal" + contraint_numbers = 1 + loose_init_const.append([name, values, types, contraint_numbers]) + + name = "C20" + values = -0.13 + types = "Equal" + contraint_numbers = 1 + loose_init_const.append([name, values, types, contraint_numbers]) + + name = "constant" + values = -6.941462080282776 + types = "Equal" + contraint_numbers = 1 + loose_init_const.append([name, values, types, contraint_numbers]) + + # add loose constraints + name = "C19" + values = 0.5 + types = "LessThanEqual" + contraint_numbers = 2 + loose_init_const.append([name, values, types, contraint_numbers]) + + name = "C20" + values = -0.8 + types = "LessThanEqual" + contraint_numbers = 2 + loose_init_const.append([name, values, types, contraint_numbers]) + + name = "constant" + values = -10 + types = "LessThanEqual" + contraint_numbers = 2 + loose_init_const.append([name, values, types, contraint_numbers]) + + name = "C12" + values = 2 + types = "LessThanEqual" + contraint_numbers = 3 + loose_init_const.append([name, values, types, contraint_numbers]) + + name = "C13" + values = -3 + types = "LessThanEqual" + contraint_numbers = 3 + loose_init_const.append([name, values, types, contraint_numbers]) + + name = "constant" + values = -36 + types = "LessThanEqual" + contraint_numbers = 3 + loose_init_const.append([name, values, types, contraint_numbers]) + + linear_constraints2 = h2o.H2OFrame(loose_init_const) + linear_constraints2.set_names(["names", "values", "types", "constraint_numbers"]) + # GLM model with with GLM coefficients set to GLM model coefficients built without constraints + h2o_glm_optimal_init = glm(family="binomial", lambda_=0.0, seed=12345, remove_collinear_columns=True,solver="irlsm", + linear_constraints=linear_constraints2, init_optimal_glm=True) + h2o_glm_optimal_init.train(x=predictors, y=response, training_frame=train) + init_logloss = h2o_glm_optimal_init.model_performance()._metric_json['logloss'] + print("logloss with constraints and coefficients initialized with glm model built without constraints: {0}".format(init_logloss)) + print(glm.getConstraintsInfo(h2o_glm_optimal_init)) + + # GLM model with with GLM coefficients set to random GLM model coefficients + random_coef = [0.9740393731418461, 0.9021970400494406, 0.8337282995102272, 0.20588758679724872, 0.12522385214612453, + 0.6390730524643073, 0.7055779213989253, 0.9004255614099713, 0.4075431157767999, 0.161093231584713, + 0.15250197544465616, 0.7172682822215489, 0.60836236371404, 0.07086628306822396, 0.263719138602719, + 0.16102036359390437, 0.0065987448849305075, 0.5881312311814277, 0.7836567678399617, 0.9104401158881326, + 0.8432891635016235, 0.033440093086177236, 0.8514611306363931, 0.2855332934628241, 0.36525972112514427, + 0.7526593301495519, 0.9963694184200753, 0.5614168317678196, 0.7950126291921057, 0.6212978800904426, + 0.176936615687169, 0.8817788599562331, 0.13699370230879637, 0.5754950980437555, 0.1507294463182668, + 0.23409699287029495, 0.6949148063429461, 0.47140569181488556, 0.1470896240551064, 0.8475557222612405, + 0.05957485472498203, 0.07490903723892406, 0.8412381196460251, 0.26874846387453943, 0.13669341206289243, + 0.8525684329438777, 0.46716360402752777, 0.8522055745422484, 0.3129394551398561, 0.908966336417204, + 0.26259461196353984, 0.07245314277889847, 0.41429401839807156, 0.22772860293274222, 0.26662443208488784, + 0.9875655504027848, 0.5832266083052889, 0.24205847206862052, 0.9843760682096272, 0.16269008279311103, + 0.4941250734508458, 0.5446841276322587, 0.19222703209695946, 0.9232239752817498, 0.8824688635063289, + 0.224690851359456, 0.5809304720756304, 0.36863807988348585] + constraint_eta0 = [0.1258925] + constraint_tau = [50, 60] + constraint_alpha = [0.1] + constraint_beta = [0.9] + constraint_c0 = [10] # initial value + h2o_glm_random_init = utils_for_glm_tests.constraint_glm_gridsearch(train, predictors, response, solver="IRLSM", + family="binomial", + linear_constraints=linear_constraints2, + startval=random_coef, + constraint_eta0=constraint_eta0, + constraint_tau=constraint_tau, + constraint_alpha=constraint_alpha, + constraint_beta=constraint_beta, + constraint_c0=constraint_c0, return_best=False) + init_random_logloss = h2o_glm_random_init.model_performance()._metric_json['logloss'] + print("logloss with constraints and coefficients initialized random initial values: {0}, number of iterations" + " taken to build the model: {1}".format(init_random_logloss, utils_for_glm_tests.find_glm_iterations(h2o_glm_random_init))) + print(glm.getConstraintsInfo(h2o_glm_random_init)) + + # GLM model with GLM coefficients with default initialization + h2o_glm_default_init = utils_for_glm_tests.constraint_glm_gridsearch(train, predictors, response, solver="IRLSM", + family="binomial", + linear_constraints=linear_constraints2, + constraint_eta0=constraint_eta0, + constraint_tau=constraint_tau, + constraint_alpha=constraint_alpha, + constraint_beta=constraint_beta, + constraint_c0=constraint_c0, return_best=False) + default_init_logloss = h2o_glm_default_init.model_performance()._metric_json['logloss'] + print("logloss with constraints and default coefficients initialization: {0}, number of iterations" + " taken to build the model: {1}".format(default_init_logloss, utils_for_glm_tests.find_glm_iterations(h2o_glm_default_init))) + print(glm.getConstraintsInfo(h2o_glm_default_init)) + + assert abs(logloss-init_logloss)<2e-6, "logloss from optimal GLM {0} and logloss from GLM with loose constraints " \ + "and initialized with optimal GLM {1} should equal but is not." \ + "".format(logloss, init_logloss) + assert logloss<=init_random_logloss, "logloss from optimal GLM {0} should be lower than GLM with constraints " \ + "and with random initial coefficients {1} but is" \ + " not.".format(logloss, init_random_logloss) + assert logloss<=default_init_logloss, "logloss from optimal GLM {0} should be less than GLM with constraints " \ + "and with default initial coefficients {1} but is" \ + " not.".format(logloss, default_init_logloss) + + +if __name__ == "__main__": + pyunit_utils.standalone_test(test_constraints_binomial) +else: + test_constraints_binomial() diff --git a/h2o-py/tests/testdir_algos/glm/pyunit_GH_6722_glm_coefNames.py b/h2o-py/tests/testdir_algos/glm/pyunit_GH_6722_glm_coefNames.py new file mode 100644 index 000000000000..e7a692d0e859 --- /dev/null +++ b/h2o-py/tests/testdir_algos/glm/pyunit_GH_6722_glm_coefNames.py @@ -0,0 +1,33 @@ +import sys +sys.path.insert(1,"../../../") +import h2o +from tests import pyunit_utils +from h2o.estimators.glm import H2OGeneralizedLinearEstimator as glm + +# This test is used to obtain the coefficient names that can be used to specify constraints for constrained GLM. +def test_glm_coefNames(): + h2o_data = h2o.import_file( + path=pyunit_utils.locate("smalldata/glm_test/gaussian_20cols_10000Rows.csv")) + enum_columns = ["C1", "C2", "C3", "C4", "C5", "C6", "C7", "C8", "C9", "C10"] + for cname in enum_columns: + h2o_data[cname] = h2o_data[cname].asfactor() + myY = "C21" + myX = h2o_data.names.remove(myY) + model = glm(max_iterations=0) + model.train(x=myX, y=myY, training_frame=h2o_data) + original_coef_names = model.coef_names() + + model2 = glm() + model2.train(x=myX, y=myY, training_frame=h2o_data) + names_model = list(model2.coef().keys()) + names_model.remove('Intercept') + + # both coefficients should equal + assert original_coef_names == names_model, "Expected coefficients: {0}, actual: {1}".format(names_model, + original_coef_names) + + +if __name__ == "__main__": + pyunit_utils.standalone_test(test_glm_coefNames) +else: + test_glm_coefNames() diff --git a/h2o-py/tests/testdir_algos/glm/pyunit_GH_6722_light_tight_beta_equality_lessthan_constraints_binomial.py b/h2o-py/tests/testdir_algos/glm/pyunit_GH_6722_light_tight_beta_equality_lessthan_constraints_binomial.py new file mode 100644 index 000000000000..97ddf83bbe29 --- /dev/null +++ b/h2o-py/tests/testdir_algos/glm/pyunit_GH_6722_light_tight_beta_equality_lessthan_constraints_binomial.py @@ -0,0 +1,212 @@ +import h2o +from h2o.estimators.glm import H2OGeneralizedLinearEstimator as glm +from tests import pyunit_utils +from tests.pyunit_utils import utils_for_glm_tests + +def test_light_tight_linear_constraints_only_gaussian(): + ''' + Test constrained GLM with beta, equality and less than and equal to constraints. The constraints are not very + tight. However, coefficients from GLM built without constraints won't be able to satisfied the constraints. + Constrained GLM models are built with coefficients initialized with coefficients from GLM built without constraints, + default coefficients and random coefficients. + ''' + train = h2o.import_file(path=pyunit_utils.locate("smalldata/glm_test/binomial_20_cols_10KRows.csv")) + for ind in range(10): + train[ind] = train[ind].asfactor() + train["C21"] = train["C21"].asfactor() + response = "C21" + predictors = list(range(0,20)) + + light_tight_constraints = [] # this constraint is satisfied by default coefficient initialization + # add beta constraints + bc = [] + name = "C11" + lower_bound = -7 + upper_bound = 0 + bc.append([name, lower_bound, upper_bound]) + + name = "C18" + lower_bound = 7.5 + upper_bound = 8 + bc.append([name, lower_bound, upper_bound]) + + name = "C15" + lower_bound = -4.5 + upper_bound = 0 + bc.append([name, lower_bound, upper_bound]) + + name = "C16" + lower_bound = -9 + upper_bound = 0.3 + bc.append([name, lower_bound, upper_bound]) + + beta_constraints = h2o.H2OFrame(bc) + beta_constraints.set_names(["names", "lower_bounds", "upper_bounds"]) + + h2o_glm = glm(family="binomial", lambda_=0.0, solver="irlsm", seed=12345, standardize=True) + h2o_glm.train(x=predictors, y=response, training_frame=train) + logloss = h2o_glm.model_performance()._metric_json['logloss'] + print("logloss with no constraints: {0}".format(logloss)) + + # add light tight constraints + name = "C19" + values = 0.5 + types = "Equal" + contraint_numbers = 0 + light_tight_constraints.append([name, values, types, contraint_numbers]) + + name = "C10.1" + values = -0.3 + types = "Equal" + contraint_numbers = 0 + light_tight_constraints.append([name, values, types, contraint_numbers]) + + name = "constant" + values = -1.00 + types = "Equal" + contraint_numbers = 0 + light_tight_constraints.append([name, values, types, contraint_numbers]) + + name = "C18" + values = 0.75 + types = "Equal" + contraint_numbers = 1 + light_tight_constraints.append([name, values, types, contraint_numbers]) + + name = "C20" + values = -0.13 + types = "Equal" + contraint_numbers = 1 + light_tight_constraints.append([name, values, types, contraint_numbers]) + + name = "constant" + values = -6.9 + types = "Equal" + contraint_numbers = 1 + light_tight_constraints.append([name, values, types, contraint_numbers]) + + # add loose constraints + name = "C19" + values = 0.5 + types = "LessThanEqual" + contraint_numbers = 2 + light_tight_constraints.append([name, values, types, contraint_numbers]) + + name = "C20" + values = -0.8 + types = "LessThanEqual" + contraint_numbers = 2 + light_tight_constraints.append([name, values, types, contraint_numbers]) + + name = "constant" + values = -6 + types = "LessThanEqual" + contraint_numbers = 2 + light_tight_constraints.append([name, values, types, contraint_numbers]) + + name = "C12" + values = 2 + types = "LessThanEqual" + contraint_numbers = 3 + light_tight_constraints.append([name, values, types, contraint_numbers]) + + name = "C13" + values = -3 + types = "LessThanEqual" + contraint_numbers = 3 + light_tight_constraints.append([name, values, types, contraint_numbers]) + + name = "constant" + values = -21 + types = "LessThanEqual" + contraint_numbers = 3 + light_tight_constraints.append([name, values, types, contraint_numbers]) + + linear_constraints2 = h2o.H2OFrame(light_tight_constraints) + linear_constraints2.set_names(["names", "values", "types", "constraint_numbers"]) + + # GLM model with GLM coefficients with default initialization + constraint_eta0 = [0.1258925] + constraint_tau = [1.2, 1.5] + constraint_alpha = [0.1] + constraint_beta = [0.9] + constraint_c0 = [5, 10] # initial value + # GLM model with with GLM coefficients set to GLM model coefficients built without constraints + h2o_glm_optimal_init = utils_for_glm_tests.constraint_glm_gridsearch(train, predictors, response, solver="IRLSM", + family="binomial", + linear_constraints=linear_constraints2, + beta_constraints=beta_constraints, + init_optimal_glm=True, + constraint_eta0=constraint_eta0, + constraint_tau=constraint_tau, + constraint_alpha=constraint_alpha, + constraint_beta=constraint_beta, + constraint_c0=constraint_c0, + return_best=False, epsilon=0.5) + optimal_init_logloss = h2o_glm_optimal_init.model_performance()._metric_json['logloss'] + print("logloss with optimal GLM coefficient initializaiton: {0}, number of iterations taken to build the model: " + "{1}".format(optimal_init_logloss, utils_for_glm_tests.find_glm_iterations(h2o_glm_optimal_init))) + print(glm.getConstraintsInfo(h2o_glm_optimal_init)) + + h2o_glm_default_init = utils_for_glm_tests.constraint_glm_gridsearch(train, predictors, response, solver="IRLSM", + family="binomial", + linear_constraints=linear_constraints2, + beta_constraints=beta_constraints, + init_optimal_glm=False, + constraint_eta0=constraint_eta0, + constraint_tau=constraint_tau, + constraint_alpha=constraint_alpha, + constraint_beta=constraint_beta, + constraint_c0=constraint_c0, + return_best=False, epsilon=0.5) + default_init_logloss = h2o_glm_default_init.model_performance()._metric_json['logloss'] + print("logloss with default GLM coefficient initializaiton: {0}, number of iterations taken to build the model: " + "{1}".format(default_init_logloss, utils_for_glm_tests.find_glm_iterations(h2o_glm_default_init))) + print(glm.getConstraintsInfo(h2o_glm_default_init)) + random_coef = [0.9740393731418461, 0.9021970400494406, 0.8337282995102272, 0.20588758679724872, 0.12522385214612453, + 0.6390730524643073, 0.7055779213989253, 0.9004255614099713, 0.4075431157767999, 0.161093231584713, + 0.15250197544465616, 0.7172682822215489, 0.60836236371404, 0.07086628306822396, 0.263719138602719, + 0.16102036359390437, 0.0065987448849305075, 0.5881312311814277, 0.7836567678399617, 0.9104401158881326, + 0.8432891635016235, 0.033440093086177236, 0.8514611306363931, 0.2855332934628241, 0.36525972112514427, + 0.7526593301495519, 0.9963694184200753, 0.5614168317678196, 0.7950126291921057, 0.6212978800904426, + 0.176936615687169, 0.8817788599562331, 0.13699370230879637, 0.5754950980437555, 0.1507294463182668, + 0.23409699287029495, 0.6949148063429461, 0.47140569181488556, 0.1470896240551064, 0.8475557222612405, + 0.05957485472498203, 0.07490903723892406, 0.8412381196460251, 0.26874846387453943, 0.13669341206289243, + 0.8525684329438777, 0.46716360402752777, 0.8522055745422484, 0.3129394551398561, 0.908966336417204, + 0.26259461196353984, 0.07245314277889847, 0.41429401839807156, 0.22772860293274222, 0.26662443208488784, + 0.9875655504027848, 0.5832266083052889, 0.24205847206862052, 0.9843760682096272, 0.16269008279311103, + 0.4941250734508458, 0.5446841276322587, 0.19222703209695946, 0.9232239752817498, 0.8824688635063289, + 0.224690851359456, 0.5809304720756304, 0.36863807988348585] + + h2o_glm_random_init = utils_for_glm_tests.constraint_glm_gridsearch(train, predictors, response, solver="IRLSM", + family="binomial", + linear_constraints=linear_constraints2, + beta_constraints=beta_constraints, + startval=random_coef, + init_optimal_glm=False, + constraint_eta0=constraint_eta0, + constraint_tau=constraint_tau, + constraint_alpha=constraint_alpha, + constraint_beta=constraint_beta, + constraint_c0=constraint_c0, + return_best=False, epsilon=0.5) + random_init_logloss = h2o_glm_random_init.model_performance()._metric_json['logloss'] + print("logloss with random GLM coefficient initializaiton: {0}, number of iterations taken to build the model: " + "{1}".format(random_init_logloss, utils_for_glm_tests.find_glm_iterations(h2o_glm_random_init))) + print(glm.getConstraintsInfo(h2o_glm_random_init)) + + assert logloss <= optimal_init_logloss, "logloss from optimal GLM {0} should be lower than logloss from GLM with light tight" \ + " constraints and initialized with optimal GLM {1} but is not.".format(logloss, optimal_init_logloss) + + assert logloss <= default_init_logloss, "logloss from optimal GLM {0} should be lower than logloss from GLM with light tight" \ + " constraints and initialized with default coefficients GLM {1} but is " \ + "not.".format(logloss, default_init_logloss) + + assert logloss <= random_init_logloss, "logloss from optimal GLM {0} should be lower than logloss from GLM with light tight" \ + " constraints and initialized with random coefficients GLM {1} but is " \ + "not.".format(logloss, random_init_logloss) + +if __name__ == "__main__": + pyunit_utils.standalone_test(test_light_tight_linear_constraints_only_gaussian) +else: + test_light_tight_linear_constraints_only_gaussian() diff --git a/h2o-py/tests/testdir_algos/glm/pyunit_GH_6722_light_tight_equality_lessthan_constraints_binomial.py b/h2o-py/tests/testdir_algos/glm/pyunit_GH_6722_light_tight_equality_lessthan_constraints_binomial.py new file mode 100644 index 000000000000..96146d98234a --- /dev/null +++ b/h2o-py/tests/testdir_algos/glm/pyunit_GH_6722_light_tight_equality_lessthan_constraints_binomial.py @@ -0,0 +1,190 @@ +import h2o +from h2o.estimators.glm import H2OGeneralizedLinearEstimator as glm +from tests import pyunit_utils +from tests.pyunit_utils import utils_for_glm_tests + +def test_light_tight_linear_constraints_only_gaussian(): + ''' + Test constrained GLM with equality and less than and equal to constraints. The constraints are not very + tight. However, coefficients from GLM built without constraints won't be able to satisfied the constraints. + Constrained GLM models are built with coefficients initialized with coefficients from GLM built without constraints, + default coefficients and random coefficients. + ''' + train = h2o.import_file(path=pyunit_utils.locate("smalldata/glm_test/binomial_20_cols_10KRows.csv")) + for ind in range(10): + train[ind] = train[ind].asfactor() + train["C21"] = train["C21"].asfactor() + response = "C21" + predictors = list(range(0,20)) + + light_tight_constraints = [] # this constraint is satisfied by default coefficient initialization + + h2o_glm = glm(family="binomial", lambda_=0.0, solver="irlsm", seed=12345, standardize=True) + h2o_glm.train(x=predictors, y=response, training_frame=train) + logloss = h2o_glm.model_performance()._metric_json['logloss'] + print("logloss with no constraints: {0}".format(logloss)) + + # add light tight constraints + name = "C19" + values = 0.5 + types = "Equal" + contraint_numbers = 0 + light_tight_constraints.append([name, values, types, contraint_numbers]) + + name = "C10.1" + values = -0.3 + types = "Equal" + contraint_numbers = 0 + light_tight_constraints.append([name, values, types, contraint_numbers]) + + name = "constant" + values = -1.00 + types = "Equal" + contraint_numbers = 0 + light_tight_constraints.append([name, values, types, contraint_numbers]) + + name = "C18" + values = 0.75 + types = "Equal" + contraint_numbers = 1 + light_tight_constraints.append([name, values, types, contraint_numbers]) + + name = "C20" + values = -0.13 + types = "Equal" + contraint_numbers = 1 + light_tight_constraints.append([name, values, types, contraint_numbers]) + + name = "constant" + values = -6.9 + types = "Equal" + contraint_numbers = 1 + light_tight_constraints.append([name, values, types, contraint_numbers]) + + # add loose constraints + name = "C19" + values = 0.5 + types = "LessThanEqual" + contraint_numbers = 2 + light_tight_constraints.append([name, values, types, contraint_numbers]) + + name = "C20" + values = -0.8 + types = "LessThanEqual" + contraint_numbers = 2 + light_tight_constraints.append([name, values, types, contraint_numbers]) + + name = "constant" + values = -6 + types = "LessThanEqual" + contraint_numbers = 2 + light_tight_constraints.append([name, values, types, contraint_numbers]) + + name = "C12" + values = 2 + types = "LessThanEqual" + contraint_numbers = 3 + light_tight_constraints.append([name, values, types, contraint_numbers]) + + name = "C13" + values = -3 + types = "LessThanEqual" + contraint_numbers = 3 + light_tight_constraints.append([name, values, types, contraint_numbers]) + + name = "constant" + values = -21 + types = "LessThanEqual" + contraint_numbers = 3 + light_tight_constraints.append([name, values, types, contraint_numbers]) + linear_constraints2 = h2o.H2OFrame(light_tight_constraints) + linear_constraints2.set_names(["names", "values", "types", "constraint_numbers"]) + + + linear_constraints2 = h2o.H2OFrame(light_tight_constraints) + linear_constraints2.set_names(["names", "values", "types", "constraint_numbers"]) + + # GLM model with GLM coefficients with default initialization + constraint_eta0 = [0.1258925] + constraint_tau = [1.2] + constraint_alpha = [0.1] + constraint_beta = [0.9] + constraint_c0 = [10, 20] # initial value + # GLM model with with GLM coefficients set to GLM model coefficients built without constraints + h2o_glm_optimal_init = utils_for_glm_tests.constraint_glm_gridsearch(train, predictors, response, solver="IRLSM", + family="binomial", + linear_constraints=linear_constraints2, + init_optimal_glm=True, + constraint_eta0=constraint_eta0, + constraint_tau=constraint_tau, + constraint_alpha=constraint_alpha, + constraint_beta=constraint_beta, + constraint_c0=constraint_c0, + return_best=False) + optimal_init_logloss = h2o_glm_optimal_init.model_performance()._metric_json['logloss'] + print("logloss with optimal GLM coefficient initializaiton: {0}, number of iterations taken to build the model: " + "{1}".format(optimal_init_logloss, utils_for_glm_tests.find_glm_iterations(h2o_glm_optimal_init))) + print(glm.getConstraintsInfo(h2o_glm_optimal_init)) + + h2o_glm_default_init = utils_for_glm_tests.constraint_glm_gridsearch(train, predictors, response, solver="IRLSM", + family="binomial", + linear_constraints=linear_constraints2, + init_optimal_glm=False, + constraint_eta0=constraint_eta0, + constraint_tau=constraint_tau, + constraint_alpha=constraint_alpha, + constraint_beta=constraint_beta, + constraint_c0=constraint_c0, + return_best=False, + epsilon=5e-1) + default_init_logloss = h2o_glm_default_init.model_performance()._metric_json['logloss'] + print("logloss with default GLM coefficient initializaiton: {0}, number of iterations taken to build the model: " + "{1}".format(default_init_logloss, utils_for_glm_tests.find_glm_iterations(h2o_glm_default_init))) + print(glm.getConstraintsInfo(h2o_glm_default_init)) + random_coef = [0.9740393731418461, 0.9021970400494406, 0.8337282995102272, 0.20588758679724872, 0.12522385214612453, + 0.6390730524643073, 0.7055779213989253, 0.9004255614099713, 0.4075431157767999, 0.161093231584713, + 0.15250197544465616, 0.7172682822215489, 0.60836236371404, 0.07086628306822396, 0.263719138602719, + 0.16102036359390437, 0.0065987448849305075, 0.5881312311814277, 0.7836567678399617, 0.9104401158881326, + 0.8432891635016235, 0.033440093086177236, 0.8514611306363931, 0.2855332934628241, 0.36525972112514427, + 0.7526593301495519, 0.9963694184200753, 0.5614168317678196, 0.7950126291921057, 0.6212978800904426, + 0.176936615687169, 0.8817788599562331, 0.13699370230879637, 0.5754950980437555, 0.1507294463182668, + 0.23409699287029495, 0.6949148063429461, 0.47140569181488556, 0.1470896240551064, 0.8475557222612405, + 0.05957485472498203, 0.07490903723892406, 0.8412381196460251, 0.26874846387453943, 0.13669341206289243, + 0.8525684329438777, 0.46716360402752777, 0.8522055745422484, 0.3129394551398561, 0.908966336417204, + 0.26259461196353984, 0.07245314277889847, 0.41429401839807156, 0.22772860293274222, 0.26662443208488784, + 0.9875655504027848, 0.5832266083052889, 0.24205847206862052, 0.9843760682096272, 0.16269008279311103, + 0.4941250734508458, 0.5446841276322587, 0.19222703209695946, 0.9232239752817498, 0.8824688635063289, + 0.224690851359456, 0.5809304720756304, 0.36863807988348585] + + h2o_glm_random_init = utils_for_glm_tests.constraint_glm_gridsearch(train, predictors, response, solver="IRLSM", + family="binomial", + linear_constraints=linear_constraints2, + startval=random_coef, + init_optimal_glm=False, + constraint_eta0=constraint_eta0, + constraint_tau=constraint_tau, + constraint_alpha=constraint_alpha, + constraint_beta=constraint_beta, + constraint_c0=constraint_c0, + return_best=False, + epsilon=5e-1) + random_init_logloss = h2o_glm_random_init.model_performance()._metric_json['logloss'] + print("logloss with random GLM coefficient initializaiton: {0}, number of iterations taken to build the model: " + "{1}".format(random_init_logloss, utils_for_glm_tests.find_glm_iterations(h2o_glm_random_init))) + print(glm.getConstraintsInfo(h2o_glm_random_init)) + + assert logloss <= optimal_init_logloss, "logloss from optimal GLM {0} should be lower than logloss from GLM with light tight" \ + " constraints and initialized with optimal GLM {1} but is not.".format(logloss, optimal_init_logloss) + + assert logloss <= default_init_logloss, "logloss from optimal GLM {0} should be lower than logloss from GLM with light tight" \ + " constraints and initialized with default coefficients GLM {1} but is " \ + "not.".format(logloss, default_init_logloss) + + assert logloss <= random_init_logloss, "logloss from optimal GLM {0} should be lower than logloss from GLM with light tight" \ + " constraints and initialized with random coefficients GLM {1} but is " \ + "not.".format(logloss, random_init_logloss) + +if __name__ == "__main__": + pyunit_utils.standalone_test(test_light_tight_linear_constraints_only_gaussian) +else: + test_light_tight_linear_constraints_only_gaussian() diff --git a/h2o-py/tests/testdir_algos/glm/pyunit_GH_6722_light_tight_linear_constraints_only_binomial.py b/h2o-py/tests/testdir_algos/glm/pyunit_GH_6722_light_tight_linear_constraints_only_binomial.py new file mode 100644 index 000000000000..103086f4ca6f --- /dev/null +++ b/h2o-py/tests/testdir_algos/glm/pyunit_GH_6722_light_tight_linear_constraints_only_binomial.py @@ -0,0 +1,206 @@ +import h2o +from h2o.estimators.glm import H2OGeneralizedLinearEstimator as glm +from tests import pyunit_utils +from tests.pyunit_utils import utils_for_glm_tests + +def test_light_tight_linear_constraints_only_gaussian(): + ''' + Test constrained GLM with less than and equal to constraints. The constraints are not very + tight. However, coefficients from GLM built without constraints won't be able to satisfied the constraints. + Constrained GLM models are built with coefficients initialized with coefficients from GLM built without constraints, + default coefficients and random coefficients. + ''' + train = h2o.import_file(path=pyunit_utils.locate("smalldata/glm_test/binomial_20_cols_10KRows.csv")) + for ind in range(10): + train[ind] = train[ind].asfactor() + train["C21"] = train["C21"].asfactor() + response = "C21" + predictors = list(range(0,20)) + + light_tight_constraints = [] # this constraint is satisfied by default coefficient initialization + + h2o_glm = glm(family="binomial", lambda_=0.0, solver="irlsm", seed=12345, standardize=True) + h2o_glm.train(x=predictors, y=response, training_frame=train) + logloss = h2o_glm.model_performance()._metric_json['logloss'] + print("logloss with no constraints: {0}".format(logloss)) + + # add light tight constraints + name = "C1.1" + values = 0.5 + types = "LessThanEqual" + contraint_numbers = 0 + light_tight_constraints.append([name, values, types, contraint_numbers]) + + name = "C2.1" + values = -0.25 + types = "LessThanEqual" + contraint_numbers = 0 + light_tight_constraints.append([name, values, types, contraint_numbers]) + + name = "constant" + values = -1.9 + types = "LessThanEqual" + contraint_numbers = 0 + light_tight_constraints.append([name, values, types, contraint_numbers]) + + name = "C4.1" + values = 1.5 + types = "LessThanEqual" + contraint_numbers = 1 + light_tight_constraints.append([name, values, types, contraint_numbers]) + + name = "C17" + values = 3 + types = "LessThanEqual" + contraint_numbers = 1 + light_tight_constraints.append([name, values, types, contraint_numbers]) + + name = "C15" + values = -2 + types = "LessThanEqual" + contraint_numbers = 1 + light_tight_constraints.append([name, values, types, contraint_numbers]) + + name = "constant" + values = -11 + types = "LessThanEqual" + contraint_numbers = 1 + light_tight_constraints.append([name, values, types, contraint_numbers]) + + name = "C12" + values = -0.5 + types = "LessThanEqual" + contraint_numbers = 2 + light_tight_constraints.append([name, values, types, contraint_numbers]) + + name = "C13" + values = -1.5 + types = "LessThanEqual" + contraint_numbers = 2 + light_tight_constraints.append([name, values, types, contraint_numbers]) + + name = "C14" + values = 2 + types = "LessThanEqual" + contraint_numbers = 2 + light_tight_constraints.append([name, values, types, contraint_numbers]) + + name = "constant" + values = -7.5 + types = "LessThanEqual" + contraint_numbers = 2 + light_tight_constraints.append([name, values, types, contraint_numbers]) + + name = "C11" + values = 0.25 + types = "LessThanEqual" + contraint_numbers = 3 + light_tight_constraints.append([name, values, types, contraint_numbers]) + + name = "C18" + values = -0.5 + types = "LessThanEqual" + contraint_numbers = 3 + light_tight_constraints.append([name, values, types, contraint_numbers]) + + name = "C19" + values = 0.75 + types = "LessThanEqual" + contraint_numbers = 3 + light_tight_constraints.append([name, values, types, contraint_numbers]) + + name = "constant" + values = 10 + types = "LessThanEqual" + contraint_numbers = 3 + light_tight_constraints.append([name, values, types, contraint_numbers]) + + linear_constraints2 = h2o.H2OFrame(light_tight_constraints) + linear_constraints2.set_names(["names", "values", "types", "constraint_numbers"]) + + # GLM model with GLM coefficients with default initialization + constraint_eta0 = [0.1258925] + constraint_tau = [1.2, 1.5] + constraint_alpha = [0.1] + constraint_beta = [0.9] + constraint_c0 = [1.2, 5] # initial value + # GLM model with with GLM coefficients set to GLM model coefficients built without constraints + h2o_glm_optimal_init = utils_for_glm_tests.constraint_glm_gridsearch(train, predictors, response, solver="IRLSM", + family="binomial", + linear_constraints=linear_constraints2, + init_optimal_glm=True, + constraint_eta0=constraint_eta0, + constraint_tau=constraint_tau, + constraint_alpha=constraint_alpha, + constraint_beta=constraint_beta, + constraint_c0=constraint_c0, + return_best=False) + optimal_init_logloss = h2o_glm_optimal_init.model_performance()._metric_json['logloss'] + print("logloss with optimal GLM coefficient initializaiton: {0}, number of iterations taken to build the model: " + "{1}".format(optimal_init_logloss, utils_for_glm_tests.find_glm_iterations(h2o_glm_optimal_init))) + print(glm.getConstraintsInfo(h2o_glm_optimal_init)) + print("All constraints satisfied: {0}".format(glm.allConstraintsPassed(h2o_glm_optimal_init))) + + h2o_glm_default_init = utils_for_glm_tests.constraint_glm_gridsearch(train, predictors, response, solver="IRLSM", + family="binomial", + linear_constraints=linear_constraints2, + init_optimal_glm=False, + constraint_eta0=constraint_eta0, + constraint_tau=constraint_tau, + constraint_alpha=constraint_alpha, + constraint_beta=constraint_beta, + constraint_c0=constraint_c0, + return_best=False) + default_init_logloss = h2o_glm_default_init.model_performance()._metric_json['logloss'] + print("logloss with default GLM coefficient initializaiton: {0}, number of iterations taken to build the model: " + "{1}".format(default_init_logloss, utils_for_glm_tests.find_glm_iterations(h2o_glm_default_init))) + print(glm.getConstraintsInfo(h2o_glm_default_init)) + print("All constraints satisfied: {0}".format(glm.allConstraintsPassed(h2o_glm_default_init))) + + random_coef = [0.9740393731418461, 0.9021970400494406, 0.8337282995102272, 0.20588758679724872, 0.12522385214612453, + 0.6390730524643073, 0.7055779213989253, 0.9004255614099713, 0.4075431157767999, 0.161093231584713, + 0.15250197544465616, 0.7172682822215489, 0.60836236371404, 0.07086628306822396, 0.263719138602719, + 0.16102036359390437, 0.0065987448849305075, 0.5881312311814277, 0.7836567678399617, 0.9104401158881326, + 0.8432891635016235, 0.033440093086177236, 0.8514611306363931, 0.2855332934628241, 0.36525972112514427, + 0.7526593301495519, 0.9963694184200753, 0.5614168317678196, 0.7950126291921057, 0.6212978800904426, + 0.176936615687169, 0.8817788599562331, 0.13699370230879637, 0.5754950980437555, 0.1507294463182668, + 0.23409699287029495, 0.6949148063429461, 0.47140569181488556, 0.1470896240551064, 0.8475557222612405, + 0.05957485472498203, 0.07490903723892406, 0.8412381196460251, 0.26874846387453943, 0.13669341206289243, + 0.8525684329438777, 0.46716360402752777, 0.8522055745422484, 0.3129394551398561, 0.908966336417204, + 0.26259461196353984, 0.07245314277889847, 0.41429401839807156, 0.22772860293274222, 0.26662443208488784, + 0.9875655504027848, 0.5832266083052889, 0.24205847206862052, 0.9843760682096272, 0.16269008279311103, + 0.4941250734508458, 0.5446841276322587, 0.19222703209695946, 0.9232239752817498, 0.8824688635063289, + 0.224690851359456, 0.5809304720756304, 0.36863807988348585] + + h2o_glm_random_init = utils_for_glm_tests.constraint_glm_gridsearch(train, predictors, response, solver="IRLSM", + family="binomial", + linear_constraints=linear_constraints2, + startval=random_coef, + init_optimal_glm=False, + constraint_eta0=constraint_eta0, + constraint_tau=constraint_tau, + constraint_alpha=constraint_alpha, + constraint_beta=constraint_beta, + constraint_c0=constraint_c0, + return_best=False) + random_init_logloss = h2o_glm_random_init.model_performance()._metric_json['logloss'] + print("logloss with random GLM coefficient initializaiton: {0}, number of iterations taken to build the model: " + "{1}".format(random_init_logloss, utils_for_glm_tests.find_glm_iterations(h2o_glm_random_init))) + print(glm.getConstraintsInfo(h2o_glm_random_init)) + print("All constraints satisfied: {0}".format(glm.allConstraintsPassed(h2o_glm_random_init))) + + assert logloss <= optimal_init_logloss, "logloss from optimal GLM {0} should be lower than logloss from GLM with light tight" \ + " constraints and initialized with optimal GLM {1} but is not.".format(logloss, optimal_init_logloss) + + assert logloss <= default_init_logloss, "logloss from optimal GLM {0} should be lower than logloss from GLM with light tight" \ + " constraints and initialized with default coefficients GLM {1} but is " \ + "not.".format(logloss, default_init_logloss) + + assert logloss <= random_init_logloss, "logloss from optimal GLM {0} should be lower than logloss from GLM with light tight" \ + " constraints and initialized with random coefficients GLM {1} but is " \ + "not.".format(logloss, random_init_logloss) + +if __name__ == "__main__": + pyunit_utils.standalone_test(test_light_tight_linear_constraints_only_gaussian) +else: + test_light_tight_linear_constraints_only_gaussian() diff --git a/h2o-py/tests/testdir_algos/glm/pyunit_GH_6722_linear_constraints_error.py b/h2o-py/tests/testdir_algos/glm/pyunit_GH_6722_linear_constraints_error.py new file mode 100644 index 000000000000..fda0353d34cb --- /dev/null +++ b/h2o-py/tests/testdir_algos/glm/pyunit_GH_6722_linear_constraints_error.py @@ -0,0 +1,31 @@ +import h2o +from h2o.estimators.glm import H2OGeneralizedLinearEstimator +from tests import pyunit_utils + +# this test will try to specify a linear constraints of only one coefficients and this should throw an error. +def test_bad_linear_constraints(): + h2o_data = h2o.import_file( + path=pyunit_utils.locate("smalldata/glm_test/gaussian_20cols_10000Rows.csv")) + enum_columns = ["C1", "C2", "C3", "C4", "C5", "C6", "C7", "C8", "C9", "C10"] + for cname in enum_columns: + h2o_data[cname] = h2o_data[cname].asfactor() + myY = "C21" + myX = h2o_data.names.remove(myY) + + dictLinearBounds = {'names': ["C11", "constant"], 'types':['Equal', 'Equal'], 'values': [0.5, -1.5], + 'constraint_numbers': [0, 0]} + linearConstraints = h2o.H2OFrame(dictLinearBounds) + linearConstraints = linearConstraints[["names", "types", "values", "constraint_numbers"]] + try: + model = H2OGeneralizedLinearEstimator(linear_constraints = linearConstraints, solver="irlsm") + model.train(x=myX, y=myY, training_frame=h2o_data) + print("Should have thrown an error....") + except Exception as e: + print(e.args[0]) + assert 'Linear constraint must have at least two coefficients' in e.args[0] + + +if __name__ == "__main__": + pyunit_utils.standalone_test(test_bad_linear_constraints) +else: + test_bad_linear_constraints() diff --git a/h2o-py/tests/testdir_algos/glm/pyunit_GH_6722_loose_beta_linear_constraints_binomial.py b/h2o-py/tests/testdir_algos/glm/pyunit_GH_6722_loose_beta_linear_constraints_binomial.py new file mode 100644 index 000000000000..3a75c301cc65 --- /dev/null +++ b/h2o-py/tests/testdir_algos/glm/pyunit_GH_6722_loose_beta_linear_constraints_binomial.py @@ -0,0 +1,162 @@ +import h2o +from h2o.estimators.glm import H2OGeneralizedLinearEstimator as glm +from tests import pyunit_utils +from tests.pyunit_utils import utils_for_glm_tests + +def test_constraints_binomial(): + ''' + check and make sure coefficients close to GLM built without constraints are generated with loose constraints + that are satisfied with coefficients from GLM without constraints. Only beta and less than and equal to + constraints are present. + ''' + train = h2o.import_file(path=pyunit_utils.locate("smalldata/glm_test/binomial_20_cols_10KRows.csv")) + for ind in range(10): + train[ind] = train[ind].asfactor() + train["C21"] = train["C21"].asfactor() + response = "C21" + predictors = list(range(0,20)) + + loose_init_const = [] # this constraint is satisfied by default coefficient initialization + bc = [] + name = "C11" + lower_bound = -8 + upper_bound = 0 + bc.append([name, lower_bound, upper_bound]) + + name = "C18" + lower_bound = 6 + upper_bound = 8 + bc.append([name, lower_bound, upper_bound]) + + name = "C15" + lower_bound = -9 + upper_bound = 0 + bc.append([name, lower_bound, upper_bound]) + + name = "C16" + lower_bound = -20 + upper_bound = 0.3 + bc.append([name, lower_bound, upper_bound]) + + beta_constraints = h2o.H2OFrame(bc) + beta_constraints.set_names(["names", "lower_bounds", "upper_bounds"]) + h2o_glm = glm(family="binomial", remove_collinear_columns=True, lambda_=0.0, solver="irlsm", seed=12345, + standardize=True) + h2o_glm.train(x=predictors, y=response, training_frame=train) + logloss = h2o_glm.model_performance()._metric_json['logloss'] + print("logloss with no constraints: {0}".format(logloss)) + + # add loose constraints + name = "C19" + values = 0.5 + types = "LessThanEqual" + contraint_numbers = 0 + loose_init_const.append([name, values, types, contraint_numbers]) + + name = "C20" + values = -0.8 + types = "LessThanEqual" + contraint_numbers = 0 + loose_init_const.append([name, values, types, contraint_numbers]) + + name = "constant" + values = -10 # 490 + types = "LessThanEqual" + contraint_numbers = 0 + loose_init_const.append([name, values, types, contraint_numbers]) + + name = "C12" + values = 2 + types = "LessThanEqual" + contraint_numbers = 1 + loose_init_const.append([name, values, types, contraint_numbers]) + + name = "C13" + values = -3 + types = "LessThanEqual" + contraint_numbers = 1 + loose_init_const.append([name, values, types, contraint_numbers]) + + name = "constant" + values = -36 + types = "LessThanEqual" + contraint_numbers = 1 + loose_init_const.append([name, values, types, contraint_numbers]) + + linear_constraints2 = h2o.H2OFrame(loose_init_const) + linear_constraints2.set_names(["names", "values", "types", "constraint_numbers"]) + # GLM model with with GLM coefficients set to GLM model coefficients built without constraints + h2o_glm_optimal_init = glm(family="binomial", lambda_=0.0, seed=12345, remove_collinear_columns=True,solver="irlsm", + linear_constraints=linear_constraints2, beta_constraints=beta_constraints, + init_optimal_glm=True) + h2o_glm_optimal_init.train(x=predictors, y=response, training_frame=train) + init_logloss = h2o_glm_optimal_init.model_performance()._metric_json['logloss'] + print("logloss with constraints and coefficients initialized with glm model built without constraints: {0}".format(init_logloss)) + print(glm.getConstraintsInfo(h2o_glm_optimal_init)) + + # GLM model with with GLM coefficients set to random GLM model coefficients + random_coef = [0.9740393731418461, 0.9021970400494406, 0.8337282995102272, 0.20588758679724872, 0.12522385214612453, + 0.6390730524643073, 0.7055779213989253, 0.9004255614099713, 0.4075431157767999, 0.161093231584713, + 0.15250197544465616, 0.7172682822215489, 0.60836236371404, 0.07086628306822396, 0.263719138602719, + 0.16102036359390437, 0.0065987448849305075, 0.5881312311814277, 0.7836567678399617, 0.9104401158881326, + 0.8432891635016235, 0.033440093086177236, 0.8514611306363931, 0.2855332934628241, 0.36525972112514427, + 0.7526593301495519, 0.9963694184200753, 0.5614168317678196, 0.7950126291921057, 0.6212978800904426, + 0.176936615687169, 0.8817788599562331, 0.13699370230879637, 0.5754950980437555, 0.1507294463182668, + 0.23409699287029495, 0.6949148063429461, 0.47140569181488556, 0.1470896240551064, 0.8475557222612405, + 0.05957485472498203, 0.07490903723892406, 0.8412381196460251, 0.26874846387453943, 0.13669341206289243, + 0.8525684329438777, 0.46716360402752777, 0.8522055745422484, 0.3129394551398561, 0.908966336417204, + 0.26259461196353984, 0.07245314277889847, 0.41429401839807156, 0.22772860293274222, 0.26662443208488784, + 0.9875655504027848, 0.5832266083052889, 0.24205847206862052, 0.9843760682096272, 0.16269008279311103, + 0.4941250734508458, 0.5446841276322587, 0.19222703209695946, 0.9232239752817498, 0.8824688635063289, + 0.224690851359456, 0.5809304720756304, 0.36863807988348585] + constraint_eta0 = [0.01, 0.1258925] + constraint_tau = [2, 50] + constraint_alpha = [0.01] + constraint_beta = [0.5] + constraint_c0 = [2] # initial value + # GLM model with GLM coefficients with default initialization + h2o_glm_random_init = utils_for_glm_tests.constraint_glm_gridsearch(train, predictors, response, solver="IRLSM", + family="binomial", + beta_constraints=beta_constraints, + linear_constraints=linear_constraints2, + startval=random_coef, + constraint_eta0=constraint_eta0, + constraint_tau=constraint_tau, + constraint_alpha=constraint_alpha, + constraint_beta=constraint_beta, + constraint_c0=constraint_c0) + init_random_logloss = h2o_glm_random_init.model_performance()._metric_json['logloss'] + print("logloss with constraints and coefficients initialized random initial values: {0}, number of iterations" + " taken to build the model: {1}".format(init_random_logloss, utils_for_glm_tests.find_glm_iterations(h2o_glm_random_init))) + print(glm.getConstraintsInfo(h2o_glm_random_init)) + + # GLM model with GLM coefficients with default initialization + h2o_glm_default_init = utils_for_glm_tests.constraint_glm_gridsearch(train, predictors, response, solver="IRLSM", + family="binomial", + beta_constraints=beta_constraints, + linear_constraints=linear_constraints2, + constraint_eta0=constraint_eta0, + constraint_tau=constraint_tau, + constraint_alpha=constraint_alpha, + constraint_beta=constraint_beta, + constraint_c0=constraint_c0) + default_init_logloss = h2o_glm_default_init.model_performance()._metric_json['logloss'] + print("logloss with constraints and default coefficients initialization: {0}, number of iterations" + " taken to build the model: {1}".format(default_init_logloss, utils_for_glm_tests.find_glm_iterations(h2o_glm_default_init))) + print(glm.getConstraintsInfo(h2o_glm_default_init)) + + + assert logloss<=init_logloss, "logloss from optimal GLM {0} should be less than logloss {1} from GLM with loose " \ + "constraints and initialized with optimal GLM {1} but is not.".format(logloss, init_logloss) + assert abs(logloss-init_random_logloss) < 1e-6 or logloss <= init_random_logloss, \ + "logloss from optimal GLM {0} should be smaller than or close to GLM with constraints and with random initial " \ + "coefficients {1} but is not.".format(logloss, init_random_logloss) + assert abs(logloss-default_init_logloss) or logloss <= default_init_logloss, \ + "logloss from optimal GLM {0} should be smaller than or close to GLM with constraints and with default initial" \ + " coefficients {1} but is not.".format(logloss, default_init_logloss) + + +if __name__ == "__main__": + pyunit_utils.standalone_test(test_constraints_binomial) +else: + test_constraints_binomial() diff --git a/h2o-py/tests/testdir_algos/glm/pyunit_GH_6722_loose_only_linear_constraints_binomial.py b/h2o-py/tests/testdir_algos/glm/pyunit_GH_6722_loose_only_linear_constraints_binomial.py new file mode 100644 index 000000000000..af40e6503b38 --- /dev/null +++ b/h2o-py/tests/testdir_algos/glm/pyunit_GH_6722_loose_only_linear_constraints_binomial.py @@ -0,0 +1,140 @@ +import h2o +from h2o.estimators.glm import H2OGeneralizedLinearEstimator as glm +from tests import pyunit_utils +from tests.pyunit_utils import utils_for_glm_tests + +def test_constraints_binomial(): + ''' + check and make sure coefficients close to GLM built without constraints are generated with loose constraints + that are satisfied with coefficients from GLM without constraints. Only less than and equal to + constraints are present. + ''' + train = h2o.import_file(path=pyunit_utils.locate("smalldata/glm_test/binomial_20_cols_10KRows.csv")) + for ind in range(10): + train[ind] = train[ind].asfactor() + train["C21"] = train["C21"].asfactor() + response = "C21" + predictors = list(range(0,20)) + + loose_init_const = [] # this constraint is satisfied by default coefficient initialization + + h2o_glm = glm(family="binomial", remove_collinear_columns=True, lambda_=0.0, solver="irlsm", seed=12345, + standardize=True) + h2o_glm.train(x=predictors, y=response, training_frame=train) + logloss = h2o_glm.model_performance()._metric_json['logloss'] + print("logloss with no constraints: {0}".format(logloss)) + + # add loose constraints + name = "C19" + values = 0.5 + types = "LessThanEqual" + contraint_numbers = 0 + loose_init_const.append([name, values, types, contraint_numbers]) + + name = "C20" + values = -0.8 + types = "LessThanEqual" + contraint_numbers = 0 + loose_init_const.append([name, values, types, contraint_numbers]) + + name = "constant" + values = -10 + types = "LessThanEqual" + contraint_numbers = 0 + loose_init_const.append([name, values, types, contraint_numbers]) + + name = "C12" + values = 2 + types = "LessThanEqual" + contraint_numbers = 1 + loose_init_const.append([name, values, types, contraint_numbers]) + + name = "C13" + values = -3 + types = "LessThanEqual" + contraint_numbers = 1 + loose_init_const.append([name, values, types, contraint_numbers]) + + name = "constant" + values = -36 + types = "LessThanEqual" + contraint_numbers = 1 + loose_init_const.append([name, values, types, contraint_numbers]) + + linear_constraints2 = h2o.H2OFrame(loose_init_const) + linear_constraints2.set_names(["names", "values", "types", "constraint_numbers"]) + + # GLM model with with GLM coefficients set to GLM model coefficients built without constraints + h2o_glm_optimal_init = glm(family="binomial", remove_collinear_columns=True, lambda_=0.0, solver="irlsm", + linear_constraints=linear_constraints2, seed=12345, init_optimal_glm=True) + h2o_glm_optimal_init.train(x=predictors, y=response, training_frame=train) + init_logloss = h2o_glm_optimal_init.model_performance()._metric_json['logloss'] + print("logloss with optimal GLM coefficient initializaiton: {0}".format(init_logloss)) + print(glm.getConstraintsInfo(h2o_glm_optimal_init)) + + # GLM model with random initialization + random_coef = [0.9740393731418461, 0.9021970400494406, 0.8337282995102272, 0.20588758679724872, 0.12522385214612453, + 0.6390730524643073, 0.7055779213989253, 0.9004255614099713, 0.4075431157767999, 0.161093231584713, + 0.15250197544465616, 0.7172682822215489, 0.60836236371404, 0.07086628306822396, 0.263719138602719, + 0.16102036359390437, 0.0065987448849305075, 0.5881312311814277, 0.7836567678399617, 0.9104401158881326, + 0.8432891635016235, 0.033440093086177236, 0.8514611306363931, 0.2855332934628241, 0.36525972112514427, + 0.7526593301495519, 0.9963694184200753, 0.5614168317678196, 0.7950126291921057, 0.6212978800904426, + 0.176936615687169, 0.8817788599562331, 0.13699370230879637, 0.5754950980437555, 0.1507294463182668, + 0.23409699287029495, 0.6949148063429461, 0.47140569181488556, 0.1470896240551064, 0.8475557222612405, + 0.05957485472498203, 0.07490903723892406, 0.8412381196460251, 0.26874846387453943, 0.13669341206289243, + 0.8525684329438777, 0.46716360402752777, 0.8522055745422484, 0.3129394551398561, 0.908966336417204, + 0.26259461196353984, 0.07245314277889847, 0.41429401839807156, 0.22772860293274222, 0.26662443208488784, + 0.9875655504027848, 0.5832266083052889, 0.24205847206862052, 0.9843760682096272, 0.16269008279311103, + 0.4941250734508458, 0.5446841276322587, 0.19222703209695946, 0.9232239752817498, 0.8824688635063289, + 0.224690851359456, 0.5809304720756304, 0.36863807988348585] + constraint_eta0 = [0.1] + constraint_tau = [5] + constraint_alpha = [0.01] + constraint_beta = [0.5] + constraint_c0 = [5, 10] # initial value + h2o_glm_random_init = utils_for_glm_tests.constraint_glm_gridsearch(train, predictors, response, solver="IRLSM", + family="binomial", + linear_constraints=linear_constraints2, + startval=random_coef, + constraint_eta0=constraint_eta0, + constraint_tau=constraint_tau, + constraint_alpha=constraint_alpha, + constraint_beta=constraint_beta, + constraint_c0=constraint_c0) + random_init_logloss = h2o_glm_random_init.model_performance()._metric_json['logloss'] + print("logloss with random coefficient initializaiton: {0}, number of iterations taken to build the model: " + "{1}".format(random_init_logloss, utils_for_glm_tests.find_glm_iterations(h2o_glm_random_init))) + print(glm.getConstraintsInfo(h2o_glm_random_init)) + + # GLM model with GLM coefficients with default initialization + h2o_glm_default_init = utils_for_glm_tests.constraint_glm_gridsearch(train, predictors, response, solver="IRLSM", + family="binomial", + linear_constraints=linear_constraints2, + constraint_eta0=constraint_eta0, + constraint_tau=constraint_tau, + constraint_alpha=constraint_alpha, + constraint_beta=constraint_beta, + constraint_c0=constraint_c0) + default_init_logloss = h2o_glm_default_init.model_performance()._metric_json['logloss'] + print("logloss with default coefficient initializaiton: {0}, number of iterations taken to build the model: " + "{1}".format(default_init_logloss, utils_for_glm_tests.find_glm_iterations(h2o_glm_default_init))) + print(glm.getConstraintsInfo(h2o_glm_default_init)) + + # since the constraints are loose, performance of GLM model without linear constraints and GLM model with linear + # constraint and initialized with optimal GLM model coefficients should equal. We will compare the logloss + assert abs(logloss-init_logloss)<1e-6, "logloss from optimal GLM {0} and logloss from GLM with loose constraints " \ + "and initialized with optimal GLM {1} should equal but is not." \ + "".format(logloss, init_logloss) + + assert abs(logloss - random_init_logloss) < 1e-6, "logloss from optimal GLM {0} should equal to GLM with " \ + "loose constraints and with random initial coefficients {1}" \ + " but is not.".format(logloss, random_init_logloss) + # both should get similar results because constraints is very loose + assert abs(logloss - default_init_logloss) < 1e-6, "logloss from optimal GLM {0} should equal to GLM with " \ + "loose constraints and with default initial coefficients {1}" \ + " but is not.".format(logloss, default_init_logloss) + +if __name__ == "__main__": + pyunit_utils.standalone_test(test_constraints_binomial) +else: + test_constraints_binomial() diff --git a/h2o-py/tests/testdir_algos/glm/pyunit_GH_6722_redundant_constraints.py b/h2o-py/tests/testdir_algos/glm/pyunit_GH_6722_redundant_constraints.py new file mode 100644 index 000000000000..56a9c052625b --- /dev/null +++ b/h2o-py/tests/testdir_algos/glm/pyunit_GH_6722_redundant_constraints.py @@ -0,0 +1,207 @@ +import sys +sys.path.insert(1,"../../../") +import h2o +from tests import pyunit_utils +from h2o.estimators.glm import H2OGeneralizedLinearEstimator as glm + +def test_redundant_constraints(): + d = h2o.import_file(path=pyunit_utils.locate("smalldata/glm_test/gaussian_20cols_10000Rows.csv")) + enum_cols = [0,1,2,3,4,5,6,7,8,9] + for colInd in enum_cols: + d[colInd] = d[colInd].asfactor() + + tight_constraints = [] # this constraint is satisfied by default coefficient initialization + # add light tight constraints + name = "C1.1" + values = 0.5 + types = "LessThanEqual" + contraint_numbers = 0 + tight_constraints.append([name, values, types, contraint_numbers]) + + name = "C1.3" + values = 1.0 + types = "LessThanEqual" + contraint_numbers = 0 + tight_constraints.append([name, values, types, contraint_numbers]) + + name = "constant" + values = -3 + types = "LessThanEqual" + contraint_numbers = 0 + tight_constraints.append([name, values, types, contraint_numbers]) + + name = "C2.3" + values = 3 + types = "LessThanEqual" + contraint_numbers = 1 + tight_constraints.append([name, values, types, contraint_numbers]) + + name = "C11" + values = -4 + types = "LessThanEqual" + contraint_numbers = 1 + tight_constraints.append([name, values, types, contraint_numbers]) + + name = "C12" + values = 0.5 + types = "LessThanEqual" + contraint_numbers = 1 + tight_constraints.append([name, values, types, contraint_numbers]) + + name = "C13" + values = 0.1 + types = "Equal" + contraint_numbers = 2 + tight_constraints.append([name, values, types, contraint_numbers]) + + name = "C14" + values = -0.2 + types = "Equal" + contraint_numbers = 2 + tight_constraints.append([name, values, types, contraint_numbers]) + + name = "C15" + values = 2 + types = "LessThanEqual" + contraint_numbers = 3 + tight_constraints.append([name, values, types, contraint_numbers]) + + name = "C16" + values = -0.1 + types = "LessThanEqual" + contraint_numbers = 3 + tight_constraints.append([name, values, types, contraint_numbers]) + + name = "C17" + values = -0.4 + types = "LessThanEqual" + contraint_numbers = 3 + tight_constraints.append([name, values, types, contraint_numbers]) + + name = "constant" + values = 0.8 + types = "LessThanEqual" + contraint_numbers = 3 + tight_constraints.append([name, values, types, contraint_numbers]) + + name = "C1.4" + values = 0.1 + types = "Equal" + contraint_numbers = 4 + tight_constraints.append([name, values, types, contraint_numbers]) + + name = "C2.1" + values = 0.7 + types = "Equal" + contraint_numbers = 4 + tight_constraints.append([name, values, types, contraint_numbers]) + + name = "constant" + values = -1.1 + types = "Equal" + contraint_numbers = 4 + tight_constraints.append([name, values, types, contraint_numbers]) + + + name = "C2.1" + values = 0.1 + types = "Equal" + contraint_numbers = 2 + tight_constraints.append([name, values, types, contraint_numbers]) + + name = "C14" + values = 2 + types = "Equal" + contraint_numbers = 5 + tight_constraints.append([name, values, types, contraint_numbers]) + + name = "C18" + values = 0.5 + types = "Equal" + contraint_numbers = 5 + tight_constraints.append([name, values, types, contraint_numbers]) + + name = "C2.2" + values = -0.3 + types = "Equal" + contraint_numbers = 5 + tight_constraints.append([name, values, types, contraint_numbers]) + + name = "C11" + values = 0.5 + types = "Equal" + contraint_numbers = 6 + tight_constraints.append([name, values, types, contraint_numbers]) + + name = "C13" + values = -1.5 + types = "Equal" + contraint_numbers = 6 + tight_constraints.append([name, values, types, contraint_numbers]) + + name = "constant" + values = -0.3 + types = "Equal" + contraint_numbers = 6 + tight_constraints.append([name, values, types, contraint_numbers]) + + name = "C15" + values = 4 + types = "LessThanEqual" + contraint_numbers = 7 + tight_constraints.append([name, values, types, contraint_numbers]) + + name = "C16" + values = -0.2 + types = "LessThanEqual" + contraint_numbers = 7 + tight_constraints.append([name, values, types, contraint_numbers]) + + name = "C17" + values = -0.8 + types = "LessThanEqual" + contraint_numbers = 7 + tight_constraints.append([name, values, types, contraint_numbers]) + + name = "constant" + values = -0.8 + types = "LessThanEqual" + contraint_numbers = 7 + tight_constraints.append([name, values, types, contraint_numbers]) + + name = "C11" + values = 1.5 + types = "Equal" + contraint_numbers = 8 + tight_constraints.append([name, values, types, contraint_numbers]) + + name = "C13" + values = -4.5 + types = "Equal" + contraint_numbers = 8 + tight_constraints.append([name, values, types, contraint_numbers]) + + name = "constant" + values = -0.9 + types = "Equal" + contraint_numbers = 8 + tight_constraints.append([name, values, types, contraint_numbers]) + + linear_constraints = h2o.H2OFrame(tight_constraints) + linear_constraints.set_names(["names", "values", "types", "constraint_numbers"]) + + try: + m = glm(family='gaussian', max_iterations=1, linear_constraints=linear_constraints, solver="irlsm", + lambda_=0.0) + m.train(training_frame=d,y= "C21") + assert False, "Should have thrown exception of redundant constraints" + except Exception as ex: + print(ex) + temp = str(ex) + assert ("redundant and possibly conflicting linear constraints" in temp), "Wrong exception was received." + print("redundant constraint test passed!") + +if __name__ == "__main__": + pyunit_utils.standalone_test(test_redundant_constraints) +else: + test_redundant_constraints() diff --git a/h2o-py/tests/testdir_algos/glm/pyunit_GH_6722_separate_linear_beta_gaussian.py b/h2o-py/tests/testdir_algos/glm/pyunit_GH_6722_separate_linear_beta_gaussian.py new file mode 100644 index 000000000000..6bcfc2fbcf45 --- /dev/null +++ b/h2o-py/tests/testdir_algos/glm/pyunit_GH_6722_separate_linear_beta_gaussian.py @@ -0,0 +1,113 @@ +import h2o +from h2o.estimators.glm import H2OGeneralizedLinearEstimator as glm +from tests import pyunit_utils +from tests.pyunit_utils import utils_for_glm_tests + +def test_separate_linear_beta_gaussian(): + ''' + This test will check that when separate_linear_beta=True, those coefficients should be within the beta constraint + range. + ''' + h2o_data = h2o.import_file( + path=pyunit_utils.locate("smalldata/glm_test/gaussian_20cols_10000Rows.csv")) + enum_columns = ["C1", "C2", "C3", "C4", "C5", "C6", "C7", "C8", "C9", "C10"] + for cname in enum_columns: + h2o_data[cname] = h2o_data[cname].asfactor() + myY = "C21" + myX = h2o_data.names.remove(myY) + + linear_constraints = [] # this constraint is satisfied by default coefficient initialization + name = "C1.2" + values = 1 + types = "Equal" + contraint_numbers = 0 + linear_constraints.append([name, values, types, contraint_numbers]) + + name = "C11" + values = 1 + types = "Equal" + contraint_numbers = 0 + linear_constraints.append([name, values, types, contraint_numbers]) + + name = "constant" + values = 13.56 + types = "Equal" + contraint_numbers = 0 + linear_constraints.append([name, values, types, contraint_numbers]) + + name = "C5.2" + values = 1 + types = "LessThanEqual" + contraint_numbers = 1 + linear_constraints.append([name, values, types, contraint_numbers]) + + name = "C12" + values = 1 + types = "LessThanEqual" + contraint_numbers = 1 + linear_constraints.append([name, values, types, contraint_numbers]) + + name = "C15" + values = 1 + types = "LessThanEqual" + contraint_numbers = 1 + linear_constraints.append([name, values, types, contraint_numbers]) + + name = "constant" + values = -5 + types = "LessThanEqual" + contraint_numbers = 1 + linear_constraints.append([name, values, types, contraint_numbers]) + + linear_constraints2 = h2o.H2OFrame(linear_constraints) + linear_constraints2.set_names(["names", "values", "types", "constraint_numbers"]) + + bc = [] + name = "C1.1" + c1p1LowerBound = -36 + c1p1UpperBound=-35 + bc.append([name, c1p1LowerBound, c1p1UpperBound]) + + name = "C5.2" + c5p2LowerBound=-14 + c5p2UpperBound=-13 + bc.append([name, c5p2LowerBound, c5p2UpperBound]) + + name = "C11" + c11LowerBound=25 + c11UpperBound=26 + bc.append([name, c11LowerBound, c11UpperBound]) + + name = "C15" + c15LowerBound=14 + c15UpperBound=15 + bc.append([name, c15LowerBound, c15UpperBound]) + + beta_constraints = h2o.H2OFrame(bc) + beta_constraints.set_names(["names", "lower_bounds", "upper_bounds"]) + + m_sep = glm(family='gaussian', linear_constraints=linear_constraints2, solver="irlsm", lambda_=0.0, + beta_constraints=beta_constraints, separate_linear_beta=True, constraint_eta0=0.1, constraint_tau=10, + constraint_alpha=0.01, constraint_beta=0.9, constraint_c0=100) + m_sep.train(training_frame=h2o_data,x=myX, y=myY) + coef_sep = m_sep.coef() + + # check coefficients under beta constraints are within limits for when separate_linear_beta = True compared to the + # case when separate_linear_beta = False + # check C1.1 + assert coef_sep["C1.1"] >= c1p1LowerBound and coef_sep["C1.1"] <= c1p1UpperBound, \ + "Coefficient C1.1: {0} should be between -36, and -35 but is not!".format(coef_sep["C1.1"]) + # check C5.2 + assert coef_sep["C5.2"] >= c5p2LowerBound and coef_sep["C5.2"] <= c5p2UpperBound, \ + "Coefficient C5.2: {0} should be between -14, and -13 but is not!".format(coef_sep["C5.2"]) + # check C11 + assert coef_sep["C11"] >= c11LowerBound and coef_sep["C11"] <= c11UpperBound, \ + "Coefficient C11: {0} should be between 25, and 26 but is not!".format(coef_sep["C11"]) + # check C15 + assert coef_sep["C15"] >= c15LowerBound and coef_sep["C15"] <= c15UpperBound, \ + "Coefficient C15: {0} should be between 14, and 15 but is not!".format(coef_sep["C15"]) + +if __name__ == "__main__": + pyunit_utils.standalone_test(test_separate_linear_beta_gaussian) +else: + test_separate_linear_beta_gaussian() diff --git a/h2o-py/tests/testdir_algos/glm/pyunit_GH_6722_tight_beta_equality_linear_constraints_binomial.py b/h2o-py/tests/testdir_algos/glm/pyunit_GH_6722_tight_beta_equality_linear_constraints_binomial.py new file mode 100644 index 000000000000..d7befd7da108 --- /dev/null +++ b/h2o-py/tests/testdir_algos/glm/pyunit_GH_6722_tight_beta_equality_linear_constraints_binomial.py @@ -0,0 +1,322 @@ +import h2o +from h2o.estimators.glm import H2OGeneralizedLinearEstimator as glm +from tests import pyunit_utils +from tests.pyunit_utils import utils_for_glm_tests + +def test_light_tight_linear_constraints_only_gaussian(): + ''' + Test constrained GLM with beta, equality and less than and equal to constraints. The constraints are very + tight and coefficients from GLM built without constraints won't be able to satisfied the constraints. + Constrained GLM models are built with coefficients initialized with coefficients from GLM built without constraints, + default coefficients and random coefficients. + ''' + train = h2o.import_file(path=pyunit_utils.locate("smalldata/glm_test/binomial_20_cols_10KRows.csv")) + for ind in range(10): + train[ind] = train[ind].asfactor() + train["C21"] = train["C21"].asfactor() + response = "C21" + predictors = list(range(0,20)) + # add beta constraints + bc = [] + name = "C11" + lower_bound = -3.5 + upper_bound = 0 + bc.append([name, lower_bound, upper_bound]) + + name = "C18" + lower_bound = 6 + upper_bound = 7 + bc.append([name, lower_bound, upper_bound]) + + name = "C15" + lower_bound = -9 + upper_bound = -6 + bc.append([name, lower_bound, upper_bound]) + + name = "C16" + lower_bound = -20 + upper_bound = -10 + bc.append([name, lower_bound, upper_bound]) + + beta_constraints = h2o.H2OFrame(bc) + beta_constraints.set_names(["names", "lower_bounds", "upper_bounds"]) + + tight_constraints = [] # this constraint is satisfied by default coefficient initialization + + h2o_glm = glm(family="binomial", lambda_=0.0, solver="irlsm", seed=12345, standardize=True) + h2o_glm.train(x=predictors, y=response, training_frame=train) + logloss = h2o_glm.model_performance()._metric_json['logloss'] + print("logloss with no constraints: {0}".format(logloss)) + + # add tight constraints + name = "C1.1" + values = 0.5 + types = "LessThanEqual" + contraint_numbers = 0 + tight_constraints.append([name, values, types, contraint_numbers]) + + name = "C2.1" + values = -0.25 + types = "LessThanEqual" + contraint_numbers = 0 + tight_constraints.append([name, values, types, contraint_numbers]) + + name = "constant" + values = -1 + types = "LessThanEqual" + contraint_numbers = 0 + tight_constraints.append([name, values, types, contraint_numbers]) + + name = "C4.1" + values = 1.5 + types = "LessThanEqual" + contraint_numbers = 1 + tight_constraints.append([name, values, types, contraint_numbers]) + + name = "C17" + values = 3 + types = "LessThanEqual" + contraint_numbers = 1 + tight_constraints.append([name, values, types, contraint_numbers]) + + name = "C15" + values = -2 + types = "LessThanEqual" + contraint_numbers = 1 + tight_constraints.append([name, values, types, contraint_numbers]) + + name = "constant" + values = -5 + types = "LessThanEqual" + contraint_numbers = 1 + tight_constraints.append([name, values, types, contraint_numbers]) + + name = "C12" + values = -0.5 + types = "LessThanEqual" + contraint_numbers = 2 + tight_constraints.append([name, values, types, contraint_numbers]) + + name = "C13" + values = -1.5 + types = "LessThanEqual" + contraint_numbers = 2 + tight_constraints.append([name, values, types, contraint_numbers]) + + name = "C14" + values = 2 + types = "LessThanEqual" + contraint_numbers = 2 + tight_constraints.append([name, values, types, contraint_numbers]) + + name = "constant" + values = -3 + types = "LessThanEqual" + contraint_numbers = 2 + tight_constraints.append([name, values, types, contraint_numbers]) + + name = "C11" + values = 0.25 + types = "LessThanEqual" + contraint_numbers = 3 + tight_constraints.append([name, values, types, contraint_numbers]) + + name = "C18" + values = -0.5 + types = "LessThanEqual" + contraint_numbers = 3 + tight_constraints.append([name, values, types, contraint_numbers]) + + name = "C19" + values = 0.75 + types = "LessThanEqual" + contraint_numbers = 3 + tight_constraints.append([name, values, types, contraint_numbers]) + + name = "constant" + values = 5 + types = "LessThanEqual" + contraint_numbers = 3 + tight_constraints.append([name, values, types, contraint_numbers]) + + name = "C19" + values = 0.5 + types = "Equal" + contraint_numbers = 4 + tight_constraints.append([name, values, types, contraint_numbers]) + + name = "C10.1" + values = -0.3 + types = "Equal" + contraint_numbers = 4 + tight_constraints.append([name, values, types, contraint_numbers]) + + name = "constant" + values = -0.25 + types = "Equal" + contraint_numbers = 4 + tight_constraints.append([name, values, types, contraint_numbers]) + + name = "C18" + values = 0.75 + types = "Equal" + contraint_numbers = 5 + tight_constraints.append([name, values, types, contraint_numbers]) + + name = "C20" + values = -0.13 + types = "Equal" + contraint_numbers = 5 + tight_constraints.append([name, values, types, contraint_numbers]) + + name = "constant" + values = -1.5 + types = "Equal" + contraint_numbers = 5 + tight_constraints.append([name, values, types, contraint_numbers]) + + name = "C19" + values = 0.5 + types = "Equal" + contraint_numbers = 4 + tight_constraints.append([name, values, types, contraint_numbers]) + + name = "C10.1" + values = -0.3 + types = "Equal" + contraint_numbers = 4 + tight_constraints.append([name, values, types, contraint_numbers]) + + name = "constant" + values = -0.5 + types = "Equal" + contraint_numbers = 4 + tight_constraints.append([name, values, types, contraint_numbers]) + + name = "C18" + values = 0.75 + types = "Equal" + contraint_numbers = 5 + tight_constraints.append([name, values, types, contraint_numbers]) + + name = "C20" + values = -0.13 + types = "Equal" + contraint_numbers = 5 + tight_constraints.append([name, values, types, contraint_numbers]) + + name = "constant" + values = -3 + types = "Equal" + contraint_numbers = 5 + tight_constraints.append([name, values, types, contraint_numbers]) + + linear_constraints2 = h2o.H2OFrame(tight_constraints) + linear_constraints2.set_names(["names", "values", "types", "constraint_numbers"]) + + # GLM model with GLM coefficients with default initialization + constraint_eta0 = [0.001, 0.12589] + constraint_tau = [1.2, 5] + constraint_alpha = [0.01, 0.1] + constraint_beta = [0.001, 0.5] + constraint_c0 = [20, 30] # initial value + + h2o_glm_default_init = utils_for_glm_tests.constraint_glm_gridsearch(train, predictors, response, solver="IRLSM", + family="binomial", + linear_constraints=linear_constraints2, + beta_constraints=beta_constraints, + init_optimal_glm=False, + constraint_eta0=constraint_eta0, + constraint_tau=constraint_tau, + constraint_alpha=constraint_alpha, + constraint_beta=constraint_beta, + constraint_c0=constraint_c0, + return_best=False, epsilon=20) + default_init_logloss = h2o_glm_default_init.model_performance()._metric_json['logloss'] + print("logloss with default GLM coefficient initializaiton: {0}, number of iterations taken to build the model: " + "{1}".format(default_init_logloss, utils_for_glm_tests.find_glm_iterations(h2o_glm_default_init))) + + # GLM model with with GLM coefficients set to GLM model coefficients built without constraints + h2o_glm_optimal_init = utils_for_glm_tests.constraint_glm_gridsearch(train, predictors, response, solver="IRLSM", + family="binomial", + linear_constraints=linear_constraints2, + beta_constraints=beta_constraints, + init_optimal_glm=True, + constraint_eta0=constraint_eta0, + constraint_tau=constraint_tau, + constraint_alpha=constraint_alpha, + constraint_beta=constraint_beta, + constraint_c0=constraint_c0, + return_best=False, + epsilon=20) + optimal_init_logloss = h2o_glm_optimal_init.model_performance()._metric_json['logloss'] + print("logloss with optimal GLM coefficient initializaiton: {0}, number of iterations taken to build the model: " + "{1}".format(optimal_init_logloss, utils_for_glm_tests.find_glm_iterations(h2o_glm_optimal_init))) + print(glm.getConstraintsInfo(h2o_glm_optimal_init)) + print("All constraints satisfied: {0}".format(glm.allConstraintsPassed(h2o_glm_optimal_init))) + + h2o_glm_default_init = utils_for_glm_tests.constraint_glm_gridsearch(train, predictors, response, solver="IRLSM", + family="binomial", + linear_constraints=linear_constraints2, + beta_constraints=beta_constraints, + init_optimal_glm=False, + constraint_eta0=constraint_eta0, + constraint_tau=constraint_tau, + constraint_alpha=constraint_alpha, + constraint_beta=constraint_beta, + constraint_c0=constraint_c0, + return_best=False, epsilon=20) + default_init_logloss = h2o_glm_default_init.model_performance()._metric_json['logloss'] + print("logloss with default GLM coefficient initializaiton: {0}, number of iterations taken to build the model: " + "{1}".format(default_init_logloss, utils_for_glm_tests.find_glm_iterations(h2o_glm_default_init))) + print(glm.getConstraintsInfo(h2o_glm_default_init)) + print("All constraints satisfied: {0}".format(glm.allConstraintsPassed(h2o_glm_default_init))) + + random_coef = [0.9740393731418461, 0.9021970400494406, 0.8337282995102272, 0.20588758679724872, 0.12522385214612453, + 0.6390730524643073, 0.7055779213989253, 0.9004255614099713, 0.4075431157767999, 0.161093231584713, + 0.15250197544465616, 0.7172682822215489, 0.60836236371404, 0.07086628306822396, 0.263719138602719, + 0.16102036359390437, 0.0065987448849305075, 0.5881312311814277, 0.7836567678399617, 0.9104401158881326, + 0.8432891635016235, 0.033440093086177236, 0.8514611306363931, 0.2855332934628241, 0.36525972112514427, + 0.7526593301495519, 0.9963694184200753, 0.5614168317678196, 0.7950126291921057, 0.6212978800904426, + 0.176936615687169, 0.8817788599562331, 0.13699370230879637, 0.5754950980437555, 0.1507294463182668, + 0.23409699287029495, 0.6949148063429461, 0.47140569181488556, 0.1470896240551064, 0.8475557222612405, + 0.05957485472498203, 0.07490903723892406, 0.8412381196460251, 0.26874846387453943, 0.13669341206289243, + 0.8525684329438777, 0.46716360402752777, 0.8522055745422484, 0.3129394551398561, 0.908966336417204, + 0.26259461196353984, 0.07245314277889847, 0.41429401839807156, 0.22772860293274222, 0.26662443208488784, + 0.9875655504027848, 0.5832266083052889, 0.24205847206862052, 0.9843760682096272, 0.16269008279311103, + 0.4941250734508458, 0.5446841276322587, 0.19222703209695946, 0.9232239752817498, 0.8824688635063289, + 0.224690851359456, 0.5809304720756304, 0.36863807988348585] + + h2o_glm_random_init = utils_for_glm_tests.constraint_glm_gridsearch(train, predictors, response, solver="IRLSM", + family="binomial", + linear_constraints=linear_constraints2, + beta_constraints=beta_constraints, + startval=random_coef, + init_optimal_glm=False, + constraint_eta0=constraint_eta0, + constraint_tau=constraint_tau, + constraint_alpha=constraint_alpha, + constraint_beta=constraint_beta, + constraint_c0=constraint_c0, + return_best=False, epsilon=20) + random_init_logloss = h2o_glm_random_init.model_performance()._metric_json['logloss'] + print("logloss with random GLM coefficient initializaiton: {0}, number of iterations taken to build the model: " + "{1}".format(random_init_logloss, utils_for_glm_tests.find_glm_iterations(h2o_glm_random_init))) + print(glm.getConstraintsInfo(h2o_glm_random_init)) + print("All constraints satisfied: {0}".format(glm.allConstraintsPassed(h2o_glm_random_init))) + + assert logloss <= optimal_init_logloss, "logloss from optimal GLM {0} should be lower than logloss from GLM with light tight" \ + " constraints and initialized with optimal GLM {1} but is not.".format(logloss, optimal_init_logloss) + + assert logloss <= default_init_logloss, "logloss from optimal GLM {0} should be lower than logloss from GLM with light tight" \ + " constraints and initialized with default coefficients GLM {1} but is " \ + "not.".format(logloss, default_init_logloss) + + assert logloss <= random_init_logloss, "logloss from optimal GLM {0} should be lower than logloss from GLM with light tight" \ + " constraints and initialized with random coefficients GLM {1} but is " \ + "not.".format(logloss, random_init_logloss) + +if __name__ == "__main__": + pyunit_utils.standalone_test(test_light_tight_linear_constraints_only_gaussian) +else: + test_light_tight_linear_constraints_only_gaussian() diff --git a/h2o-py/tests/testdir_algos/glm/pyunit_GH_6722_tight_equality_linear_constraints_binomial.py b/h2o-py/tests/testdir_algos/glm/pyunit_GH_6722_tight_equality_linear_constraints_binomial.py new file mode 100644 index 000000000000..1f4888b195a3 --- /dev/null +++ b/h2o-py/tests/testdir_algos/glm/pyunit_GH_6722_tight_equality_linear_constraints_binomial.py @@ -0,0 +1,242 @@ +import h2o +from h2o.estimators.glm import H2OGeneralizedLinearEstimator as glm +from tests import pyunit_utils +from tests.pyunit_utils import utils_for_glm_tests + +def test_light_tight_linear_constraints_only_gaussian(): + ''' + Test constrained GLM with equality and less than and equal to constraints. The constraints are very + tight and coefficients from GLM built without constraints won't be able to satisfied the constraints. + Constrained GLM models are built with coefficients initialized with coefficients from GLM built without constraints, + default coefficients and random coefficients. + ''' + train = h2o.import_file(path=pyunit_utils.locate("smalldata/glm_test/binomial_20_cols_10KRows.csv")) + for ind in range(10): + train[ind] = train[ind].asfactor() + train["C21"] = train["C21"].asfactor() + response = "C21" + predictors = list(range(0,20)) + + light_tight_constraints = [] # this constraint is satisfied by default coefficient initialization + + h2o_glm = glm(family="binomial", lambda_=0.0, solver="irlsm", seed=12345, standardize=True) + h2o_glm.train(x=predictors, y=response, training_frame=train) + logloss = h2o_glm.model_performance()._metric_json['logloss'] + print("logloss with no constraints: {0}".format(logloss)) + + # add tight constraints + name = "C1.1" + values = 0.5 + types = "LessThanEqual" + contraint_numbers = 0 + light_tight_constraints.append([name, values, types, contraint_numbers]) + + name = "C2.1" + values = -0.25 + types = "LessThanEqual" + contraint_numbers = 0 + light_tight_constraints.append([name, values, types, contraint_numbers]) + + name = "constant" + values = -1 + types = "LessThanEqual" + contraint_numbers = 0 + light_tight_constraints.append([name, values, types, contraint_numbers]) + + name = "C4.1" + values = 1.5 + types = "LessThanEqual" + contraint_numbers = 1 + light_tight_constraints.append([name, values, types, contraint_numbers]) + + name = "C17" + values = 3 + types = "LessThanEqual" + contraint_numbers = 1 + light_tight_constraints.append([name, values, types, contraint_numbers]) + + name = "C15" + values = -2 + types = "LessThanEqual" + contraint_numbers = 1 + light_tight_constraints.append([name, values, types, contraint_numbers]) + + name = "constant" + values = -5 + types = "LessThanEqual" + contraint_numbers = 1 + light_tight_constraints.append([name, values, types, contraint_numbers]) + + name = "C12" + values = -0.5 + types = "LessThanEqual" + contraint_numbers = 2 + light_tight_constraints.append([name, values, types, contraint_numbers]) + + name = "C13" + values = -1.5 + types = "LessThanEqual" + contraint_numbers = 2 + light_tight_constraints.append([name, values, types, contraint_numbers]) + + name = "C14" + values = 2 + types = "LessThanEqual" + contraint_numbers = 2 + light_tight_constraints.append([name, values, types, contraint_numbers]) + + name = "constant" + values = -3 + types = "LessThanEqual" + contraint_numbers = 2 + light_tight_constraints.append([name, values, types, contraint_numbers]) + + name = "C11" + values = 0.25 + types = "LessThanEqual" + contraint_numbers = 3 + light_tight_constraints.append([name, values, types, contraint_numbers]) + + name = "C18" + values = -0.5 + types = "LessThanEqual" + contraint_numbers = 3 + light_tight_constraints.append([name, values, types, contraint_numbers]) + + name = "C19" + values = 0.75 + types = "LessThanEqual" + contraint_numbers = 3 + light_tight_constraints.append([name, values, types, contraint_numbers]) + + name = "constant" + values = 5 + types = "LessThanEqual" + contraint_numbers = 3 + light_tight_constraints.append([name, values, types, contraint_numbers]) + + name = "C19" + values = 0.5 + types = "Equal" + contraint_numbers = 4 + light_tight_constraints.append([name, values, types, contraint_numbers]) + + name = "C10.1" + values = -0.3 + types = "Equal" + contraint_numbers = 4 + light_tight_constraints.append([name, values, types, contraint_numbers]) + + name = "constant" + values = -0.5 + types = "Equal" + contraint_numbers = 4 + light_tight_constraints.append([name, values, types, contraint_numbers]) + + name = "C18" + values = 0.75 + types = "Equal" + contraint_numbers = 5 + light_tight_constraints.append([name, values, types, contraint_numbers]) + + name = "C20" + values = -0.13 + types = "Equal" + contraint_numbers = 5 + light_tight_constraints.append([name, values, types, contraint_numbers]) + + name = "constant" + values = -3 + types = "Equal" + contraint_numbers = 5 + light_tight_constraints.append([name, values, types, contraint_numbers]) + + linear_constraints2 = h2o.H2OFrame(light_tight_constraints) + linear_constraints2.set_names(["names", "values", "types", "constraint_numbers"]) + + # GLM model with GLM coefficients with default initialization + constraint_eta0 = [0.001] + constraint_tau = [2] + constraint_alpha = [0.001] + constraint_beta = [0.001] + constraint_c0 = [1.5, 5] # initial value + # GLM model with with GLM coefficients set to GLM model coefficients built without constraints + h2o_glm_optimal_init = utils_for_glm_tests.constraint_glm_gridsearch(train, predictors, response, solver="IRLSM", + family="binomial", + linear_constraints=linear_constraints2, + init_optimal_glm=True, + constraint_eta0=constraint_eta0, + constraint_tau=constraint_tau, + constraint_alpha=constraint_alpha, + constraint_beta=constraint_beta, + constraint_c0=constraint_c0, + return_best=False) + optimal_init_logloss = h2o_glm_optimal_init.model_performance()._metric_json['logloss'] + print("logloss with optimal GLM coefficient initializaiton: {0}, number of iterations taken to build the model: " + "{1}".format(optimal_init_logloss, utils_for_glm_tests.find_glm_iterations(h2o_glm_optimal_init))) + print(glm.getConstraintsInfo(h2o_glm_optimal_init)) + print("All constraints satisfied: {0}".format(glm.allConstraintsPassed(h2o_glm_optimal_init))) + + h2o_glm_default_init = utils_for_glm_tests.constraint_glm_gridsearch(train, predictors, response, solver="IRLSM", + family="binomial", + linear_constraints=linear_constraints2, + init_optimal_glm=False, + constraint_eta0=constraint_eta0, + constraint_tau=constraint_tau, + constraint_alpha=constraint_alpha, + constraint_beta=constraint_beta, + constraint_c0=constraint_c0, + return_best=False) + default_init_logloss = h2o_glm_default_init.model_performance()._metric_json['logloss'] + print("logloss with default GLM coefficient initializaiton: {0}, number of iterations taken to build the model: " + "{1}".format(default_init_logloss, utils_for_glm_tests.find_glm_iterations(h2o_glm_default_init))) + print(glm.getConstraintsInfo(h2o_glm_default_init)) + print("All constraints satisfied: {0}".format(glm.allConstraintsPassed(h2o_glm_default_init))) + + random_coef = [0.9740393731418461, 0.9021970400494406, 0.8337282995102272, 0.20588758679724872, 0.12522385214612453, + 0.6390730524643073, 0.7055779213989253, 0.9004255614099713, 0.4075431157767999, 0.161093231584713, + 0.15250197544465616, 0.7172682822215489, 0.60836236371404, 0.07086628306822396, 0.263719138602719, + 0.16102036359390437, 0.0065987448849305075, 0.5881312311814277, 0.7836567678399617, 0.9104401158881326, + 0.8432891635016235, 0.033440093086177236, 0.8514611306363931, 0.2855332934628241, 0.36525972112514427, + 0.7526593301495519, 0.9963694184200753, 0.5614168317678196, 0.7950126291921057, 0.6212978800904426, + 0.176936615687169, 0.8817788599562331, 0.13699370230879637, 0.5754950980437555, 0.1507294463182668, + 0.23409699287029495, 0.6949148063429461, 0.47140569181488556, 0.1470896240551064, 0.8475557222612405, + 0.05957485472498203, 0.07490903723892406, 0.8412381196460251, 0.26874846387453943, 0.13669341206289243, + 0.8525684329438777, 0.46716360402752777, 0.8522055745422484, 0.3129394551398561, 0.908966336417204, + 0.26259461196353984, 0.07245314277889847, 0.41429401839807156, 0.22772860293274222, 0.26662443208488784, + 0.9875655504027848, 0.5832266083052889, 0.24205847206862052, 0.9843760682096272, 0.16269008279311103, + 0.4941250734508458, 0.5446841276322587, 0.19222703209695946, 0.9232239752817498, 0.8824688635063289, + 0.224690851359456, 0.5809304720756304, 0.36863807988348585] + + h2o_glm_random_init = utils_for_glm_tests.constraint_glm_gridsearch(train, predictors, response, solver="IRLSM", + family="binomial", + linear_constraints=linear_constraints2, + startval=random_coef, + init_optimal_glm=False, + constraint_eta0=constraint_eta0, + constraint_tau=constraint_tau, + constraint_alpha=constraint_alpha, + constraint_beta=constraint_beta, + constraint_c0=constraint_c0, + return_best=False) + random_init_logloss = h2o_glm_random_init.model_performance()._metric_json['logloss'] + print("logloss with random GLM coefficient initializaiton: {0}, number of iterations taken to build the model: " + "{1}".format(random_init_logloss, utils_for_glm_tests.find_glm_iterations(h2o_glm_random_init))) + print(glm.getConstraintsInfo(h2o_glm_random_init)) + print("All constraints satisfied: {0}".format(glm.allConstraintsPassed(h2o_glm_random_init))) + + assert logloss <= optimal_init_logloss, "logloss from optimal GLM {0} should be lower than logloss from GLM with light tight" \ + " constraints and initialized with optimal GLM {1} but is not.".format(logloss, optimal_init_logloss) + + assert logloss <= default_init_logloss, "logloss from optimal GLM {0} should be lower than logloss from GLM with light tight" \ + " constraints and initialized with default coefficients GLM {1} but is " \ + "not.".format(logloss, default_init_logloss) + + assert logloss <= random_init_logloss, "logloss from optimal GLM {0} should be lower than logloss from GLM with light tight" \ + " constraints and initialized with random coefficients GLM {1} but is " \ + "not.".format(logloss, random_init_logloss) + +if __name__ == "__main__": + pyunit_utils.standalone_test(test_light_tight_linear_constraints_only_gaussian) +else: + test_light_tight_linear_constraints_only_gaussian() diff --git a/h2o-py/tests/testdir_algos/glm/pyunit_GH_6722_tight_linear_constraints_only_binomial.py b/h2o-py/tests/testdir_algos/glm/pyunit_GH_6722_tight_linear_constraints_only_binomial.py new file mode 100644 index 000000000000..d8668bad9776 --- /dev/null +++ b/h2o-py/tests/testdir_algos/glm/pyunit_GH_6722_tight_linear_constraints_only_binomial.py @@ -0,0 +1,206 @@ +import h2o +from h2o.estimators.glm import H2OGeneralizedLinearEstimator as glm +from tests import pyunit_utils +from tests.pyunit_utils import utils_for_glm_tests + +def test_light_tight_linear_constraints_only_gaussian(): + ''' + Test constrained GLM with less than and equal to constraints. The constraints are very + tight and coefficients from GLM built without constraints won't be able to satisfied the constraints. + Constrained GLM models are built with coefficients initialized with coefficients from GLM built without constraints, + default coefficients and random coefficients. + ''' + train = h2o.import_file(path=pyunit_utils.locate("smalldata/glm_test/binomial_20_cols_10KRows.csv")) + for ind in range(10): + train[ind] = train[ind].asfactor() + train["C21"] = train["C21"].asfactor() + response = "C21" + predictors = list(range(0,20)) + + tight_constraints = [] # this constraint is satisfied by default coefficient initialization + + h2o_glm = glm(family="binomial", lambda_=0.0, solver="irlsm", seed=12345, standardize=True) + h2o_glm.train(x=predictors, y=response, training_frame=train) + logloss = h2o_glm.model_performance()._metric_json['logloss'] + print("logloss with no constraints: {0}".format(logloss)) + + # add light tight constraints + name = "C1.1" + values = 0.5 + types = "LessThanEqual" + contraint_numbers = 0 + tight_constraints.append([name, values, types, contraint_numbers]) + + name = "C2.1" + values = -0.25 + types = "LessThanEqual" + contraint_numbers = 0 + tight_constraints.append([name, values, types, contraint_numbers]) + + name = "constant" + values = -1 + types = "LessThanEqual" + contraint_numbers = 0 + tight_constraints.append([name, values, types, contraint_numbers]) + + name = "C4.1" + values = 1.5 + types = "LessThanEqual" + contraint_numbers = 1 + tight_constraints.append([name, values, types, contraint_numbers]) + + name = "C17" + values = 3 + types = "LessThanEqual" + contraint_numbers = 1 + tight_constraints.append([name, values, types, contraint_numbers]) + + name = "C15" + values = -2 + types = "LessThanEqual" + contraint_numbers = 1 + tight_constraints.append([name, values, types, contraint_numbers]) + + name = "constant" + values = -5 + types = "LessThanEqual" + contraint_numbers = 1 + tight_constraints.append([name, values, types, contraint_numbers]) + + name = "C12" + values = -0.5 + types = "LessThanEqual" + contraint_numbers = 2 + tight_constraints.append([name, values, types, contraint_numbers]) + + name = "C13" + values = -1.5 + types = "LessThanEqual" + contraint_numbers = 2 + tight_constraints.append([name, values, types, contraint_numbers]) + + name = "C14" + values = 2 + types = "LessThanEqual" + contraint_numbers = 2 + tight_constraints.append([name, values, types, contraint_numbers]) + + name = "constant" + values = -3 + types = "LessThanEqual" + contraint_numbers = 2 + tight_constraints.append([name, values, types, contraint_numbers]) + + name = "C11" + values = 0.25 + types = "LessThanEqual" + contraint_numbers = 3 + tight_constraints.append([name, values, types, contraint_numbers]) + + name = "C18" + values = -0.5 + types = "LessThanEqual" + contraint_numbers = 3 + tight_constraints.append([name, values, types, contraint_numbers]) + + name = "C19" + values = 0.75 + types = "LessThanEqual" + contraint_numbers = 3 + tight_constraints.append([name, values, types, contraint_numbers]) + + name = "constant" + values = 5 + types = "LessThanEqual" + contraint_numbers = 3 + tight_constraints.append([name, values, types, contraint_numbers]) + + linear_constraints2 = h2o.H2OFrame(tight_constraints) + linear_constraints2.set_names(["names", "values", "types", "constraint_numbers"]) + + # GLM model with GLM coefficients with default initialization + constraint_eta0 = [0.1258925] + constraint_tau = [1.2, 5] + constraint_alpha = [0.1] + constraint_beta = [0.9] + constraint_c0 = [10, 12] # initial value + # GLM model with with GLM coefficients set to GLM model coefficients built without constraints + h2o_glm_optimal_init = utils_for_glm_tests.constraint_glm_gridsearch(train, predictors, response, solver="IRLSM", + family="binomial", + linear_constraints=linear_constraints2, + init_optimal_glm=True, + constraint_eta0=constraint_eta0, + constraint_tau=constraint_tau, + constraint_alpha=constraint_alpha, + constraint_beta=constraint_beta, + constraint_c0=constraint_c0, + return_best=False) + optimal_init_logloss = h2o_glm_optimal_init.model_performance()._metric_json['logloss'] + print("logloss with optimal GLM coefficient initializaiton: {0}, number of iterations taken to build the model: " + "{1}".format(optimal_init_logloss, utils_for_glm_tests.find_glm_iterations(h2o_glm_optimal_init))) + print(glm.getConstraintsInfo(h2o_glm_optimal_init)) + print("All constraints satisfied: {0}".format(glm.allConstraintsPassed(h2o_glm_optimal_init))) + + h2o_glm_default_init = utils_for_glm_tests.constraint_glm_gridsearch(train, predictors, response, solver="IRLSM", + family="binomial", + linear_constraints=linear_constraints2, + init_optimal_glm=False, + constraint_eta0=constraint_eta0, + constraint_tau=constraint_tau, + constraint_alpha=constraint_alpha, + constraint_beta=constraint_beta, + constraint_c0=constraint_c0, + return_best=False) + default_init_logloss = h2o_glm_default_init.model_performance()._metric_json['logloss'] + print("logloss with default GLM coefficient initializaiton: {0}, number of iterations taken to build the model: " + "{1}".format(default_init_logloss, utils_for_glm_tests.find_glm_iterations(h2o_glm_default_init))) + print(glm.getConstraintsInfo(h2o_glm_default_init)) + print("All constraints satisfied: {0}".format(glm.allConstraintsPassed(h2o_glm_default_init))) + + random_coef = [0.9740393731418461, 0.9021970400494406, 0.8337282995102272, 0.20588758679724872, 0.12522385214612453, + 0.6390730524643073, 0.7055779213989253, 0.9004255614099713, 0.4075431157767999, 0.161093231584713, + 0.15250197544465616, 0.7172682822215489, 0.60836236371404, 0.07086628306822396, 0.263719138602719, + 0.16102036359390437, 0.0065987448849305075, 0.5881312311814277, 0.7836567678399617, 0.9104401158881326, + 0.8432891635016235, 0.033440093086177236, 0.8514611306363931, 0.2855332934628241, 0.36525972112514427, + 0.7526593301495519, 0.9963694184200753, 0.5614168317678196, 0.7950126291921057, 0.6212978800904426, + 0.176936615687169, 0.8817788599562331, 0.13699370230879637, 0.5754950980437555, 0.1507294463182668, + 0.23409699287029495, 0.6949148063429461, 0.47140569181488556, 0.1470896240551064, 0.8475557222612405, + 0.05957485472498203, 0.07490903723892406, 0.8412381196460251, 0.26874846387453943, 0.13669341206289243, + 0.8525684329438777, 0.46716360402752777, 0.8522055745422484, 0.3129394551398561, 0.908966336417204, + 0.26259461196353984, 0.07245314277889847, 0.41429401839807156, 0.22772860293274222, 0.26662443208488784, + 0.9875655504027848, 0.5832266083052889, 0.24205847206862052, 0.9843760682096272, 0.16269008279311103, + 0.4941250734508458, 0.5446841276322587, 0.19222703209695946, 0.9232239752817498, 0.8824688635063289, + 0.224690851359456, 0.5809304720756304, 0.36863807988348585] + + h2o_glm_random_init = utils_for_glm_tests.constraint_glm_gridsearch(train, predictors, response, solver="IRLSM", + family="binomial", + linear_constraints=linear_constraints2, + startval=random_coef, + init_optimal_glm=False, + constraint_eta0=constraint_eta0, + constraint_tau=constraint_tau, + constraint_alpha=constraint_alpha, + constraint_beta=constraint_beta, + constraint_c0=constraint_c0, + return_best=False) + random_init_logloss = h2o_glm_random_init.model_performance()._metric_json['logloss'] + print("logloss with random GLM coefficient initializaiton: {0}, number of iterations taken to build the model: " + "{1}".format(random_init_logloss, utils_for_glm_tests.find_glm_iterations(h2o_glm_random_init))) + print(glm.getConstraintsInfo(h2o_glm_random_init)) + print("All constraints satisfied: {0}".format(glm.allConstraintsPassed(h2o_glm_random_init))) + + assert logloss <= optimal_init_logloss, "logloss from optimal GLM {0} should be lower than logloss from GLM with light tight" \ + " constraints and initialized with optimal GLM {1} but is not.".format(logloss, optimal_init_logloss) + + assert logloss <= default_init_logloss, "logloss from optimal GLM {0} should be lower than logloss from GLM with light tight" \ + " constraints and initialized with default coefficients GLM {1} but is " \ + "not.".format(logloss, default_init_logloss) + + assert logloss <= random_init_logloss, "logloss from optimal GLM {0} should be lower than logloss from GLM with light tight" \ + " constraints and initialized with random coefficients GLM {1} but is " \ + "not.".format(logloss, random_init_logloss) + +if __name__ == "__main__": + pyunit_utils.standalone_test(test_light_tight_linear_constraints_only_gaussian) +else: + test_light_tight_linear_constraints_only_gaussian() diff --git a/h2o-py/tests/testdir_algos/glm/pyunit_PUBDEV_7730_plot_pr_xval.py b/h2o-py/tests/testdir_algos/glm/pyunit_PUBDEV_7730_plot_pr_xval.py index 17e05528225f..c7bda99d2da7 100644 --- a/h2o-py/tests/testdir_algos/glm/pyunit_PUBDEV_7730_plot_pr_xval.py +++ b/h2o-py/tests/testdir_algos/glm/pyunit_PUBDEV_7730_plot_pr_xval.py @@ -21,11 +21,11 @@ def glm_pr_plot_test(): test_data = data_frames[1] # build model with CV but no validation dataset - cv_model = glm(family='binomial',alpha=[0.1,0.5,0.9], nfolds = 3, fold_assignment="modulo") + cv_model = glm(family='binomial',alpha=[0.1,0.5,0.9], nfolds = 3, fold_assignment="modulo", seed=12345) cv_model.train(training_frame=training_data,x=myX,y=myY, validation_frame=test_data) fn = "pr_plot_train_valid_cx.png" perf = cv_model.model_performance(xval=True) - perf.plot(type="pr", server=True, save_to_file=fn) + perf.plot(type="pr", server=True, save_plot_path=fn) if os.path.isfile(fn): os.remove(fn) diff --git a/h2o-r/h2o-package/R/glm.R b/h2o-r/h2o-package/R/glm.R index 4c59645bd341..48b09a7fbea4 100644 --- a/h2o-r/h2o-package/R/glm.R +++ b/h2o-r/h2o-package/R/glm.R @@ -31,7 +31,7 @@ #' @param random_columns random columns indices for HGLM. #' @param ignore_const_cols \code{Logical}. Ignore constant columns. Defaults to TRUE. #' @param score_each_iteration \code{Logical}. Whether to score during each iteration of model training. Defaults to FALSE. -#' @param score_iteration_interval Perform scoring for every score_iteration_interval iterations Defaults to -1. +#' @param score_iteration_interval Perform scoring for every score_iteration_interval iterations. Defaults to -1. #' @param offset_column Offset column. This will be added to the combination of columns before applying the link function. #' @param weights_column Column with observation weights. Giving some observation a weight of zero is equivalent to excluding it from #' the dataset; giving an observation a relative weight of 2 is equivalent to repeating that row twice. Negative @@ -46,7 +46,7 @@ #' @param rand_family Random Component Family array. One for each random component. Only support gaussian for now. Must be one of: #' "[gaussian]". #' @param tweedie_variance_power Tweedie variance power Defaults to 0. -#' @param tweedie_link_power Tweedie link power Defaults to 1. +#' @param tweedie_link_power Tweedie link power. Defaults to 1. #' @param theta Theta Defaults to 1e-10. #' @param solver AUTO will set the solver based on given data and the other parameters. IRLSM is fast on on problems with small #' number of predictors and for lambda-search with L1 penalty, L_BFGS scales better for datasets with many @@ -56,35 +56,34 @@ #' represents Lasso regression, a value of 0 produces Ridge regression, and anything in between specifies the #' amount of mixing between the two. Default value of alpha is 0 when SOLVER = 'L-BFGS'; 0.5 otherwise. #' @param lambda Regularization strength -#' @param lambda_search \code{Logical}. Use lambda search starting at lambda max, given lambda is then interpreted as lambda min +#' @param lambda_search \code{Logical}. Use lambda search starting at lambda max, given lambda is then interpreted as lambda min. #' Defaults to FALSE. -#' @param early_stopping \code{Logical}. Stop early when there is no more relative improvement on train or validation (if provided) +#' @param early_stopping \code{Logical}. Stop early when there is no more relative improvement on train or validation (if provided). #' Defaults to TRUE. #' @param nlambdas Number of lambdas to be used in a search. Default indicates: If alpha is zero, with lambda search set to True, #' the value of nlamdas is set to 30 (fewer lambdas are needed for ridge regression) otherwise it is set to 100. #' Defaults to -1. -#' @param standardize \code{Logical}. Standardize numeric columns to have zero mean and unit variance Defaults to TRUE. +#' @param standardize \code{Logical}. Standardize numeric columns to have zero mean and unit variance. Defaults to TRUE. #' @param missing_values_handling Handling of missing values. Either MeanImputation, Skip or PlugValues. Must be one of: "MeanImputation", #' "Skip", "PlugValues". Defaults to MeanImputation. #' @param plug_values Plug Values (a single row frame containing values that will be used to impute missing values of the -#' training/validation frame, use with conjunction missing_values_handling = PlugValues) -#' @param compute_p_values \code{Logical}. Request p-values computation, p-values work only with IRLSM solver and no regularization -#' Defaults to FALSE. +#' training/validation frame, use with conjunction missing_values_handling = PlugValues). +#' @param compute_p_values \code{Logical}. Request p-values computation, p-values work only with IRLSM solver. Defaults to FALSE. #' @param dispersion_parameter_method Method used to estimate the dispersion parameter for Tweedie, Gamma and Negative Binomial only. Must be one #' of: "deviance", "pearson", "ml". Defaults to pearson. #' @param init_dispersion_parameter Only used for Tweedie, Gamma and Negative Binomial GLM. Store the initial value of dispersion parameter. If -#' fix_dispersion_parameter is set, this value will be used in the calculation of p-values.Default to 1.0. -#' Defaults to 1. -#' @param remove_collinear_columns \code{Logical}. In case of linearly dependent columns, remove some of the dependent columns Defaults to FALSE. +#' fix_dispersion_parameter is set, this value will be used in the calculation of p-values. Defaults to 1. +#' @param remove_collinear_columns \code{Logical}. In case of linearly dependent columns, remove the dependent columns. Defaults to FALSE. #' @param intercept \code{Logical}. Include constant term in the model Defaults to TRUE. -#' @param non_negative \code{Logical}. Restrict coefficients (not intercept) to be non-negative Defaults to FALSE. -#' @param max_iterations Maximum number of iterations Defaults to -1. +#' @param non_negative \code{Logical}. Restrict coefficients (not intercept) to be non-negative. Defaults to FALSE. +#' @param max_iterations Maximum number of iterations. Value should >=1. A value of 0 is only set when only the model coefficient +#' names and model coefficient dimensions are needed. Defaults to -1. #' @param objective_epsilon Converge if objective value changes less than this. Default (of -1.0) indicates: If lambda_search is set to #' True the value of objective_epsilon is set to .0001. If the lambda_search is set to False and lambda is equal #' to zero, the value of objective_epsilon is set to .000001, for any other value of lambda the default value of #' objective_epsilon is set to .0001. Defaults to -1. -#' @param beta_epsilon Converge if beta changes less (using L-infinity norm) than beta esilon, ONLY applies to IRLSM solver -#' Defaults to 0.0001. +#' @param beta_epsilon Converge if beta changes less (using L-infinity norm) than beta esilon. ONLY applies to IRLSM solver. Defaults +#' to 0.0001. #' @param gradient_epsilon Converge if objective changes less (using L-infinity norm) than this, ONLY applies to L-BFGS solver. Default #' (of -1.0) indicates: If lambda_search is set to False and lambda is equal to zero, the default value of #' gradient_epsilon is equal to .000001, otherwise the default value is .0001. If lambda_search is set to True, @@ -92,17 +91,18 @@ #' @param link Link function. Must be one of: "family_default", "identity", "logit", "log", "inverse", "tweedie", "ologit". #' Defaults to family_default. #' @param rand_link Link function array for random component in HGLM. Must be one of: "[identity]", "[family_default]". -#' @param startval double array to initialize fixed and random coefficients for HGLM, coefficients for GLM. +#' @param startval double array to initialize fixed and random coefficients for HGLM, coefficients for GLM. If standardize is +#' true, the standardized coefficients should be used. Otherwise, use the regular coefficients. #' @param calc_like \code{Logical}. if true, will return likelihood function value. Defaults to FALSE. -#' @param HGLM \code{Logical}. If set to true, will return HGLM model. Otherwise, normal GLM model will be returned Defaults -#' to FALSE. +#' @param HGLM \code{Logical}. If set to true, will return HGLM model. Otherwise, normal GLM model will be returned. +#' Defaults to FALSE. #' @param prior Prior probability for y==1. To be used only for logistic regression iff the data has been sampled and the mean #' of response does not reflect reality. Defaults to -1. #' @param cold_start \code{Logical}. Only applicable to multiple alpha/lambda values. If false, build the next model for next set #' of alpha/lambda values starting from the values provided by current model. If true will start GLM model from #' scratch. Defaults to FALSE. #' @param lambda_min_ratio Minimum lambda used in lambda search, specified as a ratio of lambda_max (the smallest lambda that drives all -#' coefficients to zero). Default indicates: if the number of observations is greater than the number of +#' coefficients to zero). Default indicates: if the number of observations is greater than the number of #' variables, then lambda_min_ratio is set to 0.0001; if the number of observations is less than the number of #' variables, then lambda_min_ratio is set to 0.01. Defaults to -1. #' @param beta_constraints Beta constraints @@ -111,7 +111,7 @@ #' max_active_predictors is set to 5000 otherwise it is set to 100000000. Defaults to -1. #' @param interactions A list of predictor column indices to interact. All pairwise combinations will be computed for the list. #' @param interaction_pairs A list of pairwise (first order) column interactions. -#' @param obj_reg Likelihood divider in objective value computation, default (of -1.0) will set it to 1/nobs Defaults to -1. +#' @param obj_reg Likelihood divider in objective value computation, default (of -1.0) will set it to 1/nobs. Defaults to -1. #' @param stopping_rounds Early stopping based on convergence of stopping_metric. Stop if simple moving average of length k of the #' stopping_metric does not improve for k:=stopping_rounds scoring events (0 to disable) Defaults to 0. #' @param stopping_metric Metric to use for early stopping (AUTO: logloss for classification, deviance for regression and anomaly_score @@ -155,6 +155,21 @@ #' @param influence If set to dfbetas will calculate the difference in beta when a datarow is included and excluded in the #' dataset. Must be one of: "dfbetas". #' @param gainslift_bins Gains/Lift table number of bins. 0 means disabled.. Default value -1 means automatic binning. Defaults to -1. +#' @param linear_constraints Linear constraints: used to specify linear constraints involving more than one coefficients in standard form. +#' It is only supported for solver IRLSM. It contains four columns: names (strings for coefficient names or +#' constant), values, types ( strings of 'Equal' or 'LessThanEqual'), constraint_numbers (0 for first linear +#' constraint, 1 for second linear constraint, ...). +#' @param init_optimal_glm \code{Logical}. If true, will initialize coefficients with values derived from GLM runs without linear +#' constraints. Only available for linear constraints. Defaults to FALSE. +#' @param separate_linear_beta \code{Logical}. If true, will keep the beta constraints and linear constraints separate. After new +#' coefficients are found, first beta constraints will be applied followed by the application of linear +#' constraints. Note that the beta constraints in this case will not be part of the objective function. If +#' false, will combine the beta and linear constraints. Defaults to FALSE. +#' @param constraint_eta0 For constrained GLM only. It affects the setting of eta_k+1=eta_0/power(ck+1, alpha). Defaults to 0.1258925. +#' @param constraint_tau For constrained GLM only. It affects the setting of c_k+1=tau*c_k. Defaults to 10. +#' @param constraint_alpha For constrained GLM only. It affects the setting of eta_k = eta_0/pow(c_0, alpha). Defaults to 0.1. +#' @param constraint_beta For constrained GLM only. It affects the setting of eta_k+1 = eta_k/pow(c_k, beta). Defaults to 0.9. +#' @param constraint_c0 For constrained GLM only. It affects the initial setting of epsilon_k = 1/c_0. Defaults to 10. #' @return A subclass of \code{\linkS4class{H2OModel}} is returned. The specific subclass depends on the machine #' learning task at hand (if it's binomial classification, then an \code{\linkS4class{H2OBinomialModel}} is #' returned, if it's regression then a \code{\linkS4class{H2ORegressionModel}} is returned). The default print- @@ -276,7 +291,15 @@ h2o.glm <- function(x, fix_tweedie_variance_power = TRUE, dispersion_learning_rate = 0.5, influence = c("dfbetas"), - gainslift_bins = -1) + gainslift_bins = -1, + linear_constraints = NULL, + init_optimal_glm = FALSE, + separate_linear_beta = FALSE, + constraint_eta0 = 0.1258925, + constraint_tau = 10, + constraint_alpha = 0.1, + constraint_beta = 0.9, + constraint_c0 = 10) { # Validate required training_frame first and other frame args: should be a valid key or an H2OFrame object training_frame <- .validate.H2OFrame(training_frame, required=TRUE) @@ -459,6 +482,22 @@ h2o.glm <- function(x, parms$influence <- influence if (!missing(gainslift_bins)) parms$gainslift_bins <- gainslift_bins + if (!missing(linear_constraints)) + parms$linear_constraints <- linear_constraints + if (!missing(init_optimal_glm)) + parms$init_optimal_glm <- init_optimal_glm + if (!missing(separate_linear_beta)) + parms$separate_linear_beta <- separate_linear_beta + if (!missing(constraint_eta0)) + parms$constraint_eta0 <- constraint_eta0 + if (!missing(constraint_tau)) + parms$constraint_tau <- constraint_tau + if (!missing(constraint_alpha)) + parms$constraint_alpha <- constraint_alpha + if (!missing(constraint_beta)) + parms$constraint_beta <- constraint_beta + if (!missing(constraint_c0)) + parms$constraint_c0 <- constraint_c0 if( !missing(interactions) ) { # interactions are column names => as-is @@ -563,6 +602,14 @@ h2o.glm <- function(x, dispersion_learning_rate = 0.5, influence = c("dfbetas"), gainslift_bins = -1, + linear_constraints = NULL, + init_optimal_glm = FALSE, + separate_linear_beta = FALSE, + constraint_eta0 = 0.1258925, + constraint_tau = 10, + constraint_alpha = 0.1, + constraint_beta = 0.9, + constraint_c0 = 10, segment_columns = NULL, segment_models_id = NULL, parallelism = 1) @@ -750,6 +797,22 @@ h2o.glm <- function(x, parms$influence <- influence if (!missing(gainslift_bins)) parms$gainslift_bins <- gainslift_bins + if (!missing(linear_constraints)) + parms$linear_constraints <- linear_constraints + if (!missing(init_optimal_glm)) + parms$init_optimal_glm <- init_optimal_glm + if (!missing(separate_linear_beta)) + parms$separate_linear_beta <- separate_linear_beta + if (!missing(constraint_eta0)) + parms$constraint_eta0 <- constraint_eta0 + if (!missing(constraint_tau)) + parms$constraint_tau <- constraint_tau + if (!missing(constraint_alpha)) + parms$constraint_alpha <- constraint_alpha + if (!missing(constraint_beta)) + parms$constraint_beta <- constraint_beta + if (!missing(constraint_c0)) + parms$constraint_c0 <- constraint_c0 if( !missing(interactions) ) { # interactions are column names => as-is diff --git a/h2o-r/h2o-package/R/models.R b/h2o-r/h2o-package/R/models.R index dcff26351981..7bc4b6d85e49 100755 --- a/h2o-r/h2o-package/R/models.R +++ b/h2o-r/h2o-package/R/models.R @@ -2676,6 +2676,90 @@ h2o.coef_with_p_values <- function(object) { } } +#' +#' Return the GLM linear constraints descriptions, constraints values, constraints bounds and whether the constraints +#' satisfied the bounds (true) or not (false) +#' +#' @param object An \linkS4class{H2OModel} object. +#' @examples +#' \dontrun{ +#' library(h2o) +#' h2o.init() +#' +#' f <- "https://s3.amazonaws.com/h2o-public-test-data/smalldata/junit/cars_20mpg.csv" +#' cars <- h2o.importFile(f) +#' predictors <- c("displacement", "power", "weight", "acceleration", "year") +#' response <- "acceleration" +#' colnames <- c("power", "weight", "constant") +#' values <- c(0.5, 1.0, 100) +#' types <- c("lessthanequal", "lessthanequal", "lessthanequal") +#' numbers <- c(0, 0, 0) +#' con <- data.frame(names=colnames, values=values, types=types, constraint_numbers=numbers) +#' cars_model <- h2o.glm(y=response, solver="irlsm", +#' x=predictors, +#' training_frame = cars, +#' linear_constraints=as.h2o(con), +#' lambda=0.0, +#' family="gaussian") +#' print(h2o.get_constraints_info(cars_model)) +#' } +#' @export +h2o.get_constraints_info <- function(object) { + if (is(object, "H2OModel") && object@algorithm %in% c("glm")) { + if (is.null(object@parameters$linear_constraints)) { + stop("GLM linear constraints information is only available where there are linear constraints specified + in the parameter linear_constraints!") + } else { + object@model$linear_constraints_table + } + } else { + stop("get_constraints_info is only available for GLM models with linear constraints specified in the parameter + linear_constraints!.") + } +} + +#' +#' Return the TRUE if all constraints are satisfied for a constraint GLM model and FALSE otherwise. If you want to +#' check which constraint failed, use h2o.get_constraints_info method. +#' +#' @param object An \linkS4class{H2OModel} object. +#' @examples +#' \dontrun{ +#' library(h2o) +#' h2o.init() +#' +#' f <- "https://s3.amazonaws.com/h2o-public-test-data/smalldata/junit/cars_20mpg.csv" +#' cars <- h2o.importFile(f) +#' predictors <- c("displacement", "power", "weight", "acceleration", "year") +#' response <- "acceleration" +#' colnames <- c("power", "weight", "constant") +#' values <- c(0.5, 1.0, 100) +#' types <- c("lessthanequal", "lessthanequal", "lessthanequal") +#' numbers <- c(0, 0, 0) +#' con <- data.frame(names=colnames, values=values, types=types, constraint_numbers=numbers) +#' cars_model <- h2o.glm(y=response, solver="irlsm", +#' x=predictors, +#' training_frame = cars, +#' linear_constraints=as.h2o(con), +#' lambda=0.0, +#' family="gaussian") +#' print(h2o.all_constraints_passed(cars_model)) +#' } +#' @export +h2o.all_constraints_passed <- function(object) { + if (is(object, "H2OModel") && object@algorithm %in% c("glm")) { + if (is.null(object@parameters$linear_constraints)) { + stop("h2o.all_constraints_passed is only available where there are linear constraints specified + in the parameter linear_constraints!") + } else { + object@model$all_constraints_satisfied + } + } else { + stop("h2o.all_constraints_passed is only available for GLM models with linear constraints specified in the + parameter linear_constraints!.") + } +} + #' #' Return the variable inflation factors associated with numerical predictors for GLM models. #' @@ -2969,6 +3053,41 @@ extract_scoring_history <- function(model, value) { } } +#' +#' Return the GLM coefficient names without building the actual GLM model by setting max_iterations=0. +#' +#' @param object an \linkS4class{H2OModel} object. +#' +#' @examples +#' \dontrun{ +#' library(h2o) +#' h2o.init() +#' +#' f <- "https://s3.amazonaws.com/h2o-public-test-data/smalldata/junit/cars_20mpg.csv" +#' cars <- h2o.importFile(f) +#' predictors <- c("displacement", "power", "weight", "acceleration", "year") +#' response <- "cylinders" +#' cars_glm <- h2o.glm(balance_classes = TRUE, +#' seed = 1234, +#' x = predictors, +#' y = response, +#' training_frame = cars, +#' max_iterations=0) +#' h2o.coef_names(cars_glm) +#' } +#' @export +h2o.coef_names <- function(object) { + if (is(object, "H2OModel") && + (object@algorithm %in% c("glm"))) { + coef_names = object@model$coefficient_names + if (object@allparameters$intercept) { # intercept exist + return(coef_names[-length(coef_names)]) + } else { + return(coef_names) + } + } +} + #' #' Return coefficients fitted on the standardized data (requires standardize = True, which is on by default). These coefficients can be used to evaluate variable importance. #' diff --git a/h2o-r/h2o-package/pkgdown/_pkgdown.yml b/h2o-r/h2o-package/pkgdown/_pkgdown.yml index 1139222ea4e3..3b66ce2221d1 100644 --- a/h2o-r/h2o-package/pkgdown/_pkgdown.yml +++ b/h2o-r/h2o-package/pkgdown/_pkgdown.yml @@ -43,6 +43,7 @@ reference: - h2o.aggregator - h2o.aic - h2o.all + - h2o.all_constraints_passed - h2o.anomaly - h2o.any - h2o.anyFactor @@ -71,6 +72,7 @@ reference: - h2o.clusterIsUp - h2o.clusterStatus - h2o.coef + - h2o.coef_names - h2o.coef_norm - h2o.coef_with_p_values - h2o.colnames @@ -132,6 +134,7 @@ reference: - h2o.generic - h2o.genericModel - h2o.get_automl + - h2o.get_constraints_info - h2o.get_knot_locations - h2o.get_leaderboard - h2o.get_ntrees_actual diff --git a/h2o-r/tests/testdir_algos/glm/runit_GH_6722_redundant_constraints.R b/h2o-r/tests/testdir_algos/glm/runit_GH_6722_redundant_constraints.R new file mode 100644 index 000000000000..d41fd01ded8c --- /dev/null +++ b/h2o-r/tests/testdir_algos/glm/runit_GH_6722_redundant_constraints.R @@ -0,0 +1,31 @@ +setwd(normalizePath(dirname(R.utils::commandArgs(asValues=TRUE)$"f"))) +source("../../../scripts/h2o-r-test-setup.R") +############################################################### +#### Test redundant constraints are caught ####### +############################################################### + +test_constraints_redundant <- function() { + result = tryCatch({ + training_frame <<- h2o.importFile(locate("smalldata/glm_test/gaussian_20cols_10000Rows.csv")) + training_frame[,1] <- as.factor(training_frame[,1]) + training_frame[,2] <- as.factor(training_frame[,2]) + training_frame[,3] <- as.factor(training_frame[,3]) + training_frame[,4] <- as.factor(training_frame[,4]) + training_frame[,5] <- as.factor(training_frame[,5]) + training_frame[,6] <- as.factor(training_frame[,6]) + training_frame[,7] <- as.factor(training_frame[,7]) + training_frame[,8] <- as.factor(training_frame[,8]) + training_frame[,9] <- as.factor(training_frame[,9]) + training_frame[,10] <- as.factor(training_frame[,10]) + linearConstraints <<- h2o.importFile(locate("smalldata/glm_test/linearConstraint3.csv")) + + x=c(1:20) + model <- h2o.glm(x = x, y=21, training_frame=training_frame, max_iterations=1, linear_constraints=linearConstraints, solver="IRLSM") + }, error = function(e) { + print("***") + print(e) + expect_true(grepl("redundant and possibly conflicting linear constraints:", e)) + }) +} + +doTest("GLM Test: Detect redundant constraint specification", test_constraints_redundant) diff --git a/h2o-r/tests/testdir_algos/glm/runit_gh_6722_glm_coefficient_names.R b/h2o-r/tests/testdir_algos/glm/runit_gh_6722_glm_coefficient_names.R new file mode 100644 index 000000000000..560ca4507f89 --- /dev/null +++ b/h2o-r/tests/testdir_algos/glm/runit_gh_6722_glm_coefficient_names.R @@ -0,0 +1,30 @@ +setwd(normalizePath(dirname(R.utils::commandArgs(asValues=TRUE)$"f"))) +source("../../../scripts/h2o-r-test-setup.R") + +# This test is used to test that we can obtain glm model coefficients without actually running the GLM model +# building process by setting max_iterations=0. Coefficient names with and without building the models +# are extracted and compared and they should be the same. +test <- function(h) { + training_frame <- h2o.importFile(locate("smalldata/glm_test/gaussian_20cols_10000Rows.csv")) + training_frame[,1] <- as.factor(training_frame[,1]) + training_frame[,2] <- as.factor(training_frame[,2]) + training_frame[,3] <- as.factor(training_frame[,3]) + training_frame[,4] <- as.factor(training_frame[,4]) + training_frame[,5] <- as.factor(training_frame[,5]) + training_frame[,6] <- as.factor(training_frame[,6]) + training_frame[,7] <- as.factor(training_frame[,7]) + training_frame[,8] <- as.factor(training_frame[,8]) + training_frame[,9] <- as.factor(training_frame[,9]) + training_frame[,10] <- as.factor(training_frame[,10]) + x=c(1:20) + model <- h2o.glm(x = x, y=21, training_frame=training_frame, max_iterations=0) + coeffs <- h2o.coef_names(model) + + model2 <- h2o.glm(x = x, y=21, training_frame=training_frame) + coeffs2 <- model2@model$coefficients_table$names + coeffs2NoIntercept <- coeffs2[-1] + + expect_equal(coeffs, coeffs2NoIntercept) +} + +doTest("GLM extract model coefficients", test) From a3ae44d4d2517e63f52794c45e22b804e9b07b3e Mon Sep 17 00:00:00 2001 From: Adam Valenta Date: Tue, 30 Apr 2024 16:56:16 +0200 Subject: [PATCH 4/4] GH-16154 devops changed url (#16185) * Currently the jenkins link not contain a node name. Its not a point to determine labels by that. * change node label also for PR pipelines - related to 3253c98b3bf35d1e50eb788f TODO: delete the node labels logic if all the builds run ok --- scripts/jenkins/groovy/buildConfig.groovy | 18 ++++-------------- scripts/jenkins/jenkinsfiles/Jenkinsfile | 2 +- 2 files changed, 5 insertions(+), 15 deletions(-) diff --git a/scripts/jenkins/groovy/buildConfig.groovy b/scripts/jenkins/groovy/buildConfig.groovy index d3ea58ba1fd4..65d21237ea50 100644 --- a/scripts/jenkins/groovy/buildConfig.groovy +++ b/scripts/jenkins/groovy/buildConfig.groovy @@ -99,7 +99,7 @@ class BuildConfig { changesMap[COMPONENT_HADOOP] = buildHadoop changedPythonTests = detectPythonTestChanges(changes) - nodeLabels = NodeLabels.findByBuildURL(context.env.BUILD_URL) + nodeLabels = NodeLabels.LABELS_C1 supportedXGBEnvironments = [ 'centos7.3': [ [name: 'CentOS 7.3 Minimal', dockerfile: 'xgb/centos/Dockerfile-centos-minimal', fromImage: 'centos:7.3.1611', targetName: XGB_TARGET_MINIMAL, nodeLabel: getDefaultNodeLabel()], @@ -369,13 +369,13 @@ class BuildConfig { } static enum NodeLabels { - LABELS_C1('docker && !mr-0xc8', 'mr-0xc9', 'gpu && !2gpu', 'mr-0xk10'), //master or nightly build - LABELS_B4('docker', 'docker', 'gpu && !2gpu', 'docker') //PR build + LABELS_C1('docker && !mr-0xc8', 'mr-0xc9', 'gpu && !2gpu', 'mr-0xk10'), //master or nightly build - use only this one + LABELS_B4('docker', 'docker', 'gpu && !2gpu', 'docker') //PR build - not used static Map LABELS_MAP = [ "c1": LABELS_C1, "g1": LABELS_C1, //mr-0xg1 was set as alias to mr-0xc1 - "b4": LABELS_B4 + "b4": LABELS_B4 // not used ] private final String defaultNodeLabel @@ -405,16 +405,6 @@ class BuildConfig { String getGPUBenchmarkNodeLabel() { return gpuBenchmarkNodeLabel } - - private static NodeLabels findByBuildURL(final String buildURL) { - final String name = buildURL.replaceAll('http://mr-0x', '').replaceAll(':8080.*', '') - - if (LABELS_MAP.containsKey(name)) { - return LABELS_MAP.get(name) - } else { - throw new IllegalArgumentException(String.format("Master %s (%s) is unknown", name, buildURL)) - } - } } } diff --git a/scripts/jenkins/jenkinsfiles/Jenkinsfile b/scripts/jenkins/jenkinsfiles/Jenkinsfile index 7a2bf78c4050..1a6d9dfcb1c9 100644 --- a/scripts/jenkins/jenkinsfiles/Jenkinsfile +++ b/scripts/jenkins/jenkinsfiles/Jenkinsfile @@ -2,7 +2,7 @@ final String MODE_PR = 'MODE_PR' final String MODE_MASTER = 'MODE_MASTER' -final String DEFAULT_NODE_LABEL = 'h2o-3 && docker && !mr-0xc8 && (!micro || micro_21)' +final String DEFAULT_NODE_LABEL = 'h2o-3' final int HEALTH_CHECK_RETRIES = 5 def defineTestStages = null