GH-15809: fixes loglikelihood and aic for glm generic model

h2o-algos/src/main/java/hex/generic/GenericModel.java

@@ -2,12 +2,14 @@
 
 import hex.*;
 import hex.genmodel.*;
+import hex.genmodel.algos.glm.GlmMojoModelBase;
 import hex.genmodel.algos.kmeans.KMeansMojoModel;
 import hex.genmodel.descriptor.ModelDescriptor;
 import hex.genmodel.descriptor.ModelDescriptorBuilder;
 import hex.genmodel.easy.EasyPredictModelWrapper;
 import hex.genmodel.easy.RowData;
 import hex.genmodel.easy.exception.PredictException;
+import hex.glm.GLMModel;
 import hex.tree.isofor.ModelMetricsAnomaly;
 import water.*;
 import water.fvec.*;
@@ -42,6 +44,7 @@ public class GenericModel extends Model<GenericModel, GenericModelParameters, Ge
      */
     private final String _algoName; 
     private final GenModelSource<?> _genModelSource;
+    private GLMModel.GLMParameters _glmParameters;
 
     /**
      * Full constructor
@@ -56,6 +59,26 @@ public GenericModel(Key<GenericModel> selfKey, GenericModelParameters parms, Gen
         if (mojoModel._modelAttributes != null && mojoModel._modelAttributes.getModelParameters() != null) {
             _parms._modelParameters = GenericModelParameters.convertParameters(mojoModel._modelAttributes.getModelParameters());
         }
+        _glmParameters = null;
+        if(_algoName.toLowerCase().contains("glm")) {
+            GlmMojoModelBase glmModel = (GlmMojoModelBase) mojoModel;
+            // create GLM parameters instance
+            _glmParameters = new GLMModel.GLMParameters(
+                    GLMModel.GLMParameters.Family.valueOf(getParamByName("family").toString()),
+                    GLMModel.GLMParameters.Link.valueOf(getParamByName("link").toString()),
+                    Arrays.stream(getParamByName("lambda").toString().trim().replaceAll("\\[", "")
+                                    .replaceAll("\\]", "").split(",\\s*"))
+                            .mapToDouble(Double::parseDouble).toArray(),
+                    Arrays.stream(getParamByName("alpha").toString().trim().replaceAll("\\[", "")
+                                    .replaceAll("\\]", "").split(",\\s*"))
+                            .mapToDouble(Double::parseDouble).toArray(),
+                    Double.parseDouble(getParamByName("tweedie_variance_power").toString()),
+                    Double.parseDouble(getParamByName("tweedie_link_power").toString()),
+                    null,
+                    Double.parseDouble(getParamByName("theta").toString()), 
+                    glmModel.getDispersionEstimated()
+            );
+        }
     }
 
     public GenericModel(Key<GenericModel> selfKey, GenericModelParameters parms, GenericModelOutput output,
@@ -133,6 +156,35 @@ protected PredictScoreResult predictScoreImpl(Frame fr, Frame adaptFrm, String d
             return super.predictScoreImpl(fr, adaptFrm, destination_key, j, computeMetrics, customMetricFunc);
     }
 
+    private Iced getParamByName(String name) {
+        return Arrays.stream(this._parms._modelParameters)
+                .filter(p -> Objects.equals(p.name, name)).findAny().get().actual_value;
+    }
+
+    @Override
+    public double aic(double likelihood) {
+        // calculate negative loglikelihood specifically  for GLM
+        if (!_algoName.equals("glm")) {
+            return Double.NaN;
+        } else {
+            long betasCount = Arrays.stream(((GlmMojoModelBase) this.genModel()).getBeta()).filter(b -> b != 0).count();
+            return -2 * likelihood + 2 * betasCount;
+        }
+    }
+
+    @Override
+    public double likelihood(double w, double y, double[] f) {
+        // calculate negative loglikelihood specifically for GLM
+        if(!_algoName.equals("glm")) {
+            return Double.NaN;
+        } else if (w == 0) {
+            return 0;
+        } else {
+            // time-consuming calculation for the final scoring for GLM model
+            return _glmParameters.likelihood(w, y, f);
+        }
+    }
+
     PredictScoreResult predictScoreMojoImpl(Frame fr, String destination_key, Job<?> j, boolean computeMetrics) {
         GenModel model = genModel();
         String[] names = model.getOutputNames();

h2o-algos/src/main/java/hex/glm/GLMModel.java

@@ -727,6 +727,11 @@ public GLMParameters(Family f, Link l, double [] lambda, double [] alpha, double
 
     public GLMParameters(Family f, Link l, double [] lambda, double [] alpha, double twVar, double twLnk, 
                          String[] interactions, double theta){
+      this(f,l,lambda,alpha,twVar,twLnk,interactions, theta, Double.NaN);
+    }
+
+    public GLMParameters(Family f, Link l, double [] lambda, double [] alpha, double twVar, double twLnk,
+                         String[] interactions, double theta, double dispersion_estimated){
       this._lambda = lambda;
       this._alpha = alpha;
       this._tweedie_variance_power = twVar;
@@ -736,7 +741,7 @@ public GLMParameters(Family f, Link l, double [] lambda, double [] alpha, double
       _link = l;
       this._theta=theta;
       this._invTheta = 1.0/theta;
-      this._dispersion_estimated = _init_dispersion_parameter;
+      this._dispersion_estimated = Double.isNaN(dispersion_estimated) ? _init_dispersion_parameter : dispersion_estimated;
     }
 
     public final double variance(double mu){

h2o-algos/src/main/java/hex/glm/GLMMojoWriter.java

@@ -39,6 +39,8 @@ protected void writeModelData() throws IOException {
 
     if (GLMModel.GLMParameters.Family.tweedie.equals(model._parms._family))
       writekv("tweedie_link_power", model._parms._tweedie_link_power);
+
+    writekv("dispersion_estimated", (model._parms._compute_p_values ? model._parms._dispersion_estimated : 1.0));
   }
 
 }

h2o-algos/src/main/java/hex/psvm/MetricBuilderPSVM.java

@@ -76,7 +76,7 @@ public ModelMetrics makeModelMetrics(Model m, Frame f, Frame frameWithWeights, F
     } else {
       auc = AUC2.emptyAUC();
     }
-    ModelMetricsBinomial mm = new ModelMetricsBinomial(m, f, _count, mse, _domain, sigma, auc, Double.NaN, null, _customMetric);
+    ModelMetricsBinomial mm = new ModelMetricsBinomial(m, f, _count, mse, _domain, sigma, auc, Double.NaN, Double.NaN, Double.NaN, null, _customMetric);
     if (m != null) m.addModelMetrics(mm);
     return mm;
   }

h2o-algos/src/main/java/hex/tree/isofor/MetricBuilderAnomalySupervised.java

@@ -35,7 +35,7 @@ public MetricBuilderAnomalySupervised(String[] domain) {
       auc = AUC2.emptyAUC();
     }
     ModelMetricsBinomial mm = new ModelMetricsBinomial(m, f, _count, mse, _domain,
-            sigma, auc, logloss, null, _customMetric);
+            sigma, auc, logloss, 0, 0, null, _customMetric);
     if (m != null) {
       m.addModelMetrics(mm);
     }

h2o-automl/src/test/java/ai/h2o/automl/ModelingStepsExecutorTest.java

@@ -361,7 +361,7 @@ protected void doSth() {
             GBMModel.GBMOutput output = new GBMModel.GBMOutput(new GBM(parms));
             Model res = new GBMModel(_job._result, parms, output);
             Frame fr = aml().getTrainingFrame();
-            output._training_metrics = new ModelMetricsRegression(res, fr, 1, 1, 1, 1, 1, 1, null);
+            output._training_metrics = new ModelMetricsRegression(res, fr, 1, 1, 1, 1, 1, 1, null, 0, 0);
             DKV.put(_job._result, res);
         }
     }

h2o-core/src/main/java/hex/Model.java

@@ -1381,7 +1381,11 @@ public double deviance(double w, double y, double f) {
   }
 
   public double likelihood(double w, double y, double[] f) {
-    return 0.0; // place holder.  This function is overridden in GLM.
+    return Double.NaN; // placeholder.  This function is overridden in GLM and GenericModel.
+  }
+
+  public double aic(double likelihood) {
+    return Double.NaN; // placeholder.  This function is overridden in GenericModel.
   }
 
   public ScoringInfo[] scoring_history() { return scoringInfo; }

h2o-core/src/main/java/hex/ModelMetricsBinomial.java

@@ -19,15 +19,19 @@
 public class ModelMetricsBinomial extends ModelMetricsSupervised {
   public final AUC2 _auc;
   public final double _logloss;
+  public final double _loglikelihood;
+  public final double _aic;
   public double _mean_per_class_error;
   public final GainsLift _gainsLift;
 
   public ModelMetricsBinomial(Model model, Frame frame, long nobs, double mse, String[] domain,
-                              double sigma, AUC2 auc, double logloss, GainsLift gainsLift,
+                              double sigma, AUC2 auc, double logloss, double loglikelihood, double aic, GainsLift gainsLift,
                               CustomMetric customMetric) {
     super(model, frame,  nobs, mse, domain, sigma, customMetric);
     _auc = auc;
     _logloss = logloss;
+    _loglikelihood = loglikelihood;
+    _aic = aic;
     _gainsLift = gainsLift;
     _mean_per_class_error = cm() == null ? Double.NaN : cm().mean_per_class_error();
   }
@@ -49,6 +53,8 @@ public String toString() {
       sb.append(" pr_auc: " + (float)_auc.pr_auc() + "\n");
     }
     sb.append(" logloss: " + (float)_logloss + "\n");
+    sb.append(" loglikelihood: " + (float)_loglikelihood + "\n");
+    sb.append(" AIC: " + (float)_aic + "\n");
     sb.append(" mean_per_class_error: " + (float)_mean_per_class_error + "\n");
     sb.append(" default threshold: " + (_auc == null ? 0.5 : (float)_auc.defaultThreshold()) + "\n");
     if (cm() != null) sb.append(" CM: " + cm().toASCII());
@@ -57,6 +63,8 @@ public String toString() {
   }
 
   public double logloss() { return _logloss; }
+  public double loglikelihood() { return _loglikelihood; }
+  public double aic() { return _aic; }
   public double mean_per_class_error() { return _mean_per_class_error; }
   @Override public AUC2 auc_obj() { return _auc; }
   @Override public ConfusionMatrix cm() {
@@ -161,6 +169,7 @@ private static class BinomialMetrics extends MRTask<BinomialMetrics> {
 
   public static class MetricBuilderBinomial<T extends MetricBuilderBinomial<T>> extends MetricBuilderSupervised<T> {
     protected double _logloss;
+    protected double _loglikelihood;
     protected AUC2.AUCBuilder _auc;
 
     public MetricBuilderBinomial( String[] domain ) { super(2,domain); _auc = new AUC2.AUCBuilder(AUC2.NBINS); }
@@ -197,6 +206,11 @@ public static class MetricBuilderBinomial<T extends MetricBuilderBinomial<T>> ex
         // Compute log loss
         _logloss += w * MathUtils.logloss(err);
       }
+
+      if(m != null && m.getClass().toString().contains("Generic")) {
+        _loglikelihood += m.likelihood(w, yact[0], ds);
+      }
+
       _count++;
       _wcount += w;
       assert !Double.isNaN(_sumsqe);
@@ -207,6 +221,7 @@ public static class MetricBuilderBinomial<T extends MetricBuilderBinomial<T>> ex
     @Override public void reduce( T mb ) {
       super.reduce(mb); // sumseq, count
       _logloss += mb._logloss;
+      _loglikelihood += mb._loglikelihood;
       _auc.reduce(mb._auc);
     }
 
@@ -256,18 +271,24 @@ private ModelMetrics makeModelMetrics(final Model m, final Frame f, final Frame
 
     private ModelMetrics makeModelMetrics(Model m, Frame f, GainsLift gl) {
       double mse = Double.NaN;
+      double loglikelihood = Double.NaN;
+      double aic = Double.NaN;
       double logloss = Double.NaN;
       double sigma = Double.NaN;
       final AUC2 auc;
       if (_wcount > 0) {
         sigma = weightedSigma();
         mse = _sumsqe / _wcount;
         logloss = _logloss / _wcount;
+        if(m != null && m.getClass().toString().contains("Generic")) {
+          loglikelihood = -1 * _loglikelihood ; // get likelihood from negative loglikelihood
+          aic = m.aic(loglikelihood);
+        }
         auc = new AUC2(_auc);
       } else {
         auc = new AUC2();
       }
-      ModelMetricsBinomial mm = new ModelMetricsBinomial(m, f, _count, mse, _domain, sigma, auc,  logloss, gl, _customMetric);
+      ModelMetricsBinomial mm = new ModelMetricsBinomial(m, f, _count, mse, _domain, sigma, auc,  logloss, loglikelihood, aic, gl, _customMetric);
       if (m!=null) m.addModelMetrics(mm);
       return mm;
     }

h2o-core/src/main/java/hex/ModelMetricsBinomialGLM.java

@@ -14,7 +14,7 @@ public ModelMetricsBinomialGLM(Model model, Frame frame, long nobs, double mse,
                                  double sigma, AUC2 auc, double logloss, double resDev, double nullDev,
                                  double aic, long nDof, long rDof, GainsLift gainsLift,
                                  CustomMetric customMetric, double loglikelihood) {
-    super(model, frame, nobs, mse, domain, sigma, auc, logloss, gainsLift, customMetric);
+    super(model, frame, nobs, mse, domain, sigma, auc, logloss, loglikelihood, aic, gainsLift, customMetric);
     _resDev = resDev;
     _nullDev = nullDev;
     _AIC = aic;
@@ -70,7 +70,7 @@ public ModelMetricsMultinomialGLM(Model model, Frame frame, long nobs, double ms
                                       double sigma, ConfusionMatrix cm, float [] hr, double logloss,
                                       double resDev, double nullDev, double aic, long nDof, long rDof,
                                       MultinomialAUC auc, CustomMetric customMetric, double loglikelihood) {
-      super(model, frame, nobs,  mse, domain, sigma, cm, hr, logloss, auc, customMetric);
+      super(model, frame, nobs,  mse, domain, sigma, cm, hr, logloss, loglikelihood, aic, auc, customMetric);
       _resDev = resDev;
       _nullDev = nullDev;
       _AIC = aic;

h2o-core/src/main/java/hex/ModelMetricsBinomialGeneric.java

@@ -16,7 +16,7 @@ public ModelMetricsBinomialGeneric(Model model, Frame frame, long nobs, double m
                                      CustomMetric customMetric, double mean_per_class_error, TwoDimTable thresholds_and_metric_scores,
                                      TwoDimTable max_criteria_and_metric_scores, TwoDimTable confusion_matrix, double r2,
                                      final String description) {
-    super(model, frame, nobs, mse, domain, sigma, auc, logloss, null, customMetric);
+    super(model, frame, nobs, mse, domain, sigma, auc, logloss, 0, 0, null, customMetric);
     _gainsLiftTable = gainsLiftTable;
     _thresholds_and_metric_scores = thresholds_and_metric_scores;
     _max_criteria_and_metric_scores = max_criteria_and_metric_scores;

h2o-core/src/main/java/hex/ModelMetricsMultinomial.java

@@ -18,14 +18,20 @@ public class ModelMetricsMultinomial extends ModelMetricsSupervised {
   public final float[] _hit_ratios;         // Hit ratios
   public final ConfusionMatrix _cm;
   public final double _logloss;
+  public final double _loglikelihood;
+  public final double _aic;
   public double _mean_per_class_error;
   public MultinomialAUC _auc;
 
-  public ModelMetricsMultinomial(Model model, Frame frame, long nobs, double mse, String[] domain, double sigma, ConfusionMatrix cm, float[] hr, double logloss, MultinomialAUC auc, CustomMetric customMetric) {
+  public ModelMetricsMultinomial(Model model, Frame frame, long nobs, double mse, String[] domain, double sigma, 
+                                 ConfusionMatrix cm, float[] hr, double logloss, double loglikelihood, double aic, 
+                                 MultinomialAUC auc, CustomMetric customMetric) {
     super(model, frame, nobs, mse, domain, sigma, customMetric);
     _cm = cm;
     _hit_ratios = hr;
     _logloss = logloss;
+    _loglikelihood = loglikelihood;
+    _aic = aic;
     _mean_per_class_error = cm==null || cm.tooLarge() ? Double.NaN : cm.mean_per_class_error();
     _auc = auc;
   }
@@ -35,6 +41,8 @@ public String toString() {
     StringBuilder sb = new StringBuilder();
     sb.append(super.toString());
     sb.append(" logloss: " + (float)_logloss + "\n");
+    sb.append(" loglikelihood: " + (float)_loglikelihood + "\n");
+    sb.append(" AIC: " + (float)_aic + "\n");
     sb.append(" mean_per_class_error: " + (float)_mean_per_class_error + "\n");
     sb.append(" hit ratios: " + Arrays.toString(_hit_ratios) + "\n");
     sb.append(" AUC: "+auc()+ "\n");
@@ -59,6 +67,8 @@ public String toString() {
   }
 
   public double logloss() { return _logloss; }
+  public double loglikelihood() { return _loglikelihood; }
+  public double aic() { return _aic; }
   public double mean_per_class_error() { return _mean_per_class_error; }
   @Override public ConfusionMatrix cm() { return _cm; }
   @Override public float[] hr() { return _hit_ratios; }
@@ -235,6 +245,7 @@ public static class MetricBuilderMultinomial<T extends MetricBuilderMultinomial<
     double[/*K*/] _hits;            // the number of hits for hitratio, length: K
     int _K;               // TODO: Let user set K
     double _logloss;
+    protected double _loglikelihood;
     boolean _calculateAuc;
     AUC2.AUCBuilder[/*nclasses*/][/*nclasses*/] _ovoAucs;
     AUC2.AUCBuilder[/*nclasses*/] _ovrAucs;
@@ -302,6 +313,11 @@ public MetricBuilderMultinomial( int nclasses, String[] domain, MultinomialAucTy
       if(_calculateAuc) {
         calculateAucsPerRow(ds, iact, w);
       }
+
+
+      if(m != null && m.getClass().toString().contains("Generic")) {
+        _loglikelihood += m.likelihood(w, yact[0], ds);
+      }
       return ds;                // Flow coding
     }
 
@@ -335,6 +351,7 @@ private void calculateAucsPerRow(double ds[], int iact, double w){
       ArrayUtils.add(_cm, mb._cm);
       _hits = ArrayUtils.add(_hits, mb._hits);
       _logloss += mb._logloss;
+      _loglikelihood += mb._loglikelihood;
       if(_calculateAuc) {
         for (int i = 0; i < _ovoAucs.length; i++) {
           _ovrAucs[i].reduce(mb._ovrAucs[i]);
@@ -350,6 +367,8 @@ private void calculateAucsPerRow(double ds[], int iact, double w){
     @Override public ModelMetrics makeModelMetrics(Model m, Frame f, Frame adaptedFrame, Frame preds) {
       double mse = Double.NaN;
       double logloss = Double.NaN;
+      double loglikelihood = Double.NaN;
+      double aic = Double.NaN;
       float[] hr = new float[_K];
       ConfusionMatrix cm = new ConfusionMatrix(_cm, _domain);
       double sigma = weightedSigma();
@@ -360,10 +379,14 @@ private void calculateAucsPerRow(double ds[], int iact, double w){
         }
         mse = _sumsqe / _wcount;
         logloss = _logloss / _wcount;
+        if(m != null && m.getClass().toString().contains("Generic")) {
+          loglikelihood = -1 * _loglikelihood ; // get likelihood from negative loglikelihood
+          aic = m.aic(loglikelihood);
+        }
       }
       MultinomialAUC auc = new MultinomialAUC(_ovrAucs,_ovoAucs, _domain, _wcount == 0, _aucType);
       ModelMetricsMultinomial mm = new ModelMetricsMultinomial(m, f, _count, mse, _domain, sigma, cm,
-                                                               hr, logloss, auc, _customMetric);
+                                                               hr, logloss, loglikelihood, aic, auc, _customMetric);
       if (m!=null) m.addModelMetrics(mm);
       return mm;
     }

h2o-core/src/main/java/hex/ModelMetricsMultinomialGeneric.java

@@ -16,7 +16,7 @@ public ModelMetricsMultinomialGeneric(Model model, Frame frame, long nobs, doubl
                                         TwoDimTable confusion_matrix, TwoDimTable hit_ratio_table, double logloss, CustomMetric customMetric,
                                         double mean_per_class_error, double r2, TwoDimTable multinomial_auc_table, TwoDimTable multinomial_aucpr_table, 
                                         MultinomialAucType type, final String description) {
-    super(model, frame, nobs, mse, domain, sigma, null, null, logloss, null, customMetric);
+    super(model, frame, nobs, mse, domain, sigma, null, null, logloss, 0, 0, null, customMetric);
     _confusion_matrix_table = confusion_matrix; 
     _hit_ratio_table = hit_ratio_table;
     _auc = new MultinomialAUC(multinomial_auc_table, multinomial_aucpr_table, domain, type);