Survival contrast sets running without crash.

agudys · agudys · commit 4ec5e59e310b · 2024-06-11T14:15:23.000+02:00
diff --git a/adaa.analytics.rules/build.gradle b/adaa.analytics.rules/build.gradle
@@ -27,7 +27,7 @@ codeQuality {
 }
 
 sourceCompatibility = 1.8
-version = '1.7.15'
+version = '1.7.16'
 
 
 jar {
diff --git a/adaa.analytics.rules/src/main/java/adaa/analytics/rules/logic/induction/ContrastClassificationFinder.java b/adaa.analytics.rules/src/main/java/adaa/analytics/rules/logic/induction/ContrastClassificationFinder.java
@@ -2,12 +2,10 @@
 
 import adaa.analytics.rules.logic.quality.IQualityMeasure;
 import adaa.analytics.rules.logic.quality.NegativeControlledMeasure;
-import adaa.analytics.rules.logic.representation.ContrastRule;
-import adaa.analytics.rules.logic.representation.MultiSet;
-import adaa.analytics.rules.logic.representation.Rule;
-import adaa.analytics.rules.logic.representation.SingletonSet;
+import adaa.analytics.rules.logic.representation.*;
 import com.rapidminer.example.Attribute;
 import com.rapidminer.example.ExampleSet;
+import com.rapidminer.example.set.SortedExampleSet;
 import com.rapidminer.example.table.NominalMapping;
 
 import java.util.Map;
@@ -34,6 +32,11 @@ public ContrastClassificationFinder(InductionParameters params) {
         params.setPruningMeasure(new NegativeControlledMeasure(params.getPruningMeasure(), params.getMaxcovNegative()));
     }
 
+    public ExampleSet preprocess(ExampleSet trainSet) {
+        super.preprocess(trainSet);
+        return trainSet; // return original one
+    }
+
 
     /**
      * Invokes grow method from the super class and verifies negative coverage requirement.
diff --git a/adaa.analytics.rules/src/main/java/adaa/analytics/rules/logic/induction/ContrastRegressionFinder.java b/adaa.analytics.rules/src/main/java/adaa/analytics/rules/logic/induction/ContrastRegressionFinder.java
@@ -32,9 +32,9 @@ public String getName() {
         @Override
         public double calculate(ExampleSet dataset, ContingencyTable ct) {
 
-            ContrastRegressionExampleSet cer = (dataset instanceof ContrastExampleSet) ? (ContrastRegressionExampleSet)dataset : null;
+            ContrastRegressionExampleSet cer = (dataset instanceof ContrastRegressionExampleSet) ? (ContrastRegressionExampleSet)dataset : null;
             if (cer == null) {
-                throw new InvalidParameterException("ContrastSurvivalRuleSet supports only ContrastRegressionExampleSet instances");
+                throw new InvalidParameterException("ContrastRegressionRuleSet supports only ContrastRegressionExampleSet instances");
             }
 
             Covering cov = (Covering)ct;
@@ -78,6 +78,11 @@ public ContrastRegressionFinder(InductionParameters params) {
         params.setMeanBasedRegression(false);
     }
 
+    public ExampleSet preprocess(ExampleSet trainSet) {
+        super.preprocess(trainSet);
+        return trainSet; // return original one
+    }
+
     /**
      * Invokes grow method from the super class and verifies negative coverage requirement.
      *
diff --git a/adaa.analytics.rules/src/main/java/adaa/analytics/rules/logic/induction/ContrastSnC.java b/adaa.analytics.rules/src/main/java/adaa/analytics/rules/logic/induction/ContrastSnC.java
@@ -40,7 +40,7 @@ public ContrastSnC(AbstractFinder finder, InductionParameters params) {
     public RuleSetBase run(ExampleSet dataset) {
 
         // make a contrast dataset
-        ContrastExampleSet ces;
+        IContrastExampleSet ces;
 
         if (factory.getType() == RuleFactory.CONTRAST_REGRESSION) {
             ces = new ContrastRegressionExampleSet((SimpleExampleSet) dataset);
@@ -84,7 +84,7 @@ public RuleSetBase run(ExampleSet dataset) {
      * @param dataset Training data set.
      * @return Rule set.
      */
-    protected void run(ContrastExampleSet dataset, ContrastRuleSet crs) {
+    protected void run(IContrastExampleSet dataset, ContrastRuleSet crs) {
         Logger.log("ContrastSnC.run()\n", Level.FINE);
 
         // try to get contrast attribute (use label if not specified)
diff --git a/adaa.analytics.rules/src/main/java/adaa/analytics/rules/logic/induction/ContrastSurvivalFinder.java b/adaa.analytics.rules/src/main/java/adaa/analytics/rules/logic/induction/ContrastSurvivalFinder.java
@@ -11,6 +11,7 @@
 import java.security.InvalidParameterException;
 import java.util.HashSet;
 import java.util.Set;
+import java.util.logging.Level;
 
 public class ContrastSurvivalFinder extends SurvivalLogRankFinder implements IPenalizedFinder {
 
@@ -27,7 +28,7 @@ public String getName() {
         @Override
         public double calculate(ExampleSet dataset, ContingencyTable ct) {
 
-            ContrastSurvivalExampleSet ces = (dataset instanceof ContrastExampleSet) ? (ContrastSurvivalExampleSet)dataset : null;
+            ContrastSurvivalExampleSet ces = (dataset instanceof ContrastSurvivalExampleSet) ? (ContrastSurvivalExampleSet)dataset : null;
             if (ces == null) {
                 throw new InvalidParameterException("ContrastSurvivalRuleSet supports only ContrastSurvivalExampleSet instances");
             }
@@ -71,6 +72,11 @@ public ContrastSurvivalFinder(InductionParameters params) {
         params.setVotingMeasure(m);
     }
 
+    public ExampleSet preprocess(ExampleSet trainSet) {
+        super.preprocess(trainSet);
+        return trainSet; // return original one
+    }
+
     /**
      * Invokes grow method from the super class and verifies negative coverage requirement.
      *
@@ -123,6 +129,79 @@ public void postprocess(
         notifyRuleReady(rule);
     }
 
+    protected boolean checkCandidate(
+            ExampleSet dataset,
+            Rule rule,
+            ConditionBase candidate,
+            Set<Integer> uncovered,
+            Set<Integer> covered,
+            ConditionEvaluation currentBest) {
+
+        try {
+
+            CompoundCondition newPremise = new CompoundCondition();
+            newPremise.getSubconditions().addAll(rule.getPremise().getSubconditions());
+            newPremise.addSubcondition(candidate);
+
+            Rule newRule = (Rule) rule.clone();
+            newRule.setPremise(newPremise);
+
+
+            Covering cov = new Covering();
+            newRule.covers(dataset, cov, cov.positives, cov.negatives);
+
+            double new_p = 0, new_n = 0;
+
+            if (dataset.getAttributes().getWeight() == null) {
+                // unweighted examples
+                new_p = SetHelper.intersectionSize(uncovered, cov.positives);
+                new_n =	SetHelper.intersectionSize(uncovered, cov.negatives);
+            } else {
+                // calculate weights of newly covered examples
+                for (int id : cov.positives) {
+                    new_p += uncovered.contains(id) ? dataset.getExample(id).getWeight() : 0;
+                }
+                for (int id : cov.negatives) {
+                    new_n += uncovered.contains(id) ? dataset.getExample(id).getWeight() : 0;
+                }
+            }
+
+            if (checkCoverage(cov.weighted_p, cov.weighted_n, new_p, new_n, dataset.size(), 0, uncovered.size(), rule.getRuleOrderNum())) {
+
+                double quality = params.getInductionMeasure().calculate(dataset, cov);
+
+                if (candidate instanceof ElementaryCondition) {
+                    ElementaryCondition ec = (ElementaryCondition) candidate;
+                    quality = modifier.modifyQuality(quality, ec.getAttribute(), cov.weighted_p, new_p);
+                }
+
+                if (quality > currentBest.quality ||
+                        (quality == currentBest.quality && (new_p > currentBest.covered || currentBest.opposite))) {
+
+                    Logger.log("\t\tCurrent best: " + candidate + " (p=" + cov.weighted_p +
+                            ", new_p=" + (double) new_p +
+                            ", P=" + cov.weighted_P +
+                            ", mean_y=" + cov.mean_y + ", mean_y2=" + cov.mean_y2 + ", stddev_y=" + cov.stddev_y +
+                            ", quality=" + quality + "\n", Level.FINEST);
+
+                    currentBest.quality = quality;
+                    currentBest.condition = candidate;
+                    currentBest.covered = new_p;
+                    currentBest.covering = cov;
+                    currentBest.opposite = (candidate instanceof ElementaryCondition) &&
+                            (((ElementaryCondition) candidate).getValueSet() instanceof SingletonSetComplement);
+
+                    //rule.setWeight(quality);
+                    return true;
+                }
+            }
+
+        } catch (Exception e) {
+            e.printStackTrace();
+        }
+        return false;
+    }
+
 
     boolean checkCoverage(double p, double n, double new_p, double new_n, double P, double N) {
                return ((new_p) >= params.getAbsoluteMinimumCovered(P)) &&
diff --git a/adaa.analytics.rules/src/main/java/adaa/analytics/rules/logic/representation/ContrastExampleSet.java b/adaa.analytics.rules/src/main/java/adaa/analytics/rules/logic/representation/ContrastExampleSet.java
@@ -11,7 +11,7 @@
 import java.util.ArrayList;
 import java.util.List;
 
-public class ContrastExampleSet extends SimpleExampleSet {
+public class ContrastExampleSet extends SimpleExampleSet implements IContrastExampleSet {
 
     protected Attribute contrastAttribute;
 
diff --git a/adaa.analytics.rules/src/main/java/adaa/analytics/rules/logic/representation/ContrastRegressionExampleSet.java b/adaa.analytics.rules/src/main/java/adaa/analytics/rules/logic/representation/ContrastRegressionExampleSet.java
@@ -12,22 +12,30 @@
 import java.util.ArrayList;
 import java.util.List;
 
-public class ContrastRegressionExampleSet extends ContrastExampleSet {
+public class ContrastRegressionExampleSet extends SortedExampleSetEx implements IContrastExampleSet {
+
+    protected Attribute contrastAttribute;
 
     /** Training set estimator. */
     protected double trainingEstimator;
 
     /** Collection of Kaplan-Meier estimators for contrast groups. */
     protected List<Double> groupEstimators = new ArrayList<Double>();
 
+    public Attribute getContrastAttribute() { return contrastAttribute; }
+
     /** Gets {@link #groupEstimators} */
     public List<Double> getGroupEstimators() { return groupEstimators; }
 
     /** Gets {@link #trainingEstimator}}. */
     public double getTrainingEstimator() { return trainingEstimator; }
 
     public ContrastRegressionExampleSet(SimpleExampleSet exampleSet) {
-        super(exampleSet);
+        super(exampleSet, exampleSet.getAttributes().getLabel(), SortedExampleSetEx.INCREASING);
+
+        contrastAttribute = (exampleSet.getAttributes().getSpecial(ContrastRule.CONTRAST_ATTRIBUTE_ROLE) == null)
+                ? exampleSet.getAttributes().getLabel()
+                : exampleSet.getAttributes().getSpecial(ContrastRule.CONTRAST_ATTRIBUTE_ROLE);
 
         String averageName = (exampleSet.getAttributes().getWeight() != null)
                 ? Statistics.AVERAGE_WEIGHTED : Statistics.AVERAGE;
@@ -57,6 +65,7 @@ public ContrastRegressionExampleSet(SimpleExampleSet exampleSet) {
 
     public ContrastRegressionExampleSet(ContrastRegressionExampleSet rhs) {
         super(rhs);
+        this.contrastAttribute = rhs.contrastAttribute;
         this.trainingEstimator = rhs.trainingEstimator;
         this.groupEstimators = rhs.groupEstimators;
     }
diff --git a/adaa.analytics.rules/src/main/java/adaa/analytics/rules/logic/representation/ContrastRegressionRuleSet.java b/adaa.analytics.rules/src/main/java/adaa/analytics/rules/logic/representation/ContrastRegressionRuleSet.java
@@ -39,7 +39,7 @@ public class ContrastRegressionRuleSet extends ContrastRuleSet {
     public ContrastRegressionRuleSet(ExampleSet exampleSet, boolean isVoting, InductionParameters params, Knowledge knowledge) {
         super(exampleSet, isVoting, params, knowledge);
 
-        ContrastRegressionExampleSet cer = (exampleSet instanceof ContrastExampleSet) ? (ContrastRegressionExampleSet)exampleSet : null;
+        ContrastRegressionExampleSet cer = (exampleSet instanceof ContrastRegressionExampleSet) ? (ContrastRegressionExampleSet)exampleSet : null;
         if (cer == null) {
             throw new InvalidParameterException("ContrastRegressionRuleSet supports only ContrastRegressionExampleSet instances");
         }
diff --git a/adaa.analytics.rules/src/main/java/adaa/analytics/rules/logic/representation/ContrastSurvivalExampleSet.java b/adaa.analytics.rules/src/main/java/adaa/analytics/rules/logic/representation/ContrastSurvivalExampleSet.java
@@ -1,34 +1,46 @@
 package adaa.analytics.rules.logic.representation;
 
+import com.rapidminer.example.Attribute;
 import com.rapidminer.example.ExampleSet;
 import com.rapidminer.example.set.AttributeValueFilterSingleCondition;
 import com.rapidminer.example.set.ConditionedExampleSet;
 import com.rapidminer.example.set.SimpleExampleSet;
+import com.rapidminer.example.set.SortedExampleSet;
 import com.rapidminer.example.table.NominalMapping;
 import com.rapidminer.operator.tools.ExpressionEvaluationException;
 
 import java.util.ArrayList;
 import java.util.List;
 
-public class ContrastSurvivalExampleSet extends ContrastExampleSet {
+public class ContrastSurvivalExampleSet extends SortedExampleSetEx implements IContrastExampleSet {
+
+    protected Attribute contrastAttribute;
 
     /** Training set estimator. */
     protected KaplanMeierEstimator trainingEstimator;
 
     /** Collection of Kaplan-Meier estimators for contrast groups. */
     protected List<KaplanMeierEstimator> groupEstimators = new ArrayList<KaplanMeierEstimator>();
 
+    public Attribute getContrastAttribute() { return contrastAttribute; }
+
     /** Gets {@link #groupEstimators} */
     public List<KaplanMeierEstimator> getGroupEstimators() { return groupEstimators; }
 
     /** Gets {@link #trainingEstimator}}. */
     public KaplanMeierEstimator getTrainingEstimator() { return trainingEstimator; }
 
     public ContrastSurvivalExampleSet(SimpleExampleSet exampleSet) {
-        super(exampleSet);
+        super(exampleSet, exampleSet.getAttributes().getSpecial(SurvivalRule.SURVIVAL_TIME_ROLE), SortedExampleSetEx.INCREASING);
+
+        contrastAttribute = (exampleSet.getAttributes().getSpecial(ContrastRule.CONTRAST_ATTRIBUTE_ROLE) == null)
+                ? exampleSet.getAttributes().getLabel()
+                : exampleSet.getAttributes().getSpecial(ContrastRule.CONTRAST_ATTRIBUTE_ROLE);
+
+        Attribute survTime = exampleSet.getAttributes().getSpecial(SurvivalRule.SURVIVAL_TIME_ROLE);
 
         // establish training survival estimator
-        trainingEstimator = new KaplanMeierEstimator(exampleSet);
+        trainingEstimator = new KaplanMeierEstimator(this);
 
         // establish contrast groups survival estimator
         try {
@@ -39,7 +51,9 @@ public ContrastSurvivalExampleSet(SimpleExampleSet exampleSet) {
                         contrastAttribute, AttributeValueFilterSingleCondition.EQUALS, mapping.mapIndex(i));
 
                 ExampleSet conditionedSet = new ConditionedExampleSet(exampleSet, cnd);
-                groupEstimators.add(new KaplanMeierEstimator(conditionedSet));
+                SortedExampleSetEx cses = new SortedExampleSetEx(conditionedSet, survTime, SortedExampleSet.INCREASING);
+
+                groupEstimators.add(new KaplanMeierEstimator(cses));
             }
 
         } catch (ExpressionEvaluationException e) {
@@ -49,6 +63,7 @@ public ContrastSurvivalExampleSet(SimpleExampleSet exampleSet) {
 
     public ContrastSurvivalExampleSet(ContrastSurvivalExampleSet rhs) {
         super(rhs);
+        this.contrastAttribute = rhs.contrastAttribute;
         this.trainingEstimator = rhs.trainingEstimator;
         this.groupEstimators = rhs.groupEstimators;
     }
diff --git a/adaa.analytics.rules/src/main/java/adaa/analytics/rules/logic/representation/ContrastSurvivalRuleSet.java b/adaa.analytics.rules/src/main/java/adaa/analytics/rules/logic/representation/ContrastSurvivalRuleSet.java
@@ -42,9 +42,9 @@ public class ContrastSurvivalRuleSet extends ContrastRuleSet {
     public ContrastSurvivalRuleSet(ExampleSet exampleSet, boolean isVoting, InductionParameters params, Knowledge knowledge) {
         super(exampleSet, isVoting, params, knowledge);
 
-        ContrastSurvivalExampleSet ces = (exampleSet instanceof ContrastExampleSet) ? (ContrastSurvivalExampleSet)exampleSet : null;
+        ContrastSurvivalExampleSet ces = (exampleSet instanceof ContrastSurvivalExampleSet) ? (ContrastSurvivalExampleSet)exampleSet : null;
         if (ces == null) {
-            throw new InvalidParameterException("ContrastSurvivalRuleSet supports only ContrastExampleSet instances");
+            throw new InvalidParameterException("ContrastSurvivalRuleSet supports only ContrastSurvivalExampleSet instances");
         }
 
         trainingEstimator = ces.getTrainingEstimator();
diff --git a/adaa.analytics.rules/src/main/java/adaa/analytics/rules/logic/representation/KaplanMeierEstimator.java b/adaa.analytics.rules/src/main/java/adaa/analytics/rules/logic/representation/KaplanMeierEstimator.java
@@ -135,7 +135,7 @@ public KaplanMeierEstimator(ExampleSet data, Set<Integer> indices) {
 
         SortedExampleSetEx set = (data instanceof SortedExampleSetEx) ? (SortedExampleSetEx)data : null;
         if (set == null) {
-            throw new InvalidParameterException("RegressionRules support only ListedExampleSet example sets");
+            throw new InvalidParameterException("KaplanMeierEstimator support only SortedExampleSetEx example sets");
         }
 
 		this.reserve(indices.size());
diff --git a/adaa.analytics.rules/src/main/java/adaa/analytics/rules/logic/representation/SortedExampleSetEx.java b/adaa.analytics.rules/src/main/java/adaa/analytics/rules/logic/representation/SortedExampleSetEx.java
@@ -42,6 +42,11 @@ public SortedExampleSetEx(SortedExampleSet exampleSet) {
         fillLabelsAndWeights();
     }
 
+    public SortedExampleSetEx(SortedExampleSetEx exampleSet) {
+        super(exampleSet);
+        fillLabelsAndWeights();
+    }
+
     protected final void fillLabelsAndWeights() {
         labels = new double[this.size()];
         labelsWeighted = new double[this.size()];

Original file line number	Diff line number	Diff line change
`@@ -27,7 +27,7 @@ codeQuality {`
`27`	`27`	`}`
`28`	`28`
`29`	`29`	`sourceCompatibility = 1.8`
`30`		`-version = '1.7.15'`
	`30`	`+version = '1.7.16'`
`31`	`31`
`32`	`32`
`33`	`33`	`jar {`
Original file line number	Diff line number	Diff line change
`@@ -39,7 +39,7 @@ public class ContrastRegressionRuleSet extends ContrastRuleSet {`
`39`	`39`	`public ContrastRegressionRuleSet(ExampleSet exampleSet, boolean isVoting, InductionParameters params, Knowledge knowledge) {`
`40`	`40`	`super(exampleSet, isVoting, params, knowledge);`
`41`	`41`
`42`		`- ContrastRegressionExampleSet cer = (exampleSet instanceof ContrastExampleSet) ? (ContrastRegressionExampleSet)exampleSet : null;`
	`42`	`+ ContrastRegressionExampleSet cer = (exampleSet instanceof ContrastRegressionExampleSet) ? (ContrastRegressionExampleSet)exampleSet : null;`
`43`	`43`	`if (cer == null) {`
`44`	`44`	`throw new InvalidParameterException("ContrastRegressionRuleSet supports only ContrastRegressionExampleSet instances");`
`45`	`45`	`}`