Skip to content

[FIX] Test and Score: Improve data errors #4738

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
lanzagar merged 1 commit into from
May 8, 2020
Merged

[FIX] Test and Score: Improve data errors #4738

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
46 changes: 22 additions & 24 deletions Orange/widgets/evaluate/owtestandscore.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,7 @@
from Orange.evaluation import Results
from Orange.preprocess.preprocess import Preprocess
import Orange.regression
from Orange.statistics.util import unique
from Orange.widgets import gui, settings, widget
from Orange.widgets.evaluate.utils import \
usable_scorers, ScoreTable, learner_name, scorer_caller
Expand Down Expand Up @@ -187,19 +188,15 @@ class Outputs:
class_selection = settings.ContextSetting(TARGET_AVERAGE)

class Error(OWWidget.Error):
train_data_empty = Msg("Train dataset is empty.")
test_data_empty = Msg("Test dataset is empty.")
class_required = Msg("Train data input requires a target variable.")
too_many_classes = Msg("Too many target variables.")
class_required_test = Msg("Test data input requires a target variable.")
too_many_folds = Msg("Number of folds exceeds the data size")
class_inconsistent = Msg("Test and train datasets "
"have different target variables.")
memory_error = Msg("Not enough memory.")
no_class_values = Msg("Target variable has no values.")
only_one_class_var_value = Msg("Target variable has only one value.")
test_data_incompatible = Msg(
"Test data may be incompatible with train data.")
train_data_error = Msg("{}")

class Warning(OWWidget.Warning):
missing_data = \
Expand Down Expand Up @@ -382,26 +379,27 @@ def set_train_data(self, data):
"""
self.cancel()
self.Information.data_sampled.clear()
self.Error.train_data_empty.clear()
self.Error.class_required.clear()
self.Error.too_many_classes.clear()
self.Error.no_class_values.clear()
self.Error.only_one_class_var_value.clear()
if data is not None and not data:
self.Error.train_data_empty()
data = None
if data:
conds = [not data.domain.class_vars,
len(data.domain.class_vars) > 1,
np.isnan(data.Y).all(),
data.domain.has_discrete_class and len(data.domain.class_var.values) == 1]
errors = [self.Error.class_required,
self.Error.too_many_classes,
self.Error.no_class_values,
self.Error.only_one_class_var_value]
for cond, error in zip(conds, errors):
self.Error.train_data_error.clear()

if data is not None:
data_errors = [
("Train dataset is empty.", len(data) == 0),
(
"Train data input requires a target variable.",
not data.domain.class_vars
),
("Too many target variables.", len(data.domain.class_vars) > 1),
("Target variable has no values.", np.isnan(data.Y).all()),
(
"Target variable has only one value.",
data.domain.has_discrete_class and len(unique(data.Y)) < 2
),
("Data has no features to learn from.", data.X.shape[1] == 0),
]

for error_msg, cond in data_errors:
if cond:
error()
self.Error.train_data_error(error_msg)
data = None
break

Expand Down
49 changes: 33 additions & 16 deletions Orange/widgets/evaluate/tests/test_owtestandscore.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -166,25 +166,38 @@ def test_one_class_value(self):
"yyyy"))
)
self.widget.n_folds = 0
self.assertFalse(self.widget.Error.only_one_class_var_value.is_shown())
self.assertFalse(self.widget.Error.train_data_error.is_shown())
self.send_signal("Data", table)
self.send_signal("Learner", MajorityLearner(), 0, wait=1000)
self.assertTrue(self.widget.Error.only_one_class_var_value.is_shown())
self.assertTrue(self.widget.Error.train_data_error.is_shown())

def test_nan_class(self):
"""
Do not crash on a data with only nan class values.
GH-2751
"""
def assertErrorShown(data, is_shown):
def test_data_errors(self):
""" Test all data_errors """
def assertErrorShown(data, is_shown, message):
self.send_signal("Data", data)
self.assertEqual(is_shown, self.widget.Error.no_class_values.is_shown())
self.assertEqual(is_shown, self.widget.Error.train_data_error.is_shown())
self.assertEqual(message, str(self.widget.Error.train_data_error))

data = Table("iris")[::30]
data.Y[:] = np.nan

for data, is_shown in zip([None, data, Table("iris")[:30]], [False, True, False]):
assertErrorShown(data, is_shown)
iris_empty_x = Table.from_table(
Domain([], data.domain.class_var), Table("iris")
)

for data, is_shown, message in zip(
[None, data, Table("iris")[:30], iris_empty_x, data[:0]],
[False, True, True, True, True],
[
"",
"Target variable has no values.",
"Target variable has only one value.",
"Data has no features to learn from.",
"Train dataset is empty."
]

):
assertErrorShown(data, is_shown, message)

def test_addon_scorers(self):
try:
Expand Down Expand Up @@ -319,15 +332,19 @@ def _test_scores(self, train_data, test_data, learner, sampling, n_folds):
self.send_signal(self.widget.Inputs.learner, learner, 0, wait=5000)
return self._retrieve_scores()

def test_scores_constant_all_same(self):
def test_scores_constant(self):
table = Table.from_list(
self.scores_domain,
list(zip(*self.scores_table_values + [list("yyyy")]))
list(zip(*self.scores_table_values + [list("yyyn")]))
)

self.assertTupleEqual(self._test_scores(
table, table, ConstantLearner(), OWTestAndScore.TestOnTest, None),
(None, 1, 1, 1, 1))
self.assertTupleEqual(
self._test_scores(
table, table[:3], ConstantLearner(),
OWTestAndScore.TestOnTest, None
),
(None, 1, 1, 1, 1)
)

def test_scores_log_reg_overfitted(self):
table = Table.from_list(
Expand Down