Skip to content

Commit fafad4c

Browse files
oakbaninchilada
authored andcommitted
feat(Audience Evaluation): Audience Logging (#156)
Summary ------- This adds logging for audience evaluation. Test plan --------- Unit tests written in - test_condition.py - test_audience.py Issues ------ - OASIS-3850
1 parent a3b46a2 commit fafad4c

File tree

9 files changed

+998
-154
lines changed

9 files changed

+998
-154
lines changed

.travis.yml

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -25,7 +25,9 @@ jobs:
2525
- stage: 'Linting'
2626
language: python
2727
python: "2.7"
28-
install: "pip install flake8"
28+
# flake8 version should be same as the version in requirements/test.txt
29+
# to avoid lint errors on CI
30+
install: "pip install flake8==3.6.0"
2931
script: "flake8"
3032
after_success: travis_terminate 0
3133
- stage: 'Integration Tests'

optimizely/decision_service.py

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
# Copyright 2017-2018, Optimizely
1+
# Copyright 2017-2019, Optimizely
22
# Licensed under the Apache License, Version 2.0 (the "License");
33
# you may not use this file except in compliance with the License.
44
# You may obtain a copy of the License at
@@ -156,7 +156,7 @@ def get_variation(self, experiment, user_id, attributes, ignore_user_profile=Fal
156156
self.logger.warning('User profile has invalid format.')
157157

158158
# Bucket user and store the new decision
159-
if not audience_helper.is_user_in_experiment(self.config, experiment, attributes):
159+
if not audience_helper.is_user_in_experiment(self.config, experiment, attributes, self.logger):
160160
self.logger.info('User "%s" does not meet conditions to be in experiment "%s".' % (
161161
user_id,
162162
experiment.key
@@ -198,7 +198,7 @@ def get_variation_for_rollout(self, rollout, user_id, attributes=None):
198198
experiment = self.config.get_experiment_from_key(rollout.experiments[idx].get('key'))
199199

200200
# Check if user meets audience conditions for targeting rule
201-
if not audience_helper.is_user_in_experiment(self.config, experiment, attributes):
201+
if not audience_helper.is_user_in_experiment(self.config, experiment, attributes, self.logger):
202202
self.logger.debug('User "%s" does not meet conditions for targeting rule %s.' % (
203203
user_id,
204204
idx + 1
@@ -226,7 +226,8 @@ def get_variation_for_rollout(self, rollout, user_id, attributes=None):
226226
everyone_else_experiment = self.config.get_experiment_from_key(rollout.experiments[-1].get('key'))
227227
if audience_helper.is_user_in_experiment(self.config,
228228
self.config.get_experiment_from_key(rollout.experiments[-1].get('key')),
229-
attributes):
229+
attributes,
230+
self.logger):
230231
# Determine bucketing ID to be used
231232
bucketing_id = self._get_bucketing_id(user_id, attributes)
232233
variation = self.bucketer.bucket(everyone_else_experiment, user_id, bucketing_id)

optimizely/helpers/audience.py

Lines changed: 35 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
# Copyright 2016, 2018, Optimizely
1+
# Copyright 2016, 2018-2019, Optimizely
22
# Licensed under the Apache License, Version 2.0 (the "License");
33
# you may not use this file except in compliance with the License.
44
# You may obtain a copy of the License at
@@ -11,26 +11,41 @@
1111
# See the License for the specific language governing permissions and
1212
# limitations under the License.
1313

14+
import json
15+
1416
from . import condition as condition_helper
1517
from . import condition_tree_evaluator
18+
from .enums import AudienceEvaluationLogs as audience_logs
1619

1720

18-
def is_user_in_experiment(config, experiment, attributes):
21+
def is_user_in_experiment(config, experiment, attributes, logger):
1922
""" Determine for given experiment if user satisfies the audiences for the experiment.
2023
2124
Args:
2225
config: project_config.ProjectConfig object representing the project.
2326
experiment: Object representing the experiment.
2427
attributes: Dict representing user attributes which will be used in determining
2528
if the audience conditions are met. If not provided, default to an empty dict.
29+
logger: Provides a logger to send log messages to.
2630
2731
Returns:
2832
Boolean representing if user satisfies audience conditions for any of the audiences or not.
2933
"""
3034

31-
# Return True in case there are no audiences
3235
audience_conditions = experiment.getAudienceConditionsOrIds()
36+
37+
logger.debug(audience_logs.EVALUATING_AUDIENCES_COMBINED.format(
38+
experiment.key,
39+
json.dumps(audience_conditions)
40+
))
41+
42+
# Return True in case there are no audiences
3343
if audience_conditions is None or audience_conditions == []:
44+
logger.info(audience_logs.AUDIENCE_EVALUATION_RESULT_COMBINED.format(
45+
experiment.key,
46+
'TRUE'
47+
))
48+
3449
return True
3550

3651
if attributes is None:
@@ -39,7 +54,7 @@ def is_user_in_experiment(config, experiment, attributes):
3954
def evaluate_custom_attr(audienceId, index):
4055
audience = config.get_audience(audienceId)
4156
custom_attr_condition_evaluator = condition_helper.CustomAttributeConditionEvaluator(
42-
audience.conditionList, attributes)
57+
audience.conditionList, attributes, logger)
4358

4459
return custom_attr_condition_evaluator.evaluate(index)
4560

@@ -49,14 +64,28 @@ def evaluate_audience(audienceId):
4964
if audience is None:
5065
return None
5166

52-
return condition_tree_evaluator.evaluate(
67+
logger.debug(audience_logs.EVALUATING_AUDIENCE.format(audienceId, audience.conditions))
68+
69+
result = condition_tree_evaluator.evaluate(
5370
audience.conditionStructure,
5471
lambda index: evaluate_custom_attr(audienceId, index)
5572
)
5673

74+
result_str = str(result).upper() if result is not None else 'UNKNOWN'
75+
logger.info(audience_logs.AUDIENCE_EVALUATION_RESULT.format(audienceId, result_str))
76+
77+
return result
78+
5779
eval_result = condition_tree_evaluator.evaluate(
5880
audience_conditions,
5981
evaluate_audience
6082
)
6183

62-
return eval_result or False
84+
eval_result = eval_result or False
85+
86+
logger.info(audience_logs.AUDIENCE_EVALUATION_RESULT_COMBINED.format(
87+
experiment.key,
88+
str(eval_result).upper()
89+
))
90+
91+
return eval_result

optimizely/helpers/condition.py

Lines changed: 124 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -12,10 +12,12 @@
1212
# limitations under the License.
1313

1414
import json
15+
import numbers
1516

1617
from six import string_types
1718

1819
from . import validator
20+
from .enums import AudienceEvaluationLogs as audience_logs
1921

2022

2123
class ConditionOperatorTypes(object):
@@ -37,20 +39,47 @@ class CustomAttributeConditionEvaluator(object):
3739

3840
CUSTOM_ATTRIBUTE_CONDITION_TYPE = 'custom_attribute'
3941

40-
def __init__(self, condition_data, attributes):
42+
def __init__(self, condition_data, attributes, logger):
4143
self.condition_data = condition_data
4244
self.attributes = attributes or {}
45+
self.logger = logger
4346

44-
def is_value_valid_for_exact_conditions(self, value):
47+
def _get_condition_json(self, index):
48+
""" Method to generate json for logging audience condition.
49+
50+
Args:
51+
index: Index of the condition.
52+
53+
Returns:
54+
String: Audience condition JSON.
55+
"""
56+
condition = self.condition_data[index]
57+
condition_log = {
58+
'name': condition[0],
59+
'value': condition[1],
60+
'type': condition[2],
61+
'match': condition[3]
62+
}
63+
64+
return json.dumps(condition_log)
65+
66+
def is_value_type_valid_for_exact_conditions(self, value):
4567
""" Method to validate if the value is valid for exact match type evaluation.
4668
4769
Args:
4870
value: Value to validate.
4971
5072
Returns:
51-
Boolean: True if value is a string type, or a boolean, or is finite. Otherwise False.
73+
Boolean: True if value is a string, boolean, or number. Otherwise False.
5274
"""
53-
if isinstance(value, string_types) or isinstance(value, bool) or validator.is_finite_number(value):
75+
# No need to check for bool since bool is a subclass of int
76+
if isinstance(value, string_types) or isinstance(value, (numbers.Integral, float)):
77+
return True
78+
79+
return False
80+
81+
def is_value_a_number(self, value):
82+
if isinstance(value, (numbers.Integral, float)) and not isinstance(value, bool):
5483
return True
5584

5685
return False
@@ -69,12 +98,32 @@ def exact_evaluator(self, index):
6998
- if the condition value or user attribute value has an invalid type.
7099
- if there is a mismatch between the user attribute type and the condition value type.
71100
"""
101+
condition_name = self.condition_data[index][0]
72102
condition_value = self.condition_data[index][1]
73-
user_value = self.attributes.get(self.condition_data[index][0])
103+
user_value = self.attributes.get(condition_name)
74104

75-
if not self.is_value_valid_for_exact_conditions(condition_value) or \
76-
not self.is_value_valid_for_exact_conditions(user_value) or \
105+
if not self.is_value_type_valid_for_exact_conditions(condition_value) or \
106+
(self.is_value_a_number(condition_value) and not validator.is_finite_number(condition_value)):
107+
self.logger.warning(audience_logs.UNKNOWN_CONDITION_VALUE.format(
108+
self._get_condition_json(index)
109+
))
110+
return None
111+
112+
if not self.is_value_type_valid_for_exact_conditions(user_value) or \
77113
not validator.are_values_same_type(condition_value, user_value):
114+
self.logger.warning(audience_logs.UNEXPECTED_TYPE.format(
115+
self._get_condition_json(index),
116+
type(user_value),
117+
condition_name
118+
))
119+
return None
120+
121+
if self.is_value_a_number(user_value) and \
122+
not validator.is_finite_number(user_value):
123+
self.logger.warning(audience_logs.INFINITE_ATTRIBUTE_VALUE.format(
124+
self._get_condition_json(index),
125+
condition_name
126+
))
78127
return None
79128

80129
return condition_value == user_value
@@ -104,10 +153,29 @@ def greater_than_evaluator(self, index):
104153
- False if the user attribute value is less than or equal to the condition value.
105154
None: if the condition value isn't finite or the user attribute value isn't finite.
106155
"""
156+
condition_name = self.condition_data[index][0]
107157
condition_value = self.condition_data[index][1]
108-
user_value = self.attributes.get(self.condition_data[index][0])
158+
user_value = self.attributes.get(condition_name)
109159

110-
if not validator.is_finite_number(condition_value) or not validator.is_finite_number(user_value):
160+
if not validator.is_finite_number(condition_value):
161+
self.logger.warning(audience_logs.UNKNOWN_CONDITION_VALUE.format(
162+
self._get_condition_json(index)
163+
))
164+
return None
165+
166+
if not self.is_value_a_number(user_value):
167+
self.logger.warning(audience_logs.UNEXPECTED_TYPE.format(
168+
self._get_condition_json(index),
169+
type(user_value),
170+
condition_name
171+
))
172+
return None
173+
174+
if not validator.is_finite_number(user_value):
175+
self.logger.warning(audience_logs.INFINITE_ATTRIBUTE_VALUE.format(
176+
self._get_condition_json(index),
177+
condition_name
178+
))
111179
return None
112180

113181
return user_value > condition_value
@@ -124,10 +192,29 @@ def less_than_evaluator(self, index):
124192
- False if the user attribute value is greater than or equal to the condition value.
125193
None: if the condition value isn't finite or the user attribute value isn't finite.
126194
"""
195+
condition_name = self.condition_data[index][0]
127196
condition_value = self.condition_data[index][1]
128-
user_value = self.attributes.get(self.condition_data[index][0])
197+
user_value = self.attributes.get(condition_name)
198+
199+
if not validator.is_finite_number(condition_value):
200+
self.logger.warning(audience_logs.UNKNOWN_CONDITION_VALUE.format(
201+
self._get_condition_json(index)
202+
))
203+
return None
204+
205+
if not self.is_value_a_number(user_value):
206+
self.logger.warning(audience_logs.UNEXPECTED_TYPE.format(
207+
self._get_condition_json(index),
208+
type(user_value),
209+
condition_name
210+
))
211+
return None
129212

130-
if not validator.is_finite_number(condition_value) or not validator.is_finite_number(user_value):
213+
if not validator.is_finite_number(user_value):
214+
self.logger.warning(audience_logs.INFINITE_ATTRIBUTE_VALUE.format(
215+
self._get_condition_json(index),
216+
condition_name
217+
))
131218
return None
132219

133220
return user_value < condition_value
@@ -144,10 +231,22 @@ def substring_evaluator(self, index):
144231
- False if the condition value is not a substring of the user attribute value.
145232
None: if the condition value isn't a string or the user attribute value isn't a string.
146233
"""
234+
condition_name = self.condition_data[index][0]
147235
condition_value = self.condition_data[index][1]
148-
user_value = self.attributes.get(self.condition_data[index][0])
236+
user_value = self.attributes.get(condition_name)
237+
238+
if not isinstance(condition_value, string_types):
239+
self.logger.warning(audience_logs.UNKNOWN_CONDITION_VALUE.format(
240+
self._get_condition_json(index),
241+
))
242+
return None
149243

150-
if not isinstance(condition_value, string_types) or not isinstance(user_value, string_types):
244+
if not isinstance(user_value, string_types):
245+
self.logger.warning(audience_logs.UNEXPECTED_TYPE.format(
246+
self._get_condition_json(index),
247+
type(user_value),
248+
condition_name
249+
))
151250
return None
152251

153252
return condition_value in user_value
@@ -175,15 +274,27 @@ def evaluate(self, index):
175274
"""
176275

177276
if self.condition_data[index][2] != self.CUSTOM_ATTRIBUTE_CONDITION_TYPE:
277+
self.logger.warning(audience_logs.UNKNOWN_CONDITION_TYPE.format(self._get_condition_json(index)))
178278
return None
179279

180280
condition_match = self.condition_data[index][3]
181281
if condition_match is None:
182282
condition_match = ConditionMatchTypes.EXACT
183283

184284
if condition_match not in self.EVALUATORS_BY_MATCH_TYPE:
285+
self.logger.warning(audience_logs.UNKNOWN_MATCH_TYPE.format(self._get_condition_json(index)))
185286
return None
186287

288+
if condition_match != ConditionMatchTypes.EXISTS:
289+
attribute_key = self.condition_data[index][0]
290+
if attribute_key not in self.attributes:
291+
self.logger.debug(audience_logs.MISSING_ATTRIBUTE_VALUE.format(self._get_condition_json(index), attribute_key))
292+
return None
293+
294+
if self.attributes.get(attribute_key) is None:
295+
self.logger.debug(audience_logs.NULL_ATTRIBUTE_VALUE.format(self._get_condition_json(index), attribute_key))
296+
return None
297+
187298
return self.EVALUATORS_BY_MATCH_TYPE[condition_match](self, index)
188299

189300

optimizely/helpers/enums.py

Lines changed: 23 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
# Copyright 2016-2018, Optimizely
1+
# Copyright 2016-2019, Optimizely
22
# Licensed under the Apache License, Version 2.0 (the "License");
33
# you may not use this file except in compliance with the License.
44
# You may obtain a copy of the License at
@@ -14,6 +14,28 @@
1414
import logging
1515

1616

17+
class AudienceEvaluationLogs(object):
18+
AUDIENCE_EVALUATION_RESULT = 'Audience "{}" evaluated to {}.'
19+
AUDIENCE_EVALUATION_RESULT_COMBINED = 'Audiences for experiment "{}" collectively evaluated to {}.'
20+
EVALUATING_AUDIENCE = 'Starting to evaluate audience "{}" with conditions: {}.'
21+
EVALUATING_AUDIENCES_COMBINED = 'Evaluating audiences for experiment "{}": {}.'
22+
INFINITE_ATTRIBUTE_VALUE = 'Audience condition "{}" evaluated to UNKNOWN because the number value ' \
23+
'for user attribute "{}" is not in the range [-2^53, +2^53].'
24+
MISSING_ATTRIBUTE_VALUE = 'Audience condition {} evaluated to UNKNOWN because no value was passed for '\
25+
'user attribute "{}".'
26+
NULL_ATTRIBUTE_VALUE = 'Audience condition "{}" evaluated to UNKNOWN because a null value was passed '\
27+
'for user attribute "{}".'
28+
UNEXPECTED_TYPE = 'Audience condition "{}" evaluated to UNKNOWN because a value of type "{}" was passed '\
29+
'for user attribute "{}".'
30+
31+
UNKNOWN_CONDITION_TYPE = 'Audience condition "{}" uses an unknown condition type. You may need to upgrade to a '\
32+
'newer release of the Optimizely SDK.'
33+
UNKNOWN_CONDITION_VALUE = 'Audience condition "{}" has an unsupported condition value. You may need to upgrade to a '\
34+
'newer release of the Optimizely SDK.'
35+
UNKNOWN_MATCH_TYPE = 'Audience condition "{}" uses an unknown match type. You may need to upgrade to a '\
36+
'newer release of the Optimizely SDK.'
37+
38+
1739
class ControlAttributes(object):
1840
BOT_FILTERING = '$opt_bot_filtering'
1941
BUCKETING_ID = '$opt_bucketing_id'

0 commit comments

Comments
 (0)