55
55
'P91' : 'Sexual Orientation' ,
56
56
'P69' :'Educated At' ,
57
57
'P102' :'Member of political party' ,
58
- 'P551' :'Residence'
58
+ 'P551' :'Residence' ,
59
+ 'P3373' : 'Sibling'
59
60
}
60
61
61
62
@@ -71,10 +72,25 @@ def getWDPPropertyLabel(propertyId):
71
72
wd2 = request_wd .get (property_url )
72
73
73
74
if wd2 .status_code == 200 :
74
- data2 = wd2 .json ()['entities' ][propertyId ]
75
- if 'en' in data2 ['labels' ]:
76
- wd_properties [propertyId ]= data2 ['labels' ]['en' ]['value' ]
77
- return wd_properties [propertyId ]
75
+ log .debug (wd2 .json ())
76
+ # data2 = wd2.json()['entities']
77
+ for k ,data2 in wd2 .json ()['entities' ].items ():
78
+ if 'en' in data2 ['labels' ]:
79
+ data2 ['label' ]= data2 ['labels' ]['en' ]['value' ]
80
+ if 'mul' in data2 ['labels' ]:
81
+ data2 ['label' ]= data2 ['labels' ]['mul' ]['value' ]
82
+ if 'label' not in data2 :
83
+ if len (data2 ['labels' ]) == 0 :
84
+ label = ''
85
+ else :
86
+ key = next (iter (data2 ['labels' ]))
87
+ data2 ['label' ] = data2 ['labels' ][key ]['value' ]
88
+
89
+ if k != propertyId :
90
+ wd_properties [k ]= data2
91
+ else :
92
+ wd_properties [propertyId ]= data2
93
+ return data2
78
94
else :
79
95
wd_properties [propertyId ]= ''
80
96
return wd_properties [propertyId ]
@@ -83,11 +99,13 @@ def getWDPPropertyLabel(propertyId):
83
99
def processWikidata (performer ,performer_update ,url ):
84
100
wikidata_id = url [30 :]
85
101
api_url = 'https://www.wikidata.org/wiki/Special:EntityData/%s.json' % (wikidata_id ,)
86
- log .debug ('about to fetch wikidata url: %s' % (api_url ,))
102
+ log .debug ('about to fetch wikidata url: %s for performer %s ' % (api_url , performer [ 'name' ] ,))
87
103
wd = request_wd .get (api_url )
88
104
if wd .status_code == 200 :
89
105
# log.debug(wd.json().keys())
90
- data = wd .json ()['entities' ][wikidata_id ]
106
+
107
+ data2 = wd .json ()['entities' ]
108
+ data = data2 [next (iter (data2 ))]
91
109
if settings ['wikidatExtraUrls' ]:
92
110
urls = []
93
111
for claim ,urlstring in wikidata_property_urls .items ():
@@ -110,65 +128,175 @@ def processWikidata(performer,performer_update,url):
110
128
performer_update ['urls' ].append (url )
111
129
performer_update ['update' ] = True
112
130
log .debug (performer_update )
131
+
113
132
if settings ['awards' ]:
133
+ won_award = []
134
+ nominated_award = []
135
+ award_totals = {}
136
+ nominated_totals = {}
114
137
# award received (P166)
115
138
# nominated for (P1411)
116
139
for prop in ['P166' ,'P1411' ]:
117
140
if prop in data ['claims' ]:
118
141
for c in data ['claims' ][prop ]:
119
- # log.debug(c)
142
+ log .debug (c )
143
+
120
144
121
145
award = {}
122
146
award_id = c ['mainsnak' ]['datavalue' ]['value' ]['id' ]
123
147
124
- award ['name' ]= getWDPPropertyLabel (award_id )
148
+ award ['wd' ]= getWDPPropertyLabel (award_id )
149
+ award ['name' ]= award ['wd' ]['label' ]
150
+ award ['label' ] = award ['wd' ]['label' ]
151
+
152
+ if prop == 'P166' :
153
+ award ['type' ]= 'award received'
154
+ elif prop == 'P1411' :
155
+ award ['type' ]= 'nominated'
156
+
157
+
158
+ if 'P1027' in award ['wd' ]['claims' ].keys ():
159
+ award ['conferred_wd' ]= getWDPPropertyLabel (award ['wd' ]['claims' ]['P1027' ][0 ]['mainsnak' ]['datavalue' ]['value' ]['id' ])
160
+ award ['conferred' ] = award ['conferred_wd' ]['label' ]
161
+ if award ['type' ]== 'award received' :
162
+ if award ['conferred' ] not in award_totals :
163
+ award_totals [award ['conferred' ]]= 0
164
+ award_totals [award ['conferred' ]] = award_totals [award ['conferred' ]] + 1
165
+ if settings ['createTag' ]:
166
+ performer_update ['tag_names' ].append (
167
+ '[%s Award Winner]' % (award ['conferred' ],))
168
+
169
+ else :
170
+ if award ['conferred' ] not in nominated_totals :
171
+ nominated_totals [award ['conferred' ]]= 0
172
+ nominated_totals [award ['conferred' ]] = nominated_totals [award ['conferred' ]] + 1
173
+ if settings ['createTag' ]:
174
+ performer_update ['tag_names' ].append ('[%s Award Nominated]' % (award ['conferred' ],))
175
+ else :
176
+ if award ['type' ]== 'award received' :
177
+ if 'unknown' not in nominated_totals :
178
+ award_totals ['unknown' ] = 0
179
+ award_totals ['unknown' ] = award_totals ['unknown' ] + 1
180
+ else :
181
+ if 'unknown' not in nominated_totals :
182
+ nominated_totals ['unknown' ]= 0
183
+ nominated_totals ['unknown' ] = nominated_totals ['unknown' ] + 1
184
+
185
+
186
+ # sublcass of, can be award for best scene
187
+ if 'P279' in award ['wd' ]['claims' ].keys ():
188
+ award ['subclass_wd' ] = getWDPPropertyLabel (award ['wd' ]['claims' ]['P279' ][0 ]['mainsnak' ]['datavalue' ]['value' ]['id' ])
189
+ award ['subclass' ] = award ['subclass_wd' ]['label' ]
190
+
191
+
192
+
125
193
126
194
if 'qualifiers' in c :
127
195
for q ,qv in c ['qualifiers' ].items ():
128
196
# point in time
129
- # log.debug('q=%s qv=%s'% (q,qv,))
197
+ log .debug ('q=%s qv=%s' % (q ,qv ,))
130
198
if q == 'P585' :
131
199
if len (qv )> 0 :
132
200
award ['time' ]= qv [0 ]['datavalue' ]['value' ]['time' ][1 :5 ]
133
- # Subject of (the event name
201
+ # Subject of (the event name)
134
202
if q == 'P805' :
135
- award ['venue' ] = getWDPPropertyLabel (qv [0 ]['datavalue' ]['value' ]['id' ])
203
+ award ['venue_wd' ] = getWDPPropertyLabel (qv [0 ]['datavalue' ]['value' ]['id' ])
204
+ award ['venue' ] = award ['venue_wd' ]['label' ]
205
+ # Award Rationale
206
+ if q == 'P6208' :
207
+ award ['name' ]= qv [0 ]['datavalue' ]['value' ]['text' ]
208
+
209
+
136
210
137
211
if award :
138
- log .info ('award: %s' % (award ,))
212
+ # log.info('award: %s' % (award,))
139
213
if 'custom_fields' not in performer_update :
140
- performer_update ['custom_fields' ]= {'full' :performer ['custom_fields' ]}
214
+ performer_update ['custom_fields' ]= {'full' :performer ['custom_fields' ].copy ()}
215
+
141
216
award_name = award ['name' ]
142
- award_value = award ['name' ]
217
+ award [ ' award_value' ] = award ['name' ]
143
218
if 'venue' in award and 'time' in award :
144
- award_value = '%s - %s: %s' % (award ['time' ], award ['venue' ],award ['name' ],)
219
+ award [ ' award_value' ] = '%s - %s: %s' % (award ['time' ], award ['venue' ],award ['name' ],)
145
220
elif 'time' in award :
146
- award_value = '%s: %s' % (award ['time' ],award ['name' ],)
221
+ award [ ' award_value' ] = '%s: %s' % (award ['time' ],award ['name' ],)
147
222
elif 'venue' in award :
148
- award_value = '%s: %s' % (award ['venue' ],award ['name' ],)
149
- if prop == 'P1411' :
150
- award_value = '%s - Nominated' % award_value
223
+ award ['award_value' ]= '%s: %s' % (award ['venue' ],award ['name' ],)
224
+
225
+
226
+ if award ['type' ]== 'award received' :
227
+ won_award .append (award )
228
+ if award ['type' ]== 'nominated' :
229
+ award ['award_value' ]= '%s - Nominated' % award ['award_value' ]
230
+ nominated_award .append (award )
151
231
if award_name not in performer_update ['custom_fields' ]['full' ]:
152
- performer_update ['custom_fields' ]['full' ][award_name ]= award_value
232
+
233
+ performer_update ['custom_fields' ]['full' ][award_name ]= award ['award_value' ]
153
234
performer_update ['update' ] = True
154
- log .debug (performer_update )
155
- if settings ['createTag' ]:
156
- if prop == 'P166' :
157
- performer_update ['tag_names' ].append ('[Award Winner]' )
158
- performer_update ['update' ] = True
159
- elif prop == 'P1411' :
160
- performer_update ['tag_names' ].append ('[Award Nominated]' )
161
- performer_update ['update' ] = True
235
+ else :
236
+ if award ['award_value' ] not in performer_update ['custom_fields' ]['full' ][award_name ]:
237
+ tmp = performer_update ['custom_fields' ]['full' ][award_name ].split (', ' )
238
+ tmp .append (award ['award_value' ])
239
+ performer_update ['custom_fields' ]['full' ][award_name ]= ', ' .join (sorted (tmp ,reverse = True ))
240
+ performer_update ['update' ] = True
241
+ #check what type of
242
+ log .debug (award ['wd' ])
243
+
244
+ log .debug (performer )
245
+ if award_totals :
246
+ performer_update ['custom_fields' ]['full' ]['award totals' ] = ', ' .join (
247
+ [ "%s: %s" % (k ,v ,) for k ,v in award_totals .items ()])
248
+ performer_update ['update' ] = True
249
+ if nominated_totals :
250
+ performer_update ['custom_fields' ]['full' ]['nominated totals' ] = ', ' .join (
251
+ ["%s: %s" % (k , v ,) for k , v in nominated_totals .items ()])
252
+ performer_update ['update' ] = True
253
+ if won_award :
254
+ # performer_update['custom_fields']['full']['json_awards'] = json.dumps([x[for x in won_award])
255
+ performer_update ['custom_fields' ]['full' ]['Awards Won' ] = ', ' .join (
256
+ [x ['award_value' ] for x in won_award ])
257
+ if settings ['createTag' ]:
258
+ performer_update ['tag_names' ].append ('[Award Winner]' )
259
+ performer_update ['update' ] = True
260
+ if nominated_award :
261
+ # performer_update['custom_fields']['full']['json_nominated'] = json.dumps(nominated_award)
262
+ performer_update ['custom_fields' ]['full' ]['Awards Nominated' ] = ', ' .join (
263
+ [x ['award_value' ] for x in nominated_award ])
264
+ if settings ['createTag' ]:
265
+ performer_update ['tag_names' ].append ('[Award Nominated]' )
266
+ performer_update ['update' ] = True
267
+ # if settings['createTag']:
268
+ # if 'P31' in award['wd']['claims']:
269
+ # for c in award['wd']['claims']['P31']:
270
+ # log.debug('c %s' % (c,))
271
+ # # avn Award Q824540
272
+ # if c['mainsnak']['datavalue']['value']['id']=='Q824540':
273
+ # log.debug('---------------')
274
+ # if prop=='P166':
275
+ # performer_update['tag_names'].append('[AVN Award Winner]')
276
+ # performer_update['update'] = True
277
+ # elif prop=='P1411':
278
+ # performer_update['tag_names'].append('[AVN Award Nominated]')
279
+ # performer_update['update'] = True
280
+ #
281
+ # if settings['createTag']:
282
+ # if prop=='P166':
283
+ # performer_update['tag_names'].append('[Award Winner]')
284
+ # performer_update['update'] = True
285
+ # elif prop=='P1411':
286
+ # performer_update['tag_names'].append('[Award Nominated]')
287
+ # performer_update['update'] = True
162
288
if settings ['otherInfo' ]:
163
289
for claim , label in wikidata_field_properties .items ():
164
290
if claim in data ['claims' ]:
165
291
claim_values = []
166
292
for c in data ['claims' ][claim ]:
167
293
# log.debug(c)
168
- claim_values .append (getWDPPropertyLabel (c ['mainsnak' ]['datavalue' ]['value' ]['id' ]))
294
+ # some bad data, used th have a property but no longer, maybe it got deleted and this is pointing to the delted item, 'unknown value'
295
+ if 'datavalue' in c ['mainsnak' ]:
296
+ claim_values .append (getWDPPropertyLabel (c ['mainsnak' ]['datavalue' ]['value' ]['id' ])['label' ])
169
297
if len (claim_values )> 0 :
170
298
if 'custom_fields' not in performer_update :
171
- performer_update ['custom_fields' ] = {'full' : performer ['custom_fields' ]}
299
+ performer_update ['custom_fields' ] = {'full' : performer ['custom_fields' ]. copy () }
172
300
if label not in performer_update ['custom_fields' ]['full' ]:
173
301
performer_update ['update' ] = True
174
302
performer_update ['custom_fields' ]['full' ][label ] = ', ' .join (claim_values )
@@ -178,30 +306,38 @@ def processWikidata(performer,performer_update,url):
178
306
def processPerformer (performer ):
179
307
180
308
performer_update = {'id' :performer ['id' ],'update' :False ,"tag_names" :[]}
181
- # log.debug(performer)
309
+ log .debug (performer )
182
310
for u in performer ['urls' ]:
183
311
if u .startswith ('https://www.wikidata.org' ) and settings ['processWikidata' ]:
184
312
processWikidata (performer ,performer_update ,u )
313
+
185
314
if performer_update ['update' ]:
186
- log . debug ( 'needs update' )
315
+ needs_update = False
187
316
performer_update .pop ('update' )
188
317
performer_update ['tag_ids' ]= [x ['id' ] for x in performer ['tags' ]]
189
318
for t in performer_update ['tag_names' ]:
190
319
tt = stash .find_tag (t , create = True )
191
320
if tt ['id' ] not in performer_update ['tag_ids' ]:
192
321
performer_update ['tag_ids' ].append (tt ['id' ])
322
+ needs_update = True
193
323
performer_update .pop ('tag_names' )
194
324
195
325
if settings ['schema' ] < 71 :
196
326
log .info ('your version of stash does not support custom fields, a new version of stash should be released soon' )
197
327
# other features will still work for other versions
198
328
performer_update .pop ('custom_fields' )
199
- log .info ('updating performer: %s' % (performer_update ,))
200
- stash .update_performer (performer_update )
329
+ else :
330
+ if not performer ['custom_fields' ]== performer_update ['custom_fields' ]['full' ]:
331
+ needs_update = True
332
+ if needs_update :
333
+ log .info ('updating performer: %s' % (performer_update ,))
334
+ stash .update_performer (performer_update )
335
+ else :
336
+ log .debug ('no performer update needed' )
201
337
202
338
def processPerformers ():
203
339
query = {}
204
- count = stash .find_scenes (
340
+ count = stash .find_performers (
205
341
f = query ,
206
342
filter = {"per_page" : 1 },
207
343
get_count = True ,
0 commit comments