Skip to content

Commit bbd4165

Browse files
NandakaNandaka
committedMar 26, 2022
Implement #1107
·
v20240703v20220701
1 parent 0b7042b commit bbd4165

File tree

4 files changed

+126
-59
lines changed

4 files changed

+126
-59
lines changed
 

‎PixivArtist.py

+1-3
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,6 @@
11
# -*- coding: utf-8 -*-
22
# pylint: disable=I0011, C, C0302
33

4-
import re
5-
64
import demjson3
75
from bs4 import BeautifulSoup
86

@@ -20,7 +18,7 @@ class PixivArtist:
2018
isLastPage = None
2119
haveImages = None
2220
totalImages = 0
23-
__re_imageULItemsClass = re.compile(r".*\b_image-items\b.*")
21+
# __re_imageULItemsClass = re.compile(r".*\b_image-items\b.*")
2422
offset = None
2523
limit = None
2624
reference_image_id = 0

‎PixivBookmark.py

+16-30
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,7 @@
1414

1515
class PixivBookmark(object):
1616
'''Class for parsing Bookmarks'''
17-
__re_imageULItemsClass = re.compile(r".*\b_image-items\b.*")
17+
# __re_imageULItemsClass = re.compile(r".*\b_image-items\b.*")
1818

1919
@staticmethod
2020
def parseBookmark(page, root_directory, db_path, locale='', is_json=False):
@@ -89,13 +89,26 @@ def exportList(lst, filename):
8989
if not filename.endswith('.txt'):
9090
filename = filename + '.txt'
9191
writer = codecs.open(filename, 'w', encoding='utf-8')
92-
writer.write(u'###Export date: ' + str(datetime.today()) + '###\n')
92+
writer.write(f'###Export members date: {datetime.today()} ###\n')
9393
for item in lst:
9494
data = str(item.memberId)
9595
if len(item.path) > 0:
9696
data = data + ' ' + item.path
9797
writer.write(data)
98-
writer.write(u'\r\n')
98+
writer.write('\r\n')
99+
writer.write('###END-OF-FILE###')
100+
writer.close()
101+
102+
@staticmethod
103+
def export_image_list(lst, filename):
104+
if not filename.endswith('.txt'):
105+
filename = filename + '.txt'
106+
writer = codecs.open(filename, 'w', encoding='utf-8')
107+
writer.write(f'###Export images date: {datetime.today()} ###\n')
108+
for item in lst:
109+
data = str(item)
110+
writer.write(data)
111+
writer.write('\r\n')
99112
writer.write('###END-OF-FILE###')
100113
writer.close()
101114

@@ -122,31 +135,4 @@ def __ParseNewIllustBookmark(self, page):
122135
for image_id in page_json["body"]["page"]["ids"]:
123136
self.imageList.append(int(image_id))
124137

125-
# # Fix Issue#290
126-
# jsBookmarkItem = page.find(id='js-mount-point-latest-following')
127-
# if jsBookmarkItem is not None:
128-
# js = jsBookmarkItem["data-items"]
129-
# items = json.loads(js)
130-
# for item in items:
131-
# image_id = item["illustId"]
132-
# # bookmarkCount = item["bookmarkCount"]
133-
# # imageResponse = item["responseCount"]
134-
# self.imageList.append(int(image_id))
135-
# else:
136-
# result = page.find(attrs={'class': '_image-items autopagerize_page_element'}).findAll('a')
137-
# for r in result:
138-
# href = re.search(r'/artworks/(\d+)', r['href'])
139-
# if href is not None:
140-
# href = int(href.group(1))
141-
# if href not in self.imageList:
142-
# self.imageList.append(href)
143-
144138
return self.imageList
145-
146-
# def __CheckLastPage(self, page):
147-
# check = page.findAll('a', attrs={'class': '_button', 'rel': 'next'})
148-
# if len(check) > 0:
149-
# self.isLastPage = False
150-
# else:
151-
# self.isLastPage = True
152-
# return self.isLastPage

‎PixivBookmarkHandler.py

+40-14
Original file line numberDiff line numberDiff line change
@@ -3,12 +3,12 @@
33
from datetime import time
44

55
import PixivArtistHandler
6-
from PixivBrowserFactory import PixivBrowser
76
import PixivConstant
87
import PixivDownloadHandler
98
import PixivHelper
109
import PixivImageHandler
1110
from PixivBookmark import PixivBookmark
11+
from PixivBrowserFactory import PixivBrowser
1212
from PixivGroup import PixivGroup
1313

1414
# from bs4 import BeautifulSoup
@@ -122,19 +122,6 @@ def process_new_illust_from_bookmark(caller,
122122
mode = "r18"
123123
pb = br.getFollowedNewIllusts(mode, current_page=i)
124124

125-
# url = 'https://www.pixiv.net/bookmark_new_illust.php?p=' + str(i)
126-
# if config.r18mode:
127-
# url = 'https://www.pixiv.net/bookmark_new_illust_r18.php?p=' + str(i)
128-
129-
# PixivHelper.print_and_log('info', "Source URL: " + url)
130-
# page = br.open(url)
131-
# parsed_page = BeautifulSoup(page.read().decode("utf-8"), features="html5lib")
132-
# pb = PixivNewIllustBookmark(parsed_page)
133-
134-
# if not pb.haveImages:
135-
# print("No images!")
136-
# break
137-
138125
for image_id in pb.imageList:
139126
print(f"Image #{image_count}")
140127
result = PixivImageHandler.process_image(caller,
@@ -279,6 +266,45 @@ def export_bookmark(caller,
279266
raise
280267

281268

269+
def export_image_bookmark(caller,
270+
config,
271+
hide='n',
272+
start_page=1,
273+
end_page=0,
274+
tag=None,
275+
use_image_tag=False,
276+
filename='exported_images.txt'):
277+
try:
278+
print("Getting image bookmarks...")
279+
total_list = list()
280+
private_list = list()
281+
public_list = list()
282+
total_bookmark_count = 0
283+
284+
if hide == 'n':
285+
(public_list, total_bookmark_count) = get_image_bookmark(caller, config, False, start_page, end_page, tag, use_image_tag)
286+
elif hide == 'y':
287+
# public and private image bookmarks
288+
(public_list, total_bookmark_count_pub) = get_image_bookmark(caller, config, False, start_page, end_page, tag, use_image_tag)
289+
(private_list, total_bookmark_count_priv) = get_image_bookmark(caller, config, True, start_page, end_page, tag, use_image_tag)
290+
total_bookmark_count = total_bookmark_count_pub + total_bookmark_count_priv
291+
else:
292+
(private_list, total_bookmark_count) = get_image_bookmark(caller, config, True, start_page, end_page, tag, use_image_tag)
293+
total_list.extend(private_list)
294+
total_list.extend(public_list)
295+
296+
PixivBookmark.export_image_list(total_list, filename)
297+
298+
PixivHelper.print_and_log('info', f"Found {len(total_list)} of {total_bookmark_count} possible image(s) .")
299+
300+
print("Done.\n")
301+
except KeyboardInterrupt:
302+
raise
303+
except BaseException:
304+
PixivHelper.print_and_log('error', 'Error at export_image_bookmark(): {0}'.format(sys.exc_info()))
305+
raise
306+
307+
282308
def get_bookmarks(caller, config, hide, start_page=1, end_page=0, member_id=None):
283309
br: PixivBrowser = caller.__br__
284310

‎PixivUtil2.py

+69-12
Original file line numberDiff line numberDiff line change
@@ -154,7 +154,7 @@ def menu():
154154
print(' 2. Download by image_id')
155155
print(' 3. Download by tags')
156156
print(' 4. Download from list')
157-
print(' 5. Download from bookmarked artists (/bookmark.php?type=user)')
157+
print(' 5. Download from followed artists (/bookmark.php?type=user)')
158158
print(' 6. Download from bookmarked images (/bookmark.php)')
159159
print(' 7. Download from tags list')
160160
print(' 8. Download new illust from bookmarked members (/bookmark_new_illust.php)')
@@ -184,6 +184,7 @@ def menu():
184184
print(' d. Manage database')
185185
print(' e. Export online followed artist.')
186186
print(' m. Export online other\'s followed artist.')
187+
print(' p. Export online image bookmarks.')
187188
print(' i. Import list file')
188189
print(' r. Reload config.ini')
189190
print(' p. Print config.ini')
@@ -735,8 +736,8 @@ def menu_export_online_bookmark(opisvalid, args, options):
735736
filename = "export.txt"
736737

737738
if opisvalid:
738-
if len(args) > 0:
739-
filename = args[0]
739+
if options.export_filename is not None:
740+
filename = options.export_filename
740741
if options.bookmark_flag is not None:
741742
hide = options.bookmark_flag.lower()
742743
if hide not in ('y', 'n', 'o'):
@@ -759,9 +760,9 @@ def menu_export_online_user_bookmark(opisvalid, args, options):
759760
filename = "export-user.txt"
760761

761762
if opisvalid and len(args) > 0:
762-
arg = args[0]
763-
if len(args) > 1:
764-
filename = args[1]
763+
arg = args[0] # member id
764+
if options.export_filename is not None:
765+
filename = options.export_filename
765766
else:
766767
filename = f"export-user-{arg}.txt"
767768
else:
@@ -778,6 +779,52 @@ def menu_export_online_user_bookmark(opisvalid, args, options):
778779
PixivBookmarkHandler.export_bookmark(sys.modules[__name__], __config__, filename, 'n', 1, 0, member_id)
779780

780781

782+
def menu_export_from_online_image_bookmark(opisvalid, args, options):
783+
__log__.info("Export User's Image Bookmark mode (p).")
784+
start_page = 1
785+
end_page = 0
786+
hide = 'n'
787+
tag = ''
788+
use_image_tag = False
789+
filename = "Exported_images.txt"
790+
791+
if opisvalid:
792+
if len(args) > 0:
793+
tag = args[0]
794+
795+
(start_page, end_page) = get_start_and_end_page_from_options(options)
796+
if options.bookmark_flag is not None:
797+
hide = options.bookmark_flag.lower()
798+
if hide not in ('y', 'n', 'o'):
799+
PixivHelper.print_and_log("error", f"Invalid args for bookmark_flag: {options.bookmark_flag}, valid values are [y/n/o].")
800+
return
801+
use_image_tag = options.use_image_tag
802+
if options.export_filename is not None:
803+
filename = options.export_filename
804+
else:
805+
hide = input("Include Private bookmarks [y/n/o, default is no]: ").rstrip("\r") or 'n'
806+
hide = hide.lower()
807+
if hide not in ('y', 'n', 'o'):
808+
print("Invalid args: ", hide)
809+
return
810+
tag = input("Tag (press enter for all images): ").rstrip("\r") or ''
811+
(start_page, end_page) = PixivHelper.get_start_and_end_number(total_number_of_page=options.number_of_pages)
812+
if tag != '':
813+
use_image_tag = input("Use Image Tags as the filter [y/n, default is no]? ").rstrip("\r") or 'n'
814+
use_image_tag = use_image_tag.lower()
815+
use_image_tag = True if use_image_tag == 'y' else False
816+
filename = input(f"Filename (default is '{filename}'): ").rstrip("\r") or filename
817+
818+
PixivBookmarkHandler.export_image_bookmark(sys.modules[__name__],
819+
__config__,
820+
hide=hide,
821+
start_page=start_page,
822+
end_page=end_page,
823+
tag=tag,
824+
use_image_tag=use_image_tag,
825+
filename=filename)
826+
827+
781828
def menu_fanbox_download_from_list(op_is_valid, via, args, options):
782829
via_type = ""
783830
if via == PixivModelFanbox.FanboxArtist.SUPPORTING:
@@ -935,12 +982,13 @@ def menu_sketch_download_by_artist_id(opisvalid, args, options):
935982
if opisvalid and len(args) > 0:
936983
for member_id in args:
937984
try:
938-
prefix = f"[{current_member} of {len(args)}] "
985+
prefix = f"Pixiv Sketch [{current_member} of {len(args)}] "
939986
PixivSketchHandler.process_sketch_artists(sys.modules[__name__],
940987
__config__,
941988
member_id,
942989
page,
943-
end_page)
990+
end_page,
991+
title_prefix=prefix)
944992
current_member = current_member + 1
945993
except PixivException as ex:
946994
PixivHelper.print_and_log("error", f"Error when processing Pixiv Sketch:{member_id}", ex)
@@ -953,12 +1001,13 @@ def menu_sketch_download_by_artist_id(opisvalid, args, options):
9531001
PixivHelper.print_and_log('info', f"Artist IDs: {member_ids}")
9541002
for member_id in member_ids:
9551003
try:
956-
prefix = f"[{current_member} of {len(member_ids)}] "
1004+
prefix = f"Pixiv Sketch [{current_member} of {len(member_ids)}] "
9571005
PixivSketchHandler.process_sketch_artists(sys.modules[__name__],
9581006
__config__,
9591007
member_id,
9601008
page,
961-
end_page)
1009+
end_page,
1010+
title_prefix=prefix)
9621011
current_member = current_member + 1
9631012
except PixivException as ex:
9641013
PixivHelper.print_and_log("error", f"Error when processing Pixiv Sketch:{member_id}", ex)
@@ -1087,7 +1136,7 @@ def setup_option_parser():
10871136
__valid_options = ('1', '2', '3', '4', '5', '6', '7', '8', '9', '10', '11', '12', '13', '14', '15', '16', '17', '18',
10881137
'f1', 'f2', 'f3', 'f4', 'f5',
10891138
's1', 's2',
1090-
'd', 'e', 'm', 'b')
1139+
'd', 'e', 'm', 'b', 'p')
10911140
parser = OptionParser()
10921141

10931142
# need to keep the whitespace to adjust the output for --help
@@ -1115,6 +1164,7 @@ def setup_option_parser():
11151164
b - Batch Download from batch_job.json \n
11161165
e - Export online bookmark \n
11171166
m - Export online user bookmark \n
1167+
p - Export online image bookmark \n
11181168
d - Manage database''')
11191169
parser.add_option('-x', '--exit_when_done',
11201170
dest='exit_when_done',
@@ -1210,7 +1260,7 @@ def setup_option_parser():
12101260
parser.add_option('--bcl', '--bookmark_count_limit',
12111261
dest='bookmark_count_limit',
12121262
default=-1,
1213-
help='''Bookmark count limit in integer. \n
1263+
help='''Bookmark count limit in integer. \n
12141264
Used in option 3, 5, 7, and 8.''')
12151265
parser.add_option('--rm', '--rank_mode',
12161266
dest='rank_mode',
@@ -1220,6 +1270,11 @@ def setup_option_parser():
12201270
dest='rank_content',
12211271
default="all",
12221272
help='''Ranking Content Type.''')
1273+
parser.add_option('--ef', '--export_filename',
1274+
dest='export_filename',
1275+
default="export.txt",
1276+
help='''Filename for exporting members/images. \n
1277+
Used in option e, m, p''')
12231278
return parser
12241279

12251280

@@ -1285,6 +1340,8 @@ def main_loop(ewd, op_is_valid, selection, np_is_valid_local, args, options):
12851340
menu_export_online_bookmark(op_is_valid, args, options)
12861341
elif selection == 'm':
12871342
menu_export_online_user_bookmark(op_is_valid, args, options)
1343+
elif selection == 'p':
1344+
menu_export_from_online_image_bookmark(op_is_valid, args, options)
12881345
elif selection == 'd':
12891346
__dbManager__.main()
12901347
elif selection == 'r':

0 commit comments

Comments
 (0)
Please sign in to comment.