Merge pull request #189 from jkowalleck/bugfix/188

closes #188

Author: jkowalleck

HISTORY.md

@@ -4,6 +4,13 @@
 ## Unreleased
 
 
+## 2.3.1
+
+### Fix
+
+*  fixed the paging of the `Pr0gramm` ImageCrawler in `promoted=True` mode.
+
+
 ## 2.3.0
 
 ### Breaking changes

nichtparasoup/imagecrawler/pr0gramm.py

@@ -81,9 +81,10 @@ def _reset(self) -> None:
 
     def _crawl(self) -> ImageCollection:
         images = ImageCollection()
+        promoted = self._config['promoted']
         api_uri = self._get_api_uri(
             flags=1,
-            promoted=self._config['promoted'],
+            promoted=promoted,
             tags=self._config.get('tags', None),
             older=self._older)
         response_raw, api_uri = self._remote_fetcher.get_string(api_uri)
@@ -98,5 +99,5 @@ def _crawl(self) -> ImageCollection:
         if response['atEnd']:
             self.reset()
         else:
-            self._older = response['items'][-1]['id'] or None
+            self._older = response['items'][-1]['promoted' if promoted else 'id'] or None
         return images

tests/test_10_nichtparasoup/test_imagecrawler/test_pr0gramm.py

@@ -133,6 +133,8 @@ def test_older(self, older: Optional[int], older_qs: str) -> None:
         'get-flags_1-promoted_1-tags_s15000-video.json',
     '/api/items/get?flags=1&promoted=1&tags=%21%28s%3A1000%29+-%22video%22':
         'get-flags_1-promoted_1-tags_s1000-video.json',
+    '/api/items/get?flags=1&promoted=0&tags=%21%28s%3A1000%29+-%22video%22':
+        'get-flags_1-promoted_0-tags_s1000-video.json',
 }, base_dir=path_join(dirname(__file__), 'testdata_pr0gramm'))
 
 
@@ -148,6 +150,7 @@ def test_reset_done(self) -> None:
         self.assertIsNone(crawler._older)
 
 
+@ddt
 class Pr0grammCrawlTest(unittest.TestCase):
 
     def test_reset_at_end(self) -> None:
@@ -170,16 +173,17 @@ def test_no_reset_before_end(self) -> None:
         # assert
         self.assertFalse(crawler._reset_before_next_crawl)
 
-    def test_crawl_cursor(self) -> None:
+    @ddt_data((True, 503528), (False, 3652675))  # type: ignore
+    @ddt_unpack  # type: ignore
+    def test_crawl_cursor(self, promoted: bool, expected_cursor: Optional[int]) -> None:
         # arrange
-        crawler = Pr0gramm(flags=1, promoted=True, tags='!s:1000')
+        crawler = Pr0gramm(flags=1, promoted=promoted, tags='!s:1000')
         crawler._remote_fetcher = _FILE_FETCHER
         crawler._older = None
-        expected_cursor = 3639645
         # act
         crawler._crawl()
         # assert
-        self.assertEqual(crawler._older, expected_cursor)
+        self.assertEqual(crawler._older, expected_cursor, msg='promoted={!r}'.format(promoted))
 
     def test_crawl_images(self) -> None:
         # arrange