diff --git a/models.py b/models.py index 0918aad4..b0eb6163 100644 --- a/models.py +++ b/models.py @@ -615,11 +615,6 @@ def use_urls_as_ids(obj): rel_urls=self.mf2.get('rel-urls')) use_urls_as_ids(obj) - # TODO: remove once we drop superfeedr - elif self.atom: - obj = atom.atom_to_activity(self.atom)['object'] - use_urls_as_ids(obj) - else: return None diff --git a/tests/test_web.py b/tests/test_web.py index 4f0c2a93..36762f51 100644 --- a/tests/test_web.py +++ b/tests/test_web.py @@ -1,7 +1,6 @@ """Unit tests for webmention.py.""" import copy from datetime import timedelta -from unittest import skip from unittest.mock import ANY, patch from flask import g, get_flashed_messages @@ -513,63 +512,20 @@ def test_get_or_create_scripts_leading_trailing_dots(self, mock_get, mock_post): self.assert_entities_equal(user, Web.get_by_id('foo.bar')) self.assertIsNone(Web.get_by_id('..foo.bar.')) - @skip - def test_get_or_create_subscribes_superfeedr(self, mock_get, mock_post): - self.user.obj.mf2 = ACTOR_MF2_REL_FEED_URL - self.user.obj.put() - self.user.has_redirects = False - self.user.put() - - user = Web.get_or_create('user.com') - self.assertEqual('user.com', user.key.id()) - self.assert_entities_equal( - user, self.user, - ignore=['superfeedr_subscribed', 'superfeedr_subscribed_feed', 'updated']) - self.assertEqual(NOW, user.superfeedr_subscribed) - self.assertEqual('https://foo/atom', user.superfeedr_subscribed_feed) - - self.assert_req(mock_post, SUPERFEEDR_PUSH_API, data={ - 'hub.mode': 'subscribe', - 'hub.topic': 'https://foo/atom', - 'hub.callback': 'http://localhost/superfeedr/notify/user.com', - 'format': 'atom', - 'retrieve': 'true', - }, auth=ANY) - self.assertEqual(NOW, self.user.key.get().superfeedr_subscribed) - - def test_get_or_create_subscribe_error(self, mock_get, mock_post): - self.user.obj.mf2 = ACTOR_MF2_REL_FEED_URL - self.user.obj.put() - self.user.has_redirects = False - self.user.put() - - mock_post.return_value = requests_response('Nope', status=500) - - user = Web.get_or_create('user.com') - self.assert_entities_equal(user, self.user, ignore=['updated']) - self.assertIsNone(user.superfeedr_subscribed) - self.assertIsNone(user.superfeedr_subscribed_feed) - - def test_get_or_create_existing_subscribed(self, *_): - self.user.superfeedr_subscribed = NOW - self.user.put() - + @patch('oauth_dropins.webutil.appengine_config.tasks_client.create_task') + def test_get_or_create_existing_no_poll_feed_task(self, mock_create_task, _, __): user = Web.get_or_create('user.com') - self.assertEqual('user.com', user.key.id()) - self.assert_entities_equal(user, self.user) + mock_create_task.assert_not_called() - def test_get_or_create_existing_has_last_webmention(self, *_): - self.user.last_webmention_in = NOW - self.user.put() - - user = Web.get_or_create('user.com') - self.assertEqual('user.com', user.key.id()) - self.assert_entities_equal(user, self.user) + @patch('oauth_dropins.webutil.appengine_config.tasks_client.create_task') + def test_get_or_create_new_creates_poll_feed_task(self, mock_create_task, + mock_get, __): + common.RUN_TASKS_INLINE = False + mock_get.return_value = ACTOR_HTML_RESP - def test_get_or_create_existing_not_subscribed_no_feed(self, *_): - user = Web.get_or_create('user.com') - self.assertEqual('user.com', user.key.id()) - self.assert_entities_equal(user, self.user) + user = Web.get_or_create('new.com') + self.assert_task(mock_create_task, 'poll-feed', '/queue/poll-feed', + domain='new.com') def test_get_or_create_existing_opted_out(self, *_): self.user.obj.mf2['properties']['summary'] = '#nobridge' @@ -633,28 +589,6 @@ def test_make_task(self, mock_create_task, mock_get, mock_post): self.assertEqual(NOW, self.user.key.get().last_webmention_in) - @skip - def test_first_webmention_unsubscribe_superfeedr(self, mock_get, mock_post): - self.user.superfeedr_subscribed = NOW - self.user.superfeedr_subscribed_feed = 'http://feed' - self.user.put() - - mock_get.return_value = NOTE - - params = { - 'source': 'https://user.com/post', - 'target': 'https://fed.brid.gy/', - } - got = self.post('/webmention', data=params) - self.assertEqual(204, got.status_code) - - self.assertEqual(NOW, self.user.key.get().last_webmention_in) - self.assert_req(mock_post, SUPERFEEDR_PUSH_API, data={ - 'hub.mode': 'unsubscribe', - 'hub.topic': 'http://feed', - 'hub.callback': 'http://localhost/superfeedr/notify/user.com', - }, auth=ANY) - def test_no_user(self, mock_get, mock_post): orig_count = Object.query().count() @@ -1996,6 +1930,26 @@ def test_poll_feed_wrong_content_type(self, mock_get, _): self.assertEqual(200, got.status_code) self.assertIsNone(self.user.key.get().last_polled_feed) + @patch('oauth_dropins.webutil.appengine_config.tasks_client.create_task') + def test_poll_feed_last_webmention_in_noop(self, mock_create_task, mock_get, _): + common.RUN_TASKS_INLINE = False + + self.user.last_webmention_in = NOW + self.user.put() + self.user.obj.mf2 = { + **ACTOR_MF2, + 'rel-urls': { + 'https://foo/rss': {'rels': ['alternate'], 'type': rss.CONTENT_TYPE}, + }, + } + self.user.obj.put() + + got = self.post('/queue/poll-feed', data={'domain': 'user.com'}) + self.assertEqual(200, got.status_code) + self.assertIsNone(self.user.key.get().last_polled_feed) + mock_create_task.assert_not_called() + mock_get.assert_not_called() + def _test_verify(self, redirects, hcard, actor, redirects_error=None): self.user.has_redirects = False self.user.put() diff --git a/web.py b/web.py index 09823456..c1616558 100644 --- a/web.py +++ b/web.py @@ -100,8 +100,6 @@ class Web(User, Protocol): has_hcard = ndb.BooleanProperty() last_webmention_in = ndb.DateTimeProperty(tzinfo=timezone.utc) last_polled_feed = ndb.DateTimeProperty(tzinfo=timezone.utc) - superfeedr_subscribed = ndb.DateTimeProperty(tzinfo=timezone.utc) - superfeedr_subscribed_feed = ndb.StringProperty() # Originally, BF served Web users' AP actor ids on fed.brid.gy, eg # https://fed.brid.gy/snarfed.org . When we started adding new protocols, we @@ -110,6 +108,10 @@ class Web(User, Protocol): # property tracks which subdomain a given Web user's AP actor uses. ap_subdomain = ndb.StringProperty(choices=['fed', 'web'], default='web') + # OLD. some stored entities still have these; do not reuse. + # superfeedr_subscribed = ndb.DateTimeProperty(tzinfo=timezone.utc) + # superfeedr_subscribed_feed = ndb.StringProperty() + @classmethod def _get_kind(cls): return 'MagicKey' @@ -123,7 +125,7 @@ def _pre_put_hook(self): @classmethod def get_or_create(cls, id, **kwargs): - """Normalize domain, pass through, then subscribe in Superfeedr. + """Normalize domain, then pass through to :meth:`User.get_or_create`. Normalizing currently consists of lower casing and removing leading and trailing dots. @@ -135,8 +137,8 @@ def get_or_create(cls, id, **kwargs): domain = key.id().lower().strip('.') user = super().get_or_create(domain, **kwargs) - # TODO - # maybe_superfeedr_subscribe(user) + if not user.existing: + common.create_task(queue='poll-feed', domain=domain) return user @@ -586,8 +588,6 @@ def webmention_external(): if request.path == '/webmention': # exclude interactive user.last_webmention_in = util.now() user.put() - # TODO - # maybe_superfeedr_unsubscribe(user) return common.create_task('webmention', **request.form) @@ -611,23 +611,6 @@ def webmention_interactive(): return redirect('/', code=302) -def maybe_superfeedr_subscribe(user): - """Subscribes to a user's Atom or RSS feed in Superfeedr. - - Args: - user (Web) - """ - if user.superfeedr_subscribed: - logger.info(f'User {user.key.id()} already subscribed via Superfeedr') - return - elif user.has_redirects or user.last_webmention_in: - logger.info(f'User {user.key.id()} has Webfinger redirects or publishes via webmention, not subscribing via Superfeedr') - return - elif not user.obj or not user.obj.mf2: - logger.info(f"User {user.key.id()} has no mf2, can't subscribe via Superfeedr") - return - - @app.post(f'/queue/poll-feed') def poll_feed_task(): """Fetches a :class:`Web` site's feed and delivers new/updated posts. @@ -635,9 +618,13 @@ def poll_feed_task(): Params: ``domain`` (str): key id of the :class:`Web` user """ - user = Web.get_by_id(flask_util.get_required_param('domain')) - if not user: - error(f'No Web user found for domain {domain}', status=304) + domain = flask_util.get_required_param('domain') + user = Web.get_by_id(domain) + if not (user and user.obj and user.obj.mf2): + error(f'No Web user or object found for domain {domain}', status=304) + elif user.last_webmention_in: + logger.info(f'Dropping since last_webmention_in is set') + return 'OK' # discover feed URL for url, info in user.obj.mf2.get('rel-urls', {}).items(): @@ -692,13 +679,14 @@ def poll_feed_task(): authed_as=user.key.id()) # create next poll task + def clamp(delay): + return max(min(delay, MAX_FEED_POLL_PERIOD), MIN_FEED_POLL_PERIOD) + if published_deltas: - seconds = statistics.mean(t.total_seconds() for t in published_deltas) - delay = max(min(timedelta(seconds=seconds), MAX_FEED_POLL_PERIOD), - MIN_FEED_POLL_PERIOD) + delay = clamp(timedelta(seconds=statistics.mean( + t.total_seconds() for t in published_deltas))) else: - # TODO - delay = MIN_FEED_POLL_PERIOD + delay = clamp(util.now() - (user.last_polled_feed or user.created)) common.create_task(queue='poll-feed', domain=user.key.id(), delay=delay) return 'OK'