Merge pull request #20 from epandurski/master

Use DebtorInfoFetch.forced_iri instead of "ignore_cache"

migrations/versions/88cd664f51a3_.py

@@ -0,0 +1,66 @@
+"""empty message
+
+Revision ID: 88cd664f51a3
+Revises: dcb810823e04
+Create Date: 2024-09-21 19:16:59.266474
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = '88cd664f51a3'
+down_revision = 'dcb810823e04'
+branch_labels = None
+depends_on = None
+
+
+def upgrade(engine_name):
+    globals()["upgrade_%s" % engine_name]()
+
+
+def downgrade(engine_name):
+    globals()["downgrade_%s" % engine_name]()
+
+
+
+
+
+def upgrade_():
+    # ### commands auto generated by Alembic - please adjust! ###
+    with op.batch_alter_table('debtor_info_fetch', schema=None) as batch_op:
+        batch_op.add_column(sa.Column('forced_iri', sa.String(), nullable=True))
+        batch_op.drop_column('ignore_cache')
+
+    with op.batch_alter_table('fetch_debtor_info_signal', schema=None) as batch_op:
+        batch_op.add_column(sa.Column('forced_iri', sa.String(), nullable=True))
+        batch_op.drop_column('ignore_cache')
+
+    # ### end Alembic commands ###
+
+
+def downgrade_():
+    # ### commands auto generated by Alembic - please adjust! ###
+    with op.batch_alter_table('fetch_debtor_info_signal', schema=None) as batch_op:
+        batch_op.add_column(sa.Column('ignore_cache', sa.BOOLEAN(), autoincrement=False, nullable=False))
+        batch_op.drop_column('forced_iri')
+
+    with op.batch_alter_table('debtor_info_fetch', schema=None) as batch_op:
+        batch_op.add_column(sa.Column('ignore_cache', sa.BOOLEAN(), autoincrement=False, nullable=False))
+        batch_op.drop_column('forced_iri')
+
+    # ### end Alembic commands ###
+
+
+def upgrade_solver():
+    # ### commands auto generated by Alembic - please adjust! ###
+    pass
+    # ### end Alembic commands ###
+
+
+def downgrade_solver():
+    # ### commands auto generated by Alembic - please adjust! ###
+    pass
+    # ### end Alembic commands ###
+

swpt_trade/actors.py

@@ -413,7 +413,7 @@ def _on_fetch_debtor_info_signal(
     debtor_id: int,
     is_locator_fetch: bool,
     is_discovery_fetch: bool,
-    ignore_cache: bool,
+    forced_iri: Optional[str],
     recursion_level: int,
     ts: datetime,
     *args,
@@ -424,7 +424,7 @@ def _on_fetch_debtor_info_signal(
         debtor_id=debtor_id,
         is_locator_fetch=is_locator_fetch,
         is_discovery_fetch=is_discovery_fetch,
-        ignore_cache=ignore_cache,
+        forced_iri=forced_iri,
         recursion_level=recursion_level,
         ts=ts,
     )

swpt_trade/fetch_debtor_infos.py

@@ -8,7 +8,7 @@
 from datetime import datetime, timezone, timedelta
 from marshmallow import ValidationError
 from flask import current_app
-from sqlalchemy.sql.expression import and_, not_
+from sqlalchemy.sql.expression import and_, null
 from swpt_pythonlib.utils import ShardingRealm
 from swpt_pythonlib.swpt_uris import parse_debtor_uri
 from swpt_trade.extensions import db
@@ -119,7 +119,6 @@ def _process_debtor_info_fetches_batch(
                             debtor_id=peg_debtor_id,
                             is_locator_fetch=True,
                             is_discovery_fetch=False,
-                            ignore_cache=False,
                             recursion_level=recursion_level + 1,
                         )
                     )
@@ -157,7 +156,7 @@ def _query_and_resolve_pending_fetches(
             DebtorInfoDocument,
             and_(
                 DebtorInfoDocument.debtor_info_locator == DebtorInfoFetch.iri,
-                not_(DebtorInfoFetch.ignore_cache),
+                DebtorInfoFetch.forced_iri == null(),
             ),
         )
         .filter(DebtorInfoFetch.next_attempt_at <= current_ts)
@@ -256,7 +255,7 @@ def _make_https_requests(
             # Here we catch and log such errors as warnings.
             logger.warning(
                 "Caught error during request to %s",
-                fetch.iri,
+                fetch.forced_iri or fetch.iri,
                 exc_info=obj,
             )
             results.append(FetchResult(fetch=fetch))
@@ -290,7 +289,7 @@ async def _make_https_request(
         client: aiohttp.ClientSession,
         fetch: DebtorInfoFetch,
 ) -> FetchResult:
-    iri = fetch.iri
+    iri = fetch.forced_iri or fetch.iri
     try:
         if not iri.startswith("https://"):
             raise aiohttp.InvalidURL(iri)

swpt_trade/models/debtor_info.py

@@ -91,7 +91,7 @@ class DebtorInfoFetch(db.Model):
     debtor_id = db.Column(db.BigInteger, primary_key=True)
     is_locator_fetch = db.Column(db.BOOLEAN, nullable=False, default=False)
     is_discovery_fetch = db.Column(db.BOOLEAN, nullable=False, default=False)
-    ignore_cache = db.Column(db.BOOLEAN, nullable=False, default=False)
+    forced_iri = db.Column(db.String)
     recursion_level = db.Column(db.SmallInteger, nullable=False, default=0)
     inserted_at = db.Column(
         db.TIMESTAMP(timezone=True), nullable=False, default=get_now_utc

swpt_trade/models/signals.py

@@ -143,7 +143,7 @@ class __marshmallow__(Schema):
         debtor_id = fields.Integer()
         is_locator_fetch = fields.Boolean()
         is_discovery_fetch = fields.Boolean()
-        ignore_cache = fields.Boolean()
+        forced_iri = fields.String()
         recursion_level = fields.Integer()
         inserted_at = fields.DateTime(data_key="ts")
 
@@ -154,7 +154,7 @@ class __marshmallow__(Schema):
     debtor_id = db.Column(db.BigInteger, nullable=False)
     is_locator_fetch = db.Column(db.BOOLEAN, nullable=False)
     is_discovery_fetch = db.Column(db.BOOLEAN, nullable=False)
-    ignore_cache = db.Column(db.BOOLEAN, nullable=False)
+    forced_iri = db.Column(db.String)
     recursion_level = db.Column(db.SmallInteger, nullable=False)
 
     @property

swpt_trade/procedures/debtor_info.py

@@ -21,7 +21,7 @@ def schedule_debtor_info_fetch(
         debtor_id: int,
         is_locator_fetch: bool,
         is_discovery_fetch: bool,
-        ignore_cache: bool,
+        forced_iri: Optional[str],
         recursion_level: int,
         ts: datetime,
 ) -> None:
@@ -38,8 +38,8 @@ def schedule_debtor_info_fetch(
         debtor_info_fetch.is_discovery_fetch = (
             is_discovery_fetch or debtor_info_fetch.is_discovery_fetch
         )
-        debtor_info_fetch.ignore_cache = (
-            ignore_cache or debtor_info_fetch.ignore_cache
+        debtor_info_fetch.forced_iri = (
+            forced_iri or debtor_info_fetch.forced_iri
         )
         debtor_info_fetch.recursion_level = min(
             recursion_level, debtor_info_fetch.recursion_level
@@ -52,7 +52,7 @@ def schedule_debtor_info_fetch(
                     debtor_id=debtor_id,
                     is_locator_fetch=is_locator_fetch,
                     is_discovery_fetch=is_discovery_fetch,
-                    ignore_cache=ignore_cache,
+                    forced_iri=forced_iri,
                     recursion_level=recursion_level,
                 )
             )
@@ -101,7 +101,7 @@ def discover_debtor(
                         debtor_id=debtor_id,
                         is_locator_fetch=True,
                         is_discovery_fetch=False,
-                        ignore_cache=forced_refetch,
+                        forced_iri=iri if forced_refetch else None,
                         recursion_level=0,
                     )
                 )
@@ -118,7 +118,6 @@ def discover_debtor(
                     debtor_id=debtor_id,
                     is_locator_fetch=False,
                     is_discovery_fetch=True,
-                    ignore_cache=True,
                     recursion_level=0,
                 )
             )
@@ -139,7 +138,6 @@ def discover_debtor(
                 debtor_id=debtor_id,
                 is_locator_fetch=False,
                 is_discovery_fetch=True,
-                ignore_cache=True,
                 recursion_level=0,
             )
         )
@@ -185,7 +183,6 @@ def confirm_debtor(
             debtor_id=debtor_id,
             is_locator_fetch=True,
             is_discovery_fetch=False,
-            ignore_cache=True,
             recursion_level=0,
         )
     )

swpt_trade/schemas.py

@@ -37,7 +37,7 @@ class Meta:
     )
     is_locator_fetch = fields.Boolean(required=True)
     is_discovery_fetch = fields.Boolean(required=True)
-    ignore_cache = fields.Boolean(required=True)
+    forced_iri = fields.String(load_default=None)
     recursion_level = fields.Integer(
         required=True, validate=validate.Range(min=0, max=MAX_INT16)
     )

tests/test_actors.py

@@ -264,7 +264,7 @@ def test_on_fetch_debtor_info_signal(db_session, actors):
         debtor_id=D_ID,
         is_locator_fetch=True,
         is_discovery_fetch=False,
-        ignore_cache=False,
+        forced_iri=None,
         recursion_level=2,
         ts=datetime.now(tz=timezone.utc),
     )

tests/test_cli.py

@@ -133,7 +133,7 @@ def make_https_requests(fetches, **kwargs):
         debtor_id=666,
         is_locator_fetch=True,
         is_discovery_fetch=True,
-        ignore_cache=True,
+        forced_iri=None,
     )
     db.session.add(dif)
     db.session.commit()
@@ -167,7 +167,6 @@ def invoke():
     assert fetch_signals[0].debtor_id == 777
     assert fetch_signals[0].is_locator_fetch is True
     assert fetch_signals[0].is_discovery_fetch is False
-    assert fetch_signals[0].ignore_cache is False
     assert fetch_signals[0].recursion_level == 1
 
     stored_signals = m.StoreDocumentSignal.query.all()

tests/test_fetch_debtor_infos.py

@@ -42,7 +42,6 @@ def make_https_requests(fetches, **kwargs):
             debtor_id=666,
             is_locator_fetch=True,
             is_discovery_fetch=False,
-            ignore_cache=True,
         )
     )
     db.session.commit()
@@ -82,7 +81,6 @@ def make_https_requests(fetches, **kwargs):
             debtor_id=666,
             is_locator_fetch=True,
             is_discovery_fetch=False,
-            ignore_cache=False,
         )
     )
     db.session.add(
@@ -91,7 +89,6 @@ def make_https_requests(fetches, **kwargs):
             debtor_id=888,
             is_locator_fetch=True,
             is_discovery_fetch=False,
-            ignore_cache=True,
         )
     )
     db.session.commit()
@@ -133,15 +130,13 @@ def make_https_requests(fetches, **kwargs):
         debtor_id=666,
         is_locator_fetch=True,
         is_discovery_fetch=True,
-        ignore_cache=True,
     )
     dif2 = m.DebtorInfoFetch(
         iri="wrong IRI",
         debtor_id=999,
         is_locator_fetch=True,
         is_discovery_fetch=False,
-        ignore_cache=False,
-    )
+      )
     db.session.add(dif1)
     db.session.add(dif2)
     db.session.commit()
@@ -154,7 +149,6 @@ def make_https_requests(fetches, **kwargs):
     fetches[0].debtor_id == 999
     fetches[0].is_locator_fetch is True
     fetches[0].is_discovery_fetch is False
-    fetches[0].ignore_cache is False
     fetches[0].recursion_level == 0
     fetches[0].attempts_count == 1
     fetches[0].latest_attempt_at is not None
@@ -167,7 +161,6 @@ def make_https_requests(fetches, **kwargs):
     assert fetch_signals[0].debtor_id == 777
     assert fetch_signals[0].is_locator_fetch is True
     assert fetch_signals[0].is_discovery_fetch is False
-    assert fetch_signals[0].ignore_cache is False
     assert fetch_signals[0].recursion_level == 1
 
     stored_signals = m.StoreDocumentSignal.query.all()

tests/test_models.py

@@ -145,7 +145,6 @@ def test_non_smp_signals(db_session):
         debtor_id=1,
         is_locator_fetch=True,
         is_discovery_fetch=False,
-        ignore_cache=False,
         recursion_level=0,
     )
     db_session.add(signal)

tests/test_procedures.py

@@ -361,7 +361,7 @@ def test_schedule_debtor_info_fetch(db_session, current_ts):
         debtor_id=666,
         is_locator_fetch=True,
         is_discovery_fetch=False,
-        ignore_cache=False,
+        forced_iri=None,
         recursion_level=4,
         ts=current_ts,
     )
@@ -371,7 +371,6 @@ def test_schedule_debtor_info_fetch(db_session, current_ts):
     assert fetches[0].debtor_id == 666
     assert fetches[0].is_locator_fetch is True
     assert fetches[0].is_discovery_fetch is False
-    assert fetches[0].ignore_cache is False
     assert fetches[0].recursion_level == 4
     assert fetches[0].attempts_count == 0
 
@@ -381,7 +380,7 @@ def test_schedule_debtor_info_fetch(db_session, current_ts):
         debtor_id=666,
         is_locator_fetch=False,
         is_discovery_fetch=True,
-        ignore_cache=True,
+        forced_iri=None,
         recursion_level=2,
         ts=current_ts,
     )
@@ -391,7 +390,6 @@ def test_schedule_debtor_info_fetch(db_session, current_ts):
     assert fetches[0].debtor_id == 666
     assert fetches[0].is_locator_fetch is True
     assert fetches[0].is_discovery_fetch is True
-    assert fetches[0].ignore_cache is True
     assert fetches[0].recursion_level == 2
     assert fetches[0].attempts_count == 0
 
@@ -421,7 +419,6 @@ def test_discover_and_confirm_debtor(db_session, current_ts):
     assert fetch_signals[0].debtor_id == 666
     assert fetch_signals[0].is_locator_fetch is False
     assert fetch_signals[0].is_discovery_fetch is True
-    assert fetch_signals[0].ignore_cache is True
     assert fetch_signals[0].recursion_level == 0
 
     # Process the same discover message again (does nothing).
@@ -456,7 +453,6 @@ def test_discover_and_confirm_debtor(db_session, current_ts):
     assert fetch_signals[1].debtor_id == 666
     assert fetch_signals[1].is_locator_fetch is True
     assert fetch_signals[1].is_discovery_fetch is False
-    assert fetch_signals[1].ignore_cache is True
     assert fetch_signals[1].recursion_level == 0
 
     # Process another confirm message for this debtor.
@@ -478,7 +474,6 @@ def test_discover_and_confirm_debtor(db_session, current_ts):
     assert fetch_signals[2].debtor_id == 666
     assert fetch_signals[2].is_locator_fetch is True
     assert fetch_signals[2].is_discovery_fetch is False
-    assert fetch_signals[2].ignore_cache is True
     assert fetch_signals[2].recursion_level == 0
 
     # Process a very old confirm message (does nothing).
@@ -523,13 +518,11 @@ def test_discover_and_confirm_debtor(db_session, current_ts):
     assert fetch_signals[0].debtor_id == 666
     assert fetch_signals[0].is_locator_fetch is False
     assert fetch_signals[0].is_discovery_fetch is True
-    assert fetch_signals[0].ignore_cache is True
     assert fetch_signals[0].recursion_level == 0
     assert fetch_signals[1].iri == "https:/example.com/locator"
     assert fetch_signals[1].debtor_id == 666
     assert fetch_signals[1].is_locator_fetch is True
     assert fetch_signals[1].is_discovery_fetch is False
-    assert fetch_signals[1].ignore_cache is True
     assert fetch_signals[1].recursion_level == 0
 
     # Process a confirm message for another debtor.
@@ -552,7 +545,6 @@ def test_discover_and_confirm_debtor(db_session, current_ts):
     assert fetch_signals[5].debtor_id == 1234
     assert fetch_signals[5].is_locator_fetch is True
     assert fetch_signals[5].is_discovery_fetch is False
-    assert fetch_signals[5].ignore_cache is True
     assert fetch_signals[5].recursion_level == 0