Upgrade to SQLAlchemy 2

Signed-off-by: Aurélien Bompard <[email protected]>

datanommer.commands/datanommer/commands/__init__.py

@@ -16,11 +16,11 @@
 import json
 import logging
 import time
-from datetime import datetime, timedelta
+from datetime import datetime, timedelta, timezone
 
 import click
 from fedora_messaging import config as fedora_messaging_config
-from sqlalchemy import func
+from sqlalchemy import func, select
 
 import datanommer.models as m
 
@@ -83,24 +83,26 @@ def dump(config_path, since, before):
         alembic_ini=config["alembic_ini"],
     )
 
-    query = m.Message.query
+    query = select(m.Message)
     if before:
         try:
             before = datetime.fromisoformat(before)
         except ValueError:
             raise click.ClickException("Invalid date format")
 
-        query = query.filter(m.Message.timestamp <= before)
+        query = query.where(m.Message.timestamp <= before)
 
     if since:
         try:
             since = datetime.fromisoformat(since)
         except ValueError:
             raise click.ClickException("Invalid date format")
 
-        query = query.filter(m.Message.timestamp >= since)
+        query = query.where(m.Message.timestamp >= since)
 
-    results = [json.dumps(msg.as_fedora_message_dict()) for msg in query.all()]
+    results = [
+        json.dumps(msg.as_fedora_message_dict()) for msg in m.session.scalars(query)
+    ]
     click.echo(f"[{','.join(results)}]")
 
 
@@ -157,23 +159,17 @@ def stats(config_path, topic, category):
     )
 
     if topic:
+        query = select(m.Message.topic, func.count(m.Message.topic))
         if category:
-            query = m.session.query(
-                m.Message.topic, func.count(m.Message.topic)
-            ).filter(m.Message.category == category)
-        else:
-            query = m.session.query(m.Message.topic, func.count(m.Message.topic))
+            query = query.where(m.Message.category == category)
         query = query.group_by(m.Message.topic)
     else:
+        query = select(m.Message.category, func.count(m.Message.category))
         if category:
-            query = m.session.query(
-                m.Message.category, func.count(m.Message.category)
-            ).filter(m.Message.category == category)
-        else:
-            query = m.session.query(m.Message.category, func.count(m.Message.category))
+            query = query.where(m.Message.category == category)
         query = query.group_by(m.Message.category)
 
-    results = query.all()
+    results = m.session.execute(query).all()
 
     if topic:
         for topic, count in results:
@@ -295,30 +291,28 @@ def latest(config_path, topic, category, overall, timestamp, timesince, human):
     )
 
     if topic:
-        queries = [m.Message.query.filter(m.Message.topic == topic)]
+        queries = [select(m.Message).where(m.Message.topic == topic)]
 
     elif category:
-        queries = [m.Message.query.filter(m.Message.category == category)]
+        queries = [select(m.Message).where(m.Message.category == category)]
     elif not overall:
         # If no args..
-        categories = [
-            c[0]
-            for c in m.session.query(m.Message.category)
-            .distinct()
-            .order_by(m.Message.category)
-        ]
+        categories_query = (
+            select(m.Message.category).distinct().order_by(m.Message.category)
+        )
+        categories = m.session.scalars(categories_query)
         queries = [
-            m.Message.query.filter(m.Message.category == category)
+            select(m.Message).where(m.Message.category == category)
             for category in categories
         ]
     else:
         # Show only the single latest message, regardless of type.
-        queries = [m.Message.query]
+        queries = [select(m.Message)]
 
     # Only check messages from the last year to speed up queries
     a_year = timedelta(days=365)
-    earliest = datetime.utcnow() - a_year
-    queries = [q.filter(m.Message.timestamp > earliest) for q in queries]
+    earliest = datetime.now(tz=timezone.utc) - a_year
+    queries = [q.where(m.Message.timestamp > earliest) for q in queries]
 
     # Order and limit to the latest.
     queries = [q.order_by(m.Message.timestamp.desc()).limit(1) for q in queries]
@@ -337,7 +331,7 @@ def formatter(key, val):
             return f'{{"{key}": {json.dumps(val.as_fedora_message_dict())}}}'
 
     results = []
-    for result in sum((query.all() for query in queries), []):
+    for result in sum((list(m.session.scalars(query)) for query in queries), []):
         results.append(formatter(result.category, result))
 
     click.echo(f"[{','.join(results)}]")

datanommer.commands/datanommer/commands/extract_users.py

@@ -2,7 +2,7 @@
 
 import click
 from fedora_messaging.message import load_message
-from sqlalchemy import and_
+from sqlalchemy import and_, select
 
 import datanommer.models as m
 
@@ -37,11 +37,11 @@ def main(config_path, topic, category):
     if topic and category:
         raise click.UsageError("can't use both --topic and --category, choose one.")
 
-    query = m.Message.query
+    query = select(m.Message)
     if topic:
-        query = query.filter(m.Message.topic == topic)
+        query = query.where(m.Message.topic == topic)
     elif category:
-        query = query.filter(m.Message.category == category)
+        query = query.where(m.Message.category == category)
 
     query = (
         query.join(
@@ -52,11 +52,11 @@ def main(config_path, topic, category):
             ),
             isouter=True,
         )
-        .filter(m.users_assoc_table.c.msg_id.is_(None))
+        .where(m.users_assoc_table.c.msg_id.is_(None))
         .order_by(m.Message.timestamp)
     )
 
-    for message in query:
+    for message in m.session.scalars(query):
         fm_msg = load_message(
             {
                 "topic": message.topic,

datanommer.commands/tests/test_commands.py

@@ -64,6 +64,7 @@ def test_create(mocker):
 
     runner = CliRunner()
     result = runner.invoke(datanommer.commands.create, [])
+    assert result.exit_code == 0, result.output
 
     assert result.output == "Creating Datanommer database and tables\n"
     mock_model_init.assert_called_once_with(
@@ -91,6 +92,7 @@ def test_stats(datanommer_models, mock_config, mock_init):
 
     runner = CliRunner()
     result = runner.invoke(datanommer.commands.stats, [])
+    assert result.exit_code == 0, result.output
 
     assert "git has 2 entries" in result.output
     assert "fas has 1 entries" in result.output
@@ -116,6 +118,7 @@ def test_stats_topics(datanommer_models, mock_config, mock_init):
 
     runner = CliRunner()
     result = runner.invoke(datanommer.commands.stats, ["--topic"])
+    assert result.exit_code == 0, result.output
 
     assert (
         "org.fedoraproject.prod.git.receive.valgrind.master has 1 entries"
@@ -148,6 +151,7 @@ def test_stats_category_topics(datanommer_models, mock_config, mock_init):
 
     runner = CliRunner()
     result = runner.invoke(datanommer.commands.stats, ["--topic", "--category", "git"])
+    assert result.exit_code == 0, result.output
 
     assert (
         "org.fedoraproject.prod.git.receive.valgrind.master has 1 entries"
@@ -180,6 +184,7 @@ def test_stats_category(datanommer_models, mock_config, mock_init):
 
     runner = CliRunner()
     result = runner.invoke(datanommer.commands.stats, ["--category", "git"])
+    assert result.exit_code == 0, result.output
 
     assert result.output == "git has 2 entries\n"
 
@@ -196,6 +201,7 @@ def test_dump(datanommer_models, mock_config, mock_init):
 
     runner = CliRunner()
     result = runner.invoke(datanommer.commands.dump, [])
+    assert result.exit_code == 0, result.output
 
     json_object = json.loads(result.output)
 
@@ -219,6 +225,7 @@ def test_dump_before(datanommer_models, mock_config, mock_init):
 
     runner = CliRunner()
     result = runner.invoke(datanommer.commands.dump, ["--before", "2013-02-16"])
+    assert result.exit_code == 0, result.output
 
     json_object = json.loads(result.output)
 
@@ -246,6 +253,7 @@ def test_dump_since(datanommer_models, mock_config, mock_init):
 
     runner = CliRunner()
     result = runner.invoke(datanommer.commands.dump, ["--since", "2013-02-14T08:00:00"])
+    assert result.exit_code == 0, result.output
 
     json_object = json.loads(result.output)
 
@@ -276,6 +284,7 @@ def test_dump_timespan(datanommer_models, mock_config, mock_init):
         datanommer.commands.dump,
         ["--before", "2013-02-16", "--since", "2013-02-14T08:00:00"],
     )
+    assert result.exit_code == 0, result.output
 
     json_object = json.loads(result.output)
 
@@ -288,9 +297,11 @@ def test_dump_timespan(datanommer_models, mock_config, mock_init):
 def test_dump_invalid_dates(datanommer_models, mock_config, mock_init):
     runner = CliRunner()
     result = runner.invoke(datanommer.commands.dump, ["--before", "2013-02-16asdasd"])
+    assert result.exit_code > 0, result.output
     assert result.output == "Error: Invalid date format\n"
 
     result = runner.invoke(datanommer.commands.dump, ["--since", "2013-02-16asdasd"])
+    assert result.exit_code > 0, result.output
     assert result.output == "Error: Invalid date format\n"
 
 
@@ -314,6 +325,7 @@ def test_latest_overall(datanommer_models, mock_config, mock_init):
 
     runner = CliRunner()
     result = runner.invoke(datanommer.commands.latest, ["--overall"])
+    assert result.exit_code == 0, result.output
 
     json_object = json.loads(result.output)
 
@@ -343,6 +355,7 @@ def test_latest_topic(datanommer_models, mock_config, mock_init):
     result = runner.invoke(
         datanommer.commands.latest, ["--topic", "org.fedoraproject.stg.fas.user.create"]
     )
+    assert result.exit_code == 0, result.output
 
     json_object = json.loads(result.output)
 
@@ -370,6 +383,7 @@ def test_latest_category(datanommer_models, mock_config, mock_init):
 
     runner = CliRunner()
     result = runner.invoke(datanommer.commands.latest, ["--category", "fas"])
+    assert result.exit_code == 0, result.output
 
     json_object = json.loads(result.output)
 
@@ -402,6 +416,7 @@ def test_latest_timestamp_human(datanommer_models, mocker, mock_config, mock_ini
 
     runner = CliRunner()
     result = runner.invoke(datanommer.commands.latest, ["--timestamp", "--human"])
+    assert result.exit_code == 0, result.output
 
     json_object = json.loads(result.output)
 
@@ -431,6 +446,7 @@ def test_latest_timestamp(datanommer_models, mocker, mock_config, mock_init):
 
     runner = CliRunner()
     result = runner.invoke(datanommer.commands.latest, ["--timestamp"])
+    assert result.exit_code == 0, result.output
 
     json_object = json.loads(result.output)
 
@@ -465,6 +481,7 @@ def test_latest_timesince(datanommer_models, mocker, mock_config, mock_init):
 
     runner = CliRunner()
     result = runner.invoke(datanommer.commands.latest, ["--timesince"])
+    assert result.exit_code == 0, result.output
 
     json_object = json.loads(result.output)
 
@@ -501,6 +518,7 @@ def test_latest_timesince_human(datanommer_models, mock_config, mock_init, mocke
 
     runner = CliRunner()
     result = runner.invoke(datanommer.commands.latest, ["--timesince", "--human"])
+    assert result.exit_code == 0, result.output
 
     assert json.loads(result.output) == ["1 day, 0:00:00", "0:00:01"]
 
@@ -527,6 +545,7 @@ def test_latest(datanommer_models, mock_config, mock_init):
 
     runner = CliRunner()
     result = runner.invoke(datanommer.commands.latest, [])
+    assert result.exit_code == 0, result.output
 
     json_object = json.loads(result.output)
 

datanommer.commands/tests/test_extract_users.py

@@ -80,7 +80,6 @@ def test_extract_users_topic_and_category(mock_config, mock_init):
     result = runner.invoke(
         extract_users, ["--category", "bodhi", "--topic", "some.topic"]
     )
-    print(result.output)
     assert result.exit_code != 0, result.output
     assert "Error: can't use both --topic and --category, choose one." in result.output
 
@@ -92,8 +91,8 @@ def test_extract_users_no_users(datanommer_models, mock_config, mock_init):
     result = runner.invoke(extract_users)
 
     assert result.exit_code == 0, result.output
-    users_count = m.session.execute(
+    users_count = m.session.scalar(
         sa.select(sa.func.count(m.users_assoc_table.c.msg_id))
-    ).scalar_one()
+    )
     assert users_count == 0
     assert result.output == ""

datanommer.commands/tox.ini

@@ -12,6 +12,8 @@ skip_install = true
 #     poetry
 deps =
     poetry>=1.2
+env =
+    SQLALCHEMY_WARN_20=1
 commands_pre =
     poetry install --all-extras
     poetry run {toxinidir}/../tools/install-models-as-editable.sh

datanommer.consumer/tests/test_consumer.py

@@ -14,11 +14,12 @@
 # You should have received a copy of the GNU General Public License along
 # with this program.  If not, see <http://www.gnu.org/licenses/>.
 
+import dm as dm
 import pytest
 from fedora_messaging import message
+from sqlalchemy import func, select
 
 import datanommer.consumer
-import datanommer.models
 
 
 @pytest.fixture
@@ -36,15 +37,15 @@ def test_consume(datanommer_models, consumer):
     consumer = datanommer.consumer.Nommer()
 
     consumer(example_message)
-    assert datanommer.models.Message.query.count() == 1
+    assert dm.session.scalar(select(func.count(dm.Message.id))) == 1
 
 
 def test_add_exception(datanommer_models, consumer, mocker):
     example_message = message.Message(
         topic="nice.message", body={"encouragement": "You're doing great!"}
     )
 
-    datanommer.models.add = mocker.Mock(side_effect=Exception("an exception"))
+    dm.add = mocker.Mock(side_effect=Exception("an exception"))
     consumer = datanommer.consumer.Nommer()
     with pytest.raises(Exception):
         consumer(example_message)

datanommer.consumer/tox.ini

@@ -12,6 +12,8 @@ skip_install = true
 #     poetry
 deps =
     poetry>=1.2
+env =
+    SQLALCHEMY_WARN_20=1
 commands_pre =
     poetry install --all-extras
     poetry run {toxinidir}/../tools/install-models-as-editable.sh