CFbot integration

This adds our own custom replication between the CFbot and the
commitfest app. We currently keep the last CFbot runs in our database,
in an attempt to keep the queries easy and efficient. The CFbot results
are now shown on the overview page of each commitfest, as well as on the
page for each specific patch.

Author: JelteF

Diff for: media/commitfest/needs_rebase_success.svg

@@ -0,0 +1,40 @@
+# Generated by Django 3.2.25 on 2024-10-31 22:53
+
+from django.db import migrations, models
+import django.db.models.deletion
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('commitfest', '0005_history_dateindex'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='CfbotBranch',
+            fields=[
+                ('patch', models.OneToOneField(on_delete=django.db.models.deletion.CASCADE, primary_key=True, related_name='cfbot_branch', serialize=False, to='commitfest.patch')),
+                ('branch_id', models.IntegerField()),
+                ('branch_name', models.TextField()),
+                ('commit_id', models.TextField(blank=True, null=True)),
+                ('apply_url', models.TextField()),
+                ('status', models.TextField(choices=[('testing', 'Testing'), ('finished', 'Finished'), ('failed', 'Failed'), ('timeout', 'Timeout')])),
+                ('created', models.DateTimeField(auto_now_add=True)),
+                ('modified', models.DateTimeField(auto_now=True)),
+            ],
+        ),
+        migrations.CreateModel(
+            name='CfbotTask',
+            fields=[
+                ('id', models.TextField(primary_key=True, serialize=False)),
+                ('task_name', models.TextField()),
+                ('branch_id', models.IntegerField()),
+                ('position', models.IntegerField()),
+                ('status', models.TextField(choices=[('CREATED', 'Created'), ('NEEDS_APPROVAL', 'Needs Approval'), ('TRIGGERED', 'Triggered'), ('EXECUTING', 'Executing'), ('FAILED', 'Failed'), ('COMPLETED', 'Completed'), ('ABORTED', 'Aborted'), ('ERRORED', 'Errored')])),
+                ('created', models.DateTimeField(auto_now_add=True)),
+                ('modified', models.DateTimeField(auto_now=True)),
+                ('patch', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='cfbot_tasks', to='commitfest.patch')),
+            ],
+        ),
+    ]

Diff for: media/commitfest/new_failure.svg

@@ -0,0 +1,18 @@
+# Generated by Django 3.2.25 on 2024-12-06 08:38
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('commitfest', '0006_cfbot_integration'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='cfbottask',
+            name='status',
+            field=models.TextField(choices=[('CREATED', 'Created'), ('NEEDS_APPROVAL', 'Needs Approval'), ('TRIGGERED', 'Triggered'), ('EXECUTING', 'Executing'), ('FAILED', 'Failed'), ('COMPLETED', 'Completed'), ('SCHEDULED', 'Scheduled'), ('ABORTED', 'Aborted'), ('ERRORED', 'Errored')]),
+        ),
+    ]

Diff for: media/commitfest/new_success.svg

@@ -341,3 +341,43 @@ class PatchStatus(models.Model):
 class PendingNotification(models.Model):
     history = models.ForeignKey(PatchHistory, blank=False, null=False, on_delete=models.CASCADE)
     user = models.ForeignKey(User, blank=False, null=False, on_delete=models.CASCADE)
+
+
+class CfbotBranch(models.Model):
+    STATUS_CHOICES = [
+        ('testing', 'Testing'),
+        ('finished', 'Finished'),
+        ('failed', 'Failed'),
+        ('timeout', 'Timeout'),
+    ]
+
+    patch = models.OneToOneField(Patch, on_delete=models.CASCADE, related_name="cfbot_branch", primary_key=True)
+    branch_id = models.IntegerField(null=False)
+    branch_name = models.TextField(null=False)
+    commit_id = models.TextField(null=True, blank=True)
+    apply_url = models.TextField(null=False)
+    status = models.TextField(choices=STATUS_CHOICES, null=False)
+    created = models.DateTimeField(auto_now_add=True)
+    modified = models.DateTimeField(auto_now=True)
+
+class CfbotTask(models.Model):
+    STATUS_CHOICES = [
+        ('CREATED', 'Created'),
+        ('NEEDS_APPROVAL', 'Needs Approval'),
+        ('TRIGGERED', 'Triggered'),
+        ('EXECUTING', 'Executing'),
+        ('FAILED', 'Failed'),
+        ('COMPLETED', 'Completed'),
+        ('SCHEDULED', 'Scheduled'),
+        ('ABORTED', 'Aborted'),
+        ('ERRORED', 'Errored'),
+    ]
+
+    id = models.TextField(primary_key=True)
+    task_name = models.TextField(null=False)
+    patch = models.ForeignKey(Patch, on_delete=models.CASCADE, related_name="cfbot_tasks")
+    branch_id = models.IntegerField(null=False)
+    position = models.IntegerField(null=False)
+    status = models.TextField(choices=STATUS_CHOICES, null=False)
+    created = models.DateTimeField(auto_now_add=True)
+    modified = models.DateTimeField(auto_now=True)

Diff for: media/commitfest/old_failure.svg

@@ -63,6 +63,7 @@ <h3>{{p.is_open|yesno:"Active patches,Closed patches"}}</h3>
    <th>{%if p.is_open%}<a href="#" style="color:#333333;" onclick="return sortpatches(0);">Patch</a>{%if sortkey == 0%}<div style="float:right;"><i class="glyphicon glyphicon-arrow-down"></i></div>{%endif%}{%endif%}</th>
    <th>Status</th>
    <th>Ver</th>
+   <th>CI status</th>
    <th>Author</th>
    <th>Reviewers</th>
    <th>Committer</th>
@@ -86,6 +87,31 @@ <h3>{{p.is_open|yesno:"Active patches,Closed patches"}}</h3>
    <td><a href="{{p.id}}/">{{p.name}}</a></td>
    <td><span class="label label-{{p.status|patchstatuslabel}}">{{p.status|patchstatusstring}}</span></td>
    <td>{%if p.targetversion%}<span class="label label-default">{{p.targetversion}}</span>{%endif%}</td>
+   <td>
+    {%if p.cfbot_results %}
+    <a href="https://cirrus-ci.com/github/postgresql-cfbot/postgresql/cf%2F{{p.id}}" 
+       {%if p.cfbot_results.failed_task_names %}
+       title="Failed: {{p.cfbot_results.failed_task_names}}"
+        {%endif%}
+    >
+     {%if p.cfbot_results.needs_rebase %}
+     <span class="label label-warning">Needs rebase!</span>
+     {%else%}
+      {%if p.cfbot_results.failed > 0 %}
+      <span class="label label-danger">{{p.cfbot_results.failed}} failed</span>
+      {%endif%}
+      {%if p.cfbot_results.completed > 0 %}
+      <span class="label label-success">{{p.cfbot_results.completed}} succeeded</span>
+      {%endif%}
+      {%if p.cfbot_results.running > 0 %}
+      <span class="label label-info">{{p.cfbot_results.running}} running</span>
+      {%endif%}
+     {%endif%}
+    </a>
+    {% else %}
+      ?????
+    {%endif%}
+   </td>
    <td>{{p.author_names|default:''}}</td>
    <td>{{p.reviewer_names|default:''}}</td>
    <td>{{p.committer|default:''}}</td>

Diff for: media/commitfest/old_success.svg

@@ -12,6 +12,32 @@
       <th>Title</th>
       <td>{{patch.name}}</td>
     </tr>
+    <tr>
+      <th>CI (CFBot)</th>
+      <td>
+        {%if not cfbot_branch %}
+          ?????
+        {%elif not cfbot_branch.commit_id %}
+        <a href="https://cirrus-ci.com/github/postgresql-cfbot/postgresql/cf%2F{{patch.id}}">
+          <span class="label label-warning">Needs rebase!</span></a>
+        {%else%}
+        <a href="https://cirrus-ci.com/github/postgresql-cfbot/postgresql/cf%2F{{patch.id}}">
+        <span class="label label-default">Summary</span></a>
+        {%for c in cfbot_tasks %}
+            {%if c.status == 'COMPLETED'%}
+            <a href="https://cirrusci.com/task/{{c.id}}" title="{{c.task_name}}: {{c.status}}"><img src="/media/commitfest/new_success.svg"/></a>
+            {%elif c.status == 'CREATED' %}
+            <a href="https://cirrusci.com/task/{{c.id}}" title="{{c.task_name}}: {{c.status}}"><img src="/media/commitfest/waiting_to_start.svg"/></a>
+            {%elif c.status == 'EXECUTING' %}
+            <a href="https://cirrusci.com/task/{{c.id}}" title="{{c.task_name}}: {{c.status}}"><img src="/media/commitfest/running.svg"/></a>
+            {%else %}
+            <a href="https://cirrusci.com/task/{{c.id}}" title="{{c.task_name}}: {{c.status}}"><img src="/media/commitfest/new_failure.svg"/></a>
+            {%endif%}
+        {%endfor%}
+        {%endif%}
+        </a>
+      </td>
+    </tr>
     <tr>
       <th>Topic</th>
       <td>{{patch.topic}}</td>

Diff for: media/commitfest/running.svg

@@ -10,7 +10,7 @@
 
 from django.conf import settings
 
-from datetime import datetime
+from datetime import datetime, timezone
 from email.mime.text import MIMEText
 from email.utils import formatdate, make_msgid
 import json
@@ -19,7 +19,7 @@
 from pgcommitfest.mailqueue.util import send_mail, send_simple_mail
 from pgcommitfest.userprofile.util import UserWrapper
 
-from .models import CommitFest, Patch, PatchOnCommitFest, PatchHistory, Committer
+from .models import CommitFest, Patch, PatchOnCommitFest, PatchHistory, Committer, CfbotBranch, CfbotTask
 from .models import MailThread
 from .forms import PatchForm, NewPatchForm, CommentForm, CommitFestFilterForm
 from .forms import BulkEmailForm
@@ -209,11 +209,27 @@ def commitfest(request, cfid):
 
     # Let's not overload the poor django ORM
     curs = connection.cursor()
-    curs.execute("""SELECT p.id, p.name, poc.status, v.version AS targetversion, p.created, p.modified, p.lastmail, committer.username AS committer, t.topic,
+    curs.execute("""
+SELECT p.id, p.name, poc.status, v.version AS targetversion, p.created, p.modified, p.lastmail, committer.username AS committer, t.topic,
 (poc.status=ANY(%(openstatuses)s)) AS is_open,
 (SELECT string_agg(first_name || ' ' || last_name || ' (' || username || ')', ', ') FROM auth_user INNER JOIN commitfest_patch_authors cpa ON cpa.user_id=auth_user.id WHERE cpa.patch_id=p.id) AS author_names,
 (SELECT string_agg(first_name || ' ' || last_name || ' (' || username || ')', ', ') FROM auth_user INNER JOIN commitfest_patch_reviewers cpr ON cpr.user_id=auth_user.id WHERE cpr.patch_id=p.id) AS reviewer_names,
-(SELECT count(1) FROM commitfest_patchoncommitfest pcf WHERE pcf.patch_id=p.id) AS num_cfs
+(SELECT count(1) FROM commitfest_patchoncommitfest pcf WHERE pcf.patch_id=p.id) AS num_cfs,
+(
+    SELECT row_to_json(t) as cfbot_results
+    from (
+        SELECT 
+            sum((task.status = 'COMPLETED')::int) as completed,
+            sum((task.status in ('CREATED', 'SCHEDULED', 'EXECUTING'))::int) running,
+            sum((task.status in ('ABORTED', 'ERRORED', 'FAILED'))::int) failed,
+            string_agg(task.task_name, ', ') FILTER (WHERE task.status in ('ABORTED', 'ERRORED', 'FAILED')) as failed_task_names,
+            branch.commit_id IS NULL as needs_rebase
+        FROM commitfest_cfbotbranch branch
+        LEFT JOIN commitfest_cfbottask task ON task.branch_id = branch.branch_id
+        WHERE branch.patch_id=p.id
+        GROUP BY branch.commit_id
+    ) t
+)
 FROM commitfest_patch p
 INNER JOIN commitfest_patchoncommitfest poc ON poc.patch_id=p.id
 INNER JOIN commitfest_topic t ON t.id=p.topic_id
@@ -311,6 +327,12 @@ def patch(request, cfid, patchid):
     patch_commitfests = PatchOnCommitFest.objects.select_related('commitfest').filter(patch=patch).order_by('-commitfest__startdate')
     committers = Committer.objects.filter(active=True).order_by('user__last_name', 'user__first_name')
 
+    try:
+        cfbot_branch = patch.cfbot_branch
+    except CfbotBranch.DoesNotExist:
+        cfbot_branch = None
+    cfbot_tasks = patch.cfbot_tasks.order_by('position') if cfbot_branch else []
+
     # XXX: this creates a session, so find a smarter way. Probably handle
     # it in the callback and just ask the user then?
     if request.user.is_authenticated:
@@ -333,6 +355,8 @@ def patch(request, cfid, patchid):
         'cf': cf,
         'patch': patch,
         'patch_commitfests': patch_commitfests,
+        'cfbot_branch': cfbot_branch,
+        'cfbot_tasks': cfbot_tasks,
         'is_committer': is_committer,
         'is_this_committer': is_this_committer,
         'is_reviewer': is_reviewer,
@@ -788,6 +812,97 @@ def _user_and_mail(u):
     })
 
 
+@transaction.atomic
+def cfbot_ingest(message):
+    """Ingest a single message status update message receive from cfbot.  It
+     should be a Python dictionary, decoded from JSON already."""
+
+    cursor = connection.cursor()
+
+    # Every message should have a shared_secret, and it should match.
+    if message["shared_secret"] != settings.CFBOT_SECRET:
+        raise Exception("Invalid shared_secret from CFbot")
+
+    branch_status = message["branch_status"]
+    patch_id = branch_status["submission_id"]
+    branch_id = branch_status["branch_id"]
+    created = datetime.fromisoformat(branch_status["created"]).replace(tzinfo=timezone.utc)
+
+    try:
+        Patch.objects.get(pk=patch_id)
+    except Patch.DoesNotExist:
+        # If the patch doesn't exist, there's nothing to do. This should never
+        # happen in production, but on the test system it's possible because
+        # not it doesn't contain the newest patches that the CFBot knows about.
+        return
+
+    old_branch = CfbotBranch.objects.select_for_update().filter(patch_id=patch_id).first()
+    if old_branch and old_branch.branch_id != branch_id and old_branch.created.replace(tzinfo=timezone.utc) > created:
+        # This is a message for an old branch, ignore it.
+        return
+
+    # Every message should have a branch_status, which we will INSERT
+    # or UPDATE.  We do this first, because cfbot_task refers to it.
+    cursor.execute("""INSERT INTO commitfest_cfbotbranch (patch_id, branch_id,
+                                                branch_name, commit_id,
+                                                apply_url, status,
+                                                created, modified)
+                    VALUES (%s, %s, %s, %s, %s, %s, %s, %s)
+                    ON CONFLICT (patch_id) DO UPDATE
+                        SET status = EXCLUDED.status,
+                            modified = EXCLUDED.modified,
+                            branch_id = EXCLUDED.branch_id,
+                            branch_name = EXCLUDED.branch_name,
+                            commit_id = EXCLUDED.commit_id,
+                            apply_url = EXCLUDED.apply_url,
+                            created = EXCLUDED.created
+                        WHERE commitfest_cfbotbranch.modified < EXCLUDED.modified
+                        """,
+                    (patch_id,
+                    branch_id,
+                    branch_status["branch_name"],
+                    branch_status["commit_id"],
+                    branch_status["apply_url"],
+                    branch_status["status"],
+                    branch_status["created"],
+                    branch_status["modified"]))
+
+    # Most messages have a task_status.  It might be missing in rare cases, like
+    # when cfbot decides that a whole branch has timed out.  We INSERT or
+    # UPDATE.
+    if "task_status" in message:
+        task_status = message["task_status"]
+        cursor.execute("""INSERT INTO commitfest_cfbottask (id, task_name, patch_id, branch_id,
+                                               position, status,
+                                               created, modified)
+                       VALUES (%s, %s, %s, %s, %s, %s, %s, %s)
+                       ON CONFLICT (id) DO UPDATE
+                         SET status = EXCLUDED.status,
+                             modified = EXCLUDED.modified
+                       WHERE commitfest_cfbottask.modified < EXCLUDED.modified""",
+                 (task_status["task_id"],
+                  task_status["task_name"],
+                  patch_id,
+                  branch_id,
+                  task_status["position"],
+                  task_status["status"],
+                  task_status["created"],
+                  task_status["modified"]))
+
+    cursor.execute("DELETE FROM commitfest_cfbottask WHERE patch_id=%s AND branch_id != %s", (patch_id, branch_id))
+
+@csrf_exempt
+def cfbot_notify(request):
+    if request.method != 'POST':
+        return HttpResponseForbidden("Invalid method")
+
+    j = json.loads(request.body)
+    if j['shared_secret'] != settings.CFBOT_SECRET:
+        return HttpResponseForbidden("Invalid API key")
+
+    cfbot_ingest(j)
+    return HttpResponse(status=200)
+
 @csrf_exempt
 def thread_notify(request):
     if request.method != 'POST':

Diff for: media/commitfest/waiting_to_start.svg

@@ -36,6 +36,7 @@
     re_path(r'^ajax/(\w+)/$', ajax.main),
     re_path(r'^lookups/user/$', lookups.userlookup),
     re_path(r'^thread_notify/$', views.thread_notify),
+    re_path(r'^cfbot_notify/$', views.cfbot_notify),
 
     # Auth system integration
     re_path(r'^(?:account/)?login/?$', pgcommitfest.auth.login),