Refactor master cfg

.github/workflows/bbm_deploy.yml

@@ -31,6 +31,7 @@ on:
       - "locks.py"
       - "master-**"
       - "master.cfg"
+      - "master_common.py"
       - "os_info.yaml"
       - "script_templates/**"
       - "utils.py"

buildbot.tac

@@ -1,39 +1,57 @@
-import os
+# -*- python -*-
+# ex: set filetype=python:
+from pathlib import Path
 
 from twisted.application import service
+from twisted.python.log import FileLogObserver, ILogObserver
+from twisted.python.logfile import LogFile
 
 from buildbot.master import BuildMaster
 
-basedir = "."
-log_basedir = "/var/log/buildbot/"
-
-rotateLength = 20000000
+# This buildbot.tac file is a basis for all "autogen" masters.
+# The folder structure for autogen masters is:
+#
+# autogen
+# └── aarch64-master-0
+#     ├── buildbot.tac
+#     ├── master.cfg
+#     ├── master-config.yaml
+#     └── master-private.cfg
+#
+# Thus basedir is two levels below this file's position.
+buildbot_tac_dir = Path(__file__).resolve().parent
+basedir = buildbot_tac_dir.parent.parent
+
+# Hard coded as it runs in containers.
+# TODO(cvicentiu) this should come as an environment variable.
+log_basedir_path = Path("/var/log/buildbot/")
+log_basedir = log_basedir_path.as_posix()  # Kept in case buildbot uses it.
+
+rotateLength = 10000000
 maxRotatedFiles = 30
-configfile = "master.cfg"
 
-# Default umask for server
-umask = None
+master_name = buildbot_tac_dir.name
+# Last two directories. autogen and <master-name>
+cfg_from_basedir = (buildbot_tac_dir / "master.cfg").relative_to(basedir)
 
-# if this is a relocatable tac file, get the directory containing the TAC
-if basedir == ".":
-    import os
+configfile = cfg_from_basedir.as_posix()
 
-    basedir = os.path.abspath(os.path.dirname(__file__))
+# Default umask for server
+umask = None
 
 # note: this line is matched against to check that this is a buildmaster
 # directory; do not edit it.
 application = service.Application('buildmaster')  # fmt: skip
-from twisted.python.log import FileLogObserver, ILogObserver
-from twisted.python.logfile import LogFile
 
+# This logfile is monitored. It must end in .log.
 logfile = LogFile.fromFullPath(
-    os.path.join(log_basedir, "%s"),
+    str(log_basedir_path / f"{master_name}.log"),
     rotateLength=rotateLength,
     maxRotatedFiles=maxRotatedFiles,
 )
 application.setComponent(ILogObserver, FileLogObserver(logfile).emit)
 
-m = BuildMaster(basedir, configfile, umask)
+m = BuildMaster(str(basedir), configfile, umask)
 m.setServiceParent(application)
 m.log_rotation.rotateLength = rotateLength
 m.log_rotation.maxRotatedFiles = maxRotatedFiles

common_factories.py

@@ -12,6 +12,7 @@
 # Local
 from constants import MTR_ENV, SAVED_PACKAGE_BRANCHES, test_type_to_mtr_arg
 from utils import (
+    createDebRepo,
     createVar,
     dockerfile,
     getArch,
@@ -36,6 +37,7 @@
     printEnv,
     saveLogs,
     savePackageIfBranchMatch,
+    uploadDebArtifacts,
 )
 
 
@@ -626,7 +628,7 @@ def getTests(props):
     return addPostTests(f)
 
 
-def getRpmAutobakeFactory(mtrDbPool):
+def getRpmAutobakeFactory():
     f_rpm_autobake = util.BuildFactory()
     f_rpm_autobake.addStep(printEnv())
     f_rpm_autobake.addStep(
@@ -826,3 +828,117 @@ def getRpmAutobakeFactory(mtrDbPool):
         )
     )
     return f_rpm_autobake
+
+
+def getDebAutobakeFactory() -> util.BuildFactory:
+    f_deb_autobake = util.BuildFactory()
+    f_deb_autobake.addStep(printEnv())
+    f_deb_autobake.addStep(
+        steps.SetProperty(
+            property="dockerfile",
+            value=util.Interpolate("%(kw:url)s", url=dockerfile),
+            description="dockerfile",
+        )
+    )
+    f_deb_autobake.addStep(getSourceTarball())
+    # build steps
+    f_deb_autobake.addStep(
+        steps.Compile(
+            logfiles={"CMakeCache.txt": "./builddir/CMakeCache.txt"},
+            command=["debian/autobake-deb.sh"],
+            env={
+                "CCACHE_DIR": "/mnt/ccache",
+                "DEB_BUILD_OPTIONS": util.Interpolate(
+                    "parallel=%(kw:jobs)s",
+                    jobs=util.Property("jobs", default="$(getconf _NPROCESSORS_ONLN)"),
+                ),
+            },
+            description="autobake-deb.sh",
+        )
+    )
+    # upload artifacts
+    f_deb_autobake.addStep(
+        steps.SetPropertyFromCommand(
+            command="find .. -maxdepth 1 -type f", extract_fn=ls2string
+        )
+    )
+    f_deb_autobake.addStep(createDebRepo())
+    f_deb_autobake.addStep(uploadDebArtifacts())
+
+    f_deb_autobake.addStep(
+        steps.Trigger(
+            name="dockerlibrary",
+            schedulerNames=["s_dockerlibrary"],
+            waitForFinish=False,
+            updateSourceStamp=False,
+            set_properties={
+                "tarbuildnum": Property("tarbuildnum"),
+                "mariadb_version": Property("mariadb_version"),
+                "master_branch": Property("master_branch"),
+                "parentbuildername": Property("buildername"),
+            },
+            doStepIf=lambda step: hasDockerLibrary(step),
+        )
+    )
+    f_deb_autobake.addStep(
+        steps.Trigger(
+            name="release preparation",
+            schedulerNames=["s_release_prep"],
+            waitForFinish=True,
+            updateSourceStamp=False,
+            set_properties={
+                "tarbuildnum": Property("tarbuildnum"),
+                "mariadb_version": Property("mariadb_version"),
+                "master_branch": Property("master_branch"),
+                "parentbuildername": Property("buildername"),
+            },
+            doStepIf=(
+                lambda step: savePackageIfBranchMatch(step, SAVED_PACKAGE_BRANCHES)
+            ),
+        )
+    )
+    f_deb_autobake.addStep(
+        steps.Trigger(
+            name="install",
+            schedulerNames=["s_install"],
+            waitForFinish=False,
+            updateSourceStamp=False,
+            set_properties={
+                "tarbuildnum": Property("tarbuildnum"),
+                "mariadb_version": Property("mariadb_version"),
+                "master_branch": Property("master_branch"),
+                "parentbuildername": Property("buildername"),
+            },
+            doStepIf=(
+                lambda step: hasInstall(step)
+                and savePackageIfBranchMatch(step, SAVED_PACKAGE_BRANCHES)
+                and hasPackagesGenerated(step)
+            ),
+        )
+    )
+    f_deb_autobake.addStep(
+        steps.Trigger(
+            name="major-minor-upgrade",
+            schedulerNames=["s_upgrade"],
+            waitForFinish=False,
+            updateSourceStamp=False,
+            set_properties={
+                "tarbuildnum": Property("tarbuildnum"),
+                "mariadb_version": Property("mariadb_version"),
+                "master_branch": Property("master_branch"),
+                "parentbuildername": Property("buildername"),
+            },
+            doStepIf=(
+                lambda step: hasUpgrade(step)
+                and savePackageIfBranchMatch(step, SAVED_PACKAGE_BRANCHES)
+                and hasPackagesGenerated(step)
+            ),
+        )
+    )
+    f_deb_autobake.addStep(
+        steps.ShellCommand(
+            name="cleanup", command="rm -r * .* 2> /dev/null || true", alwaysRun=True
+        )
+    )
+
+    return f_deb_autobake

constants.py

@@ -21,6 +21,23 @@
     "main",
 ]
 
+# Branches with special prefixes that invoke a BB run.
+BB_TEST_BRANCHES = [
+    "bb-*",
+    "st-*",
+    "prot-*",
+    "refs/pull/*",
+    "preview-1[0-9].*",
+    "jpsn-*",
+]
+
+# A list of all branches that invoke a buildbot run.
+ALL_BB_TEST_BRANCHES = BRANCHES_MAIN + BB_TEST_BRANCHES
+
+STAGING_PROT_TEST_BRANCHES = [
+    "prot-st-*",
+]
+
 # Defines what builders report status to GitHub
 GITHUB_STATUS_BUILDERS = [
     "aarch64-macos-compile-only",

define_masters.py

@@ -2,26 +2,34 @@
 
 import os
 import shutil
+from collections import defaultdict
 
 import yaml
 
 BASE_PATH = "autogen/"
 config = {"private": {}}
-exec(open("master-private.cfg").read(), config, {})
+with open("master-private.cfg", "r") as file:
+    exec(file.read(), config, {})
+
+master_variables = config["private"]["master-variables"]
 
 with open("os_info.yaml", encoding="utf-8") as file:
     OS_INFO = yaml.safe_load(file)
 
-platforms = {}
+platforms = defaultdict(dict)
 
 for os_name in OS_INFO:
+    # TODO(cvicentiu) this should be removed, hack.
     if "install_only" in OS_INFO[os_name] and OS_INFO[os_name]["install_only"]:
         continue
+
     for arch in OS_INFO[os_name]["arch"]:
-        builder_name = arch + "-" + os_name
-        if arch not in platforms:
-            platforms[arch] = []
-        platforms[arch].append(builder_name)
+        builder_name = f"{arch}-{os_name}"
+        platforms[arch][os_name] = {
+            "image_tag": OS_INFO[os_name]["image_tag"],
+            "tags": OS_INFO[os_name]["tags"],
+            "type": OS_INFO[os_name]["type"],
+        }
 
 # Clear old configurations
 if os.path.exists(BASE_PATH):
@@ -32,33 +40,28 @@
     # If for a given architecture there are more than "max_builds" builds,
     # create multiple masters
     # "max_builds" is defined is master-private.py
-    num_masters = (
-        int(len(platforms[arch]) / config["private"]["master-variables"]["max_builds"])
-        + 1
-    )
+    num_masters = int(len(platforms[arch]) / master_variables["max_builds"]) + 1
 
     for master_id in range(num_masters):
-        dir_path = BASE_PATH + arch + "-master-" + str(master_id)
+        dir_path = f"{BASE_PATH}{arch}-master-{master_id}"
         os.makedirs(dir_path)
 
-        master_config = {}
-        master_config["builders"] = platforms[arch]
-        master_config["workers"] = config["private"]["master-variables"]["workers"][
-            arch
-        ]
-        master_config["log_name"] = (
-            "master-docker-" + arch + "-" + str(master_id) + ".log"
-        )
+        master_config = {
+            "builders": {arch: platforms[arch]},
+            "workers": master_variables["workers"][arch],
+            "log_name": f"master-docker-{arch}-{master_id}.log",
+        }
 
-        with open(dir_path + "/master-config.yaml", mode="w", encoding="utf-8") as file:
+        with open(f"{dir_path}/master-config.yaml", mode="w", encoding="utf-8") as file:
             yaml.dump(master_config, file)
 
         shutil.copyfile("master.cfg", dir_path + "/master.cfg")
         shutil.copyfile("master-private.cfg", dir_path + "/master-private.cfg")
+        shutil.copyfile("buildbot.tac", dir_path + "/buildbot.tac")
 
-        buildbot_tac = (
-            open("buildbot.tac", encoding="utf-8").read() % master_config["log_name"]
+        print(
+            "{:<20} {}".format(
+                f"Master {master_id} {arch}",
+                f'{len(master_config["builders"][arch])} builders',
+            )
         )
-        with open(dir_path + "/buildbot.tac", mode="w", encoding="utf-8") as f:
-            f.write(buildbot_tac)
-    print(arch, len(master_config["builders"]))

docker-compose/start-bbm-web.sh

@@ -10,7 +10,7 @@ err() {
   exit 1
 }
 
-cd /srv/buildbot/master/master-web || err "cd /srv/buildbot/master/master-web"
+cd /srv/buildbot/master/ || err "cd /srv/buildbot/master/"
 
 # # loop for debug
 # while true; do date && sleep 30; done
@@ -29,5 +29,4 @@ while ! nc -z 127.0.0.1 8080; do
   sleep 0.1
 done
 echo "Crossbar started"
-buildbot upgrade-master /srv/buildbot/master/master-web
-exec buildbot start --nodaemon
+exec buildbot start --nodaemon master-web

docker-compose/start.sh

@@ -11,7 +11,7 @@ err() {
 }
 
 # Make sure to pass the master name as the first argument
-cd "/srv/buildbot/master/$1" || err "cd /srv/buildbot/master/$1"
+cd "/srv/buildbot/master/" || err "cd /srv/buildbot/master/"
 
 # shellcheck disable=SC2226
 [[ -f master-private.cfg ]] || ln -s ../master-private.cfg
@@ -41,4 +41,4 @@ fi
 # loop for debug
 # while true; do date && sleep 30; done
 
-exec buildbot start --nodaemon
+exec buildbot start --nodaemon "$1"