cleanup define_masters.py and os_info.yaml

This is a larger commit that does the following: 1. os_info now specifically mentions the image_tag to use for the OS. 2. os_info specifies tags per os. 3. define_masters now reads os_info and groups by arch. 4. Make autogen buildbot.tac allow relative import 5. buildbot.tac no longer a "template" with log name filled in by master name, but a file that gets copied for each master. 6. Validate master updated to change the way each master should be started.
MariaDB · Feb 5, 2025 · 888ffda · 888ffda
1 parent 661f55c
commit 888ffda
Show file tree

Hide file tree

Showing 5 changed files with 189 additions and 120 deletions.
diff --git a/buildbot.tac b/buildbot.tac
@@ -28,10 +28,11 @@ log_basedir = "/var/log/buildbot"
 
 rotateLength = 20000000
 maxRotatedFiles = 30
+
+last_two_dirs = os.path.normpath(buildbot_tac_dir).split(os.sep)[-2:]
+master_name = last_two_dirs[-1]
 # Last two directories. autogen and <master-name>
-cfg_from_basedir = os.path.normpath(buildbot_tac_dir).split(os.sep)[-2:] + [
-    "master.cfg"
-]
+cfg_from_basedir = last_two_dirs + ["master.cfg"]
 
 configfile = os.path.join(*cfg_from_basedir)
 # Default umask for server
@@ -41,8 +42,9 @@ umask = None
 # directory; do not edit it.
 application = service.Application("buildmaster")  # fmt: skip
 
+# This logfile is monitored. It must end in .log.
 logfile = LogFile.fromFullPath(
-    os.path.join(log_basedir, "%s"),
+    os.path.join(log_basedir, f"{master_name}.log"),
     rotateLength=rotateLength,
     maxRotatedFiles=maxRotatedFiles,
 )

diff --git a/define_masters.py b/define_masters.py
@@ -2,26 +2,33 @@
 
 import os
 import shutil
+from collections import defaultdict
 
 import yaml
 
 BASE_PATH = "autogen/"
 config = {"private": {}}
-exec(open("master-private.cfg").read(), config, {})
+with open("master-private.cfg", "r") as file:
+    exec(file.read(), config, {})
+
+master_variables = config["private"]["master-variables"]
 
 with open("os_info.yaml", encoding="utf-8") as file:
     OS_INFO = yaml.safe_load(file)
 
-platforms = {}
+platforms = defaultdict(dict)
 
 for os_name in OS_INFO:
+    # TODO(cvicentiu) this should be removed, hack.
     if "install_only" in OS_INFO[os_name] and OS_INFO[os_name]["install_only"]:
         continue
+
     for arch in OS_INFO[os_name]["arch"]:
-        builder_name = arch + "-" + os_name
-        if arch not in platforms:
-            platforms[arch] = []
-        platforms[arch].append(builder_name)
+        builder_name = f"{arch}-{os_name}"
+        platforms[arch][os_name] = {
+            "image_tag": OS_INFO[os_name]["image_tag"],
+            "tags": OS_INFO[os_name]["tags"],
+        }
 
 # Clear old configurations
 if os.path.exists(BASE_PATH):
@@ -32,33 +39,23 @@
     # If for a given architecture there are more than "max_builds" builds,
     # create multiple masters
     # "max_builds" is defined is master-private.py
-    num_masters = (
-        int(len(platforms[arch]) / config["private"]["master-variables"]["max_builds"])
-        + 1
-    )
+    num_masters = int(len(platforms[arch]) / master_variables["max_builds"]) + 1
 
     for master_id in range(num_masters):
-        dir_path = BASE_PATH + arch + "-master-" + str(master_id)
+        dir_path = f"{BASE_PATH}{arch}-master-{master_id}"
         os.makedirs(dir_path)
 
-        master_config = {}
-        master_config["builders"] = platforms[arch]
-        master_config["workers"] = config["private"]["master-variables"]["workers"][
-            arch
-        ]
-        master_config["log_name"] = (
-            "master-docker-" + arch + "-" + str(master_id) + ".log"
-        )
+        master_config = {
+            "builders": {arch: platforms[arch]},
+            "workers": master_variables["workers"][arch],
+            "log_name": f"master-docker-{arch}-{master_id}.log",
+        }
 
-        with open(dir_path + "/master-config.yaml", mode="w", encoding="utf-8") as file:
+        with open(f"{dir_path}/master-config.yaml", mode="w", encoding="utf-8") as file:
             yaml.dump(master_config, file)
 
         shutil.copyfile("master.cfg", dir_path + "/master.cfg")
         shutil.copyfile("master-private.cfg", dir_path + "/master-private.cfg")
+        shutil.copyfile("buildbot.tac", dir_path + "/buildbot.tac")
 
-        buildbot_tac = (
-            open("buildbot.tac", encoding="utf-8").read() % master_config["log_name"]
-        )
-        with open(dir_path + "/buildbot.tac", mode="w", encoding="utf-8") as f:
-            f.write(buildbot_tac)
     print(arch, len(master_config["builders"]))
diff --git a/master.cfg b/master.cfg
@@ -14,7 +14,6 @@ from common_factories import (
     getSourceTarball,
 )
 from constants import (
-    BUILDERS_GALERA_MTR,
     GITHUB_STATUS_BUILDERS,
     OS_INFO,
     SAVED_PACKAGE_BRANCHES,
@@ -79,40 +78,45 @@ c["workers"] = []
 
 workers = defaultdict(list)
 
+# For each worker in master_config ['aarch64-bbw1', 2, 3, 4]
 for w_name in master_config["workers"]:
     jobs = 7
+    worker_name = w_name[:-1]  # aarch64-bbw
+    worker_id = w_name[-1]     # 1, 2, 3, 4
+
+    for arch in master_config["builders"]:
+        builders = master_config["builders"][arch]
+        for os_name in builders:
+            os_definition = builders[os_name]
+            image_tag = os_definition['image_tag']
+
+            # Skip s390x non-SLES builders on SLES host (bbw2)
+            if ("s390x" in arch
+                    and (worker_id == "2")
+                    and ("sles" not in os_name)):
+                continue
+
+            if image_tag.startswith("ubuntu"):
+                image_tag = image_tag[:-2] + "." + image_tag[-2:]
+
+            quay_name = f'{os.environ["CONTAINER_REGISTRY_URL"]}{image_tag}'
+            if arch.startswith("x86"):
+                os_name += "-i386"
+                quay_name += "-386"
+
+            base_name, name, worker_instance = createWorker(
+                worker_name,
+                worker_id,
+                os_name,
+                quay_name,
+                jobs=jobs,
+                save_packages=True,
+                shm_size="15G",
+            )
 
-    for builder in master_config["builders"]:
-        worker_name = w_name[:-1]
-        worker_id = w_name[-1]
-
-        os_name = "-".join(builder.split("-")[1:])
-        image_tag = "".join(os_name.split("-"))
-
-        # Skip s390x non-SLES builders on SLES host (bbw2)
-        if ("s390x" in builder) and (worker_id == "2") and ("sles" not in os_name):
-            continue
-
-        if image_tag.startswith("ubuntu"):
-            image_tag = image_tag[:-2] + "." + image_tag[-2:]
-
-        quay_name = os.environ["CONTAINER_REGISTRY_URL"] + image_tag
-        if builder.startswith("x86"):
-            os_name += "-i386"
-            quay_name += "-386"
-
-        base_name, name, worker_instance = createWorker(
-            worker_name,
-            worker_id,
-            os_name,
-            quay_name,
-            jobs=jobs,
-            save_packages=True,
-            shm_size="15G",
-        )
+            workers[base_name].append(name)
+            c["workers"].append(worker_instance)
 
-        workers[base_name].append(name)
-        c["workers"].append(worker_instance)
 
 ####### FACTORY CODE
 
@@ -231,77 +235,60 @@ f_deb_autobake.addStep(
 
 c["builders"] = []
 
-for builder in master_config["builders"]:
-    splits = builder.split("-")
-    arch = splits[0]
-    os_name = "-".join(splits[1:])
+for arch in master_config["builders"]:
+    builders_group = master_config["builders"][arch]
+    for os_name in builders_group:
+        worker_prefix = arch
+        worker_suffix = ''
+
+        if arch == "amd64":
+            worker_prefix = "x64"
 
-    if arch == "amd64":
-        arch = "x64"
-    worker_name = arch + "-bbw-docker-" + os_name
+        if arch == "x86":
+            worker_prefix = 'x64'
+            worker_suffix = '-i386'
+        worker_name = f'{worker_prefix}-bbw-docker-{os_name}{worker_suffix}'
 
-    if arch == "x86":
-        worker_name = "x64-bbw-docker-" + os_name + "-i386"
+        build_type = OS_INFO[os_name]["type"]
 
-    build_type = OS_INFO[os_name]["type"]
+        builder = f'{arch}-{os_name}'
 
-    # Add builder only if it's not a protected branches one
-    if builder not in GITHUB_STATUS_BUILDERS:
         tags = [os_name]
-        if arch == "s390x" and builder in BUILDERS_GALERA_MTR:
-            tags += ["experimental"]
-        if "sid" in builder or "stream-9" in builder:
-            tags += ["bleeding-edge"]
+        # Add builder only if it's not a protected branches one
+        if builder not in GITHUB_STATUS_BUILDERS:
+            c["builders"].append(
+                util.BuilderConfig(
+                    name=builder,
+                    workernames=workers[worker_name],
+                    tags=tags,
+                    collapseRequests=True,
+                    nextBuild=nextBuild,
+                    canStartBuild=canStartBuild,
+                    locks=getLocks,
+                    factory=f_quick_build,
+                )
+            )
+
+        factory_instance = (
+            f_deb_autobake if build_type != "rpm" else f_rpm_autobake
+        )
+        properties = {
+            "verbose_build": "VERBOSE=1" if arch == "ppc4le" else None,
+            "rpm_type": "".join(os_name.split("-")) if build_type == "rpm" else None
+        }
+
+        tags += [build_type, "autobake"]
+
         c["builders"].append(
             util.BuilderConfig(
-                name=builder,
+                name=f"{builder}-{build_type}-autobake",
                 workernames=workers[worker_name],
                 tags=tags,
                 collapseRequests=True,
                 nextBuild=nextBuild,
                 canStartBuild=canStartBuild,
                 locks=getLocks,
-                factory=f_quick_build,
+                properties=properties,
+                factory=factory_instance,
             )
         )
-
-    factory_instance = f_deb_autobake
-    properties = {}
-
-    if arch == "ppc64le":
-        properties["verbose_build"] = "VERBOSE=1"
-    if build_type == "rpm":
-        properties["rpm_type"] = "".join(os_name.split("-"))
-        factory_instance = f_rpm_autobake
-    tags = [os_name, build_type, "autobake"]
-    # From mariadb.org-tools/release/prep - under
-    # Dirs for buildbot.mariadb.org
-    if builder in [
-        "aarch64-openeuler-2403",
-        "amd64-openeuler-2403",
-        "s390x-ubuntu-2004",
-        "s390x-rhel-8",
-        "s390x-sles-15",
-        "ppc64le-rhel-9",
-        "s390x-rhel-9",
-        "ppc64le-ubuntu-2204",
-        "s390x-ubuntu-2204",
-        "amd64-debian-sid",
-        "aarch64-debian-sid",
-        "ppc64le-debian-sid",
-        "amd64-opensuse-1506",
-    ]:
-        tags += ["release_packages"]
-    c["builders"].append(
-        util.BuilderConfig(
-            name=builder + "-" + build_type + "-autobake",
-            workernames=workers[worker_name],
-            tags=tags,
-            collapseRequests=True,
-            nextBuild=nextBuild,
-            canStartBuild=canStartBuild,
-            locks=getLocks,
-            properties=properties,
-            factory=factory_instance,
-        )
-    )