From ed75b85aff42c6a42fd9bde7229a28b275eb0b8d Mon Sep 17 00:00:00 2001
From: Andy Georges <itkovian@gmail.com>
Date: Fri, 4 Nov 2022 14:13:19 +0100
Subject: [PATCH 01/90] patch: ugent ssh pam adoption

---
 contribs/pam_slurm_adopt/pam_slurm_adopt.c | 125 +++++++++++++++------
 1 file changed, 88 insertions(+), 37 deletions(-)

diff --git a/contribs/pam_slurm_adopt/pam_slurm_adopt.c b/contribs/pam_slurm_adopt/pam_slurm_adopt.c
index 2e40e3e6b72..44520f0c062 100644
--- a/contribs/pam_slurm_adopt/pam_slurm_adopt.c
+++ b/contribs/pam_slurm_adopt/pam_slurm_adopt.c
@@ -73,6 +73,8 @@ typedef enum {
 	CALLERID_ACTION_ALLOW,
 	CALLERID_ACTION_IGNORE,
 	CALLERID_ACTION_DENY,
+	CALLERID_ACTION_ADOPT_AND_CHECK,
+	CALLERID_ACTION_ONLY_CHECK,
 } callerid_action_t;
 
 /* module options */
@@ -86,6 +88,7 @@ static struct {
 	callerid_action_t action_unknown;
 	callerid_action_t action_adopt_failure;
 	callerid_action_t action_generic_failure;
+	callerid_action_t action_adopt;
 	log_level_t log_level;
 	char *node_name;
 	bool disable_x11;
@@ -101,6 +104,7 @@ static void _init_opts(void)
 	opts.action_unknown = CALLERID_ACTION_NEWEST;
 	opts.action_adopt_failure = CALLERID_ACTION_ALLOW;
 	opts.action_generic_failure = CALLERID_ACTION_IGNORE;
+	opts.action_adopt = CALLERID_ACTION_ADOPT_AND_CHECK;
 	opts.log_level = LOG_LEVEL_INFO;
 	opts.node_name = NULL;
 	opts.disable_x11 = false;
@@ -429,9 +433,15 @@ static int _action_unknown(pam_handle_t *pamh, struct passwd *pwd, List steps)
 	rc = _indeterminate_multiple(pamh, steps, pwd->pw_uid, &stepd);
 	if (rc == PAM_SUCCESS) {
 		info("action_unknown: Picked job %u", stepd->step_id.job_id);
-		if (_adopt_process(pamh, getpid(), stepd) == SLURM_SUCCESS) {
-			return PAM_SUCCESS;
-		}
+		if (opts.action_adopt == CALLERID_ACTION_ADOPT_AND_CHECK) {
+			if (_adopt_process(pamh, getpid(), stepd) == SLURM_SUCCESS) {
+				return PAM_SUCCESS;
+			}
+			if (opts.action_adopt_failure == CALLERID_ACTION_ALLOW)
+				return PAM_SUCCESS;
+			else
+				return PAM_PERM_DENIED;
+ 		}
 		if (opts.action_adopt_failure == CALLERID_ACTION_ALLOW)
 			return PAM_SUCCESS;
 		else
@@ -536,24 +546,27 @@ static int _try_rpc(pam_handle_t *pamh, struct passwd *pwd)
 	/* Ask the slurmd at the source IP address about this connection */
 	rc = _rpc_network_callerid(&conn, pwd->pw_name, &job_id);
 	if (rc == SLURM_SUCCESS) {
-		step_loc_t stepd;
-		memset(&stepd, 0, sizeof(stepd));
-		/* We only need the step_id struct needed to be filled in here
-		   all the rest isn't needed for the adopt.
-		*/
-		stepd.step_id.job_id = job_id;
-		stepd.step_id.step_id = SLURM_EXTERN_CONT;
-		stepd.step_id.step_het_comp = NO_VAL;
-
-		/* Adopt the process. If the adoption succeeds, return SUCCESS.
-		 * If not, maybe the adoption failed because the user hopped
-		 * into one node and was adopted into a job there that isn't on
-		 * our node here. In that case we got a bad jobid so we'll fall
-		 * through to the next action */
-		if (_adopt_process(pamh, getpid(), &stepd) == SLURM_SUCCESS)
-			return PAM_SUCCESS;
-		else
-			return PAM_IGNORE;
+		if (opts.action_adopt == CALLERID_ACTION_ADOPT_AND_CHECK) {
+			step_loc_t stepd;
+			memset(&stepd, 0, sizeof(stepd));
+			/* We only need the step_id struct needed to be filled in here
+			all the rest isn't needed for the adopt.
+			*/
+			stepd.step_id.job_id = job_id;
+			stepd.step_id.step_id = SLURM_EXTERN_CONT;
+			stepd.step_id.step_het_comp = NO_VAL;
+
+			/* Adopt the process. If the adoption succeeds, return SUCCESS.
+			* If not, maybe the adoption failed because the user hopped
+			* into one node and was adopted into a job there that isn't on
+			* our node here. In that case we got a bad jobid so we'll fall
+			* through to the next action */
+			if (_adopt_process(pamh, getpid(), &stepd) == SLURM_SUCCESS)
+				return PAM_SUCCESS;
+			else
+				return PAM_IGNORE;
+		}
+		return PAM_SUCCESS;
 	}
 
 	info("From %s port %d as %s: unable to determine source job",
@@ -645,6 +658,10 @@ static void _parse_opts(pam_handle_t *pamh, int argc, const char **argv)
 					   "unrecognized action_unknown=%s, setting to 'newest'",
 					   v);
 			}
+		} else if (!xstrncasecmp(*argv, "action_adopt=", 13)){
+			v = (char *)(13 + *argv);
+			if (!xstrncasecmp(v, "check_only", 10))
+				opts.action_adopt = CALLERID_ACTION_ONLY_CHECK;
 		} else if (!xstrncasecmp(*argv,"action_generic_failure=",23)) {
 			v = (char *)(23 + *argv);
 			if (!xstrncasecmp(v, "allow", 5))
@@ -744,15 +761,15 @@ static int check_pam_service(pam_handle_t *pamh)
  *	3) Pick a job semi-randomly (default) or skip the adoption (if
  *		configured)
  */
-PAM_EXTERN int pam_sm_acct_mgmt(pam_handle_t *pamh, int flags
-				__attribute__((unused)), int argc, const char **argv)
-{
+PAM_EXTERN int _adopt_and_or_check(pam_handle_t *pamh, int flags
+				__attribute__((unused)), int argc, const char **argv) {
 	int retval = PAM_IGNORE, rc = PAM_IGNORE, slurmrc, bufsize, user_jobs;
 	char *user_name;
 	List steps = NULL;
 	step_loc_t *stepd = NULL;
 	struct passwd pwd, *pwd_result;
 	char *buf = NULL;
+	slurm_cgroup_conf_t *cg_conf;
 
 	_init_opts();
 	_parse_opts(pamh, argc, argv);
@@ -784,6 +801,9 @@ PAM_EXTERN int pam_sm_acct_mgmt(pam_handle_t *pamh, int flags
 		pam_syslog(pamh, LOG_ERR, "No username in PAM_USER? Fail!");
 		return PAM_SESSION_ERR;
 	}
+	if (opts.action_adopt==CALLERID_ACTION_ADOPT_AND_CHECK ){
+	   cg_conf = xcgroup_get_slurm_cgroup_conf();
+	}
 
 	/* Check for an unsafe config that might lock out root. This is a very
 	 * basic check that shouldn't be 100% relied on */
@@ -870,20 +890,24 @@ PAM_EXTERN int pam_sm_acct_mgmt(pam_handle_t *pamh, int flags
 		if (opts.single_job_skip_rpc) {
 			info("Connection by user %s: user has only one job %u",
 			     user_name, stepd->step_id.job_id);
-			slurmrc = _adopt_process(pamh, getpid(), stepd);
-			/* If adoption into the only job fails, it is time to
-			 * exit. Return code is based on the
-			 * action_adopt_failure setting */
-			if (slurmrc == SLURM_SUCCESS ||
-			    (opts.action_adopt_failure ==
-			     CALLERID_ACTION_ALLOW))
+			if (opts.action_adopt == CALLERID_ACTION_ADOPT_AND_CHECK) {
+				slurmrc = _adopt_process(pamh, getpid(), stepd);
+				/* If adoption into the only job fails, it is time to
+				* exit. Return code is based on the
+				* action_adopt_failure setting */
+				if (slurmrc == SLURM_SUCCESS ||
+					(opts.action_adopt_failure ==
+					CALLERID_ACTION_ALLOW))
+					rc = PAM_SUCCESS;
+				else {
+				  send_user_msg(pamh, "Access denied by "
+						PAM_MODULE_NAME
+  				        ": failed to adopt process into cgroup, denying access because action_adopt_failure=deny");
+				  rc = PAM_PERM_DENIED;
+				}
+			} else {
 				rc = PAM_SUCCESS;
-			else {
-				send_user_msg(pamh, "Access denied by "
-					      PAM_MODULE_NAME
-					      ": failed to adopt process into cgroup, denying access because action_adopt_failure=deny");
-				rc = PAM_PERM_DENIED;
-			}
+ 			}
 			goto cleanup;
 		}
 	} else {
@@ -910,6 +934,33 @@ PAM_EXTERN int pam_sm_acct_mgmt(pam_handle_t *pamh, int flags
 	return rc;
 }
 
+/* Take control of the session, to avoid other pam modules doing the
+ * same and change e.g., cgroups.
+ */
+PAM_EXTERN int pam_sm_open_session(pam_handle_t *pamh, int flags
+				__attribute__((unused)), int argc, const char **argv)
+{
+	return _adopt_and_or_check(pamh, flags, argc, argv);
+}
+
+/* Close the session. Always succeeds, we do not need to do anything here.
+ */
+PAM_EXTERN int pam_sm_close_session(pam_handle_t *pamh, int flags
+				__attribute__((unused)), int argc, const char **argv)
+{
+	return PAM_SUCCESS;
+}
+
+
+/* Implementation for the pam_acct_mgmt API call.
+ */
+PAM_EXTERN int pam_sm_acct_mgmt(pam_handle_t *pamh, int flags
+				__attribute__((unused)), int argc, const char **argv)
+{
+	return _adopt_and_or_check(pamh, flags, argc, argv);
+}
+
+
 #ifdef PAM_STATIC
 struct pam_module _pam_slurm_adopt_modstruct = {
 	PAM_MODULE_NAME,

From 7ac52bf27d2ce32c3a55c2d55217248cd4e31f01 Mon Sep 17 00:00:00 2001
From: Andy Georges <itkovian@gmail.com>
Date: Fri, 4 Nov 2022 14:13:36 +0100
Subject: [PATCH 02/90] patch: ugent build

---
 slurm.spec | 51 +++++++++++++++++++++++++++++++++++++--------------
 1 file changed, 37 insertions(+), 14 deletions(-)

diff --git a/slurm.spec b/slurm.spec
index f0100081e34..82eeb8e7e2c 100644
--- a/slurm.spec
+++ b/slurm.spec
@@ -1,7 +1,7 @@
 Name:		slurm
 Version:	22.05.5
 %define rel	1
-Release:	%{rel}%{?dist}
+Release:        %{rel}.%{gittag}%{?dist}%{?gpu}.ug
 Summary:	Slurm Workload Manager
 
 Group:		System Environment/Base
@@ -15,7 +15,7 @@ URL:		https://slurm.schedmd.com/
 %global slurm_source_dir %{name}-%{version}-%{rel}
 %endif
 
-Source:		%{slurm_source_dir}.tar.bz2
+Source:		%{slurm_source_dir}.tar.gz
 
 # build options		.rpmmacros options	change to default action
 # ====================  ====================	========================
@@ -140,7 +140,11 @@ BuildRequires: numactl-devel
 %endif
 
 %if %{with pmix} && "%{_with_pmix}" == "--with-pmix"
+%if 0%{?fedora} || 0%{?rhel} > 7
+BuildRequires: pmix-pmi-devel
+%else
 BuildRequires: pmix
+%endif
 %global pmix_version %(rpm -q pmix --qf "%{RPMTAG_VERSION}")
 %endif
 
@@ -235,8 +239,12 @@ Summary: Slurm compute node daemon
 Group: System Environment/Base
 Requires: %{name}%{?_isa} = %{version}-%{release}
 %if %{with pmix} && "%{_with_pmix}" == "--with-pmix"
++%if 0%{?fedora} || 0%{?rhel} > 7
++Requires: pmix-pmi = %{pmix_version}
++%else
 Requires: pmix = %{pmix_version}
 %endif
+%endif
 %if %{with ucx} && "%{_with_ucx}" == "--with-ucx"
 Requires: ucx = %{ucx_version}
 %endif
@@ -256,7 +264,9 @@ database changes to slurmctld daemons on each cluster
 Summary: Slurm\'s implementation of the pmi libraries
 Group: System Environment/Base
 Requires: %{name}%{?_isa} = %{version}-%{release}
++%if ! %{with pmix}
 Conflicts: pmix-libpmi
+%endif
 %description libpmi
 Slurm\'s version of libpmi. For systems using Slurm, this version
 is preferred over the compatibility libraries shipped by the PMIx project.
@@ -317,7 +327,13 @@ BuildRequires: http-parser-devel
 %if %{defined suse_version}
 BuildRequires: libjson-c-devel
 %else
+%if 0%{?fedora} || 0%{?rhel} > 7
 BuildRequires: json-c-devel
+Requires: json-c
+%else
+BuildRequires: json-c12-devel
+Requires: json-c12
+%endif
 %endif
 %description slurmrestd
 Provides a REST interface to Slurm.
@@ -369,7 +385,7 @@ export QA_RPATHS=0x5
 # Strip out some dependencies
 
 cat > find-requires.sh <<'EOF'
-exec %{__find_requires} "$@" | egrep -v '^libpmix.so|libevent|libnvidia-ml'
+exec %{__find_requires} "$@" | egrep -v '^libpmix.so|libevent'
 EOF
 chmod +x find-requires.sh
 %global _use_internal_dependency_generator 0
@@ -387,10 +403,17 @@ install -D -m644 etc/slurmdbd.service  %{buildroot}/%{_unitdir}/slurmdbd.service
 install -D -m644 etc/slurmrestd.service  %{buildroot}/%{_unitdir}/slurmrestd.service
 %endif
 
+# in case of pmix, no conflict with pmix compat libs
+# see Cray comment below
+%if %{with pmix}
+   mkdir %{buildroot}/%{_libdir}/slurmpmi
+   mv %{buildroot}/%{_libdir}/libpmi* %{buildroot}/%{_libdir}/slurmpmi
+%endif
+
 # Do not package Slurm's version of libpmi on Cray systems in the usual location.
 # Cray's version of libpmi should be used. Move it elsewhere if the site still
 # wants to use it with other MPI stacks.
-%if %{with cray} || %{with cray_shasta}
+%if %{with cray} || %{with cray_shasta} || %{with pmix}
    mkdir %{buildroot}/%{_libdir}/slurmpmi
    mv %{buildroot}/%{_libdir}/libpmi* %{buildroot}/%{_libdir}/slurmpmi
 %endif
@@ -611,16 +634,16 @@ rm -rf %{buildroot}
 
 %files torque
 %defattr(-,root,root)
-%{_bindir}/pbsnodes
-%{_bindir}/qalter
-%{_bindir}/qdel
-%{_bindir}/qhold
-%{_bindir}/qrerun
-%{_bindir}/qrls
-%{_bindir}/qstat
-%{_bindir}/qsub
-%{_bindir}/mpiexec
-%{_bindir}/generate_pbs_nodefile
+#%{_bindir}/pbsnodes
+#%{_bindir}/qalter
+#%{_bindir}/qdel
+#%{_bindir}/qhold
+#%{_bindir}/qrerun
+#%{_bindir}/qrls
+#%{_bindir}/qstat
+#%{_bindir}/qsub
+#%{_bindir}/mpiexec
+#%{_bindir}/generate_pbs_nodefile
 %{_libdir}/slurm/job_submit_pbs.so
 %{_libdir}/slurm/spank_pbs.so
 #############################################################################

From 24a31dbdcc7e897c9f5905d34e403bdd936e0dd7 Mon Sep 17 00:00:00 2001
From: Andy Georges <itkovian@gmail.com>
Date: Fri, 4 Nov 2022 14:36:29 +0100
Subject: [PATCH 03/90] patch: gh workflow

---
 .github/workflows/build.yml | 23 +++++++++++++++++++++++
 1 file changed, 23 insertions(+)
 create mode 100644 .github/workflows/build.yml

diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
new file mode 100644
index 00000000000..b9d96649695
--- /dev/null
+++ b/.github/workflows/build.yml
@@ -0,0 +1,23 @@
+name: slurm C/C++ build
+
+on:
+  push:
+    branches: [ 20.11.ug ]
+  pull_request:
+    branches: [ 20.11.ug ]
+
+jobs:
+  build:
+
+    runs-on: ubuntu-20.04
+
+    steps:
+    - uses: actions/checkout@v2
+    - name: configure
+      run: ./configure --enable-multiple-slurmd --prefix=/tmp/slurm/
+    - name: make
+      run: make -j
+    - name: make check
+      run: make -j check
+    - name: make install
+      run: make -j install
\ No newline at end of file

From b9c1b356f32f96337e95870fcee8122d883896f3 Mon Sep 17 00:00:00 2001
From: Andy Georges <itkovian@gmail.com>
Date: Fri, 4 Nov 2022 14:41:11 +0100
Subject: [PATCH 04/90] Update build.yml

---
 .github/workflows/build.yml | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
index b9d96649695..04ee9237cb8 100644
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@@ -2,9 +2,9 @@ name: slurm C/C++ build
 
 on:
   push:
-    branches: [ 20.11.ug ]
+    branches: [ 20.11.ug, 22.05.ug ]
   pull_request:
-    branches: [ 20.11.ug ]
+    branches: [ 20.11.ug, 22.05.ug ]
 
 jobs:
   build:
@@ -20,4 +20,4 @@ jobs:
     - name: make check
       run: make -j check
     - name: make install
-      run: make -j install
\ No newline at end of file
+      run: make -j install

From b20efd0778737569905b3c47ce9c7678e16c5557 Mon Sep 17 00:00:00 2001
From: Andy Georges <itkovian@gmail.com>
Date: Fri, 4 Nov 2022 18:47:23 +0100
Subject: [PATCH 05/90] patch: protocol messages with extra fields

---
 src/common/slurm_protocol_defs.h |  3 +++
 src/common/slurm_protocol_pack.c | 22 ++++++++++++++++++++++
 2 files changed, 25 insertions(+)

diff --git a/src/common/slurm_protocol_defs.h b/src/common/slurm_protocol_defs.h
index 0415b1202a1..e4239c00278 100644
--- a/src/common/slurm_protocol_defs.h
+++ b/src/common/slurm_protocol_defs.h
@@ -1025,6 +1025,8 @@ typedef struct kill_job_msg {
 	slurm_step_id_t step_id;
 	time_t   time;		/* slurmctld's time of request */
 	char *work_dir;
+	uint32_t nnodes; /* Number of nodes allocated to the job */
+	uint16_t *job_node_cpus; /* Number of CPUs required on the nodes for the job */
 } kill_job_msg_t;
 
 typedef struct reattach_tasks_request_msg {
@@ -1064,6 +1066,7 @@ typedef struct prolog_launch_msg {
 	char *std_err;			/* pathname of stderr */
 	char *std_out;			/* pathname of stdout */
 	uint32_t uid;
+    uint16_t *job_node_cpus; /* Number of CPUs required on the nodes for the job */
 	char *user_name;		/* job's user name */
 	char *work_dir;			/* full pathname of working directory */
 	uint16_t x11;			/* X11 forwarding setup flags */
diff --git a/src/common/slurm_protocol_pack.c b/src/common/slurm_protocol_pack.c
index 1bd356687c1..307ce04882f 100644
--- a/src/common/slurm_protocol_pack.c
+++ b/src/common/slurm_protocol_pack.c
@@ -2544,6 +2544,7 @@ _pack_kill_job_msg(kill_job_msg_t * msg, buf_t *buffer, uint16_t protocol_versio
 					     protocol_version);
 		packstr_array(msg->spank_job_env, msg->spank_job_env_size,
 			      buffer);
+		pack16_array(msg->job_node_cpus, msg->nnodes, buffer);
 		pack_time(msg->start_time, buffer);
 		pack_time(msg->time, buffer);
 		packstr(msg->work_dir, buffer);
@@ -2560,6 +2561,7 @@ _pack_kill_job_msg(kill_job_msg_t * msg, buf_t *buffer, uint16_t protocol_versio
 					     protocol_version);
 		packstr_array(msg->spank_job_env, msg->spank_job_env_size,
 			      buffer);
+		pack16_array(msg->job_node_cpus, msg->nnodes, buffer);
 		pack_time(msg->start_time, buffer);
 		pack_time(msg->time, buffer);
 		packstr(msg->work_dir, buffer);
@@ -2576,6 +2578,7 @@ _pack_kill_job_msg(kill_job_msg_t * msg, buf_t *buffer, uint16_t protocol_versio
 					     protocol_version);
 		packstr_array(msg->spank_job_env, msg->spank_job_env_size,
 			      buffer);
+		pack16_array(msg->job_node_cpus, msg->nnodes, buffer);
 		pack_time(msg->start_time, buffer);
 		pack_time(msg->time, buffer);
 	}
@@ -2621,6 +2624,9 @@ _unpack_kill_job_msg(kill_job_msg_t ** msg, buf_t *buffer,
 			goto unpack_error;
 		safe_unpackstr_array(&tmp_ptr->spank_job_env,
 				     &tmp_ptr->spank_job_env_size, buffer);
+		safe_unpack16_array(&tmp_ptr->job_node_cpus,
+		             &tmp_ptr->nnodes,
+					 buffer);
 		safe_unpack_time(&tmp_ptr->start_time, buffer);
 		safe_unpack_time(&tmp_ptr->time, buffer);
 		safe_unpackstr_xmalloc(&tmp_ptr->work_dir, &uint32_tmp, buffer);
@@ -2642,6 +2648,9 @@ _unpack_kill_job_msg(kill_job_msg_t ** msg, buf_t *buffer,
 			goto unpack_error;
 		safe_unpackstr_array(&tmp_ptr->spank_job_env,
 				     &tmp_ptr->spank_job_env_size, buffer);
+		safe_unpack16_array(&tmp_ptr->job_node_cpus,
+		             &tmp_ptr->nnodes,
+					 buffer);
 		safe_unpack_time(&tmp_ptr->start_time, buffer);
 		safe_unpack_time(&tmp_ptr->time, buffer);
 		safe_unpackstr_xmalloc(&tmp_ptr->work_dir, &uint32_tmp, buffer);
@@ -2663,6 +2672,9 @@ _unpack_kill_job_msg(kill_job_msg_t ** msg, buf_t *buffer,
 			goto unpack_error;
 		safe_unpackstr_array(&tmp_ptr->spank_job_env,
 				     &tmp_ptr->spank_job_env_size, buffer);
+		safe_unpack16_array(&tmp_ptr->job_node_cpus,
+		             &tmp_ptr->nnodes,
+					 buffer);
 		safe_unpack_time(&tmp_ptr->start_time, buffer);
 		safe_unpack_time(&tmp_ptr->time, buffer);
 	}
@@ -9177,6 +9189,8 @@ static void _pack_prolog_launch_msg(prolog_launch_msg_t *msg,
 
 		packstr_array(msg->spank_job_env, msg->spank_job_env_size,
 			      buffer);
+		xassert(msg->nnodes > 0);
+        pack16_array(msg->job_node_cpus, msg->nnodes, buffer);
 		slurm_cred_pack(msg->cred, buffer, protocol_version);
 		packstr(msg->user_name, buffer);
 	} else if (protocol_version >= SLURM_MIN_PROTOCOL_VERSION) {
@@ -9203,6 +9217,8 @@ static void _pack_prolog_launch_msg(prolog_launch_msg_t *msg,
 
 		packstr_array(msg->spank_job_env, msg->spank_job_env_size,
 			      buffer);
+		xassert(msg->nnodes > 0);
+        pack16_array(msg->job_node_cpus, msg->nnodes, buffer);
 		slurm_cred_pack(msg->cred, buffer, protocol_version);
 		packstr(msg->user_name, buffer);
 	}
@@ -9252,6 +9268,9 @@ static int _unpack_prolog_launch_msg(prolog_launch_msg_t **msg,
 		safe_unpackstr_array(&launch_msg_ptr->spank_job_env,
 				     &launch_msg_ptr->spank_job_env_size,
 				     buffer);
+		safe_unpack16_array(&launch_msg_ptr->job_node_cpus,
+                    &launch_msg_ptr->nnodes,
+                    buffer);
 		if (!(launch_msg_ptr->cred = slurm_cred_unpack(buffer,
 							       protocol_version)))
 			goto unpack_error;
@@ -9293,6 +9312,9 @@ static int _unpack_prolog_launch_msg(prolog_launch_msg_t **msg,
 		safe_unpackstr_array(&launch_msg_ptr->spank_job_env,
 				     &launch_msg_ptr->spank_job_env_size,
 				     buffer);
+		safe_unpack16_array(&launch_msg_ptr->job_node_cpus,
+                    &launch_msg_ptr->nnodes,
+                    buffer);
 		if (!(launch_msg_ptr->cred = slurm_cred_unpack(buffer,
 							       protocol_version)))
 			goto unpack_error;

From ee923434e001254ca0f9d5a6ea8adc1432cc44ac Mon Sep 17 00:00:00 2001
From: Andy Georges <itkovian@gmail.com>
Date: Mon, 7 Nov 2022 10:35:15 +0100
Subject: [PATCH 06/90] patch: spank pbs

---
 src/common/slurm_protocol_defs.h             |  4 +--
 src/plugins/job_submit/pbs/spank_pbs.c       | 31 +++++++++++++++++---
 src/plugins/prep/script/prep_script_slurmd.c |  3 ++
 3 files changed, 32 insertions(+), 6 deletions(-)

diff --git a/src/common/slurm_protocol_defs.h b/src/common/slurm_protocol_defs.h
index e4239c00278..dbeffdbccb2 100644
--- a/src/common/slurm_protocol_defs.h
+++ b/src/common/slurm_protocol_defs.h
@@ -1025,8 +1025,8 @@ typedef struct kill_job_msg {
 	slurm_step_id_t step_id;
 	time_t   time;		/* slurmctld's time of request */
 	char *work_dir;
-	uint32_t nnodes; /* Number of nodes allocated to the job */
-	uint16_t *job_node_cpus; /* Number of CPUs required on the nodes for the job */
+    uint32_t nnodes; /* Number of nodes allocated to the job */
+    uint16_t *job_node_cpus; /* Number of CPUs required on the nodes for the job */
 } kill_job_msg_t;
 
 typedef struct reattach_tasks_request_msg {
diff --git a/src/plugins/job_submit/pbs/spank_pbs.c b/src/plugins/job_submit/pbs/spank_pbs.c
index 02a38c82cd0..56501652bcb 100644
--- a/src/plugins/job_submit/pbs/spank_pbs.c
+++ b/src/plugins/job_submit/pbs/spank_pbs.c
@@ -66,14 +66,28 @@ int slurm_spank_task_init(spank_t sp, int ac, char **av)
 		spank_setenv(sp, "PBS_ARRAY_ID", val, 1);
 	if (spank_getenv(sp, "SLURM_ARRAY_TASK_ID", val, sizeof(val)) ==
 	    ESPANK_SUCCESS)
-		spank_setenv(sp, "PBS_ARRAY_INDEX", val, 1);
+		spank_setenv(sp, "PBS_ARRAYID", val, 1);
 
 	if (getcwd(val, sizeof(val)))
 		spank_setenv(sp, "PBS_JOBDIR", val, 1);
 
-	if (spank_getenv(sp, "SLURM_JOB_ID", val, sizeof(val)) ==
-	    ESPANK_SUCCESS)
-		spank_setenv(sp, "PBS_JOBID", val, 1);
+	if (spank_getenv(sp, "SLURM_ARRAY_JOB_ID", val, sizeof(val)) ==
+	    ESPANK_SUCCESS) {
+        /* 20 is enough for unit32 (max index is 4M) */
+        char aid[20];
+        if (spank_getenv(sp, "SLURM_ARRAY_TASK_ID", aid, sizeof(aid)) ==
+            ESPANK_SUCCESS) {
+            /* 30k val is large enough for job id and array index */
+            strncat(val, "[", 1);
+            strncat(val, aid, sizeof(aid));
+            strncat(val, "]", 1);
+        }
+        spank_setenv(sp, "PBS_JOBID", val, 1);
+    } else {
+        if (spank_getenv(sp, "SLURM_JOB_ID", val, sizeof(val)) ==
+            ESPANK_SUCCESS)
+            spank_setenv(sp, "PBS_JOBID", val, 1);
+    }
 
 	if (spank_getenv(sp, "SLURM_JOB_NAME", val, sizeof(val)) ==
 	    ESPANK_SUCCESS)
@@ -113,6 +127,15 @@ int slurm_spank_task_init(spank_t sp, int ac, char **av)
 	if (spank_getenv(sp, "SYSTEM", val, sizeof(val)) == ESPANK_SUCCESS)
 		spank_setenv(sp, "PBS_O_SYSTEM", val, 1);
 
+	if (spank_getenv(sp, "SLURM_JOB_NUM_NODES", val, sizeof(val)) == ESPANK_SUCCESS)
+		spank_setenv(sp, "PBS_NUM_NODES", val, 1);
+
+	if (spank_getenv(sp, "SLURM_NTASKS", val, sizeof(val)) == ESPANK_SUCCESS)
+		spank_setenv(sp, "PBS_NP", val, 1);
+
+	if (spank_getenv(sp, "SLURM_NTASKS_PER_NODE", val, sizeof(val)) == ESPANK_SUCCESS)
+		spank_setenv(sp, "PBS_NUM_PPN", val, 1);
+
 	if (spank_getenv(sp, "SLURM_SUBMIT_DIR", val, sizeof(val)) ==
 	    ESPANK_SUCCESS)
 		spank_setenv(sp, "PBS_O_WORKDIR", val, 1);
diff --git a/src/plugins/prep/script/prep_script_slurmd.c b/src/plugins/prep/script/prep_script_slurmd.c
index 4621994d3e2..a0cca0643ab 100644
--- a/src/plugins/prep/script/prep_script_slurmd.c
+++ b/src/plugins/prep/script/prep_script_slurmd.c
@@ -289,6 +289,9 @@ static char **_build_env(job_env_t *job_env, slurm_cred_t *cred,
 	else
 		setenvf(&env, "SLURM_SCRIPT_CONTEXT", "prolog_slurmd");
 
+    setenvf(&env, "SLURM_JOB_NODE_CPUS", "%d", job_env->job_node_cpus);
+    setenvf(&env, "SLURM_ACTUAL_NODE_CPUS", "%d", conf->actual_cpus);
+
 	if (is_epilog && (job_env->exit_code != INFINITE)) {
 		int exit_code = 0, signal = 0;
 		if (WIFEXITED(job_env->exit_code))

From 5175e4f99a36cf8b0205fec75c5832275021f257 Mon Sep 17 00:00:00 2001
From: Andy Georges <itkovian@gmail.com>
Date: Mon, 7 Nov 2022 11:11:58 +0100
Subject: [PATCH 07/90] patch: rpc calls

---
 src/common/proc_args.h         |  6 ++++++
 src/slurmctld/node_scheduler.c | 10 +++++++++
 src/slurmd/slurmd/req.c        | 37 ++++++++++++++++++++++++++++++++++
 src/slurmd/slurmd/slurmd.h     |  1 +
 4 files changed, 54 insertions(+)

diff --git a/src/common/proc_args.h b/src/common/proc_args.h
index 5efadfbae84..300c16325ce 100644
--- a/src/common/proc_args.h
+++ b/src/common/proc_args.h
@@ -79,6 +79,12 @@ uint64_t str_to_mbytes(const char *arg);
  */
 extern char *mbytes_to_str(uint64_t mbytes);
 
+/*
+ * str_to_mbytes_pbs(): verify that arg is numeric with optional "K", "M", "G"
+ * or "T" at end and return the number in mega-bytes. Default units are B.
+ */
+long str_to_mbytes_pbs(const char *arg);
+
 /* verify that a node count in arg is of a known form (count or min-max) */
 bool verify_node_count(const char *arg, int *min_nodes, int *max_nodes);
 
diff --git a/src/slurmctld/node_scheduler.c b/src/slurmctld/node_scheduler.c
index 60abfb91373..8d3a2242ce8 100644
--- a/src/slurmctld/node_scheduler.c
+++ b/src/slurmctld/node_scheduler.c
@@ -316,6 +316,7 @@ extern void deallocate_nodes(job_record_t *job_ptr, bool timeout,
 	agent_arg_t *agent_args = NULL;
 	int down_node_cnt = 0;
 	node_record_t *node_ptr;
+	job_resources_t *job_resrcs_ptr;
 	hostlist_t hostlist = NULL;
 	uint16_t use_protocol_version = 0;
 #ifdef HAVE_FRONT_END
@@ -339,6 +340,11 @@ extern void deallocate_nodes(job_record_t *job_ptr, bool timeout,
 	if (!job_ptr->details->prolog_running)
 		hostlist = hostlist_create(NULL);
 
+	// XXX: verify
+	job_resrcs_ptr = job_ptr->job_resrcs;
+    kill_job->nnodes = job_resrcs_ptr->nhosts;
+	kill_job->job_node_cpus = job_resrcs_ptr->cpus;
+
 #ifdef HAVE_FRONT_END
 	if (job_ptr->batch_host &&
 	    (front_end_ptr = job_ptr->front_end_ptr)) {
@@ -3048,6 +3054,10 @@ extern void launch_prolog(job_record_t *job_ptr)
 
 	xassert(job_ptr->job_resrcs);
 	job_resrcs_ptr = job_ptr->job_resrcs;
+
+    prolog_msg_ptr->nnodes = job_resrcs_ptr->nhosts;
+    prolog_msg_ptr->job_node_cpus = job_resrcs_ptr->cpus;
+
 	setup_cred_arg(&cred_arg, job_ptr);
 	cred_arg.step_id.job_id = job_ptr->job_id;
 	cred_arg.step_id.step_id = SLURM_EXTERN_CONT;
diff --git a/src/slurmd/slurmd/req.c b/src/slurmd/slurmd/req.c
index 64bbcadbca8..343a0de482d 100644
--- a/src/slurmd/slurmd/req.c
+++ b/src/slurmd/slurmd/req.c
@@ -149,6 +149,8 @@ typedef struct {
 
 static void _fb_rdlock(void);
 static void _fb_rdunlock(void);
+static int  _abort_step(uint32_t job_id, uint32_t step_id);
+static char **_build_env(job_env_t *job_env, bool is_epilog);
 static void _delay_rpc(int host_inx, int host_cnt, int usec_per_rpc);
 static void _free_job_env(job_env_t *env_ptr);
 static bool _is_batch_job_finished(uint32_t job_id);
@@ -2232,9 +2234,35 @@ static int _spawn_prolog_stepd(slurm_msg_t *msg)
 	return rc;
 }
 
+static int _get_node_inx(char *hostlist)
+{
+	char *host;
+	int node_inx = -1;
+	hostset_t hset;
+
+	if (!conf->node_name)
+		return node_inx;
+
+	if ((hset = hostset_create(hostlist))) {
+		int inx = 0;
+		while ((host = hostset_shift(hset))) {
+			if (!strcmp(host, conf->node_name)) {
+				node_inx = inx;
+				free(host);
+				break;
+			}
+			inx++;
+			free(host);
+		}
+		hostset_destroy(hset);
+	}
+	return node_inx;
+}
+
 static void _rpc_prolog(slurm_msg_t *msg)
 {
 	int rc = SLURM_SUCCESS, alt_rc = SLURM_ERROR, node_id = 0;
+    int node_inx = -1;
 	prolog_launch_msg_t *req = (prolog_launch_msg_t *)msg->data;
 	job_env_t job_env;
 	bool     first_job_run;
@@ -2308,6 +2336,10 @@ static void _rpc_prolog(slurm_msg_t *msg)
 		jobid = req->job_id;
 #endif
 
+        node_inx = _get_node_inx(req->nodes);
+		debug("_rpc_prolog: _get_node_inx returned %d", node_inx);
+        job_env.job_node_cpus = (node_inx >= 0 ? req->job_node_cpus[node_inx] : 0);
+
 		if ((rc = container_g_create(jobid, req->uid)))
 			error("container_g_create(%u): %m", req->job_id);
 		else
@@ -5220,6 +5252,7 @@ _rpc_terminate_job(slurm_msg_t *msg)
 	int             nsteps = 0;
 	int		delay;
 	int		node_id = 0;
+	int node_inx = -1;
 	job_env_t       job_env;
 	uint32_t        jobid;
 
@@ -5450,6 +5483,10 @@ _rpc_terminate_job(slurm_msg_t *msg)
 	job_env.uid = req->job_uid;
 	job_env.gid = req->job_gid;
 
+    node_inx = _get_node_inx(req->nodes);
+    job_env.job_node_cpus = (node_inx >= 0 ? req->job_node_cpus[node_inx] : 0);
+    debug2("Setting job_env.job_cpu_nodes to %d", job_env.job_node_cpus);
+
 	rc = _run_epilog(&job_env, req->cred);
 	_free_job_env(&job_env);
 
diff --git a/src/slurmd/slurmd/slurmd.h b/src/slurmd/slurmd/slurmd.h
index 0458f518227..e42b7ef4521 100644
--- a/src/slurmd/slurmd/slurmd.h
+++ b/src/slurmd/slurmd/slurmd.h
@@ -73,6 +73,7 @@ typedef struct {
 	gid_t gid;
 	char *user_name;
 	char *work_dir;
+    uint16_t job_node_cpus;  /* Number of CPUs used by the job on this node */
 } job_env_t;
 
 /*

From 581de0aef41694fff9baa175dbf7a5b047b35a38 Mon Sep 17 00:00:00 2001
From: Andy Georges <itkovian@gmail.com>
Date: Mon, 7 Nov 2022 11:31:43 +0100
Subject: [PATCH 08/90] feat: build script

---
 bdist_rpm.sh | 80 ++++++++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 80 insertions(+)
 create mode 100755 bdist_rpm.sh

diff --git a/bdist_rpm.sh b/bdist_rpm.sh
new file mode 100755
index 00000000000..86a09784cd4
--- /dev/null
+++ b/bdist_rpm.sh
@@ -0,0 +1,80 @@
+#!/bin/bash
+
+set -e
+set -x
+
+echo "BUILDING GPU VERSION"
+
+VERSION=`grep "Version:.*[0-9]" slurm.spec | tr -s " " |  awk '{print $2;}'`
+RELEASE=`grep "%define rel.*[-1-9]" slurm.spec | tr -s " " | awk '{print $3}'`
+
+if [ "${RELEASE}" != "1" ]; then
+    SUFFIX=${VERSION}-${RELEASE}
+else
+    SUFFIX=${VERSION}
+fi
+
+GITTAG=$(git log --format=%ct.%h -1)
+
+rm -Rf BUILD SOURCES SPECS SRPMS RPMS BUILDROOT
+mkdir -p BUILD SOURCES SPECS SRPMS RPMS BUILDROOT
+
+git archive --format=tar.gz -o "SOURCES/slurm-${SUFFIX}.tar.gz" --prefix="slurm-${SUFFIX}/" HEAD
+cp slurm.spec "SPECS"
+
+# remove json-c12 -> use plain json-c?
+sudo yum install -y json-c-devel json-c
+if grep "release 7" /etc/redhat-release; then
+  #   json-c is only needed for restd and burstbuffer
+    sudo yum install -y json-c12-devel json-c12
+fi
+
+# TODO? also, specify python2 as python to require
+
+# commenting this out since it tries to update things :/
+#sudo yum install -y ucx-devel "pmix-devel > 3.0.0" numactl-devel hwloc-devel
+if grep "release 7" /etc/redhat-release; then
+    sudo yum install -y ucx-devel-1.5.2-1.el7.x86_64 "pmix-devel > 3.0.0" numactl-devel pmix-pmi-devel hwloc-devel-1.11.8-4.el7.x86_64
+else
+    sudo yum install -y ucx-devel-1.11.2-2.el8.x86_64 "pmix-devel > 3.1.4" numactl-devel pmix-pmi-devel hwloc-devel-2.2.0-3.el8.x86_64
+fi
+
+# there's no option to pass nvml, it is only autodetected
+# nvidia-driver-devel provides the libnividia-ml.so symlnk, the real .so.1 comes from nvidia-driver-NVML
+# cuda-nvml-dev provides the nvml.h (put is part of the cuda rpms so has cud version in the name, hence the *)
+# however, it's not in the default include paths nor the slurm hardcoded ones, so use CPATH
+# nvml api is not really cuda specific, last API is from cuda 5, so any recent cuda will do
+
+# TODO: what if more than one cuda is available/installed, then the * thingies will probably not work
+# pmix-3 as rebuild from github src.rpm includes the devel rpms in the rpm
+sudo yum remove -y cuda-nvml-dev-10-1 cuda-nvml-dev-10-2 cuda-nvml-devel-11-6
+sudo yum install -y nvidia-driver-devel nvidia-driver-NVML cuda-nvml-devel-11-6
+
+
+# glob expansion in list
+nvmls=(/usr/local/cuda*/targets/x86_64-linux/include)
+if [ "${#nvmls[@]}" -ne 1 ]; then
+    echo "0 or more than one nvml.h found: ${nvmls[@]}. Unsupported."
+    exit 1
+fi
+echo "nvml.h found in $nvmls[0]"
+
+rpmbuild --define "gittag ${GITTAG}" --define "_topdir $PWD" --with numa --with pmix --with hwloc --with mysql --with x11 --with ucx -ba SPECS/slurm.spec --define "_configure ./configure CPATH=${nvmls[0]}" --define "_smp_mflags CPATH=${nvmls[0]}"
+
+
+echo "BUILDING NON GPU VERSION"
+
+
+# there's no option to pass nvml, it is only autodetected
+# nvidia-driver-devel provides the libnividia-ml.so symlnk, the real .so.1 comes from nvidia-driver-NVML
+# cuda-nvml-dev provides the nvml.h (put is part of the cuda rpms so has cud version in the name, hence the *)
+# however, it's not in the default include paths nor the slurm hardcoded ones, so use CPATH
+# nvml api is not really cuda specific, last API is from cuda 5, so any recent cuda will do
+
+# TODO: what if more than one cuda is available/installed, then the * thingies will probably not work
+# pmix-3 as rebuild from github src.rpm includes the devel rpms in the rpm
+sudo yum remove -y nvidia-driver-devel cuda-nvml-dev-10-2 nvidia-driver-NVML cuda-nvml-dev-10-1
+sudo yum remove -y nvidia-driver-devel nvidia-driver-NVML cuda-nvml-devel-11-3-11.3.58-1.x86_64
+
+
+rpmbuild --define "gittag ${GITTAG}" --define "_topdir $PWD" --with pmix --with numa --with hwloc --with mysql --with x11 --with ucx --define "gpu .nogpu" -ba SPECS/slurm.spec

From 96864b5d78e0f2eca4c5e4ae0f90907350b612db Mon Sep 17 00:00:00 2001
From: Andy Georges <itkovian@gmail.com>
Date: Mon, 7 Nov 2022 14:12:05 +0100
Subject: [PATCH 09/90] fix: distinction between EL 8.6 and 8.x

---
 bdist_rpm.sh | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/bdist_rpm.sh b/bdist_rpm.sh
index 86a09784cd4..4686d5e49cb 100755
--- a/bdist_rpm.sh
+++ b/bdist_rpm.sh
@@ -35,8 +35,10 @@ fi
 #sudo yum install -y ucx-devel "pmix-devel > 3.0.0" numactl-devel hwloc-devel
 if grep "release 7" /etc/redhat-release; then
     sudo yum install -y ucx-devel-1.5.2-1.el7.x86_64 "pmix-devel > 3.0.0" numactl-devel pmix-pmi-devel hwloc-devel-1.11.8-4.el7.x86_64
-else
+elif grep "release 8.6" /etc/redhat-release; then
     sudo yum install -y ucx-devel-1.11.2-2.el8.x86_64 "pmix-devel > 3.1.4" numactl-devel pmix-pmi-devel hwloc-devel-2.2.0-3.el8.x86_64
+else
+    sudo yum install -y ucx-devel-1.8.1-3.el8.x86_64 "pmix-devel > 3.0.0" numactl-devel pmix-pmi-devel hwloc-devel-0:1.11.9-3.el8.x86_64
 fi
 
 # there's no option to pass nvml, it is only autodetected

From 2cfb82c6a3384fa6cb30187a9490d7f0b4ef7ca5 Mon Sep 17 00:00:00 2001
From: Andy Georges <itkovian@gmail.com>
Date: Mon, 7 Nov 2022 14:28:43 +0100
Subject: [PATCH 10/90] fix: hwloc and ucx versions for el8.4

---
 bdist_rpm.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/bdist_rpm.sh b/bdist_rpm.sh
index 4686d5e49cb..b69ca764c17 100755
--- a/bdist_rpm.sh
+++ b/bdist_rpm.sh
@@ -38,7 +38,7 @@ if grep "release 7" /etc/redhat-release; then
 elif grep "release 8.6" /etc/redhat-release; then
     sudo yum install -y ucx-devel-1.11.2-2.el8.x86_64 "pmix-devel > 3.1.4" numactl-devel pmix-pmi-devel hwloc-devel-2.2.0-3.el8.x86_64
 else
-    sudo yum install -y ucx-devel-1.8.1-3.el8.x86_64 "pmix-devel > 3.0.0" numactl-devel pmix-pmi-devel hwloc-devel-0:1.11.9-3.el8.x86_64
+    sudo yum install -y ucx-devel-1.9.0-1.el8.x86_64 "pmix-devel > 3.0.0" numactl-devel pmix-pmi-devel hwloc-devel-0:1.11.9-3.el8.x86_64
 fi
 
 # there's no option to pass nvml, it is only autodetected

From a96a7b3ddce560a16bcdc39a8a80c71767fc2120 Mon Sep 17 00:00:00 2001
From: Andy Georges <itkovian@gmail.com>
Date: Mon, 7 Nov 2022 15:29:54 +0100
Subject: [PATCH 11/90] fix: no gpu build for now

---
 bdist_rpm.sh | 20 ++++++++++----------
 1 file changed, 10 insertions(+), 10 deletions(-)

diff --git a/bdist_rpm.sh b/bdist_rpm.sh
index b69ca764c17..40765e3506d 100755
--- a/bdist_rpm.sh
+++ b/bdist_rpm.sh
@@ -49,19 +49,19 @@ fi
 
 # TODO: what if more than one cuda is available/installed, then the * thingies will probably not work
 # pmix-3 as rebuild from github src.rpm includes the devel rpms in the rpm
-sudo yum remove -y cuda-nvml-dev-10-1 cuda-nvml-dev-10-2 cuda-nvml-devel-11-6
-sudo yum install -y nvidia-driver-devel nvidia-driver-NVML cuda-nvml-devel-11-6
+#sudo yum remove -y cuda-nvml-dev-10-1 cuda-nvml-dev-10-2 cuda-nvml-devel-11-6
+#sudo yum install -y nvidia-driver-devel nvidia-driver-NVML cuda-nvml-devel-11-6
 
 
 # glob expansion in list
-nvmls=(/usr/local/cuda*/targets/x86_64-linux/include)
-if [ "${#nvmls[@]}" -ne 1 ]; then
-    echo "0 or more than one nvml.h found: ${nvmls[@]}. Unsupported."
-    exit 1
-fi
-echo "nvml.h found in $nvmls[0]"
-
-rpmbuild --define "gittag ${GITTAG}" --define "_topdir $PWD" --with numa --with pmix --with hwloc --with mysql --with x11 --with ucx -ba SPECS/slurm.spec --define "_configure ./configure CPATH=${nvmls[0]}" --define "_smp_mflags CPATH=${nvmls[0]}"
+#nvmls=(/usr/local/cuda*/targets/x86_64-linux/include)
+#if [ "${#nvmls[@]}" -ne 1 ]; then
+    #echo "0 or more than one nvml.h found: ${nvmls[@]}. Unsupported."
+    #exit 1
+#fi
+#echo "nvml.h found in $nvmls[0]"
+
+#rpmbuild --define "gittag ${GITTAG}" --define "_topdir $PWD" --with numa --with pmix --with hwloc --with mysql --with x11 --with ucx -ba SPECS/slurm.spec --define "_configure ./configure CPATH=${nvmls[0]}" --define "_smp_mflags CPATH=${nvmls[0]}"
 
 
 echo "BUILDING NON GPU VERSION"

From 511124eb9debb012e9b5933e791dd2faf2b632a8 Mon Sep 17 00:00:00 2001
From: Andy Georges <itkovian@gmail.com>
Date: Mon, 7 Nov 2022 15:44:45 +0100
Subject: [PATCH 12/90] fix: spec patch

---
 slurm.spec | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/slurm.spec b/slurm.spec
index 82eeb8e7e2c..4e248aac573 100644
--- a/slurm.spec
+++ b/slurm.spec
@@ -239,9 +239,9 @@ Summary: Slurm compute node daemon
 Group: System Environment/Base
 Requires: %{name}%{?_isa} = %{version}-%{release}
 %if %{with pmix} && "%{_with_pmix}" == "--with-pmix"
-+%if 0%{?fedora} || 0%{?rhel} > 7
-+Requires: pmix-pmi = %{pmix_version}
-+%else
+%if 0%{?fedora} || 0%{?rhel} > 7
+Requires: pmix-pmi = %{pmix_version}
+%else
 Requires: pmix = %{pmix_version}
 %endif
 %endif
@@ -264,7 +264,7 @@ database changes to slurmctld daemons on each cluster
 Summary: Slurm\'s implementation of the pmi libraries
 Group: System Environment/Base
 Requires: %{name}%{?_isa} = %{version}-%{release}
-+%if ! %{with pmix}
+%if ! %{with pmix}
 Conflicts: pmix-libpmi
 %endif
 %description libpmi

From 2b0fcefd1d84ef5dce91d6444b1e11d88334bd64 Mon Sep 17 00:00:00 2001
From: Andy Georges <itkovian@gmail.com>
Date: Mon, 7 Nov 2022 17:00:33 +0100
Subject: [PATCH 13/90] fix: type typo

---
 contribs/pam_slurm_adopt/pam_slurm_adopt.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/contribs/pam_slurm_adopt/pam_slurm_adopt.c b/contribs/pam_slurm_adopt/pam_slurm_adopt.c
index 44520f0c062..c6d06d72d92 100644
--- a/contribs/pam_slurm_adopt/pam_slurm_adopt.c
+++ b/contribs/pam_slurm_adopt/pam_slurm_adopt.c
@@ -769,7 +769,7 @@ PAM_EXTERN int _adopt_and_or_check(pam_handle_t *pamh, int flags
 	step_loc_t *stepd = NULL;
 	struct passwd pwd, *pwd_result;
 	char *buf = NULL;
-	slurm_cgroup_conf_t *cg_conf;
+	cgroup_conf_t *cg_conf;
 
 	_init_opts();
 	_parse_opts(pamh, argc, argv);

From 53d87e186c240257b0a79ff64afc93205ed5ba28 Mon Sep 17 00:00:00 2001
From: Andy Georges <itkovian@gmail.com>
Date: Mon, 7 Nov 2022 17:01:31 +0100
Subject: [PATCH 14/90] deps: install additional dependencies for building

---
 bdist_rpm.sh | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/bdist_rpm.sh b/bdist_rpm.sh
index 40765e3506d..3d8230748b2 100755
--- a/bdist_rpm.sh
+++ b/bdist_rpm.sh
@@ -41,6 +41,8 @@ else
     sudo yum install -y ucx-devel-1.9.0-1.el8.x86_64 "pmix-devel > 3.0.0" numactl-devel pmix-pmi-devel hwloc-devel-0:1.11.9-3.el8.x86_64
 fi
 
+sudo yum install munge-devel mysql-devel pam-devel readline-devel
+
 # there's no option to pass nvml, it is only autodetected
 # nvidia-driver-devel provides the libnividia-ml.so symlnk, the real .so.1 comes from nvidia-driver-NVML
 # cuda-nvml-dev provides the nvml.h (put is part of the cuda rpms so has cud version in the name, hence the *)

From 873efb9416bf7e184489079927f262d7d726099e Mon Sep 17 00:00:00 2001
From: Andy Georges <itkovian@gmail.com>
Date: Tue, 8 Nov 2022 07:49:56 +0100
Subject: [PATCH 15/90] fix: mariadb instead of mysqk

---
 bdist_rpm.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/bdist_rpm.sh b/bdist_rpm.sh
index 3d8230748b2..22fa7ccc87f 100755
--- a/bdist_rpm.sh
+++ b/bdist_rpm.sh
@@ -41,7 +41,7 @@ else
     sudo yum install -y ucx-devel-1.9.0-1.el8.x86_64 "pmix-devel > 3.0.0" numactl-devel pmix-pmi-devel hwloc-devel-0:1.11.9-3.el8.x86_64
 fi
 
-sudo yum install munge-devel mysql-devel pam-devel readline-devel
+sudo yum install munge-devel mariadb-devel pam-devel readline-devel
 
 # there's no option to pass nvml, it is only autodetected
 # nvidia-driver-devel provides the libnividia-ml.so symlnk, the real .so.1 comes from nvidia-driver-NVML

From f89c2e9bdb1548e788fae89f0653707bcab631c0 Mon Sep 17 00:00:00 2001
From: Andy Georges <itkovian@gmail.com>
Date: Tue, 8 Nov 2022 21:38:09 +0100
Subject: [PATCH 16/90] fix: pmi lib location

---
 slurm.spec | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/slurm.spec b/slurm.spec
index 4e248aac573..62f41e97921 100644
--- a/slurm.spec
+++ b/slurm.spec
@@ -413,7 +413,7 @@ install -D -m644 etc/slurmrestd.service  %{buildroot}/%{_unitdir}/slurmrestd.ser
 # Do not package Slurm's version of libpmi on Cray systems in the usual location.
 # Cray's version of libpmi should be used. Move it elsewhere if the site still
 # wants to use it with other MPI stacks.
-%if %{with cray} || %{with cray_shasta} || %{with pmix}
+%if %{with cray} || %{with cray_shasta}
    mkdir %{buildroot}/%{_libdir}/slurmpmi
    mv %{buildroot}/%{_libdir}/libpmi* %{buildroot}/%{_libdir}/slurmpmi
 %endif
@@ -625,7 +625,7 @@ rm -rf %{buildroot}
 
 %files libpmi
 %defattr(-,root,root)
-%if %{with cray} || %{with cray_shasta}
+%if %{with cray} || %{with cray_shasta} || %{with pmix}
 %{_libdir}/slurmpmi/*
 %else
 %{_libdir}/libpmi*

From 3702448c4bcc1db25c045496150584486caf9547 Mon Sep 17 00:00:00 2001
From: Andy Georges <itkovian@gmail.com>
Date: Wed, 9 Nov 2022 10:24:50 +0100
Subject: [PATCH 17/90] fix: turning gpou builds back on

---
 bdist_rpm.sh | 20 ++++++++++----------
 1 file changed, 10 insertions(+), 10 deletions(-)

diff --git a/bdist_rpm.sh b/bdist_rpm.sh
index 22fa7ccc87f..4842aedf24b 100755
--- a/bdist_rpm.sh
+++ b/bdist_rpm.sh
@@ -51,19 +51,19 @@ sudo yum install munge-devel mariadb-devel pam-devel readline-devel
 
 # TODO: what if more than one cuda is available/installed, then the * thingies will probably not work
 # pmix-3 as rebuild from github src.rpm includes the devel rpms in the rpm
-#sudo yum remove -y cuda-nvml-dev-10-1 cuda-nvml-dev-10-2 cuda-nvml-devel-11-6
-#sudo yum install -y nvidia-driver-devel nvidia-driver-NVML cuda-nvml-devel-11-6
+sudo yum remove -y cuda-nvml-dev-10-1 cuda-nvml-dev-10-2 cuda-nvml-devel-11-6
+sudo yum install -y nvidia-driver-devel nvidia-driver-NVML cuda-nvml-devel-11-6
 
 
 # glob expansion in list
-#nvmls=(/usr/local/cuda*/targets/x86_64-linux/include)
-#if [ "${#nvmls[@]}" -ne 1 ]; then
-    #echo "0 or more than one nvml.h found: ${nvmls[@]}. Unsupported."
-    #exit 1
-#fi
-#echo "nvml.h found in $nvmls[0]"
-
-#rpmbuild --define "gittag ${GITTAG}" --define "_topdir $PWD" --with numa --with pmix --with hwloc --with mysql --with x11 --with ucx -ba SPECS/slurm.spec --define "_configure ./configure CPATH=${nvmls[0]}" --define "_smp_mflags CPATH=${nvmls[0]}"
+nvmls=(/usr/local/cuda*/targets/x86_64-linux/include)
+if [ "${#nvmls[@]}" -ne 1 ]; then
+    echo "0 or more than one nvml.h found: ${nvmls[@]}. Unsupported."
+    exit 1
+fi
+echo "nvml.h found in $nvmls[0]"
+
+rpmbuild --define "gittag ${GITTAG}" --define "_topdir $PWD" --with numa --with pmix --with hwloc --with mysql --with x11 --with ucx -ba SPECS/slurm.spec --define "_configure ./configure CPATH=${nvmls[0]}" --define "_smp_mflags CPATH=${nvmls[0]}"
 
 
 echo "BUILDING NON GPU VERSION"

From 4a703f7c19c4300b6d6814dc1253c6d71e5c80f8 Mon Sep 17 00:00:00 2001
From: Andy Georges <itkovian@gmail.com>
Date: Mon, 14 Nov 2022 12:40:22 +0100
Subject: [PATCH 18/90] fix: do not try to deref before pointer is initialised

---
 src/slurmctld/node_scheduler.c | 11 ++++++-----
 1 file changed, 6 insertions(+), 5 deletions(-)

diff --git a/src/slurmctld/node_scheduler.c b/src/slurmctld/node_scheduler.c
index 8d3a2242ce8..002d4b8e7f6 100644
--- a/src/slurmctld/node_scheduler.c
+++ b/src/slurmctld/node_scheduler.c
@@ -340,10 +340,6 @@ extern void deallocate_nodes(job_record_t *job_ptr, bool timeout,
 	if (!job_ptr->details->prolog_running)
 		hostlist = hostlist_create(NULL);
 
-	// XXX: verify
-	job_resrcs_ptr = job_ptr->job_resrcs;
-    kill_job->nnodes = job_resrcs_ptr->nhosts;
-	kill_job->job_node_cpus = job_resrcs_ptr->cpus;
 
 #ifdef HAVE_FRONT_END
 	if (job_ptr->batch_host &&
@@ -507,9 +503,14 @@ extern void deallocate_nodes(job_record_t *job_ptr, bool timeout,
 	agent_args->node_count = node_count;
 
 	last_node_update = time(NULL);
-	kill_job = create_kill_job_msg(job_ptr, use_protocol_version);
+    kill_job = create_kill_job_msg(job_ptr, use_protocol_version);
 	kill_job->nodes = xstrdup(job_ptr->nodes);
 
+	// XXX: verify
+	job_resrcs_ptr = job_ptr->job_resrcs;
+    kill_job->nnodes = job_resrcs_ptr->nhosts;
+	kill_job->job_node_cpus = job_resrcs_ptr->cpus;
+
 	agent_args->msg_args = kill_job;
 	set_agent_arg_r_uid(agent_args, SLURM_AUTH_UID_ANY);
 	agent_queue_request(agent_args);

From 50ed58a241891cb339c2e50d69cd86d02316dd80 Mon Sep 17 00:00:00 2001
From: Andy Georges <itkovian@gmail.com>
Date: Fri, 27 Jan 2023 16:23:28 +0100
Subject: [PATCH 19/90] fix: the actual cgroup stuff is done anyway at a later
 point

---
 contribs/pam_slurm_adopt/pam_slurm_adopt.c | 4 ----
 1 file changed, 4 deletions(-)

diff --git a/contribs/pam_slurm_adopt/pam_slurm_adopt.c b/contribs/pam_slurm_adopt/pam_slurm_adopt.c
index bf17407a92e..11f8572abce 100644
--- a/contribs/pam_slurm_adopt/pam_slurm_adopt.c
+++ b/contribs/pam_slurm_adopt/pam_slurm_adopt.c
@@ -768,7 +768,6 @@ PAM_EXTERN int _adopt_and_or_check(pam_handle_t *pamh, int flags
 	step_loc_t *stepd = NULL;
 	struct passwd pwd, *pwd_result;
 	char *buf = NULL;
-	cgroup_conf_t *cg_conf;
 
 	_init_opts();
 	_parse_opts(pamh, argc, argv);
@@ -800,9 +799,6 @@ PAM_EXTERN int _adopt_and_or_check(pam_handle_t *pamh, int flags
 		pam_syslog(pamh, LOG_ERR, "No username in PAM_USER? Fail!");
 		return PAM_SESSION_ERR;
 	}
-	if (opts.action_adopt==CALLERID_ACTION_ADOPT_AND_CHECK ){
-	   cg_conf = xcgroup_get_slurm_cgroup_conf();
-	}
 
 	/* Check for an unsafe config that might lock out root. This is a very
 	 * basic check that shouldn't be 100% relied on */

From e3630cfbc991c9689cf81202d735a26c9c9a11c1 Mon Sep 17 00:00:00 2001
From: Andy Georges <itkovian@gmail.com>
Date: Thu, 25 May 2023 14:18:21 +0200
Subject: [PATCH 20/90] fix: also pass nnodes through the message

---
 src/common/slurm_protocol_pack.c | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/src/common/slurm_protocol_pack.c b/src/common/slurm_protocol_pack.c
index 307ce04882f..b929f08b0d5 100644
--- a/src/common/slurm_protocol_pack.c
+++ b/src/common/slurm_protocol_pack.c
@@ -2544,6 +2544,7 @@ _pack_kill_job_msg(kill_job_msg_t * msg, buf_t *buffer, uint16_t protocol_versio
 					     protocol_version);
 		packstr_array(msg->spank_job_env, msg->spank_job_env_size,
 			      buffer);
+    pack32(msg->nnodes, buffer);
 		pack16_array(msg->job_node_cpus, msg->nnodes, buffer);
 		pack_time(msg->start_time, buffer);
 		pack_time(msg->time, buffer);
@@ -2561,6 +2562,7 @@ _pack_kill_job_msg(kill_job_msg_t * msg, buf_t *buffer, uint16_t protocol_versio
 					     protocol_version);
 		packstr_array(msg->spank_job_env, msg->spank_job_env_size,
 			      buffer);
+    pack32(msg->nnodes, buffer);
 		pack16_array(msg->job_node_cpus, msg->nnodes, buffer);
 		pack_time(msg->start_time, buffer);
 		pack_time(msg->time, buffer);
@@ -2578,6 +2580,7 @@ _pack_kill_job_msg(kill_job_msg_t * msg, buf_t *buffer, uint16_t protocol_versio
 					     protocol_version);
 		packstr_array(msg->spank_job_env, msg->spank_job_env_size,
 			      buffer);
+    pack32(msg->nnodes, buffer);
 		pack16_array(msg->job_node_cpus, msg->nnodes, buffer);
 		pack_time(msg->start_time, buffer);
 		pack_time(msg->time, buffer);
@@ -2624,6 +2627,7 @@ _unpack_kill_job_msg(kill_job_msg_t ** msg, buf_t *buffer,
 			goto unpack_error;
 		safe_unpackstr_array(&tmp_ptr->spank_job_env,
 				     &tmp_ptr->spank_job_env_size, buffer);
+    safe_unpack32(&tmp_ptr->nnodes, buffer);
 		safe_unpack16_array(&tmp_ptr->job_node_cpus,
 		             &tmp_ptr->nnodes,
 					 buffer);
@@ -2648,6 +2652,7 @@ _unpack_kill_job_msg(kill_job_msg_t ** msg, buf_t *buffer,
 			goto unpack_error;
 		safe_unpackstr_array(&tmp_ptr->spank_job_env,
 				     &tmp_ptr->spank_job_env_size, buffer);
+    safe_unpack32(&tmp_ptr->nnodes, buffer);
 		safe_unpack16_array(&tmp_ptr->job_node_cpus,
 		             &tmp_ptr->nnodes,
 					 buffer);
@@ -2672,6 +2677,7 @@ _unpack_kill_job_msg(kill_job_msg_t ** msg, buf_t *buffer,
 			goto unpack_error;
 		safe_unpackstr_array(&tmp_ptr->spank_job_env,
 				     &tmp_ptr->spank_job_env_size, buffer);
+    safe_unpack32(&tmp_ptr->nnodes, buffer);
 		safe_unpack16_array(&tmp_ptr->job_node_cpus,
 		             &tmp_ptr->nnodes,
 					 buffer);

From 28f749191c4c68eb4df2e7f6f6f8cda6cbd18252 Mon Sep 17 00:00:00 2001
From: Wouter Depypere <wouter.depypere@ugent.be>
Date: Wed, 6 Sep 2023 13:44:14 +0200
Subject: [PATCH 21/90] fix dependencies for rhel 88

---
 bdist_rpm.sh | 10 ++++++----
 1 file changed, 6 insertions(+), 4 deletions(-)

diff --git a/bdist_rpm.sh b/bdist_rpm.sh
index 4842aedf24b..9cb3e673c2e 100755
--- a/bdist_rpm.sh
+++ b/bdist_rpm.sh
@@ -37,6 +37,8 @@ if grep "release 7" /etc/redhat-release; then
     sudo yum install -y ucx-devel-1.5.2-1.el7.x86_64 "pmix-devel > 3.0.0" numactl-devel pmix-pmi-devel hwloc-devel-1.11.8-4.el7.x86_64
 elif grep "release 8.6" /etc/redhat-release; then
     sudo yum install -y ucx-devel-1.11.2-2.el8.x86_64 "pmix-devel > 3.1.4" numactl-devel pmix-pmi-devel hwloc-devel-2.2.0-3.el8.x86_64
+elif grep "release 8.8" /etc/redhat-release; then
+    sudo yum install -y ucx-devel-1.13.1-2.el8.x86_64 "pmix-devel > 3.1.4" numactl-devel pmix-pmi-devel hwloc-devel-2.2.0-3.el8.x86_64
 else
     sudo yum install -y ucx-devel-1.9.0-1.el8.x86_64 "pmix-devel > 3.0.0" numactl-devel pmix-pmi-devel hwloc-devel-0:1.11.9-3.el8.x86_64
 fi
@@ -51,8 +53,8 @@ sudo yum install munge-devel mariadb-devel pam-devel readline-devel
 
 # TODO: what if more than one cuda is available/installed, then the * thingies will probably not work
 # pmix-3 as rebuild from github src.rpm includes the devel rpms in the rpm
-sudo yum remove -y cuda-nvml-dev-10-1 cuda-nvml-dev-10-2 cuda-nvml-devel-11-6
-sudo yum install -y nvidia-driver-devel nvidia-driver-NVML cuda-nvml-devel-11-6
+sudo yum remove -y cuda-nvml-dev-10-1 cuda-nvml-dev-10-2 cuda-nvml-devel-11-6 cuda-nvml-devel-12-2
+sudo yum install -y nvidia-driver-devel nvidia-driver-NVML cuda-nvml-devel-12-2
 
 
 # glob expansion in list
@@ -77,8 +79,8 @@ echo "BUILDING NON GPU VERSION"
 
 # TODO: what if more than one cuda is available/installed, then the * thingies will probably not work
 # pmix-3 as rebuild from github src.rpm includes the devel rpms in the rpm
-sudo yum remove -y nvidia-driver-devel cuda-nvml-dev-10-2 nvidia-driver-NVML cuda-nvml-dev-10-1
-sudo yum remove -y nvidia-driver-devel nvidia-driver-NVML cuda-nvml-devel-11-3-11.3.58-1.x86_64
+sudo yum remove -y nvidia-driver-devel cuda-nvml-dev-10-2 nvidia-driver-NVML cuda-nvml-dev-10-1 cuda-nvml-devel-11-6 cuda-nvml-devel-12-2
+sudo yum remove -y nvidia-driver-NVML cuda-nvml-devel-11-3-11.3.58-1.x86_64
 
 
 rpmbuild --define "gittag ${GITTAG}" --define "_topdir $PWD" --with pmix --with numa --with hwloc --with mysql --with x11 --with ucx --define "gpu .nogpu" -ba SPECS/slurm.spec

From ae1fe106ef4377d08808520b416d5bf71491e885 Mon Sep 17 00:00:00 2001
From: Wouter Depypere <wouter.depypere@ugent.be>
Date: Thu, 7 Sep 2023 10:34:37 +0200
Subject: [PATCH 22/90] add assumeyes to install command

---
 bdist_rpm.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/bdist_rpm.sh b/bdist_rpm.sh
index 9cb3e673c2e..14e65131683 100755
--- a/bdist_rpm.sh
+++ b/bdist_rpm.sh
@@ -43,7 +43,7 @@ else
     sudo yum install -y ucx-devel-1.9.0-1.el8.x86_64 "pmix-devel > 3.0.0" numactl-devel pmix-pmi-devel hwloc-devel-0:1.11.9-3.el8.x86_64
 fi
 
-sudo yum install munge-devel mariadb-devel pam-devel readline-devel
+sudo yum install -y munge-devel mariadb-devel pam-devel readline-devel
 
 # there's no option to pass nvml, it is only autodetected
 # nvidia-driver-devel provides the libnividia-ml.so symlnk, the real .so.1 comes from nvidia-driver-NVML

From ecc1404f422379dbd9f2acc3edec73d3fdea8300 Mon Sep 17 00:00:00 2001
From: Wouter Depypere <wouter.depypere@ugent.be>
Date: Mon, 11 Sep 2023 05:29:43 +0200
Subject: [PATCH 23/90] bump version of pmix

---
 bdist_rpm.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/bdist_rpm.sh b/bdist_rpm.sh
index 14e65131683..3acb455a256 100755
--- a/bdist_rpm.sh
+++ b/bdist_rpm.sh
@@ -38,7 +38,7 @@ if grep "release 7" /etc/redhat-release; then
 elif grep "release 8.6" /etc/redhat-release; then
     sudo yum install -y ucx-devel-1.11.2-2.el8.x86_64 "pmix-devel > 3.1.4" numactl-devel pmix-pmi-devel hwloc-devel-2.2.0-3.el8.x86_64
 elif grep "release 8.8" /etc/redhat-release; then
-    sudo yum install -y ucx-devel-1.13.1-2.el8.x86_64 "pmix-devel > 3.1.4" numactl-devel pmix-pmi-devel hwloc-devel-2.2.0-3.el8.x86_64
+    sudo yum install -y ucx-devel-1.13.1-2.el8.x86_64 "pmix-devel > 4.2.6" numactl-devel pmix-pmi-devel hwloc-devel-2.2.0-3.el8.x86_64
 else
     sudo yum install -y ucx-devel-1.9.0-1.el8.x86_64 "pmix-devel > 3.0.0" numactl-devel pmix-pmi-devel hwloc-devel-0:1.11.9-3.el8.x86_64
 fi

From 2e42a38631f68f944bcb50dcd1ff9508679128a8 Mon Sep 17 00:00:00 2001
From: Wouter Depypere <wouter.depypere@ugent.be>
Date: Mon, 11 Sep 2023 12:09:48 +0200
Subject: [PATCH 24/90] fix pmix version and drop pmix-pmi

---
 bdist_rpm.sh | 2 +-
 slurm.spec   | 8 --------
 2 files changed, 1 insertion(+), 9 deletions(-)

diff --git a/bdist_rpm.sh b/bdist_rpm.sh
index 14e65131683..6bad7122a1f 100755
--- a/bdist_rpm.sh
+++ b/bdist_rpm.sh
@@ -38,7 +38,7 @@ if grep "release 7" /etc/redhat-release; then
 elif grep "release 8.6" /etc/redhat-release; then
     sudo yum install -y ucx-devel-1.11.2-2.el8.x86_64 "pmix-devel > 3.1.4" numactl-devel pmix-pmi-devel hwloc-devel-2.2.0-3.el8.x86_64
 elif grep "release 8.8" /etc/redhat-release; then
-    sudo yum install -y ucx-devel-1.13.1-2.el8.x86_64 "pmix-devel > 3.1.4" numactl-devel pmix-pmi-devel hwloc-devel-2.2.0-3.el8.x86_64
+    sudo yum install -y ucx-devel-1.13.1-2.el8.x86_64 "pmix-devel >= 4.2.6" numactl-devel hwloc-devel-2.2.0-3.el8.x86_64
 else
     sudo yum install -y ucx-devel-1.9.0-1.el8.x86_64 "pmix-devel > 3.0.0" numactl-devel pmix-pmi-devel hwloc-devel-0:1.11.9-3.el8.x86_64
 fi
diff --git a/slurm.spec b/slurm.spec
index 44481577730..fdc7e8eb068 100644
--- a/slurm.spec
+++ b/slurm.spec
@@ -140,11 +140,7 @@ BuildRequires: numactl-devel
 %endif
 
 %if %{with pmix} && "%{_with_pmix}" == "--with-pmix"
-%if 0%{?fedora} || 0%{?rhel} > 7
-BuildRequires: pmix-pmi-devel
-%else
 BuildRequires: pmix
-%endif
 %global pmix_version %(rpm -q pmix --qf "%{RPMTAG_VERSION}")
 %endif
 
@@ -239,12 +235,8 @@ Summary: Slurm compute node daemon
 Group: System Environment/Base
 Requires: %{name}%{?_isa} = %{version}-%{release}
 %if %{with pmix} && "%{_with_pmix}" == "--with-pmix"
-%if 0%{?fedora} || 0%{?rhel} > 7
-Requires: pmix-pmi = %{pmix_version}
-%else
 Requires: pmix = %{pmix_version}
 %endif
-%endif
 %if %{with ucx} && "%{_with_ucx}" == "--with-ucx"
 Requires: ucx = %{ucx_version}
 %endif

From cb92e5e7a81623615a4772342f4889c846f45700 Mon Sep 17 00:00:00 2001
From: Andy Georges <itkovian@gmail.com>
Date: Fri, 5 Jan 2024 15:36:41 +0100
Subject: [PATCH 25/90] fix: dependency versions for el9

---
 bdist_rpm.sh | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/bdist_rpm.sh b/bdist_rpm.sh
index 6bad7122a1f..44ea814c45d 100755
--- a/bdist_rpm.sh
+++ b/bdist_rpm.sh
@@ -39,6 +39,8 @@ elif grep "release 8.6" /etc/redhat-release; then
     sudo yum install -y ucx-devel-1.11.2-2.el8.x86_64 "pmix-devel > 3.1.4" numactl-devel pmix-pmi-devel hwloc-devel-2.2.0-3.el8.x86_64
 elif grep "release 8.8" /etc/redhat-release; then
     sudo yum install -y ucx-devel-1.13.1-2.el8.x86_64 "pmix-devel >= 4.2.6" numactl-devel hwloc-devel-2.2.0-3.el8.x86_64
+elif grep "release 9.8" /etc/redhat-release; then
+    sudo yum install -y ucx-devel-1.13.1-2.el9.x86_64 "pmix-devel >= 3.2.3" numactl-devel hwloc-devel-2.4.1-5.el9.x86_64
 else
     sudo yum install -y ucx-devel-1.9.0-1.el8.x86_64 "pmix-devel > 3.0.0" numactl-devel pmix-pmi-devel hwloc-devel-0:1.11.9-3.el8.x86_64
 fi

From b9716a20dba8626a7316cdf9ce0e2378d3796544 Mon Sep 17 00:00:00 2001
From: Andy Georges <itkovian@gmail.com>
Date: Fri, 5 Jan 2024 15:50:33 +0100
Subject: [PATCH 26/90] fix: release 9 will do

---
 bdist_rpm.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/bdist_rpm.sh b/bdist_rpm.sh
index 44ea814c45d..58f16b04fab 100755
--- a/bdist_rpm.sh
+++ b/bdist_rpm.sh
@@ -39,7 +39,7 @@ elif grep "release 8.6" /etc/redhat-release; then
     sudo yum install -y ucx-devel-1.11.2-2.el8.x86_64 "pmix-devel > 3.1.4" numactl-devel pmix-pmi-devel hwloc-devel-2.2.0-3.el8.x86_64
 elif grep "release 8.8" /etc/redhat-release; then
     sudo yum install -y ucx-devel-1.13.1-2.el8.x86_64 "pmix-devel >= 4.2.6" numactl-devel hwloc-devel-2.2.0-3.el8.x86_64
-elif grep "release 9.8" /etc/redhat-release; then
+elif grep "release 9" /etc/redhat-release; then
     sudo yum install -y ucx-devel-1.13.1-2.el9.x86_64 "pmix-devel >= 3.2.3" numactl-devel hwloc-devel-2.4.1-5.el9.x86_64
 else
     sudo yum install -y ucx-devel-1.9.0-1.el8.x86_64 "pmix-devel > 3.0.0" numactl-devel pmix-pmi-devel hwloc-devel-0:1.11.9-3.el8.x86_64

From 41833b2274030b3750f9fabb6db022fe7324ed16 Mon Sep 17 00:00:00 2001
From: Andy Georges <itkovian@gmail.com>
Date: Thu, 11 Jan 2024 15:37:39 +0100
Subject: [PATCH 27/90] fix: home built pmix

---
 bdist_rpm.sh | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/bdist_rpm.sh b/bdist_rpm.sh
index 58f16b04fab..8089b90c547 100755
--- a/bdist_rpm.sh
+++ b/bdist_rpm.sh
@@ -39,8 +39,8 @@ elif grep "release 8.6" /etc/redhat-release; then
     sudo yum install -y ucx-devel-1.11.2-2.el8.x86_64 "pmix-devel > 3.1.4" numactl-devel pmix-pmi-devel hwloc-devel-2.2.0-3.el8.x86_64
 elif grep "release 8.8" /etc/redhat-release; then
     sudo yum install -y ucx-devel-1.13.1-2.el8.x86_64 "pmix-devel >= 4.2.6" numactl-devel hwloc-devel-2.2.0-3.el8.x86_64
-elif grep "release 9" /etc/redhat-release; then
-    sudo yum install -y ucx-devel-1.13.1-2.el9.x86_64 "pmix-devel >= 3.2.3" numactl-devel hwloc-devel-2.4.1-5.el9.x86_64
+elif grep "release 9.2" /etc/redhat-release; then
+     sudo yum install -y ucx-devel-1.13.1-2.el9.x86_64 "pmix-devel >= 4.2.7" numactl-devel "hwloc-devel >= 2.4.1-5"
 else
     sudo yum install -y ucx-devel-1.9.0-1.el8.x86_64 "pmix-devel > 3.0.0" numactl-devel pmix-pmi-devel hwloc-devel-0:1.11.9-3.el8.x86_64
 fi

From c757d05eb57cb9f28faccaeb3bfc8269143b2f42 Mon Sep 17 00:00:00 2001
From: Andy Georges <itkovian@gmail.com>
Date: Wed, 21 Feb 2024 20:31:27 +0100
Subject: [PATCH 28/90] fix: merge non-conflict inroduced duplicate code

---
 src/common/slurm_protocol_pack.c | 66 ++++----------------------------
 src/slurmctld/node_scheduler.c   |  2 +
 src/slurmd/slurmd/req.c          |  5 ++-
 3 files changed, 12 insertions(+), 61 deletions(-)

diff --git a/src/common/slurm_protocol_pack.c b/src/common/slurm_protocol_pack.c
index 3ca9e7e63bd..a7b2812cff1 100644
--- a/src/common/slurm_protocol_pack.c
+++ b/src/common/slurm_protocol_pack.c
@@ -2845,31 +2845,6 @@ _pack_kill_job_msg(kill_job_msg_t * msg, buf_t *buffer, uint16_t protocol_versio
 		packstr(msg->nodes, buffer);
 		packstr_array(msg->spank_job_env, msg->spank_job_env_size,
 			      buffer);
-    pack32(msg->nnodes, buffer);
-		pack16_array(msg->job_node_cpus, msg->nnodes, buffer);
-		pack_time(msg->start_time, buffer);
-		pack_time(msg->time, buffer);
-		packstr(msg->work_dir, buffer);
-	} else if (protocol_version >= SLURM_MIN_PROTOCOL_VERSION) {
-		if (msg->cred) {
-			pack8(1, buffer);
-			slurm_cred_pack(msg->cred, buffer, protocol_version);
-		} else
-			pack8(0, buffer);
-		packstr(msg->details, buffer);
-		pack32(msg->derived_ec, buffer);
-		pack32(msg->exit_code, buffer);
-		gres_prep_pack(msg->job_gres_prep, buffer, protocol_version);
-		pack_step_id(&msg->step_id, buffer, protocol_version);
-		pack32(msg->het_job_id, buffer);
-		pack32(msg->job_state, buffer);
-		pack32(msg->job_uid, buffer);
-		pack32(msg->job_gid, buffer);
-		packstr(msg->nodes, buffer);
-		select_g_select_jobinfo_pack(NULL, buffer,
-					     protocol_version);
-		packstr_array(msg->spank_job_env, msg->spank_job_env_size,
-			      buffer);
     pack32(msg->nnodes, buffer);
 		pack16_array(msg->job_node_cpus, msg->nnodes, buffer);
 		pack_time(msg->start_time, buffer);
@@ -2973,36 +2948,9 @@ _unpack_kill_job_msg(kill_job_msg_t ** msg, buf_t *buffer,
 					 buffer);
 		safe_unpack_time(&tmp_ptr->start_time, buffer);
 		safe_unpack_time(&tmp_ptr->time, buffer);
-		safe_unpackstr_xmalloc(&tmp_ptr->work_dir, &uint32_tmp, buffer);
-	} else if (protocol_version >= SLURM_MIN_PROTOCOL_VERSION) {
-		dynamic_plugin_data_t *select_jobinfo;
-		tmp_ptr->exit_code = INFINITE;
-		if (gres_prep_unpack(&tmp_ptr->job_gres_prep,
-				     buffer, protocol_version))
-			goto unpack_error;
-		if (unpack_step_id_members(&tmp_ptr->step_id, buffer,
-					   protocol_version) != SLURM_SUCCESS)
-			goto unpack_error;
-		safe_unpack32(&tmp_ptr->het_job_id, buffer);
-		safe_unpack32(&tmp_ptr->job_state, buffer);
-		safe_unpack32(&tmp_ptr->job_uid, buffer);
-		safe_unpack32(&tmp_ptr->job_gid, buffer);
-		safe_unpackstr_xmalloc(&tmp_ptr->nodes, &uint32_tmp, buffer);
-		if (select_g_select_jobinfo_unpack(&select_jobinfo,
-						   buffer, protocol_version))
-			goto unpack_error;
-		select_g_select_jobinfo_free(select_jobinfo);
-		safe_unpackstr_array(&tmp_ptr->spank_job_env,
-				     &tmp_ptr->spank_job_env_size, buffer);
-    safe_unpack32(&tmp_ptr->nnodes, buffer);
-		safe_unpack16_array(&tmp_ptr->job_node_cpus,
-		             &tmp_ptr->nnodes,
-					 buffer);
-		safe_unpack_time(&tmp_ptr->start_time, buffer);
-		safe_unpack_time(&tmp_ptr->time, buffer);
-		safe_unpackstr_xmalloc(&tmp_ptr->work_dir, &uint32_tmp, buffer);
-	}
-
+		safe_unpackstr(&tmp_ptr->work_dir, buffer);
+  }
+	
 	return SLURM_SUCCESS;
 
 unpack_error:
@@ -8567,8 +8515,8 @@ static int _unpack_prolog_launch_msg(slurm_msg_t *smsg, buf_t *buffer)
 		safe_unpackstr_array(&msg->spank_job_env,
 				     &msg->spank_job_env_size,
 				     buffer);
-		safe_unpack16_array(&launch_msg_ptr->job_node_cpus,
-                    &launch_msg_ptr->nnodes,
+		safe_unpack16_array(&msg->job_node_cpus,
+                    &msg->nnodes,
                     buffer);
 		if (!(msg->cred = slurm_cred_unpack(buffer,
 						    smsg->protocol_version)))
@@ -8601,8 +8549,8 @@ static int _unpack_prolog_launch_msg(slurm_msg_t *smsg, buf_t *buffer)
 		safe_unpackstr_array(&msg->spank_job_env,
 				     &msg->spank_job_env_size,
 				     buffer);
-		safe_unpack16_array(&launch_msg_ptr->job_node_cpus,
-                    &launch_msg_ptr->nnodes,
+		safe_unpack16_array(&msg->job_node_cpus,
+                    &msg->nnodes,
                     buffer);
 		if (!(msg->cred = slurm_cred_unpack(buffer,
 						    smsg->protocol_version)))
diff --git a/src/slurmctld/node_scheduler.c b/src/slurmctld/node_scheduler.c
index 4a8e85e0fc3..92616673922 100644
--- a/src/slurmctld/node_scheduler.c
+++ b/src/slurmctld/node_scheduler.c
@@ -310,6 +310,8 @@ extern void deallocate_nodes(job_record_t *job_ptr, bool timeout,
 	hostlist_t *hostlist = NULL;
 	uint16_t use_protocol_version = 0;
 	uint16_t msg_flags = 0;
+  job_resources_t	*job_resrcs_ptr = NULL;
+
 #ifdef HAVE_FRONT_END
 	front_end_record_t *front_end_ptr;
 #endif
diff --git a/src/slurmd/slurmd/req.c b/src/slurmd/slurmd/req.c
index ab7bc5745a0..91491532931 100644
--- a/src/slurmd/slurmd/req.c
+++ b/src/slurmd/slurmd/req.c
@@ -2234,7 +2234,7 @@ static int _get_node_inx(char *hostlist)
 {
 	char *host;
 	int node_inx = -1;
-	hostset_t hset;
+	hostset_t *hset;
 
 	if (!conf->node_name)
 		return node_inx;
@@ -2262,6 +2262,7 @@ static void _rpc_prolog(slurm_msg_t *msg)
 	job_env_t job_env;
 	bool     first_job_run;
 	uint32_t jobid;
+  int node_inx = 0;
 
 	if (req == NULL)
 		return;
@@ -2327,7 +2328,7 @@ static void _rpc_prolog(slurm_msg_t *msg)
 		jobid = req->job_id;
 #endif
 
-        node_inx = _get_node_inx(req->nodes);
+    node_inx = _get_node_inx(req->nodes);
 		debug("_rpc_prolog: _get_node_inx returned %d", node_inx);
         job_env.job_node_cpus = (node_inx >= 0 ? req->job_node_cpus[node_inx] : 0);
 

From 2591e5d9cc0d8ff8e4d947a141b3678d0d2ebd34 Mon Sep 17 00:00:00 2001
From: Andy Georges <itkovian@gmail.com>
Date: Fri, 23 Feb 2024 14:23:26 +0100
Subject: [PATCH 29/90] fix: pmi libs location

---
 slurm.spec | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/slurm.spec b/slurm.spec
index d9d49ce15ad..261dc610bf4 100644
--- a/slurm.spec
+++ b/slurm.spec
@@ -668,7 +668,7 @@ rm -rf %{buildroot}
 
 %files libpmi
 %defattr(-,root,root)
-%if %{with cray} || %{with cray_shasta} || %{with pmix}
+%if %{with cray} || %{with cray_shasta}
 %{_libdir}/slurmpmi/*
 %else
 %{_libdir}/libpmi*

From 53314fac3cfe157c5bd2c3466cabc15ef4a6eb2b Mon Sep 17 00:00:00 2001
From: Andy Georges <itkovian@gmail.com>
Date: Mon, 2 Sep 2024 21:18:22 +0200
Subject: [PATCH 30/90] bump: dependency versions for RHEL 9.4

---
 bdist_rpm.sh | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/bdist_rpm.sh b/bdist_rpm.sh
index 8089b90c547..1377e7d075e 100755
--- a/bdist_rpm.sh
+++ b/bdist_rpm.sh
@@ -41,6 +41,8 @@ elif grep "release 8.8" /etc/redhat-release; then
     sudo yum install -y ucx-devel-1.13.1-2.el8.x86_64 "pmix-devel >= 4.2.6" numactl-devel hwloc-devel-2.2.0-3.el8.x86_64
 elif grep "release 9.2" /etc/redhat-release; then
      sudo yum install -y ucx-devel-1.13.1-2.el9.x86_64 "pmix-devel >= 4.2.7" numactl-devel "hwloc-devel >= 2.4.1-5"
+elif grep "release 9.4" /etc/redhat-release; then
+     sudo yum install -y ucx-devel-1.15.0-2.el9.x86_64 "pmix-devel >= 4.2.7" numactl-devel "hwloc-devel >= 2.4.1-5"
 else
     sudo yum install -y ucx-devel-1.9.0-1.el8.x86_64 "pmix-devel > 3.0.0" numactl-devel pmix-pmi-devel hwloc-devel-0:1.11.9-3.el8.x86_64
 fi

From 6e2aa0bb933577d1d0376f09c7601d6ed02a6967 Mon Sep 17 00:00:00 2001
From: Andy Georges <itkovian@gmail.com>
Date: Mon, 2 Sep 2024 21:34:41 +0200
Subject: [PATCH 31/90] fix: unused variables removed

---
 src/slurmd/slurmd/req.c | 5 +----
 1 file changed, 1 insertion(+), 4 deletions(-)

diff --git a/src/slurmd/slurmd/req.c b/src/slurmd/slurmd/req.c
index f7bd40c50d1..e25304a0e48 100644
--- a/src/slurmd/slurmd/req.c
+++ b/src/slurmd/slurmd/req.c
@@ -2466,10 +2466,6 @@ static void _rpc_prolog(slurm_msg_t *msg)
 {
 	int rc = SLURM_SUCCESS;
 	prolog_launch_msg_t *req = msg->data;
-	job_env_t job_env;
-	bool     first_job_run;
-	uint32_t jobid;
-    int node_inx = 0;
 
 	if (req == NULL)
 		return;
@@ -2518,6 +2514,7 @@ static void _rpc_prolog(slurm_msg_t *msg)
 	if (!(slurm_conf.prolog_flags & PROLOG_FLAG_RUN_IN_JOB)) {
 		int node_id = 0;
 		job_env_t job_env;
+		int node_inx = 0;
 
 #ifndef HAVE_FRONT_END
 		/* It is always 0 for front end systems */

From 896fc9a79bbde8ccdaa5711e71c45f215922e953 Mon Sep 17 00:00:00 2001
From: Andy Georges <itkovian@gmail.com>
Date: Mon, 2 Sep 2024 21:46:20 +0200
Subject: [PATCH 32/90] fix: undefined var

---
 src/slurmd/slurmd/req.c | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/src/slurmd/slurmd/req.c b/src/slurmd/slurmd/req.c
index e25304a0e48..371aeef7c3f 100644
--- a/src/slurmd/slurmd/req.c
+++ b/src/slurmd/slurmd/req.c
@@ -40,6 +40,7 @@
 \*****************************************************************************/
 
 #include "config.h"
+#include <stdint.h>
 
 #define _GNU_SOURCE	/* for setresuid() */
 
@@ -5538,6 +5539,7 @@ _rpc_terminate_job(slurm_msg_t *msg)
 	if (!(slurm_conf.prolog_flags & PROLOG_FLAG_RUN_IN_JOB)) {
 		int node_id = 0;
 		job_env_t job_env;
+		int node_inx = -1;
 #ifndef HAVE_FRONT_END
 		/* It is always 0 for front end systems */
 		node_id = nodelist_find(req->nodes, conf->node_name);

From 11507ff8859120f365572dc6f3a65ed8832d50ed Mon Sep 17 00:00:00 2001
From: Andy Georges <itkovian@gmail.com>
Date: Mon, 2 Sep 2024 22:05:06 +0200
Subject: [PATCH 33/90] fix: define __USE_XOPEN_EXTENDED for nftw usage

---
 src/slurmd/slurmd/req.c | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/slurmd/slurmd/req.c b/src/slurmd/slurmd/req.c
index 371aeef7c3f..af3a1d84449 100644
--- a/src/slurmd/slurmd/req.c
+++ b/src/slurmd/slurmd/req.c
@@ -43,6 +43,7 @@
 #include <stdint.h>
 
 #define _GNU_SOURCE	/* for setresuid() */
+#define __USE_XOPEN_EXTENDED
 
 #include <ctype.h>
 #include <fcntl.h>

From cb3f02729e61a93541d3ecca6ca75225582a4fda Mon Sep 17 00:00:00 2001
From: Andy Georges <itkovian@gmail.com>
Date: Wed, 2 Oct 2024 17:02:21 +0200
Subject: [PATCH 34/90] fix: ditch build id stuff

---
 slurm.spec | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/slurm.spec b/slurm.spec
index c8f91d4479a..6b387559c83 100644
--- a/slurm.spec
+++ b/slurm.spec
@@ -15,6 +15,8 @@ URL:		https://slurm.schedmd.com/
 %global slurm_source_dir %{name}-%{version}-%{rel}
 %endif
 
+%define _build_id_links none
+
 Source:		%{slurm_source_dir}.tar.gz
 
 # build options		.rpmmacros options	change to default action

From 1673154afd85d6bbe425efa598ba64cd1c7b0738 Mon Sep 17 00:00:00 2001
From: Andy Georges <itkovian@gmail.com>
Date: Wed, 2 Oct 2024 17:02:58 +0200
Subject: [PATCH 35/90] feat: VUB build script :)

---
 build.sh | 130 +++++++++++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 130 insertions(+)
 create mode 100644 build.sh

diff --git a/build.sh b/build.sh
new file mode 100644
index 00000000000..8a779d50174
--- /dev/null
+++ b/build.sh
@@ -0,0 +1,130 @@
+#!/bin/bash
+
+set -e  # exit whenever command in pipeline fails
+set -x  # print commands as they are executed
+
+VERSION=`grep "Version:.*[0-9]" slurm.spec | tr -s " " |  awk '{print $2;}'`
+RELEASE=`grep "%define rel.*[-1-9]" slurm.spec | tr -s " " | awk '{print $3}'`
+
+if [ "${RELEASE}" != "1" ]; then
+    SUFFIX=${VERSION}-${RELEASE}
+else
+    SUFFIX=${VERSION}
+fi
+
+GITTAG=$(git log --format=%ct.%h -1)
+
+SCRIPT=$(readlink -f "${BASH_SOURCE[0]}")
+ORIGIN=$(dirname "$SCRIPT")
+
+# which version to download from github
+SLURM_VERSION=${VERSION:-24.05.3}
+UPSTREAM_REL=${UPSTREAM_REL:-1}
+
+# which release should be used for our RPMs
+OUR_RELEASE=${RELEASE:-1}
+
+# NVML
+# allow _empty_ version, which is used in pipeline
+
+if grep "release 8.8" /etc/redhat-release; then
+   NVIDIA_DRIVER=${NVIDIA_DRIVER-550.90.07}
+   NVDRV_NVML_PKG="nvidia-driver-NVML${NVIDIA_DRIVER:+-$NVIDIA_DRIVER}"
+   CUDA_VERSION=${CUDA_VERSION:-12.6}
+   CUDA_NVML_PKG="cuda-nvml-devel-${CUDA_VERSION//./-}"
+elif grep "release 9.4" /etc/redhat-release; then
+    NVDRV_NVML_PKG="libnvidia-ml"
+    CUDA_VERSION=${CUDA_VERSION:-12.6}
+    CUDA_NVML_PKG="cuda-nvml-devel-${CUDA_VERSION//./-}"
+fi
+
+# Prepare directory structure
+rm -Rf $ORIGIN/rpmbuild/ $ORIGIN/dist/
+mkdir -p $ORIGIN/rpmbuild/{BUILD,RPMS,SRPMS,SOURCES} $ORIGIN/dist
+echo "Building source tarball"
+
+# archive git repo
+echo "archive suffix: ${SUFFIX}"
+git archive --format=tar.gz -o "rpmbuild/SOURCES/slurm-${SUFFIX}.tar.gz" --prefix="slurm-${SUFFIX}/" HEAD
+
+cp slurm.spec "SPECS"
+
+# Patch sources
+#for src_patch in $ORIGIN/src-patches/*.patch; do
+    #echo "Patching $src_patch"
+    #git am $src_patch
+#done
+
+# Patch spec file
+#for spec_patch in $ORIGIN/spec-patches/*.patch; do
+    #echo "Patching $spec_patch"
+    #patch -p1 -b -i $spec_patch
+#done
+
+# Install dependencies
+# see https://slurm.schedmd.com/quickstart_admin.html#build_install
+
+echo "Installing specfile requires"
+# this goes first because it might install undesired stuff related to `--with` options
+sudo dnf -y builddep slurm.spec
+
+echo "Installing dependencies"
+# - features: basic
+sudo dnf -y install lua-devel mariadb-devel lz4-devel
+# - features: authentication (MUNGE: yes, JWT: yes, PAM: yes)
+sudo dnf -y install munge-devel libjwt-devel pam-devel
+# - features: slurmrestd
+sudo dnf -y install http-parser-devel json-c-devel libyaml-devel
+# - features: Nvidia NVML
+sudo dnf -y autoremove cuda-nvml-* nvidia-driver-NVML-* nvidia-driver* libnvidia-ml*
+sudo dnf -y install "$CUDA_NVML_PKG" "$NVDRV_NVML_PKG"
+# - plugins: MPI
+sudo dnf -y install pmix ucx-devel
+# - plugins: cgroup/v2
+# see https://slurm.schedmd.com/cgroup_v2.html
+sudo dnf -y install kernel-headers dbus-devel
+# - plugins: task/cgroup, task/affinity
+sudo dnf -y install hwloc-devel numactl-devel
+# - plugins: acct_gather_profile/hdf5
+sudo dnf -y install hdf5-devel
+
+# Build defines
+RPM_DEFINES=( --define "gittag ${GITTAG}" --define "_topdir $ORIGIN/rpmbuild" )
+
+# Build options
+SLURM_BUILDOPTS=( --with slurmrestd --without debug )
+# basic features
+SLURM_BUILDOPTS+=( --with lua --with mysql --with x11 )
+# plugins
+SLURM_BUILDOPTS+=( --with numa --with hwloc --with pmix --with ucx )
+# authentication
+SLURM_BUILDOPTS+=( --with pam --with jwt )
+
+echo "Running rpmbuild (without nvml)"
+rpmbuild -ba "${RPM_DEFINES[@]}" "${SLURM_BUILDOPTS[@]}" --without nvml \
+    slurm.spec 2>&1 | tee rpmbuild-without-nvml.out
+
+
+echo "Doing rpm rebuild (without nvml)"
+for rpm in $ORIGIN/rpmbuild/RPMS/x86_64/slurm-*$SUFFIX*.rpm ; do
+    rpmrebuild --release=$GITTAG.$OUR_RELEASE.nogpu$(rpm -E '%dist') -d $ORIGIN/dist -p $rpm
+done
+
+exit 0
+
+echo "Running rpmbuild (with nvml)"
+RPM_DEFINES+=( --define "_cuda_version $CUDA_VERSION" )
+rpmbuild -ba "${RPM_DEFINES[@]}" "${SLURM_BUILDOPTS[@]}" --with nvml \
+    slurm.spec 2>&1 | tee rpmbuild-with-nvml.out
+
+echo "Doing rpm rebuild (with nvml)"
+for rpm in $ORIGIN/rpmbuild/RPMS/x86_64/slurm-*$SUFFIX*.rpm ; do
+    rpmrebuild --release=$OUR_RELEASE.$(rpm -E '%dist') -d $ORIGIN/dist -p $rpm
+done
+
+# strip out torque binaries/wrapper from slurm-torque
+rpmrebuild -d $ORIGIN/dist --change-spec-files="sed '/\(pbsnodes\|mpiexec\|bin\/q.\+\)/d'" -p $ORIGIN/dist/x86_64/slurm-torque-*-$GITTAG.$OUR_RELEASE.nogpu.$(rpm -E '%dist').*.rpm
+rpmrebuild -d $ORIGIN/dist --change-spec-files="sed '/\(pbsnodes\|mpiexec\|bin\/q.\+\)/d'" -p $ORIGIN/dist/x86_64/slurm-torque-*-$GITTAG.$OUR_RELEASE.$(rpm -E '%dist').*.rpm
+
+# get the RPMs out of the subdirectories
+find $ORIGIN/dist/ -type f -name '*.rpm' -print0 | xargs -0 -I{} mv {} $ORIGIN/dist/

From 1571f62688c37fbcb20c0f8ad6b2e13a5790ac56 Mon Sep 17 00:00:00 2001
From: Andy Georges <itkovian@gmail.com>
Date: Wed, 2 Oct 2024 17:08:04 +0200
Subject: [PATCH 36/90] fix: UGent naming scheme

---
 build.sh | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/build.sh b/build.sh
index 8a779d50174..7544e7cedec 100644
--- a/build.sh
+++ b/build.sh
@@ -22,7 +22,7 @@ SLURM_VERSION=${VERSION:-24.05.3}
 UPSTREAM_REL=${UPSTREAM_REL:-1}
 
 # which release should be used for our RPMs
-OUR_RELEASE=${RELEASE:-1}
+OUR_RELEASE=${OUR_RELEASE:-1}
 
 # NVML
 # allow _empty_ version, which is used in pipeline
@@ -107,10 +107,9 @@ rpmbuild -ba "${RPM_DEFINES[@]}" "${SLURM_BUILDOPTS[@]}" --without nvml \
 
 echo "Doing rpm rebuild (without nvml)"
 for rpm in $ORIGIN/rpmbuild/RPMS/x86_64/slurm-*$SUFFIX*.rpm ; do
-    rpmrebuild --release=$GITTAG.$OUR_RELEASE.nogpu$(rpm -E '%dist') -d $ORIGIN/dist -p $rpm
+    rpmrebuild --release=${OUR_RELEASE}.${GITTAG}.$(rpm -E '%dist').nogpu.ug -d $ORIGIN/dist -p $rpm
 done
 
-exit 0
 
 echo "Running rpmbuild (with nvml)"
 RPM_DEFINES+=( --define "_cuda_version $CUDA_VERSION" )
@@ -119,7 +118,7 @@ rpmbuild -ba "${RPM_DEFINES[@]}" "${SLURM_BUILDOPTS[@]}" --with nvml \
 
 echo "Doing rpm rebuild (with nvml)"
 for rpm in $ORIGIN/rpmbuild/RPMS/x86_64/slurm-*$SUFFIX*.rpm ; do
-    rpmrebuild --release=$OUR_RELEASE.$(rpm -E '%dist') -d $ORIGIN/dist -p $rpm
+    rpmrebuild --release=${OUR_RELEASE}.${GITTAG}.$(rpm -E '%dist').ug -d $ORIGIN/dist -p $rpm
 done
 
 # strip out torque binaries/wrapper from slurm-torque

From 3808301b06879325bc02ad2954b25d8c4c1fe2dd Mon Sep 17 00:00:00 2001
From: Andy Georges <itkovian@gmail.com>
Date: Wed, 2 Oct 2024 18:18:27 +0200
Subject: [PATCH 37/90] fix: release env var

---
 build.sh | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/build.sh b/build.sh
index 7544e7cedec..de0fbf85cec 100644
--- a/build.sh
+++ b/build.sh
@@ -22,7 +22,7 @@ SLURM_VERSION=${VERSION:-24.05.3}
 UPSTREAM_REL=${UPSTREAM_REL:-1}
 
 # which release should be used for our RPMs
-OUR_RELEASE=${OUR_RELEASE:-1}
+OUR_RELEASE=${RELEASE:-1}
 
 # NVML
 # allow _empty_ version, which is used in pipeline
@@ -107,7 +107,7 @@ rpmbuild -ba "${RPM_DEFINES[@]}" "${SLURM_BUILDOPTS[@]}" --without nvml \
 
 echo "Doing rpm rebuild (without nvml)"
 for rpm in $ORIGIN/rpmbuild/RPMS/x86_64/slurm-*$SUFFIX*.rpm ; do
-    rpmrebuild --release=${OUR_RELEASE}.${GITTAG}.$(rpm -E '%dist').nogpu.ug -d $ORIGIN/dist -p $rpm
+    rpmrebuild --release=${OUR_RELEASE}.${GITTAG}$(rpm -E '%dist').nogpu.ug -d $ORIGIN/dist -p $rpm
 done
 
 
@@ -118,12 +118,12 @@ rpmbuild -ba "${RPM_DEFINES[@]}" "${SLURM_BUILDOPTS[@]}" --with nvml \
 
 echo "Doing rpm rebuild (with nvml)"
 for rpm in $ORIGIN/rpmbuild/RPMS/x86_64/slurm-*$SUFFIX*.rpm ; do
-    rpmrebuild --release=${OUR_RELEASE}.${GITTAG}.$(rpm -E '%dist').ug -d $ORIGIN/dist -p $rpm
+    rpmrebuild --release=${OUR_RELEASE}.${GITTAG}$(rpm -E '%dist').ug -d $ORIGIN/dist -p $rpm
 done
 
 # strip out torque binaries/wrapper from slurm-torque
-rpmrebuild -d $ORIGIN/dist --change-spec-files="sed '/\(pbsnodes\|mpiexec\|bin\/q.\+\)/d'" -p $ORIGIN/dist/x86_64/slurm-torque-*-$GITTAG.$OUR_RELEASE.nogpu.$(rpm -E '%dist').*.rpm
-rpmrebuild -d $ORIGIN/dist --change-spec-files="sed '/\(pbsnodes\|mpiexec\|bin\/q.\+\)/d'" -p $ORIGIN/dist/x86_64/slurm-torque-*-$GITTAG.$OUR_RELEASE.$(rpm -E '%dist').*.rpm
+rpmrebuild -d $ORIGIN/dist --change-spec-files="sed '/\(pbsnodes\|mpiexec\|bin\/q.\+\)/d'" -p $ORIGIN/dist/x86_64/slurm-torque-*-${OUR_RELEASE}.${GITTAG}$(rpm -E '%dist').nogpu.ug*.rpm
+rpmrebuild -d $ORIGIN/dist --change-spec-files="sed '/\(pbsnodes\|mpiexec\|bin\/q.\+\)/d'" -p $ORIGIN/dist/x86_64/slurm-torque-*-${OUR_RELEASE}.${GITTAG}$(rpm -E '%dist').ug.*.rpm
 
 # get the RPMs out of the subdirectories
 find $ORIGIN/dist/ -type f -name '*.rpm' -print0 | xargs -0 -I{} mv {} $ORIGIN/dist/

From 68d701e3dd5484c2f6bcf35215f968433d3ca06b Mon Sep 17 00:00:00 2001
From: Andy Georges <itkovian@gmail.com>
Date: Wed, 2 Oct 2024 18:18:37 +0200
Subject: [PATCH 38/90] fix: move to VUB build script

---
 bdist_rpm.sh | 85 +---------------------------------------------------
 build.sh     |  0
 2 files changed, 1 insertion(+), 84 deletions(-)
 mode change 100644 => 100755 build.sh

diff --git a/bdist_rpm.sh b/bdist_rpm.sh
index 1377e7d075e..294a641273b 100755
--- a/bdist_rpm.sh
+++ b/bdist_rpm.sh
@@ -3,88 +3,5 @@
 set -e
 set -x
 
-echo "BUILDING GPU VERSION"
 
-VERSION=`grep "Version:.*[0-9]" slurm.spec | tr -s " " |  awk '{print $2;}'`
-RELEASE=`grep "%define rel.*[-1-9]" slurm.spec | tr -s " " | awk '{print $3}'`
-
-if [ "${RELEASE}" != "1" ]; then
-    SUFFIX=${VERSION}-${RELEASE}
-else
-    SUFFIX=${VERSION}
-fi
-
-GITTAG=$(git log --format=%ct.%h -1)
-
-rm -Rf BUILD SOURCES SPECS SRPMS RPMS BUILDROOT
-mkdir -p BUILD SOURCES SPECS SRPMS RPMS BUILDROOT
-
-git archive --format=tar.gz -o "SOURCES/slurm-${SUFFIX}.tar.gz" --prefix="slurm-${SUFFIX}/" HEAD
-cp slurm.spec "SPECS"
-
-# remove json-c12 -> use plain json-c?
-sudo yum install -y json-c-devel json-c
-if grep "release 7" /etc/redhat-release; then
-  #   json-c is only needed for restd and burstbuffer
-    sudo yum install -y json-c12-devel json-c12
-fi
-
-# TODO? also, specify python2 as python to require
-
-# commenting this out since it tries to update things :/
-#sudo yum install -y ucx-devel "pmix-devel > 3.0.0" numactl-devel hwloc-devel
-if grep "release 7" /etc/redhat-release; then
-    sudo yum install -y ucx-devel-1.5.2-1.el7.x86_64 "pmix-devel > 3.0.0" numactl-devel pmix-pmi-devel hwloc-devel-1.11.8-4.el7.x86_64
-elif grep "release 8.6" /etc/redhat-release; then
-    sudo yum install -y ucx-devel-1.11.2-2.el8.x86_64 "pmix-devel > 3.1.4" numactl-devel pmix-pmi-devel hwloc-devel-2.2.0-3.el8.x86_64
-elif grep "release 8.8" /etc/redhat-release; then
-    sudo yum install -y ucx-devel-1.13.1-2.el8.x86_64 "pmix-devel >= 4.2.6" numactl-devel hwloc-devel-2.2.0-3.el8.x86_64
-elif grep "release 9.2" /etc/redhat-release; then
-     sudo yum install -y ucx-devel-1.13.1-2.el9.x86_64 "pmix-devel >= 4.2.7" numactl-devel "hwloc-devel >= 2.4.1-5"
-elif grep "release 9.4" /etc/redhat-release; then
-     sudo yum install -y ucx-devel-1.15.0-2.el9.x86_64 "pmix-devel >= 4.2.7" numactl-devel "hwloc-devel >= 2.4.1-5"
-else
-    sudo yum install -y ucx-devel-1.9.0-1.el8.x86_64 "pmix-devel > 3.0.0" numactl-devel pmix-pmi-devel hwloc-devel-0:1.11.9-3.el8.x86_64
-fi
-
-sudo yum install -y munge-devel mariadb-devel pam-devel readline-devel
-
-# there's no option to pass nvml, it is only autodetected
-# nvidia-driver-devel provides the libnividia-ml.so symlnk, the real .so.1 comes from nvidia-driver-NVML
-# cuda-nvml-dev provides the nvml.h (put is part of the cuda rpms so has cud version in the name, hence the *)
-# however, it's not in the default include paths nor the slurm hardcoded ones, so use CPATH
-# nvml api is not really cuda specific, last API is from cuda 5, so any recent cuda will do
-
-# TODO: what if more than one cuda is available/installed, then the * thingies will probably not work
-# pmix-3 as rebuild from github src.rpm includes the devel rpms in the rpm
-sudo yum remove -y cuda-nvml-dev-10-1 cuda-nvml-dev-10-2 cuda-nvml-devel-11-6 cuda-nvml-devel-12-2
-sudo yum install -y nvidia-driver-devel nvidia-driver-NVML cuda-nvml-devel-12-2
-
-
-# glob expansion in list
-nvmls=(/usr/local/cuda*/targets/x86_64-linux/include)
-if [ "${#nvmls[@]}" -ne 1 ]; then
-    echo "0 or more than one nvml.h found: ${nvmls[@]}. Unsupported."
-    exit 1
-fi
-echo "nvml.h found in $nvmls[0]"
-
-rpmbuild --define "gittag ${GITTAG}" --define "_topdir $PWD" --with numa --with pmix --with hwloc --with mysql --with x11 --with ucx -ba SPECS/slurm.spec --define "_configure ./configure CPATH=${nvmls[0]}" --define "_smp_mflags CPATH=${nvmls[0]}"
-
-
-echo "BUILDING NON GPU VERSION"
-
-
-# there's no option to pass nvml, it is only autodetected
-# nvidia-driver-devel provides the libnividia-ml.so symlnk, the real .so.1 comes from nvidia-driver-NVML
-# cuda-nvml-dev provides the nvml.h (put is part of the cuda rpms so has cud version in the name, hence the *)
-# however, it's not in the default include paths nor the slurm hardcoded ones, so use CPATH
-# nvml api is not really cuda specific, last API is from cuda 5, so any recent cuda will do
-
-# TODO: what if more than one cuda is available/installed, then the * thingies will probably not work
-# pmix-3 as rebuild from github src.rpm includes the devel rpms in the rpm
-sudo yum remove -y nvidia-driver-devel cuda-nvml-dev-10-2 nvidia-driver-NVML cuda-nvml-dev-10-1 cuda-nvml-devel-11-6 cuda-nvml-devel-12-2
-sudo yum remove -y nvidia-driver-NVML cuda-nvml-devel-11-3-11.3.58-1.x86_64
-
-
-rpmbuild --define "gittag ${GITTAG}" --define "_topdir $PWD" --with pmix --with numa --with hwloc --with mysql --with x11 --with ucx --define "gpu .nogpu" -ba SPECS/slurm.spec
+./build.sh
diff --git a/build.sh b/build.sh
old mode 100644
new mode 100755

From 95bee20b6df694e019d043a129af6b2d966bf1af Mon Sep 17 00:00:00 2001
From: Andy Georges <itkovian@gmail.com>
Date: Wed, 2 Oct 2024 18:23:53 +0200
Subject: [PATCH 39/90] fix: move final RPMs to where clusterbuildrpm expects
 them

---
 build.sh | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/build.sh b/build.sh
index de0fbf85cec..1b52f0c0934 100755
--- a/build.sh
+++ b/build.sh
@@ -126,4 +126,5 @@ rpmrebuild -d $ORIGIN/dist --change-spec-files="sed '/\(pbsnodes\|mpiexec\|bin\/
 rpmrebuild -d $ORIGIN/dist --change-spec-files="sed '/\(pbsnodes\|mpiexec\|bin\/q.\+\)/d'" -p $ORIGIN/dist/x86_64/slurm-torque-*-${OUR_RELEASE}.${GITTAG}$(rpm -E '%dist').ug.*.rpm
 
 # get the RPMs out of the subdirectories
-find $ORIGIN/dist/ -type f -name '*.rpm' -print0 | xargs -0 -I{} mv {} $ORIGIN/dist/
+find rpmbuild -type f -name "*.rpm" -exec rm {} ";"
+find $ORIGIN/dist/ -type f -name '*.rpm' -print0 | xargs -0 -I{} mv {} rpmbuild/RPMS/x86_64/

From d310bf3c86c7479271aadd09e962e663dbb8ab60 Mon Sep 17 00:00:00 2001
From: Andy Georges <itkovian@gmail.com>
Date: Wed, 2 Oct 2024 19:30:06 +0200
Subject: [PATCH 40/90] fix: dependency versions

---
 build.sh | 23 +++++++++++++++++++++--
 1 file changed, 21 insertions(+), 2 deletions(-)

diff --git a/build.sh b/build.sh
index 1b52f0c0934..bc2a9453852 100755
--- a/build.sh
+++ b/build.sh
@@ -68,6 +68,25 @@ echo "Installing specfile requires"
 # this goes first because it might install undesired stuff related to `--with` options
 sudo dnf -y builddep slurm.spec
 
+# dependecy versions
+
+if grep "release 8.8" /etc/redhat-release; then
+    UCX_VERSION="1.13.1-2.el8.x86_64"
+    PMIX_VERSION=">= 4.2.6"
+    HWLOC_VERSION=">= 2.2.0-3"
+elif grep "release 9.2" /etc/redhat-release; then
+    UCX_VERSION="1.13.1-2.el9.x86_64"
+    PMIX_VERSION=">= 4.2.7"
+    HWLOC_VERSION=">= 2.4.1-5"
+elif grep "release 9.4" /etc/redhat-release; then
+    UCX_VERSION="1.15.0-2.el9.x86_64"
+    PMIX_VERSION=">= 4.2.7"
+    HWLOC_VERSION=">= 2.4.1-5"
+else
+    echo "unsupported OS release"
+    exit 1
+fi
+
 echo "Installing dependencies"
 # - features: basic
 sudo dnf -y install lua-devel mariadb-devel lz4-devel
@@ -79,12 +98,12 @@ sudo dnf -y install http-parser-devel json-c-devel libyaml-devel
 sudo dnf -y autoremove cuda-nvml-* nvidia-driver-NVML-* nvidia-driver* libnvidia-ml*
 sudo dnf -y install "$CUDA_NVML_PKG" "$NVDRV_NVML_PKG"
 # - plugins: MPI
-sudo dnf -y install pmix ucx-devel
+sudo dnf -y install pmix "pmix-devel ${PMIX_VERSION}"  "ucx-devel-${UCX_VERSION}"
 # - plugins: cgroup/v2
 # see https://slurm.schedmd.com/cgroup_v2.html
 sudo dnf -y install kernel-headers dbus-devel
 # - plugins: task/cgroup, task/affinity
-sudo dnf -y install hwloc-devel numactl-devel
+sudo dnf -y install "hwloc-devel ${HWLOC_VERSION}" numactl-devel
 # - plugins: acct_gather_profile/hdf5
 sudo dnf -y install hdf5-devel
 

From d668f9e61d7e5c1dec0f030d1d7132c87df971e9 Mon Sep 17 00:00:00 2001
From: Andy Georges <itkovian@gmail.com>
Date: Thu, 3 Oct 2024 15:20:14 +0200
Subject: [PATCH 41/90] fix: re-enable build-ids

---
 slurm.spec | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/slurm.spec b/slurm.spec
index 6b387559c83..c808073cedb 100644
--- a/slurm.spec
+++ b/slurm.spec
@@ -15,7 +15,8 @@ URL:		https://slurm.schedmd.com/
 %global slurm_source_dir %{name}-%{version}-%{rel}
 %endif
 
-%define _build_id_links none
+#  We do not need this, but then duplicate file warning will pop up
+#  %define _build_id_links none
 
 Source:		%{slurm_source_dir}.tar.gz
 

From 5880b0e3f712c8f1c4d8a02545dc53cc137ad812 Mon Sep 17 00:00:00 2001
From: Andy Georges <itkovian@gmail.com>
Date: Thu, 14 Nov 2024 17:49:24 +0100
Subject: [PATCH 42/90] fix: cleanup duplicate code

---
 src/common/slurm_protocol_pack.c | 65 +-------------------------------
 1 file changed, 2 insertions(+), 63 deletions(-)

diff --git a/src/common/slurm_protocol_pack.c b/src/common/slurm_protocol_pack.c
index e245263b12e..3622cc0e217 100644
--- a/src/common/slurm_protocol_pack.c
+++ b/src/common/slurm_protocol_pack.c
@@ -2531,36 +2531,11 @@ _pack_kill_job_msg(kill_job_msg_t * msg, buf_t *buffer, uint16_t protocol_versio
 		packstr(msg->nodes, buffer);
 		packstr_array(msg->spank_job_env, msg->spank_job_env_size,
 			      buffer);
-    pack32(msg->nnodes, buffer);
+        pack32(msg->nnodes, buffer);
 		pack16_array(msg->job_node_cpus, msg->nnodes, buffer);
 		pack_time(msg->start_time, buffer);
 		pack_time(msg->time, buffer);
 		packstr(msg->work_dir, buffer);
-	} else if (protocol_version >= SLURM_MIN_PROTOCOL_VERSION) {
-		if (msg->cred) {
-			pack8(1, buffer);
-			slurm_cred_pack(msg->cred, buffer, protocol_version);
-		} else
-			pack8(0, buffer);
-		packstr(msg->details, buffer);
-		pack32(msg->derived_ec, buffer);
-		pack32(msg->exit_code, buffer);
-		gres_prep_pack(msg->job_gres_prep, buffer, protocol_version);
-		pack_step_id(&msg->step_id, buffer, protocol_version);
-		pack32(msg->het_job_id, buffer);
-		pack32(msg->job_state, buffer);
-		pack32(msg->job_uid, buffer);
-		pack32(msg->job_gid, buffer);
-		packstr(msg->nodes, buffer);
-		select_g_select_jobinfo_pack(NULL, buffer,
-					     protocol_version);
-		packstr_array(msg->spank_job_env, msg->spank_job_env_size,
-			      buffer);
-		pack32(msg->nnodes, buffer);
-        pack16_array(msg->job_node_cpus, msg->nnodes, buffer);
-		pack_time(msg->start_time, buffer);
-		pack_time(msg->time, buffer);
-		packstr(msg->work_dir, buffer);
 	}
 }
 
@@ -2600,49 +2575,13 @@ _unpack_kill_job_msg(kill_job_msg_t ** msg, buf_t *buffer,
 		safe_unpackstr(&tmp_ptr->nodes, buffer);
 		safe_unpackstr_array(&tmp_ptr->spank_job_env,
 				     &tmp_ptr->spank_job_env_size, buffer);
-    safe_unpack32(&tmp_ptr->nnodes, buffer);
+        safe_unpack32(&tmp_ptr->nnodes, buffer);
 		safe_unpack16_array(&tmp_ptr->job_node_cpus,
 		             &tmp_ptr->nnodes,
 					 buffer);
 		safe_unpack_time(&tmp_ptr->start_time, buffer);
 		safe_unpack_time(&tmp_ptr->time, buffer);
 		safe_unpackstr(&tmp_ptr->work_dir, buffer);
-	} else if (protocol_version >= SLURM_MIN_PROTOCOL_VERSION) {
-		dynamic_plugin_data_t *select_jobinfo;
-		safe_unpack8(&uint8_tmp, buffer);
-		if (uint8_tmp) {
-			tmp_ptr->cred = slurm_cred_unpack(buffer,
-							  protocol_version);
-			if (!tmp_ptr->cred)
-				goto unpack_error;
-		}
-		safe_unpackstr(&tmp_ptr->details, buffer);
-		safe_unpack32(&tmp_ptr->derived_ec, buffer);
-		safe_unpack32(&tmp_ptr->exit_code, buffer);
-		if (gres_prep_unpack(&tmp_ptr->job_gres_prep,
-				     buffer, protocol_version))
-			goto unpack_error;
-		if (unpack_step_id_members(&tmp_ptr->step_id, buffer,
-					   protocol_version))
-			goto unpack_error;
-		safe_unpack32(&tmp_ptr->het_job_id, buffer);
-		safe_unpack32(&tmp_ptr->job_state, buffer);
-		safe_unpack32(&tmp_ptr->job_uid, buffer);
-		safe_unpack32(&tmp_ptr->job_gid, buffer);
-		safe_unpackstr(&tmp_ptr->nodes, buffer);
-		if (select_g_select_jobinfo_unpack(&select_jobinfo,
-						   buffer, protocol_version))
-			goto unpack_error;
-		select_g_select_jobinfo_free(select_jobinfo);
-		safe_unpackstr_array(&tmp_ptr->spank_job_env,
-				     &tmp_ptr->spank_job_env_size, buffer);
-		safe_unpack32(&tmp_ptr->nnodes, buffer);
-        safe_unpack16_array(&tmp_ptr->job_node_cpus,
-            &tmp_ptr->nnodes,
-	    	buffer);
-	    safe_unpack_time(&tmp_ptr->start_time, buffer);
-		safe_unpack_time(&tmp_ptr->time, buffer);
-		safe_unpackstr(&tmp_ptr->work_dir, buffer);
 	}
 
 	return SLURM_SUCCESS;

From 4ce1b8d24d59bce6dcb674795add262fd9d77ee0 Mon Sep 17 00:00:00 2001
From: Andy Georges <itkovian@gmail.com>
Date: Fri, 15 Nov 2024 11:46:14 +0100
Subject: [PATCH 43/90] fix: pack and unpack the job_node_cpus array also for
 24.05

---
 src/common/slurm_protocol_pack.c | 9 +++++++--
 1 file changed, 7 insertions(+), 2 deletions(-)

diff --git a/src/common/slurm_protocol_pack.c b/src/common/slurm_protocol_pack.c
index 3622cc0e217..7e02604ea29 100644
--- a/src/common/slurm_protocol_pack.c
+++ b/src/common/slurm_protocol_pack.c
@@ -8251,6 +8251,8 @@ static void _pack_prolog_launch_msg(const slurm_msg_t *smsg, buf_t *buffer)
 
 		packstr_array(msg->spank_job_env, msg->spank_job_env_size,
 			      buffer);
+		xassert(msg->nnodes > 0);
+        pack16_array(msg->job_node_cpus, msg->nnodes, buffer);
 		slurm_cred_pack(msg->cred, buffer, smsg->protocol_version);
 
 		if (msg->job_ptr_buf) {
@@ -8285,7 +8287,7 @@ static void _pack_prolog_launch_msg(const slurm_msg_t *smsg, buf_t *buffer)
 		packstr_array(msg->spank_job_env, msg->spank_job_env_size,
 			      buffer);
 		xassert(msg->nnodes > 0);
-    pack16_array(msg->job_node_cpus, msg->nnodes, buffer);
+        pack16_array(msg->job_node_cpus, msg->nnodes, buffer);
 		slurm_cred_pack(msg->cred, buffer, smsg->protocol_version);
 	} else if (smsg->protocol_version >= SLURM_MIN_PROTOCOL_VERSION) {
 		gres_prep_pack(msg->job_gres_prep, buffer,
@@ -8311,7 +8313,7 @@ static void _pack_prolog_launch_msg(const slurm_msg_t *smsg, buf_t *buffer)
 		packstr_array(msg->spank_job_env, msg->spank_job_env_size,
 			      buffer);
 		xassert(msg->nnodes > 0);
-    pack16_array(msg->job_node_cpus, msg->nnodes, buffer);
+        pack16_array(msg->job_node_cpus, msg->nnodes, buffer);
 		slurm_cred_pack(msg->cred, buffer, smsg->protocol_version);
 		packstr(msg->user_name_deprecated, buffer);
 	}
@@ -8346,6 +8348,9 @@ static int _unpack_prolog_launch_msg(slurm_msg_t *smsg, buf_t *buffer)
 		safe_unpackstr_array(&msg->spank_job_env,
 				     &msg->spank_job_env_size,
 				     buffer);
+		safe_unpack16_array(&msg->job_node_cpus,
+                    &msg->nnodes,
+                    buffer);
 		if (!(msg->cred = slurm_cred_unpack(buffer,
 						    smsg->protocol_version)))
 			goto unpack_error;

From 27d83b999a631ca6a15106c975c86c428fdd9a1a Mon Sep 17 00:00:00 2001
From: Stephen Kendall <stephen@schedmd.com>
Date: Tue, 19 Nov 2024 10:50:06 -0600
Subject: [PATCH 44/90] Docs - Add multiple runtime example to containers page

Ticket 18572
---
 doc/html/containers.shtml | 58 ++++++++++++++++++++++++++++++++++++++-
 1 file changed, 57 insertions(+), 1 deletion(-)

diff --git a/doc/html/containers.shtml b/doc/html/containers.shtml
index fcbb184fa97..e8cea0253bd 100644
--- a/doc/html/containers.shtml
+++ b/doc/html/containers.shtml
@@ -353,6 +353,62 @@ exit $rc
 </pre>
 </p>
 
+<h3 id="multiple-runtimes">Handling multiple runtimes
+<a class="slurm_link" href="#multiple-runtimes"></a>
+</h3>
+
+<p>If you wish to accommodate multiple runtimes in your environment,
+it is possible to do so with a bit of extra setup. This section outlines one
+possible way to do so:</p>
+
+<ol>
+<li>Create a generic oci.conf that calls a wrapper script
+<pre>
+IgnoreFileConfigJson=true
+RunTimeRun="/opt/slurm-oci/run %b %m %u %U %n %j %s %t %@"
+RunTimeKill="kill -s SIGTERM %p"
+RunTimeDelete="kill -s SIGKILL %p"
+</pre>
+</li>
+<li>Create the wrapper script to check for user-specific run configuration
+(e.g., /opt/slurm-oci/run)
+<pre>
+#!/bin/bash
+if [[ -e ~/.slurm-oci-run ]]; then
+	~/.slurm-oci-run "$@"
+else
+	/opt/slurm-oci/slurm-oci-run-default "$@"
+fi
+</pre>
+</li>
+<li>Create a generic run configuration to use as the default
+(e.g., /opt/slurm-oci/slurm-oci-run-default)
+<pre>
+#!/bin/bash --login
+# Parse
+CONTAINER="$1"
+SPOOL_DIR="$2"
+USER_NAME="$3"
+USER_ID="$4"
+NODE_NAME="$5"
+JOB_ID="$6"
+STEP_ID="$7"
+TASK_ID="$8"
+shift 8 # subsequent arguments are the command to run in the container
+# Run
+apptainer run --bind /var/spool --containall "$CONTAINER" "$@"
+</pre>
+</li>
+<li>Add executable permissions to both scripts
+<pre>chmod +x /opt/slurm-oci/run /opt/slurm-oci/slurm-oci-run-default</pre>
+</li>
+</ol>
+
+<p>Once this is done, users may create a script at '~/.slurm-oci-run' if
+they wish to customize the container run process, such as using a different
+container runtime. Users should model this file after the default
+'/opt/slurm-oci/slurm-oci-run-default'</p>
+
 <h2 id="testing">Testing OCI runtime outside of Slurm
 <a class="slurm_link" href="#testing"></a>
 </h2>
@@ -875,6 +931,6 @@ Overview slides of Sarus are
 
 <hr size=4 width="100%">
 
-<p style="text-align:center;">Last modified 08 October 2024</p>
+<p style="text-align:center;">Last modified 19 November 2024</p>
 
 <!--#include virtual="footer.txt"-->

From 57e3a1241844f82dcedfc7b2f5e000599d6779a1 Mon Sep 17 00:00:00 2001
From: Stephen Kendall <stephen@schedmd.com>
Date: Tue, 19 Nov 2024 13:02:20 -0700
Subject: [PATCH 45/90] Docs - Reword FAQ entry on clock synchronization

Ticket 21464
---
 doc/html/faq.shtml | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/doc/html/faq.shtml b/doc/html/faq.shtml
index 9f7d09449e2..4d53e124d4f 100644
--- a/doc/html/faq.shtml
+++ b/doc/html/faq.shtml
@@ -1231,9 +1231,9 @@ that node may be rendered unusable, but no other harm will result.</p>
 
 <p><a id="clock"><b>Do I need to maintain synchronized
 clocks on the cluster?</b></a><br>
-In general, yes. Having inconsistent clocks may cause nodes to
-be unusable. Slurm log files should contain references to
-expired credentials. For example:</p>
+In general, yes. Having inconsistent clocks may cause nodes to be unusable and
+generate errors in Slurm log files regarding expired credentials. For example:
+</p>
 <pre>
 error: Munge decode failed: Expired credential
 ENCODED: Wed May 12 12:34:56 2008
@@ -2438,6 +2438,6 @@ dset TV::parallel_configs {
 }
 !-->
 
-<p style="text-align:center;">Last modified 07 November 2024</p>
+<p style="text-align:center;">Last modified 19 November 2024</p>
 
 <!--#include virtual="footer.txt"-->

From a6bbb70aaf77e86034597948fbd10036a30f7bf0 Mon Sep 17 00:00:00 2001
From: Stephen Kendall <stephen@schedmd.com>
Date: Wed, 9 Oct 2024 11:03:25 -0600
Subject: [PATCH 46/90] Docs - update v0.0.41 descriptions indicating format
 lists are ignored

Ticket 20302
---
 src/plugins/data_parser/v0.0.41/parsers.c | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/src/plugins/data_parser/v0.0.41/parsers.c b/src/plugins/data_parser/v0.0.41/parsers.c
index 9d37e8d7dbe..ee8edb9ba21 100644
--- a/src/plugins/data_parser/v0.0.41/parsers.c
+++ b/src/plugins/data_parser/v0.0.41/parsers.c
@@ -8423,7 +8423,7 @@ static const parser_t PARSER_ARRAY(OPENAPI_WARNING)[] = {
 static const parser_t PARSER_ARRAY(INSTANCE_CONDITION)[] = {
 	add_parse(CSV_STRING_LIST, cluster_list, "cluster", "CSV clusters list"),
 	add_parse(CSV_STRING_LIST, extra_list, "extra", "CSV extra list"),
-	add_parse(CSV_STRING_LIST, format_list, "format", "CSV format list"),
+	add_parse(CSV_STRING_LIST, format_list, "format", "Ignored; process JSON manually to control output format"),
 	add_parse(CSV_STRING_LIST, instance_id_list, "instance_id", "CSV instance_id list"),
 	add_parse(CSV_STRING_LIST, instance_type_list, "instance_type", "CSV instance_type list"),
 	add_parse(STRING, node_list, "node_list", "Ranged node string"),
@@ -8501,7 +8501,7 @@ static const parser_t PARSER_ARRAY(JOB_CONDITION)[] = {
 	add_flags(JOB_CONDITION_DB_FLAGS, db_flags),
 	add_parse(INT32, exitcode, "exit_code", "Job exit code (numeric)"),
 	add_flags(JOB_CONDITION_FLAGS, flags),
-	add_parse(CSV_STRING_LIST, format_list, "format", "CSV format list"),
+	add_parse(CSV_STRING_LIST, format_list, "format", "Ignored; process JSON manually to control output format"),
 	add_parse(GROUP_ID_STRING_LIST, groupid_list, "groups", "CSV group list"),
 	add_parse(CSV_STRING_LIST, jobname_list, "job_name", "CSV job name list"),
 	add_parse(UINT32_NO_VAL, nodes_max, "nodes_max", "Maximum number of nodes"),
@@ -8531,7 +8531,7 @@ static const parser_t PARSER_ARRAY(JOB_CONDITION)[] = {
 static const parser_t PARSER_ARRAY(QOS_CONDITION)[] = {
 	add_parse(CSV_STRING_LIST, description_list, "description", "CSV description list"),
 	add_parse(QOS_ID_STRING_CSV_LIST, id_list, "id", "CSV QOS id list"),
-	add_parse(CSV_STRING_LIST, format_list, "format", "CSV format list"),
+	add_parse(CSV_STRING_LIST, format_list, "format", "Ignored; process JSON manually to control output format"),
 	add_parse(QOS_NAME_CSV_LIST, name_list, "name", "CSV QOS name list"),
 	add_parse_bit_flag_array(slurmdb_qos_cond_t, QOS_PREEMPT_MODES, false, preempt_mode, "preempt_mode", "PreemptMode used when jobs in this QOS are preempted"),
 	add_parse(BOOL16, with_deleted, "with_deleted", "Include deleted QOS"),
@@ -8590,7 +8590,7 @@ static const parser_t PARSER_ARRAY(ASSOC_CONDITION)[] = {
 	add_parse(CSV_STRING_LIST, acct_list, "account", "CSV accounts list"),
 	add_parse(CSV_STRING_LIST, cluster_list, "cluster", "CSV clusters list"),
 	add_parse(QOS_ID_STRING_CSV_LIST, def_qos_id_list, "default_qos", "CSV QOS list"),
-	add_parse(CSV_STRING_LIST, format_list, "format", "CSV format list"),
+	add_parse(CSV_STRING_LIST, format_list, "format", "Ignored; process JSON manually to control output format"),
 	add_parse(ASSOC_ID_STRING_CSV_LIST, id_list, "id", "CSV id list"),
 	add_parse(BOOL16, only_defs, "only_defaults", "Filter to only defaults"),
 	add_parse(CSV_STRING_LIST, parent_acct_list, "parent_account", "CSV names of parent account"),
@@ -8658,7 +8658,7 @@ static const parser_t PARSER_ARRAY(OPENAPI_WCKEY_PARAM)[] = {
 	add_parser(slurmdb_wckey_cond_t, mtype, false, field, 0, path, desc)
 static const parser_t PARSER_ARRAY(WCKEY_CONDITION)[] = {
 	add_parse(CSV_STRING_LIST, cluster_list, "cluster", "CSV cluster name list"),
-	add_parse(CSV_STRING_LIST, format_list, "format", "CSV format name list"),
+	add_parse(CSV_STRING_LIST, format_list, "format", "Ignored; process JSON manually to control output format"),
 	add_parse(CSV_STRING_LIST, id_list, "id", "CSV id list"),
 	add_parse(CSV_STRING_LIST, name_list, "name", "CSV name list"),
 	add_parse(BOOL16, only_defs, "only_defaults", "Only query defaults"),
@@ -8716,7 +8716,7 @@ static const parser_t PARSER_ARRAY(CLUSTER_CONDITION)[] = {
 	add_parse(STRING_LIST, cluster_list, "cluster", "CSV cluster list"),
 	add_parse(STRING_LIST, federation_list, "federation", "CSV federation list"),
 	add_parse_bit_flag_array(slurmdb_cluster_cond_t, CLUSTER_REC_FLAGS, false, flags, "flags", "Query flags"),
-	add_parse(STRING_LIST, format_list, "format", "CSV format list"),
+	add_parse(STRING_LIST, format_list, "format", "Ignored; process JSON manually to control output format"),
 	add_parse(STRING_LIST, rpc_version_list, "rpc_version", "CSV RPC version list"),
 	add_parse(TIMESTAMP, usage_end, "usage_end", "Usage end (UNIX timestamp)"),
 	add_parse(TIMESTAMP, usage_start, "usage_start", "Usage start (UNIX timestamp)"),

From c8f00b6298aedb7c6074e502732a4093c1c1cff2 Mon Sep 17 00:00:00 2001
From: Stephen Kendall <stephen@schedmd.com>
Date: Wed, 9 Oct 2024 11:04:42 -0600
Subject: [PATCH 47/90] Docs - update v0.0.40 descriptions indicating format
 lists are ignored

Ticket 20302
---
 src/plugins/data_parser/v0.0.40/parsers.c | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/src/plugins/data_parser/v0.0.40/parsers.c b/src/plugins/data_parser/v0.0.40/parsers.c
index 54f1ef33636..fe0437de58b 100644
--- a/src/plugins/data_parser/v0.0.40/parsers.c
+++ b/src/plugins/data_parser/v0.0.40/parsers.c
@@ -8230,7 +8230,7 @@ static const parser_t PARSER_ARRAY(OPENAPI_WARNING)[] = {
 static const parser_t PARSER_ARRAY(INSTANCE_CONDITION)[] = {
 	add_parse(CSV_STRING_LIST, cluster_list, "cluster", "CSV clusters list"),
 	add_parse(CSV_STRING_LIST, extra_list, "extra", "CSV extra list"),
-	add_parse(CSV_STRING_LIST, format_list, "format", "CSV format list"),
+	add_parse(CSV_STRING_LIST, format_list, "format", "Ignored; process JSON manually to control output format"),
 	add_parse(CSV_STRING_LIST, instance_id_list, "instance_id", "CSV instance_id list"),
 	add_parse(CSV_STRING_LIST, instance_type_list, "instance_type", "CSV instance_type list"),
 	add_parse(STRING, node_list, "node_list", "Ranged node string"),
@@ -8305,7 +8305,7 @@ static const parser_t PARSER_ARRAY(JOB_CONDITION)[] = {
 	add_flags(JOB_CONDITION_DB_FLAGS, db_flags),
 	add_parse(INT32, exitcode, "exit_code", "Job exit code (numeric)"),
 	add_flags(JOB_CONDITION_FLAGS, flags),
-	add_parse(CSV_STRING_LIST, format_list, "format", "CSV format list"),
+	add_parse(CSV_STRING_LIST, format_list, "format", "Ignored; process JSON manually to control output format"),
 	add_parse(GROUP_ID_STRING_LIST, groupid_list, "groups", "CSV group list"),
 	add_parse(CSV_STRING_LIST, jobname_list, "job_name", "CSV job name list"),
 	add_parse(UINT32_NO_VAL, nodes_max, "nodes_max", "Maximum number of nodes"),
@@ -8335,7 +8335,7 @@ static const parser_t PARSER_ARRAY(JOB_CONDITION)[] = {
 static const parser_t PARSER_ARRAY(QOS_CONDITION)[] = {
 	add_parse(CSV_STRING_LIST, description_list, "description", "CSV description list"),
 	add_parse(QOS_ID_STRING_CSV_LIST, id_list, "id", "CSV QOS id list"),
-	add_parse(CSV_STRING_LIST, format_list, "format", "CSV format list"),
+	add_parse(CSV_STRING_LIST, format_list, "format", "Ignored; process JSON manually to control output format"),
 	add_parse(QOS_NAME_CSV_LIST, name_list, "name", "CSV QOS name list"),
 	add_parse_bit_flag_array(slurmdb_qos_cond_t, QOS_PREEMPT_MODES, false, preempt_mode, "preempt_mode", "PreemptMode used when jobs in this QOS are preempted"),
 	add_parse(BOOL16, with_deleted, "with_deleted", "Include deleted QOS"),
@@ -8394,7 +8394,7 @@ static const parser_t PARSER_ARRAY(ASSOC_CONDITION)[] = {
 	add_parse(CSV_STRING_LIST, acct_list, "account", "CSV accounts list"),
 	add_parse(CSV_STRING_LIST, cluster_list, "cluster", "CSV clusters list"),
 	add_parse(QOS_ID_STRING_CSV_LIST, def_qos_id_list, "default_qos", "CSV QOS list"),
-	add_parse(CSV_STRING_LIST, format_list, "format", "CSV format list"),
+	add_parse(CSV_STRING_LIST, format_list, "format", "Ignored; process JSON manually to control output format"),
 	add_parse(ASSOC_ID_STRING_CSV_LIST, id_list, "id", "CSV id list"),
 	add_parse(BOOL16, only_defs, "only_defaults", "Filter to only defaults"),
 	add_parse(CSV_STRING_LIST, parent_acct_list, "parent_account", "CSV names of parent account"),
@@ -8462,7 +8462,7 @@ static const parser_t PARSER_ARRAY(OPENAPI_WCKEY_PARAM)[] = {
 	add_parser(slurmdb_wckey_cond_t, mtype, false, field, 0, path, desc)
 static const parser_t PARSER_ARRAY(WCKEY_CONDITION)[] = {
 	add_parse(CSV_STRING_LIST, cluster_list, "cluster", "CSV cluster name list"),
-	add_parse(CSV_STRING_LIST, format_list, "format", "CSV format name list"),
+	add_parse(CSV_STRING_LIST, format_list, "format", "Ignored; process JSON manually to control output format"),
 	add_parse(CSV_STRING_LIST, id_list, "id", "CSV id list"),
 	add_parse(CSV_STRING_LIST, name_list, "name", "CSV name list"),
 	add_parse(BOOL16, only_defs, "only_defaults", "Only query defaults"),
@@ -8525,7 +8525,7 @@ static const parser_t PARSER_ARRAY(CLUSTER_CONDITION)[] = {
 	add_parse(STRING_LIST, cluster_list, "cluster", "CSV cluster list"),
 	add_parse(STRING_LIST, federation_list, "federation", "CSV federation list"),
 	add_parse_bit_flag_array(slurmdb_cluster_cond_t, CLUSTER_REC_FLAGS, false, flags, "flags", "Query flags"),
-	add_parse(STRING_LIST, format_list, "format", "CSV format list"),
+	add_parse(STRING_LIST, format_list, "format", "Ignored; process JSON manually to control output format"),
 	add_parse(STRING_LIST, rpc_version_list, "rpc_version", "CSV RPC version list"),
 	add_parse(TIMESTAMP, usage_end, "usage_end", "Usage end (UNIX timestamp)"),
 	add_parse(TIMESTAMP, usage_start, "usage_start", "Usage start (UNIX timestamp)"),

From bfcb9e3959ce140a59d7b9f0adc7aedb31318df2 Mon Sep 17 00:00:00 2001
From: Ben Roberts <ben@schedmd.com>
Date: Mon, 21 Oct 2024 11:59:52 -0500
Subject: [PATCH 48/90] Docs - Add srun example about exclusive access with
 number of tasks

Ticket 21193

Signed-off-by: Stephen Kendall <stephen@schedmd.com>
---
 doc/man/man1/srun.1 | 31 ++++++++++++++++++++++++++++++-
 1 file changed, 30 insertions(+), 1 deletion(-)

diff --git a/doc/man/man1/srun.1 b/doc/man/man1/srun.1
index 3f6b128bc9f..fc2ff9d03f3 100644
--- a/doc/man/man1/srun.1
+++ b/doc/man/man1/srun.1
@@ -1,4 +1,4 @@
-.TH srun "1" "Slurm Commands" "October 2024" "Slurm Commands"
+.TH srun "1" "Slurm Commands" "November 2024" "Slurm Commands"
 
 .SH "NAME"
 srun \- Run parallel jobs
@@ -4647,6 +4647,35 @@ This behavior can be changed by adding \fBSelectTypeParameters=CR_Pack_Nodes\fR
 to your slurm.conf. The logic to pack nodes will allow job steps to start on
 a single node without having to explicitly request a single node.
 
+.TP
+\fBExample 11:\fR
+This example demonstrates that adding the \fB\-\-exclusive\fR flag to job
+allocation requests can give different results based on whether you also
+request a certain number of tasks.
+
+Requesting exclusive access with no additional requirements will allow the
+process to access all the CPUs on the allocated node.
+.nf
+$ srun \-l \-\-exclusive  bash \-c 'grep Cpus_allowed_list /proc/self/status'
+0: Cpus_allowed_list:   0\-23
+.fi
+
+Adding a request for a certain number of tasks will cause each task to only
+have access to a single CPU.
+.nf
+$ srun \-l \-\-exclusive \-n2 bash \-c 'grep Cpus_allowed_list /proc/self/status'
+0: Cpus_allowed_list:   0
+1: Cpus_allowed_list:   12
+.fi
+
+You can define the number of CPUs per task if you want to give them access to
+more than one CPU.
+.nf
+$ srun \-l \-\-exclusive \-n2 \-\-cpus\-per\-task=12 bash \-c 'grep Cpus_allowed_list /proc/self/status'
+0: Cpus_allowed_list:   0\-5,12\-17
+1: Cpus_allowed_list:   6\-11,18\-23
+.fi
+
 .SH "COPYING"
 Copyright (C) 2006\-2007 The Regents of the University of California.
 Produced at Lawrence Livermore National Laboratory (cf, DISCLAIMER).

From d34898f2c38995c49190c6bfb7e4c37a00caea62 Mon Sep 17 00:00:00 2001
From: Stephen Kendall <stephen@schedmd.com>
Date: Thu, 14 Nov 2024 10:20:16 -0700
Subject: [PATCH 49/90] Docs - standardize strict_order and reorder_count
 descriptions

Ticket 21311
---
 doc/man/man5/slurm.conf.5 | 13 ++++++++-----
 1 file changed, 8 insertions(+), 5 deletions(-)

diff --git a/doc/man/man5/slurm.conf.5 b/doc/man/man5/slurm.conf.5
index 165c7e2a6e4..981b790d750 100644
--- a/doc/man/man5/slurm.conf.5
+++ b/doc/man/man5/slurm.conf.5
@@ -2719,8 +2719,10 @@ plugin.
 .TP
 \fBreorder_count=#\fR
 Specify how many attempts should be made in reordering preemptable jobs to
-minimize the count of jobs preempted.
+minimize the total number of jobs that will be preempted.
 The default value is 1. High values may adversely impact performance.
+Changes to the order of jobs on these attempts can be enabled with
+\fBstrict_order\fR.
 The logic to support this option is only available in the select/cons_tres
 plugin.
 .IP
@@ -2735,10 +2737,11 @@ otherwise a SIGTERM will be sent to the tasks.
 
 .TP
 \fBstrict_order\fR
-If set, then execute extra logic in an attempt to preempt only the lowest
-priority jobs.
-It may be desirable to set this configuration parameter when there are multiple
-priorities of preemptable jobs.
+When reordering preemptable jobs, place the most recently tested job at the
+front of the list since we are certain that it actually added resources needed
+by the new job. This ensures that with enough reorder attempts, the minimum
+possible number of jobs will be preempted.
+See also \fBreorder_count=#\fR.
 The logic to support this option is only available in the select/cons_tres
 plugin.
 .IP

From 51a1b5b707959deca01b85b816e3ba3075285e18 Mon Sep 17 00:00:00 2001
From: Nathan Rini <nate@schedmd.com>
Date: Wed, 6 Nov 2024 13:18:35 -0700
Subject: [PATCH 50/90] data_parser/v0.0.41 - Add field_parser to dump()

When dumping PARSER_MODEL_ARRAY, track the parser for the field and the
final parser for the field's contents.

Ticket 21341
---
 src/plugins/data_parser/v0.0.41/api.c     |  2 +-
 src/plugins/data_parser/v0.0.41/parsing.c | 19 +++++++++++--------
 src/plugins/data_parser/v0.0.41/parsing.h | 11 ++++++-----
 3 files changed, 18 insertions(+), 14 deletions(-)

diff --git a/src/plugins/data_parser/v0.0.41/api.c b/src/plugins/data_parser/v0.0.41/api.c
index 5b213dcd49f..7e665ecb43a 100644
--- a/src/plugins/data_parser/v0.0.41/api.c
+++ b/src/plugins/data_parser/v0.0.41/api.c
@@ -95,7 +95,7 @@ extern int data_parser_p_dump(args_t *args, data_parser_type_t type, void *src,
 		return ESLURM_NOT_SUPPORTED;
 	}
 
-	return dump(src, src_bytes, parser, dst, args);
+	return dump(src, src_bytes, NULL, parser, dst, args);
 }
 
 extern int data_parser_p_parse(args_t *args, data_parser_type_t type, void *dst,
diff --git a/src/plugins/data_parser/v0.0.41/parsing.c b/src/plugins/data_parser/v0.0.41/parsing.c
index c6db4704e18..ea342fc39d3 100644
--- a/src/plugins/data_parser/v0.0.41/parsing.c
+++ b/src/plugins/data_parser/v0.0.41/parsing.c
@@ -1244,8 +1244,9 @@ static int _foreach_dump_list(void *obj, void *arg)
 	xassert(args->parser->ptr_offset == NO_VAL);
 
 	/* we don't know the size of the items in the list */
-	if (dump(&obj, NO_VAL, find_parser_by_type(args->parser->list_type),
-		 item, args->args))
+	if (dump(&obj, NO_VAL, NULL,
+		 find_parser_by_type(args->parser->list_type), item,
+		 args->args))
 		return -1;
 
 	return 0;
@@ -1327,7 +1328,7 @@ static int _dump_pointer(const parser_t *const parser, void *src, data_t *dst,
 		return SLURM_SUCCESS;
 	}
 
-	return dump(*ptr, NO_VAL, pt, dst, args);
+	return dump(*ptr, NO_VAL, NULL, pt, dst, args);
 }
 
 static int _dump_nt_array(const parser_t *const parser, void *src, data_t *dst,
@@ -1345,7 +1346,7 @@ static int _dump_nt_array(const parser_t *const parser, void *src, data_t *dst,
 			return SLURM_SUCCESS;
 
 		for (int i = 0; !rc && array[i]; i++) {
-			rc = dump(array[i], NO_VAL,
+			rc = dump(array[i], NO_VAL, NULL,
 				  find_parser_by_type(parser->array_type),
 				  data_list_append(dst), args);
 		}
@@ -1369,7 +1370,7 @@ static int _dump_nt_array(const parser_t *const parser, void *src, data_t *dst,
 			if (done)
 				break;
 
-			rc = dump(ptr, NO_VAL,
+			rc = dump(ptr, NO_VAL, NULL,
 				  find_parser_by_type(parser->array_type),
 				  data_list_append(dst), args);
 		}
@@ -1518,7 +1519,8 @@ static int _dump_linked(args_t *args, const parser_t *const array,
 		 array->ptr_offset, (uintptr_t) dst, array->key,
 		 (uintptr_t) dst);
 
-	rc = dump(src, NO_VAL, find_parser_by_type(parser->type), dst, args);
+	rc = dump(src, NO_VAL, parser, find_parser_by_type(parser->type), dst,
+		  args);
 
 	log_flag(DATA, "END: dumping %s parser %s->%s(0x%" PRIxPTR ") for %s(0x%" PRIxPTR ")->%s(+%zd) for data(0x%" PRIxPTR ")/%s(0x%" PRIxPTR ")",
 		 parser->obj_type_string, array->type_string,
@@ -1545,8 +1547,9 @@ static void _check_dump(const parser_t *const parser, data_t *dst, args_t *args)
 	}
 }
 
-extern int dump(void *src, ssize_t src_bytes, const parser_t *const parser,
-		data_t *dst, args_t *args)
+extern int dump(void *src, ssize_t src_bytes,
+		const parser_t *const field_parser,
+		const parser_t *const parser, data_t *dst, args_t *args)
 {
 	int rc;
 
diff --git a/src/plugins/data_parser/v0.0.41/parsing.h b/src/plugins/data_parser/v0.0.41/parsing.h
index 94e325dde1a..f1b34a1554e 100644
--- a/src/plugins/data_parser/v0.0.41/parsing.h
+++ b/src/plugins/data_parser/v0.0.41/parsing.h
@@ -59,11 +59,12 @@
 #undef DATA_DUMP
 #undef DATA_PARSE
 
-extern int dump(void *src, ssize_t src_bytes, const parser_t *const parser,
-		data_t *dst, args_t *args);
-#define DUMP(type, src, dst, args)                                            \
-	dump(&src, sizeof(src), find_parser_by_type(DATA_PARSER_##type), dst, \
-	     args)
+extern int dump(void *src, ssize_t src_bytes,
+		const parser_t *const field_parser,
+		const parser_t *const parser, data_t *dst, args_t *args);
+#define DUMP(type, src, dst, args)                                             \
+	dump(&src, sizeof(src), NULL, find_parser_by_type(DATA_PARSER_##type), \
+	     dst, args)
 
 extern int parse(void *dst, ssize_t dst_bytes, const parser_t *const parser,
 		 data_t *src, args_t *args, data_t *parent_path);

From ab82257bdd267b385543b32c9eef08d14ca320ea Mon Sep 17 00:00:00 2001
From: Nathan Rini <nate@schedmd.com>
Date: Wed, 6 Nov 2024 13:23:41 -0700
Subject: [PATCH 51/90] data_parser/v0.0.41 - Pass field_parser to
 _dump_pointer()

Ticket 21341
---
 src/plugins/data_parser/v0.0.41/parsing.c | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/src/plugins/data_parser/v0.0.41/parsing.c b/src/plugins/data_parser/v0.0.41/parsing.c
index ea342fc39d3..a71c62c90de 100644
--- a/src/plugins/data_parser/v0.0.41/parsing.c
+++ b/src/plugins/data_parser/v0.0.41/parsing.c
@@ -1287,7 +1287,8 @@ static int _dump_list(const parser_t *const parser, void *src, data_t *dst,
 	return SLURM_SUCCESS;
 }
 
-static int _dump_pointer(const parser_t *const parser, void *src, data_t *dst,
+static int _dump_pointer(const parser_t *const field_parser,
+			 const parser_t *const parser, void *src, data_t *dst,
 			 args_t *args)
 {
 	const parser_t *pt = find_parser_by_type(parser->pointer_type);
@@ -1620,7 +1621,7 @@ extern int dump(void *src, ssize_t src_bytes,
 		verify_parser_not_sliced(parser);
 		xassert(data_get_type(dst) == DATA_TYPE_NULL);
 
-		rc = _dump_pointer(parser, src, dst, args);
+		rc = _dump_pointer(field_parser, parser, src, dst, args);
 		break;
 	case PARSER_MODEL_NT_PTR_ARRAY:
 	case PARSER_MODEL_NT_ARRAY:

From 7c9cc563adfa2008b35a584eb213363015725339 Mon Sep 17 00:00:00 2001
From: Nathan Rini <nate@schedmd.com>
Date: Wed, 6 Nov 2024 13:25:39 -0700
Subject: [PATCH 52/90] data_parser/v0.0.41 - Dump non-required fields as null

Avoid breaking generated clients by dumping '{}' instead of 'null' for
NULL pointers fields for PARSER_MODEL_ARRAY structs when field is marked
as required=false and clients consider 'null' a valid value but will
error for an empty dictionary '{}'.

Ticket 21341
---
 src/plugins/data_parser/v0.0.41/parsing.c | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/src/plugins/data_parser/v0.0.41/parsing.c b/src/plugins/data_parser/v0.0.41/parsing.c
index a71c62c90de..ff5bae5b924 100644
--- a/src/plugins/data_parser/v0.0.41/parsing.c
+++ b/src/plugins/data_parser/v0.0.41/parsing.c
@@ -1304,10 +1304,11 @@ static int _dump_pointer(const parser_t *const field_parser,
 		while (pt->pointer_type)
 			pt = find_parser_by_type(pt->pointer_type);
 
-		if (parser->allow_null_pointer) {
+		if (parser->allow_null_pointer ||
+		    (field_parser && !field_parser->required)) {
 			xassert(data_get_type(dst) == DATA_TYPE_NULL);
 		} else if ((pt->model == PARSER_MODEL_ARRAY) ||
-		    (pt->obj_openapi == OPENAPI_FORMAT_OBJECT)) {
+			   (pt->obj_openapi == OPENAPI_FORMAT_OBJECT)) {
 			/*
 			 * OpenAPI clients can't handle a null instead of an
 			 * object. Work around by placing an empty dictionary

From 637f251eac8c9139bd4986fc0cc369da005a09f0 Mon Sep 17 00:00:00 2001
From: Nathan Rini <nate@schedmd.com>
Date: Wed, 13 Nov 2024 10:34:41 -0700
Subject: [PATCH 53/90] Add NEWS entry for last 3 commits

Ticket 21341
---
 NEWS | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/NEWS b/NEWS
index a6152e0a41a..1cade4d5f2f 100644
--- a/NEWS
+++ b/NEWS
@@ -14,6 +14,10 @@ documents those changes that are of interest to users and administrators.
     removal of a dynamic node.
  -- gpu/nvml - Attempt loading libnvidia-ml.so.1 as a fallback for failure in
     loading libnvidia-ml.so.
+ -- slurmrestd - Fix populating non-required object fields of objects as '{}' in
+    JSON/YAML instead of 'null' causing compiled OpenAPI clients to reject
+    the response to 'GET /slurm/v0.0.40/jobs' due to validation failure of
+    '.jobs[].job_resources'.
 
 * Changes in Slurm 24.05.4
 ==========================

From 2ff6059881800694c3331c3905b2619e2af42e80 Mon Sep 17 00:00:00 2001
From: Nathan Rini <nate@schedmd.com>
Date: Thu, 14 Nov 2024 12:08:00 -0700
Subject: [PATCH 54/90] Testsuite - Avoid false failure in test_112_41

Ticket 21341
---
 testsuite/python/tests/test_112_41.py | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/testsuite/python/tests/test_112_41.py b/testsuite/python/tests/test_112_41.py
index 1d529b1fc7c..f65b39ff7e7 100644
--- a/testsuite/python/tests/test_112_41.py
+++ b/testsuite/python/tests/test_112_41.py
@@ -1001,9 +1001,11 @@ def test_jobs(slurm, slurmdb):
     resp = slurm.slurm_v0041_post_job(str(jobid), v0041_job_desc_msg=job)
     assert not len(resp.warnings)
     assert not len(resp.errors)
-    for result in resp.results:
-        assert result.job_id == jobid
-        assert result.error_code == 0
+    # Not all changes populate "results" field
+    if resp.results is not None:
+        for result in resp.results:
+            assert result.job_id == jobid
+            assert result.error_code == 0
 
     resp = slurm.slurm_v0041_get_job(str(jobid))
     assert len(resp.warnings) == 0

From 7efb6bb443327c89f695d027d1f61a538c82bd83 Mon Sep 17 00:00:00 2001
From: Nathan Rini <nate@schedmd.com>
Date: Thu, 14 Nov 2024 12:08:41 -0700
Subject: [PATCH 55/90] Testsuite - Avoid xfail in test_112_41

Ticket 21341
---
 testsuite/python/tests/test_112_41.py | 2 --
 1 file changed, 2 deletions(-)

diff --git a/testsuite/python/tests/test_112_41.py b/testsuite/python/tests/test_112_41.py
index f65b39ff7e7..cfdf67726af 100644
--- a/testsuite/python/tests/test_112_41.py
+++ b/testsuite/python/tests/test_112_41.py
@@ -928,8 +928,6 @@ def test_db_config(slurmdb):
     assert len(resp.errors) == 0
 
 
-# TODO: Remove xfail once bug 21341 is fixed
-@pytest.mark.xfail
 def test_jobs(slurm, slurmdb):
     from openapi_client.models.v0041_job_submit_req import V0041JobSubmitReq
     from openapi_client.models.v0041_job_desc_msg import V0041JobDescMsg

From a200dc8c397a2ae59b6ceb2494638d663d0b3478 Mon Sep 17 00:00:00 2001
From: Nathan Rini <nate@schedmd.com>
Date: Thu, 14 Nov 2024 12:14:46 -0700
Subject: [PATCH 56/90] Testsuite - Wait for slurmdbd to settle in test_112_41

Wait until slurmdbd gets the new job_name to continue.

Ticket 21341
---
 testsuite/python/tests/test_112_41.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/testsuite/python/tests/test_112_41.py b/testsuite/python/tests/test_112_41.py
index cfdf67726af..8d197dfa399 100644
--- a/testsuite/python/tests/test_112_41.py
+++ b/testsuite/python/tests/test_112_41.py
@@ -1043,8 +1043,8 @@ def test_jobs(slurm, slurmdb):
         assert len(resp.errors) == 0
         assert resp.jobs
         for job in resp.jobs:
-            if job.name == "allocation":
-                # job hasn't settled at slurmdbd yet
+            if job.name != "updated test job":
+                # job change hasn't settled at slurmdbd yet
                 requery = True
             else:
                 requery = False

From 4d0680564edc1e9316f9fddc04f417bf2f694d68 Mon Sep 17 00:00:00 2001
From: Alejandro Sanchez <alex@schedmd.com>
Date: Wed, 20 Nov 2024 21:45:20 +0100
Subject: [PATCH 57/90] Fix sstat/sattach regressions for steps in higher
 version slurmds.

The sstat regression was introduced in 20.11.0rc1 commit 906b3cc346 with
respect to functionality introduced in 17.11.10 commit dd7e9b1113.

The sattach regression was introduced in 16.05.1rc1 commit a1ea1e2a2f9.

Ticket 21465
---
 NEWS                    | 2 ++
 src/api/job_step_info.c | 3 ++-
 src/sattach/sattach.c   | 3 ++-
 3 files changed, 6 insertions(+), 2 deletions(-)

diff --git a/NEWS b/NEWS
index 1cade4d5f2f..29ca7d6f585 100644
--- a/NEWS
+++ b/NEWS
@@ -18,6 +18,8 @@ documents those changes that are of interest to users and administrators.
     JSON/YAML instead of 'null' causing compiled OpenAPI clients to reject
     the response to 'GET /slurm/v0.0.40/jobs' due to validation failure of
     '.jobs[].job_resources'.
+ -- Fix sstat/sattach protocol errors for steps on higher version slurmd's
+    (regressions since 20.11.0rc1 and 16.05.1rc1 respectively).
 
 * Changes in Slurm 24.05.4
 ==========================
diff --git a/src/api/job_step_info.c b/src/api/job_step_info.c
index 373f389145c..00458feb5b9 100644
--- a/src/api/job_step_info.c
+++ b/src/api/job_step_info.c
@@ -738,7 +738,8 @@ extern int slurm_job_step_stat(slurm_step_id_t *step_id,
 	memcpy(&req, step_id, sizeof(req));
 	memcpy(&resp_out->step_id, step_id, sizeof(resp_out->step_id));
 
-	req_msg.protocol_version = use_protocol_ver;
+	req_msg.protocol_version = MIN(SLURM_PROTOCOL_VERSION,
+				       use_protocol_ver);
 	req_msg.msg_type = REQUEST_JOB_STEP_STAT;
 	req_msg.data = &req;
 
diff --git a/src/sattach/sattach.c b/src/sattach/sattach.c
index 3c8f7b10e5d..921b6c7bcdd 100644
--- a/src/sattach/sattach.c
+++ b/src/sattach/sattach.c
@@ -408,7 +408,8 @@ static int _attach_to_tasks(slurm_step_id_t stepid,
 	slurm_msg_set_r_uid(&msg, SLURM_AUTH_UID_ANY);
 	msg.msg_type = REQUEST_REATTACH_TASKS;
 	msg.data = &reattach_msg;
-	msg.protocol_version = layout->start_protocol_ver;
+	msg.protocol_version = MIN(SLURM_PROTOCOL_VERSION,
+				   layout->start_protocol_ver);
 
 	if (layout->front_end)
 		hosts = layout->front_end;

From 4625559cd4b72df45de6b0345aa0594162a828cd Mon Sep 17 00:00:00 2001
From: Albert Gil <albert.gil@schedmd.com>
Date: Thu, 14 Nov 2024 18:10:50 +0100
Subject: [PATCH 58/90] Testsuite - Add start_slurmdbd() and stop_slurmdbd()

Ticket 21393

Signed-off-by: Nathan Prisbrey <nathan@schedmd.com>
---
 testsuite/python/lib/atf.py | 112 ++++++++++++++++++++++++------------
 1 file changed, 74 insertions(+), 38 deletions(-)

diff --git a/testsuite/python/lib/atf.py b/testsuite/python/lib/atf.py
index 334e127226d..05e58df3aa5 100644
--- a/testsuite/python/lib/atf.py
+++ b/testsuite/python/lib/atf.py
@@ -546,6 +546,45 @@ def start_slurmctld(clean=False, quiet=False):
             pytest.fail(f"Slurmctld is not running")
 
 
+def start_slurmdbd(clean=False, quiet=False):
+    """Starts the Slurm DB daemon (slurmdbd).
+
+    This function may only be used in auto-config mode.
+
+    Args:
+        clean (boolean): If True, clears previous slurmdbd state.
+        quiet (boolean): If True, logging is performed at the TRACE log level.
+
+    Returns:
+        None
+    """
+    if not properties["auto-config"]:
+        require_auto_config("wants to start slurmdbd")
+
+    if (
+        run_command_exit(
+            "sacctmgr show cluster", user=properties["slurm-user"], quiet=quiet
+        )
+        != 0
+    ):
+        # Start slurmdbd
+        results = run_command(
+            f"{properties['slurm-sbin-dir']}/slurmdbd",
+            user=properties["slurm-user"],
+            quiet=quiet,
+        )
+        if results["exit_code"] != 0:
+            pytest.fail(
+                f"Unable to start slurmdbd (rc={results['exit_code']}): {results['stderr']}"
+            )
+
+        # Verify that slurmdbd is running
+        if not repeat_command_until(
+            "sacctmgr show cluster", lambda results: results["exit_code"] == 0
+        ):
+            pytest.fail(f"Slurmdbd is not running")
+
+
 def start_slurm(clean=False, quiet=False):
     """Starts all applicable Slurm daemons.
 
@@ -574,28 +613,7 @@ def start_slurm(clean=False, quiet=False):
         get_config_parameter("AccountingStorageType", live=False, quiet=quiet)
         == "accounting_storage/slurmdbd"
     ):
-        if (
-            run_command_exit(
-                "sacctmgr show cluster", user=properties["slurm-user"], quiet=quiet
-            )
-            != 0
-        ):
-            # Start slurmdbd
-            results = run_command(
-                f"{properties['slurm-sbin-dir']}/slurmdbd",
-                user=properties["slurm-user"],
-                quiet=quiet,
-            )
-            if results["exit_code"] != 0:
-                pytest.fail(
-                    f"Unable to start slurmdbd (rc={results['exit_code']}): {results['stderr']}"
-                )
-
-            # Verify that slurmdbd is running
-            if not repeat_command_until(
-                "sacctmgr show cluster", lambda results: results["exit_code"] == 0
-            ):
-                pytest.fail(f"Slurmdbd is not running")
+        start_slurmdbd(clean, quiet)
 
     # Remove unnecessary default node0 from config to avoid being used or reserved
     output = run_command_output(
@@ -694,6 +712,39 @@ def stop_slurmctld(quiet=False):
         pytest.fail("Slurmctld is still running")
 
 
+def stop_slurmdbd(quiet=False):
+    """Stops the Slurm DB daemon (slurmdbd).
+
+    This function may only be used in auto-config mode.
+
+    Args:
+        quiet (boolean): If True, logging is performed at the TRACE log level.
+
+    Returns:
+        None
+    """
+
+    if not properties["auto-config"]:
+        require_auto_config("wants to stop slurmdbd")
+
+    # Stop slurmdbd
+    results = run_command(
+        "sacctmgr shutdown", user=properties["slurm-user"], quiet=quiet
+    )
+    if results["exit_code"] != 0:
+        failures.append(
+            f"Command \"sacctmgr shutdown\" failed with rc={results['exit_code']}"
+        )
+
+    # Verify that slurmdbd is not running (we might have to wait for rollups to complete)
+    if not repeat_until(
+        lambda: pids_from_exe(f"{properties['slurm-sbin-dir']}/slurmdbd"),
+        lambda pids: len(pids) == 0,
+        timeout=60,
+    ):
+        failures.append("Slurmdbd is still running")
+
+
 def stop_slurm(fatal=True, quiet=False):
     """Stops all applicable Slurm daemons.
 
@@ -728,22 +779,7 @@ def stop_slurm(fatal=True, quiet=False):
         get_config_parameter("AccountingStorageType", live=False, quiet=quiet)
         == "accounting_storage/slurmdbd"
     ):
-        # Stop slurmdbd
-        results = run_command(
-            "sacctmgr shutdown", user=properties["slurm-user"], quiet=quiet
-        )
-        if results["exit_code"] != 0:
-            failures.append(
-                f"Command \"sacctmgr shutdown\" failed with rc={results['exit_code']}"
-            )
-
-        # Verify that slurmdbd is not running (we might have to wait for rollups to complete)
-        if not repeat_until(
-            lambda: pids_from_exe(f"{properties['slurm-sbin-dir']}/slurmdbd"),
-            lambda pids: len(pids) == 0,
-            timeout=60,
-        ):
-            failures.append("Slurmdbd is still running")
+        stop_slurmdbd(quiet)
 
     # Stop slurmctld and slurmds
     results = run_command(

From 9da3a8273b55c1a3f38cbe8a2c5550459be5a2ad Mon Sep 17 00:00:00 2001
From: Nathan Prisbrey <nathan@schedmd.com>
Date: Thu, 14 Nov 2024 08:51:06 +0000
Subject: [PATCH 59/90] Testsuite - Rewrite test21.39 as test_102_5

Also changed how edge case is tested, eliminating the need to rely on
race conditions naturally because we manually create the condition by
stopping and starting slurmctld and slurmdbd ourselves.

Ticket 21393
---
 testsuite/README                     |   2 +-
 testsuite/expect/test21.39           | 439 ---------------------------
 testsuite/python/tests/test_102_5.py | 354 +++++++++++++++++++++
 3 files changed, 355 insertions(+), 440 deletions(-)
 delete mode 100755 testsuite/expect/test21.39
 create mode 100644 testsuite/python/tests/test_102_5.py

diff --git a/testsuite/README b/testsuite/README
index db15b3f8873..ce9d2e9d553 100644
--- a/testsuite/README
+++ b/testsuite/README
@@ -592,7 +592,6 @@ test21.35  Validate DenyOnLimit QoS flag is enforced on QoS and Associations.
 test21.36  Validate that sacctmgr lost jobs fixes lost jobs.
 test21.37  sacctmgr show and clear stats
 test21.38  sacctmgr modify limits for nested accounts with multiple users
-test21.39  sacctmgr create qos/account job and then delete account/qos
 test21.40  Test association plus partition/job QoS unique node limits enforced
 test21.41  sacctmgr update job set newwckey=
 test21.42  Test if headers returned by sacctmgr show can be used as format= specifiers
@@ -793,6 +792,7 @@ test_102_#   Testing of sacctmgr options.
 test_102_1   /commands/sacctmgr/test_federation.py
 test_102_2   /commands/sacctmgr/test_--usage.py
 test_102_3   /commands/sacctmgr/test_--json.py
+test_102_5   sacctmgr create qos/account job and then delete account/qos
 
 test_103_#   Testing of salloc options.
 =======================================
diff --git a/testsuite/expect/test21.39 b/testsuite/expect/test21.39
deleted file mode 100755
index 884d1623aad..00000000000
--- a/testsuite/expect/test21.39
+++ /dev/null
@@ -1,439 +0,0 @@
-#!/usr/bin/env expect
-############################################################################
-# Purpose: Test of Slurm functionality
-#          sacctmgr create qos/account job and then delete account/qos
-############################################################################
-# Copyright (C) 2019 SchedMD LLC.
-# Written by Nathan Rini <nate@schedmd.com>
-# All rights reserved.
-#
-# This file is part of Slurm, a resource management program.
-# For details, see <https://slurm.schedmd.com/>.
-# Please also read the included file: DISCLAIMER.
-#
-# Slurm is free software; you can redistribute it and/or modify it under
-# the terms of the GNU General Public License as published by the Free
-# Software Foundation; either version 2 of the License, or (at your option)
-# any later version.
-#
-# Slurm is distributed in the hope that it will be useful, but WITHOUT ANY
-# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
-# FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more
-# details.
-#
-# You should have received a copy of the GNU General Public License along
-# with Slurm; if not, write to the Free Software Foundation, Inc.,
-# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301  USA.
-############################################################################
-source ./globals
-source ./globals_accounting
-
-set part_name   "${test_name}_part"
-set ta1         "${test_name}-account.1"
-set ta2         "${test_name}-account.2"
-set tu1         [get_my_user_name]
-set tq1         "${test_name}-qos.1"
-set tq2         "${test_name}-qos.2"
-set job_id_1    0
-
-# account options
-array set acct_1 {}
-array set acct_2 {}
-
-# user options
-array set user_req_1 {}
-set user_req_1(Account) $ta1
-set user_req_1(Qos) "$tq1,$tq2"
-array set user_req_2 {}
-set user_req_2(Account) $ta2
-set user_req_2(Qos) "$tq1,$tq2"
-
-# qos options
-array set qos_1 {}
-array set qos_2 {}
-
-set access_err  0
-
-# Get the location of the slurm.conf file
-set config_dir [get_conf_path]
-set config_file $config_dir/slurm.conf
-
-# TODO: This test is just too error-prone to be useful (t21393)
-skip "This test is temporally disabled to avoid testsuite noise (t21393)"
-
-#
-# Verify preconditions
-#
-if {[get_config_param "AccountingStorageType"] ne "accounting_storage/slurmdbd"} {
-	skip "This test can't be run without a usable AccountStorageType"
-}
-
-if {![param_contains [get_config_param "AccountingStorageEnforce"] "associations"]} {
-	skip "This test can't be run without AccountingStorageEnforce=associations"
-}
-
-if {![param_contains [get_config_param "AccountingStorageEnforce"] "qos"]} {
-	skip "This test can't be run without AccountingStorageEnforce=qos"
-}
-
-if {[param_contains [get_config_param "JobContainerType"] "*tmpfs"]} {
-	skip "This test can't be run with JobContainerType=job_container/tmpfs"
-}
-
-if {[get_admin_level] ne "Administrator"} {
-	skip "This test can't be run without being an Accounting administrator.\nUse: sacctmgr mod user \$USER set admin=admin"
-}
-
-proc check_rc { rc } {
-	if {$rc != 0} {
-		fail "Subcommand failed with return code $rc"
-	}
-}
-
-# Create test assoc and accounts
-proc create_accounts {} {
-	global ta1 ta2 tq1 tq2 tu1 user_req_1 user_req_2
-	global qos_1 qos_2
-	log_debug "Create account and QOS"
-
-
-	# Create test assoc and accounts
-	check_rc [add_qos $tq1 [array get qos_1]]
-	check_rc [add_qos $tq2 [array get qos_2]]
-	check_rc [add_acct $ta1 [array get acct_1]]
-	check_rc [add_acct $ta2 [array get acct_2]]
-	check_rc [add_user $tu1 [array get user_req_1]]
-	check_rc [add_user $tu1 [array get user_req_2]]
-}
-
-# Cleanup test assoc and accounts
-proc cleanup_accounts {} {
-	global ta1 ta2 tq1 tq2
-
-	wait_for_account_done $ta1,$ta2
-
-	log_debug "Remove QOS: $tq1,$tq2"
-	remove_qos $tq1,$tq2
-
-	log_debug "Remove account: $ta1,$ta2"
-	remove_acct "" $ta1,$ta2
-}
-
-proc cleanup { } {
-	global job_id_1 config_file part_name
-
-	cancel_job $job_id_1
-	wait_for_part_done $part_name
-
-	cleanup_accounts
-
-	restore_conf $config_file
-	reconfigure
-}
-
-proc test_salloc { qos account num_nodes } {
-	global salloc number part_name bin_bash
-
-	set rc -12345
-	set job_id 0
-
-	spawn $salloc -p$part_name --exclusive -q$qos -A$account -N$num_nodes $bin_bash
-	expect {
-		-re "allocation ($number)" {
-			set job_id $expect_out(1,string)
-		}
-		-re "error" {
-			fail "salloc job was not submitted"
-		}
-		timeout {
-			fail "salloc not responding"
-		}
-	}
-
-	if { $job_id == 0 } {
-		fail "Submit failure"
-	}
-
-	return $job_id
-}
-
-proc check_job_reason { job_id state why } {
-	global scontrol re_word_str re_word_str
-
-	set found_why ""
-	set state_found ""
-
-	log_user 0
-	spawn $scontrol show job $job_id
-	expect {
-		-re "State=($re_word_str)" {
-			set state_found $expect_out(1,string)
-			exp_continue
-		}
-		-re "Reason=($re_word_str)" {
-			set found_why $expect_out(1,string)
-			exp_continue
-		}
-		timeout {
-			fail "scontrol not responding"
-		}
-		eof {
-			lassign [wait] pid spawnid os_error_flag rc
-		}
-	}
-
-	log_user 1
-	if { $state_found != $state } {
-		fail "Job ($job_id) state found was $state_found, expected $state"
-	}
-
-	set found_reason 0
-	foreach iwhy $why {
-		if { $found_why == $iwhy } {
-			set found_reason 1
-			break
-		}
-	}
-
-	if { !$found_reason } {
-		fail "Job ($job_id) scontrol returned Reason=$found_why instead of Reason='$why'"
-	}
-
-	log_debug "Found jobid $job_id with correct state '$state' and reason '$found_why'"
-	return 0
-}
-
-proc run_test { run_qos } {
-	global scontrol tq1 tq2 ta1 ta2 part_name
-	# TODO: Temporary globals for the extra debug info for bug10604
-	global squeue sinfo sacctmgr
-
-	if { $run_qos } {
-		set qos $tq1
-		set acct $ta2
-		set update_line "qos=$tq2"
-		set reason "InvalidQOS"
-	} else {
-		set qos $tq1
-		set acct $ta1
-		set update_line "account=$ta2"
-		set reason "InvalidAccount"
-	}
-
-	set job_id_1 [test_salloc $qos $acct 1]
-	set job_id_2 [test_salloc $qos $acct 1]
-	# TODO Temporary debug to troubleshoot bug 10604 (revert once fixed)
-	# check_rc [wait_for_job $job_id_1 "RUNNING"]
-	set rc [wait_for_job $job_id_1 "RUNNING"]
-	if {$rc} {
-		log_warn "Job never started, extra debug information for bug10604 below before the actual fail:"
-		run_command "$squeue"
-		run_command "$sinfo"
-		run_command "$scontrol show job"
-		run_command "$scontrol show node"
-		run_command "$scontrol show partition"
-		run_command "$sacctmgr show qos -p"
-		fail "Subcommand failed with exit code $rc"
-	}
-
-	if { $run_qos } {
-		# Remove test qos
-		if [remove_qos $qos] {
-			log_debug "We hit the race trying to get the job running before it hits the database before we removed the qos.  This can be expected, trying again."
-			wait_for_part_done $part_name
-			return 1
-		}
-	} else {
-		# Remove test acct
-		if [remove_acct "" $acct] {
-			log_debug "We hit the race trying to get the job running before it hits the database before we removed the account.  This can be expected, trying again."
-			wait_for_part_done $part_name
-			return 1
-		}
-	}
-
-	# Verify jobs state and reason
-	check_job_reason $job_id_1 "RUNNING" [list "None"]
-	check_job_reason $job_id_2 "PENDING" [list "$reason"]
-
-	# Update pending job to make it runnable, updating the running job isn't
-	# possible but it tests other code if you are watching the log file
-	spawn $scontrol update jobid=$job_id_1 $update_line
-	spawn $scontrol update jobid=$job_id_2 $update_line
-
-	sleep 5
-	# Check reasons after account alter
-	check_job_reason $job_id_1 "RUNNING" [list "None"]
-	check_job_reason $job_id_2 "PENDING" [list "Resources" "None"]
-
-	# Cleanup jobs
-	wait_for_part_done $part_name
-
-	return 0
-}
-
-cleanup_accounts
-create_accounts
-
-#
-# Copy slurm.conf file
-#
-save_conf $config_file
-
-# Comment out PrologFlags in the slurm.conf
-exec $bin_sed -i {s/^\(PrologFlags=\)/#\1/gI} $config_file
-
-# TODO: Temporarily increase logging to debug bug 10604 (remove once fixed)
-run_command -none "$bin_echo SlurmctldDebug=debug3 >> $config_file"
-# Allow the test's existing reconfigure call to establish these values
-
-reconfigure -fail
-
-delete_part $part_name
-
-if [create_part $part_name 1] {
-	fail "Unable to create partition ($part_name)"
-}
-
-for {set i 0} {$i < 2} {incr i} {
-	set cnt 0
-	set rc 1
-	# First lets test against removing an account and then qos
-	# from a running and pending job since partition only has 1 so
-	# one of these should run and the second should pend
-	while { $rc && ($cnt < 10) } {
-		set rc [run_test $i]
-		incr cnt
-	}
-	if { $rc } {
-		set credential [expr {$i == 0 ? "account" : "qos"}]
-		fail "Too many ($cnt) failures trying to remove $credential from job"
-	}
-}
-
-log_info "Testing that PD jobs that lost their QOS will not lunch until restored"
-
-# Return to clean slate
-cleanup_accounts
-
-# Add test qos
-run_command -fail "$sacctmgr -vi add qos $tq1"
-
-# Add test account
-run_command -fail "$sacctmgr -vi add account name=$ta1 set qos=normal,$tq1"
-
-# Add test user to account
-run_command -fail "$sacctmgr -vi add user name=$tu1 account=$ta1"
-
-# Check what we just added to make sure it is there
-if {![regexp -line "^\\S+\\|$ta1\\|$tu1\\|normal,$tq1" [run_command_output "$sacctmgr -n -P show assoc format=cluster,account,user,qos"]]} {
-	fail "Association with the right account, user and qos was not found"
-}
-
-# Submit a delayed job that wants to run in the test qos
-regexp {Submitted batch job (\d+)} [run_command_output -fail "$sbatch --begin=now+2 --qos=$tq1 -A $ta1 --wrap \"$srun sleep 10\" -o none -e none"] {} job_id_1
-
-# The first job should get queued
-wait_for_job -fail $job_id_1 "PENDING"
-
-# Remove the test qos out from under the job
-run_command "$sacctmgr -vi modify account name=$ta1 user=$tu1 set qos=normal"
-
-# Submit a second delayed job requesting the test qos. It should be rejected
-if {![run_command_status -none "$sbatch --begin=now+2 --qos=$tq1 -A $ta1 --wrap \"$srun sleep 10\" -o none -e none"]} {
-	fail "Job submitted with unassociated qos should have failed"
-} else {
-	log_debug "The preceding job failure was expected"
-}
-
-# Wait for the first job to pend with Reason=InvalidQOS
-set condition_matched false
-wait_for {$condition_matched} {
-	set scontrol_out [run_command_output -fail "$scontrol -o show job $job_id_1"]
-	regexp {JobState=([^ ]+) Reason=([^ ]+)} $scontrol_out {} job_state reason
-	if {$job_state eq "PENDING" && $reason eq "InvalidQOS"} {
-		set condition_matched true
-	}
-}
-subtest {$condition_matched} "Job ($job_id_1) should be PD with Reason=InvalidQOS" "JobState=$job_state, Reason=$reason"
-
-# Add back the test qos
-run_command -fail "$sacctmgr -vi modify account name=$ta1 user=$tu1 set qos=normal,$tq1"
-
-# Wait for the first job to begin running
-wait_for_job -fail $job_id_1 "RUNNING"
-
-log_info "Testing that PD jobs that lost their Account will not resume until restored"
-
-# Return to clean slate
-cleanup_accounts
-
-# Add test qos
-run_command -fail "$sacctmgr -vi add qos $tq1"
-
-# Add test accounts
-run_command -fail "$sacctmgr -vi add account name=$ta1 set qos=normal,$tq1"
-run_command -fail "$sacctmgr -vi add account name=$ta2 set qos=normal,$tq1"
-
-# Add test user to accounts
-run_command -fail "$sacctmgr -vi add user name=$tu1 account=$ta1"
-run_command -fail "$sacctmgr -vi add user name=$tu1 account=$ta2"
-
-# Check what we just added to make sure it is there
-if {![regexp -line "^\\S+\\|$ta1\\|$tu1\\|normal,$tq1" [run_command_output "$sacctmgr -n -P show assoc format=cluster,account,user,qos"]]} {
-	fail "QOS ($tq1) has not been added to account ($ta1)"
-}
-if {![regexp -line "^\\S+\\|$ta2\\|$tu1\\|normal,$tq1" [run_command_output "$sacctmgr -n -P show assoc format=cluster,account,user,qos"]]} {
-	fail "QOS ($tq1) has not been added to account ($ta2)"
-}
-
-# We may have to try this multiple times since we are relying on a race
-# condition. If the idle job gets written to the job accounting records
-# before the account is deleted, the account deletion will fail saying
-# that it cannot delete an account with active jobs.
-wait_for -fail {$delete_account_status == 0} {
-	# Submit a delayed job that wants to run with the test qos and test account 2
-	regexp {Submitted batch job (\d+)} [run_command_output -fail "$sbatch --begin=now+2 --qos=$tq1 -A $ta2 --wrap \"$srun sleep 10\" -o none -e none"] {} job_id_1
-
-	# The first job should get queued
-	wait_for_job -fail $job_id_1 "PENDING"
-
-	# Remove the test account out from under the job
-	set delete_account_results [run_command "$sacctmgr -vi delete account name=$ta2"]
-	set delete_account_status [dict get $delete_account_results exit_code]
-	set delete_account_output [dict get $delete_account_results output]
-
-	# Mitigation if the account deletion fails
-	if {$delete_account_status != 0} {
-		if [regexp {Error with request: Job\(s\) active} $delete_account_output] {
-			cancel_job $job_id_1
-		} else {
-			fail "Failure deleting account ($ta2): $delete_account_output"
-		}
-	}
-}
-
-# Submit a second delayed job requesting test account 2. It should be rejected
-if {![run_command_status -none "$sbatch --begin=now+2 --qos=$tq1 -A $ta2 --wrap \"$srun sleep 10\" -o none -e none"]} {
-	fail "Job submitted with unassociated account should have failed"
-} else {
-	log_debug "The preceding job failure was expected"
-}
-
-# Wait for the first job to pend with Reason=InvalidAccount
-set condition_matched false
-wait_for {$condition_matched} {
-	set scontrol_out [run_command_output -fail "$scontrol -o show job $job_id_1"]
-	regexp {JobState=([^ ]+) Reason=([^ ]+)} $scontrol_out {} job_state reason
-	if {$job_state eq "PENDING" && $reason eq "InvalidAccount"} {
-		set condition_matched true
-	}
-}
-subtest {$condition_matched} "Job ($job_id_1) should be PD with Reason=InvalidAccount" "JobState=$job_state, Reason=$reason"
-
-# Add back the test account
-run_command -fail "$sacctmgr -vi add account name=$ta2 set qos=normal,$tq1"
-run_command -fail "$sacctmgr -vi add user name=$tu1 account=$ta2"
-
-# Wait for the first job to begin running
-wait_for_job -fail $job_id_1 "RUNNING"
diff --git a/testsuite/python/tests/test_102_5.py b/testsuite/python/tests/test_102_5.py
new file mode 100644
index 00000000000..9fa8bf91036
--- /dev/null
+++ b/testsuite/python/tests/test_102_5.py
@@ -0,0 +1,354 @@
+############################################################################
+# Copyright (C) SchedMD LLC.
+############################################################################
+import atf
+import pytest
+
+# Global variables
+qos1 = "qos1"
+qos2 = "qos2"
+acct1 = "acct1"
+acct2 = "acct2"
+acct = "acct"
+
+
+@pytest.fixture(scope="module", autouse=True)
+def setup():
+    """Test setup with required configurations."""
+    atf.require_auto_config("Manually creating and deleting qoses and accounts")
+    atf.require_config_parameter("AccountingStorageType", "accounting_storage/slurmdbd")
+    atf.require_config_parameter_includes("AccountingStorageEnforce", "associations")
+    atf.require_config_parameter_includes("AccountingStorageEnforce", "qos")
+    atf.require_slurm_running()
+
+
+@pytest.fixture(scope="function", autouse=True)
+def setup_db():
+    # Create test QOS and account
+    atf.run_command(
+        f"sacctmgr -i add qos {qos1},{qos2}",
+        user=atf.properties["slurm-user"],
+        fatal=True,
+    )
+    atf.run_command(
+        f"sacctmgr -i add account {acct},{acct1},{acct2}",
+        user=atf.properties["slurm-user"],
+        fatal=True,
+    )
+    atf.run_command(
+        f"sacctmgr -i add user {atf.get_user_name()} DefaultAccount={acct} account={acct1},{acct2} qos=normal,{qos1},{qos2}",
+        user=atf.properties["slurm-user"],
+        fatal=True,
+    )
+    yield
+
+    atf.cancel_all_jobs(fatal=True)
+
+    atf.run_command(
+        f"sacctmgr -i remove user {atf.get_user_name()} {acct1},{acct2}",
+        user=atf.properties["slurm-user"],
+        quiet=True,
+    )
+    atf.run_command(
+        f"sacctmgr -i remove account {acct1},{acct2}",
+        user=atf.properties["slurm-user"],
+        quiet=True,
+    )
+    atf.run_command(
+        f"sacctmgr -i remove qos {qos1},{qos2}",
+        user=atf.properties["slurm-user"],
+        quiet=True,
+    )
+
+
+def submit_job_with(extra_params, xfail=False, fatal=False):
+    """Submit a job with specified extra params."""
+    return atf.submit_job_sbatch(
+        f"{extra_params} -N1 --wrap='sleep 300'", xfail=xfail, fatal=fatal
+    )
+
+
+def test_qos_removal_single():
+    """Test that removal of a single QOS:
+    1. Marks pending jobs with InvalidQOS
+    2. Rejects new job submissions using removed QOS
+    """
+
+    # Stop slurmdbd and submit job
+    atf.stop_slurmdbd(quiet=True)
+    job_id = submit_job_with(f"--qos={qos1}", fatal=True)
+
+    # Stop slurmctld, start slurmdbd, remove QOS
+    atf.stop_slurmctld(quiet=True)
+    atf.start_slurmdbd(quiet=True)
+    atf.run_command(
+        f"sacctmgr -i remove qos {qos1}",
+        user=atf.properties["slurm-user"],
+        fatal=True,
+    )
+
+    # TODO: Unnecessary for 24.05+ (Bug 21393)
+    atf.run_command(
+        f"sacctmgr -i modify user {atf.get_user_name()} set qos={qos2}",
+        user=atf.properties["slurm-user"],
+        fatal=True,
+    )
+
+    # Start slurmctld and verify job state/reason
+    atf.start_slurmctld(quiet=True)
+
+    assert atf.wait_for_job_state(
+        job_id, "PENDING", desired_reason="InvalidQOS"
+    ), f"Job {job_id} not in PENDING state with InvalidQOS reason"
+
+    # Try to submit a new job with removed QOS - should be rejected
+    assert (
+        submit_job_with(f"--qos={qos1}", xfail=True) == 0
+    ), f"Job submission with removed QOS {qos1} should have failed but got job id {job_id2}"
+
+    # Submit a job with remaining QOS - should succeed
+    assert (
+        submit_job_with(f"--qos={qos2}") != 0
+    ), f"Job submission with valid QOS {qos2} should have succeeded"
+
+
+def test_qos_removal_multiple():
+    """Test QOS removal when user has multiple QOS access:
+    1. Verifies jobs with removed QOS get marked InvalidQOS
+    2. Verifies jobs with remaining QOS stay valid
+    3. Verifies new job submissions with removed QOS are rejected
+    4. Verifies new job submissions with remaining QOS succeed
+    """
+
+    # Stop slurmdbd and submit jobs with different QOSs
+    atf.stop_slurmdbd(quiet=True)
+    job_id1 = submit_job_with(f"--qos={qos1}", fatal=True)
+    job_id2 = submit_job_with(f"--qos={qos2}", fatal=True)
+
+    # Stop slurmctld, start slurmdbd, remove first QOS
+    atf.stop_slurmctld(quiet=True)
+    atf.start_slurmdbd(quiet=True)
+    atf.run_command(
+        f"sacctmgr -i remove qos {qos1}",
+        user=atf.properties["slurm-user"],
+        fatal=True,
+    )
+
+    # TODO: Unnecessary for 24.05+ (Bug 21393)
+    atf.run_command(
+        f"sacctmgr -i modify user {atf.get_user_name()} set qos={qos2}",
+        user=atf.properties["slurm-user"],
+        fatal=True,
+    )
+
+    # Start slurmctld and verify jobs state/reason
+    atf.start_slurmctld(quiet=True)
+
+    # First job should be PENDING with InvalidQOS
+    assert atf.wait_for_job_state(
+        job_id1, "PENDING", desired_reason="InvalidQOS"
+    ), f"Job {job_id1} not in PENDING state and InvalidQOS reason"
+
+    # Second job should stay PENDING with different reason
+    assert atf.wait_for_job_state(
+        job_id2, "PENDING", timeout=10
+    ), f"Job {job_id2} should be in PENDING state"
+    assert (
+        atf.get_job_parameter(job_id2, "Reason") != "InvalidQOS"
+    ), "The second job whose QOS was not deleted should not be pending due to 'InvalidQOS'"
+
+    # Try to submit a new job with removed QOS - should be rejected
+    assert (
+        submit_job_with(f"--qos={qos1}", xfail=True) == 0
+    ), f"Job submission with removed QOS {qos1} should have failed"
+
+    # Submit a job with remaining QOS - should succeed
+    assert (
+        submit_job_with(f"--qos={qos2}") != 0
+    ), f"Job submission with valid QOS {qos2} should have succeeded"
+
+
+def test_qos_removal_running_vs_pending():
+    """Test QOS removal impact on running vs pending jobs:
+    1. Submit two jobs with same QOS - one running, one pending
+    2. Remove the QOS
+    3. Verify running job continues running
+    4. Verify pending job gets marked with InvalidQOS
+    """
+
+    # Stop slurmdbd and submit jobs - use exclusive to ensure only one can run
+    atf.stop_slurmdbd(quiet=True)
+    job_id1 = submit_job_with(f"--qos={qos1} --exclusive", fatal=True)
+    job_id2 = submit_job_with(f"--qos={qos1} --exclusive", fatal=True)
+
+    # Wait for first job to start running
+    assert atf.wait_for_job_state(
+        job_id1, "RUNNING"
+    ), f"Job {job_id1} never started running"
+
+    # Verify second job is pending (due to exclusive)
+    assert atf.wait_for_job_state(
+        job_id2, "PENDING"
+    ), f"Job {job_id2} should be pending"
+
+    # Stop slurmctld, start slurmdbd, remove QOS
+    atf.stop_slurmctld(quiet=True)
+    atf.start_slurmdbd(quiet=True)
+    atf.run_command(
+        f"sacctmgr -i remove qos {qos1}",
+        user=atf.properties["slurm-user"],
+        fatal=True,
+    )
+
+    # TODO: Unnecessary for 24.05+ (Bug 21393)
+    atf.run_command(
+        f"sacctmgr -i modify user {atf.get_user_name()} set qos={qos2}",
+        user=atf.properties["slurm-user"],
+        fatal=True,
+    )
+
+    # Start slurmctld and verify jobs state/reason
+    atf.start_slurmctld(quiet=True)
+
+    # Running job should continue running
+    assert atf.wait_for_job_state(
+        job_id1, "RUNNING"
+    ), f"Previously running job {job_id1} should stay RUNNING"
+    assert (
+        atf.get_job_parameter(job_id1, "Reason") == "None"
+    ), f"Running job {job_id1} should have 'None' as reason"
+
+    # Pending job should be marked with InvalidQOS
+    assert atf.wait_for_job_state(
+        job_id2, "PENDING", desired_reason="InvalidQOS"
+    ), f"Pending job {job_id2} should be PENDING with InvalidQOS reason"
+
+
+def test_account_removal_single():
+    """Test that removal of a single account:
+    1. Marks pending jobs with InvalidAccount
+    2. Rejects new job submissions using removed account
+    """
+
+    # Stop slurmdbd and submit job
+    atf.stop_slurmdbd(quiet=True)
+    job_id = submit_job_with(f"--account={acct1}", fatal=True)
+
+    # Stop slurmctld, start slurmdbd, remove account
+    atf.stop_slurmctld(quiet=True)
+    atf.start_slurmdbd(quiet=True)
+    atf.run_command(
+        f"sacctmgr -i remove account {acct1}",
+        user=atf.properties["slurm-user"],
+        fatal=True,
+    )
+
+    # Start slurmctld and verify job state/reason
+    atf.start_slurmctld(quiet=True)
+
+    assert atf.wait_for_job_state(
+        job_id, "PENDING", desired_reason="InvalidAccount"
+    ), f"Job {job_id} not in PENDING state and InvalidAccount reason"
+
+    # Try to submit a new job with removed account - should be rejected
+    assert (
+        submit_job_with(f"--account={acct1}", xfail=True) == 0
+    ), f"Job submission with removed account {acct1} should have failed"
+
+
+def test_account_removal_multiple():
+    """Test account removal when user has multiple account access:
+    1. Verifies jobs with removed account get marked InvalidAccount
+    2. Verifies jobs with remaining account stay valid
+    3. Verifies new job submissions with removed account are rejected
+    4. Verifies new job submissions with remaining account succeed
+    """
+
+    # Stop slurmdbd and submit jobs with different accounts
+    atf.stop_slurmdbd(quiet=True)
+    job_id1 = submit_job_with(f"--account={acct1}", fatal=True)
+    job_id2 = submit_job_with(f"--account={acct2}", fatal=True)
+
+    # Stop slurmctld, start slurmdbd, remove first account
+    atf.stop_slurmctld(quiet=True)
+    atf.start_slurmdbd(quiet=True)
+    atf.run_command(
+        f"sacctmgr -i remove account {acct1}",
+        user=atf.properties["slurm-user"],
+        fatal=True,
+    )
+
+    # Start slurmctld and verify jobs state/reason
+    atf.start_slurmctld(quiet=True)
+
+    # First job should be PENDING with InvalidAccount
+    assert atf.wait_for_job_state(
+        job_id1, "PENDING", desired_reason="InvalidAccount"
+    ), f"Job {job_id1} not in PENDING state and InvalidAccount reason"
+
+    # Second job should stay PENDING with different reason
+    assert atf.wait_for_job_state(
+        job_id2, "PENDING", timeout=10
+    ), f"Job {job_id2} should be in PENDING state"
+    assert (
+        atf.get_job_parameter(job_id2, "Reason") != "InvalidAccount"
+    ), "The second job whose account was not deleted should not be pending due to 'InvalidAccount'"
+
+    # Try to submit a new job with removed account - should be rejected
+    assert (
+        submit_job_with(f"--account={acct1}", xfail=True) == 0
+    ), f"Job submission with removed account {acct1} should have failed"
+
+    # Submit a job with remaining account - should succeed
+    assert (
+        submit_job_with(f"--account={acct2}") != 0
+    ), f"Job submission with valid account {acct2} should have succeeded"
+
+
+def test_account_removal_running_vs_pending():
+    """Test account removal impact on running vs pending jobs:
+    1. Submit two jobs with same account - one running, one pending
+    2. Remove the account
+    3. Verify running job continues running
+    4. Verify pending job gets marked with InvalidAccount
+    """
+
+    # Stop slurmdbd and submit jobs - use exclusive to ensure only one can run
+    atf.stop_slurmdbd(quiet=True)
+    job_id1 = submit_job_with(f"--account={acct1} --exclusive", fatal=True)
+    job_id2 = submit_job_with(f"--account={acct1} --exclusive", fatal=True)
+
+    # Wait for first job to start running
+    assert atf.wait_for_job_state(
+        job_id1, "RUNNING"
+    ), f"Job {job_id1} never started running"
+
+    # Verify second job is pending (due to exclusive)
+    assert atf.wait_for_job_state(
+        job_id2, "PENDING"
+    ), f"Job {job_id2} should be pending"
+
+    # Stop slurmctld, start slurmdbd, remove account
+    atf.stop_slurmctld(quiet=True)
+    atf.start_slurmdbd(quiet=True)
+    atf.run_command(
+        f"sacctmgr -i remove account {acct1}",
+        user=atf.properties["slurm-user"],
+        fatal=True,
+    )
+
+    # Start slurmctld and verify jobs state/reason
+    atf.start_slurmctld(quiet=True)
+
+    # Running job should continue running
+    assert atf.wait_for_job_state(
+        job_id1, "RUNNING"
+    ), f"Previously running job {job_id1} should stay RUNNING"
+    assert (
+        atf.get_job_parameter(job_id1, "Reason") == "None"
+    ), f"Running job {job_id1} should have 'None' as reason"
+
+    # Pending job should be marked with InvalidAccount
+    assert atf.wait_for_job_state(
+        job_id2, "PENDING", desired_reason="InvalidAccount"
+    ), f"Pending job {job_id2} should be PENDING with InvalidAccount reason"

From 949b550230d9756324aca714f85f63a45bff2214 Mon Sep 17 00:00:00 2001
From: Nathan Rini <nate@schedmd.com>
Date: Wed, 13 Nov 2024 10:11:41 -0700
Subject: [PATCH 60/90] Avoid slurmd crashing after failing to unpack
 SlurmdSpoolDir/cred_state

If a newer version of slurmd has been started and upgraded the file formats
in SlurmdSpoolDir, trying to start an older version of slurmd can result in
an unpack failure. The cred_job_list and/or cred_state_list would then be
NULL. Calling list_delete_all() on a NULL list crashes. Instead of crashing,
log a warning and return.

Issue 50154, Ticket 21432
---
 NEWS                             |  3 +++
 src/slurmd/slurmd/cred_context.c | 18 ++++++++++++++++--
 2 files changed, 19 insertions(+), 2 deletions(-)

diff --git a/NEWS b/NEWS
index 29ca7d6f585..fc583d5587b 100644
--- a/NEWS
+++ b/NEWS
@@ -20,6 +20,9 @@ documents those changes that are of interest to users and administrators.
     '.jobs[].job_resources'.
  -- Fix sstat/sattach protocol errors for steps on higher version slurmd's
     (regressions since 20.11.0rc1 and 16.05.1rc1 respectively).
+ -- slurmd - Avoid a crash when starting slurmd version 24.05 with
+    SlurmdSpoolDir files that have been upgraded to a newer major version of
+    Slurm. Log warnings instead.
 
 * Changes in Slurm 24.05.4
 ==========================
diff --git a/src/slurmd/slurmd/cred_context.c b/src/slurmd/slurmd/cred_context.c
index 998c24cf838..8df077ffc65 100644
--- a/src/slurmd/slurmd/cred_context.c
+++ b/src/slurmd/slurmd/cred_context.c
@@ -124,7 +124,14 @@ static job_state_t *_find_job_state(uint32_t jobid)
 
 static void _clear_expired_job_states(void)
 {
-	time_t now = time(NULL);
+	time_t now;
+
+	if (!cred_job_list) {
+		warning("No cred_job_list, unable to clear expired job states");
+		return;
+	}
+
+	now = time(NULL);
 	list_delete_all(cred_job_list, _list_find_expired_job_state, &now);
 }
 
@@ -140,7 +147,14 @@ static int _list_find_expired_cred_state(void *x, void *key)
 
 static void _clear_expired_credential_states(void)
 {
-	time_t now = time(NULL);
+	time_t now;
+
+	if (!cred_state_list) {
+		warning("No cred_state_list, unable to clear expired credential states");
+		return;
+	}
+
+	now = time(NULL);
 	list_delete_all(cred_state_list, _list_find_expired_cred_state, &now);
 }
 

From c6eed2f29b11e4175c79e2f77c6ffc6fb4087af9 Mon Sep 17 00:00:00 2001
From: Brian Christiansen <brian@schedmd.com>
Date: Mon, 25 Nov 2024 23:25:45 -0700
Subject: [PATCH 61/90] Fix race condition in stepmgr step completion handling

Ticket 21122

Signed-Off-By: Scott Hilton <scott@schedmd.com>
---
 NEWS                        |  1 +
 src/slurmd/slurmstepd/req.c | 58 +++++++++++++++++++++----------------
 2 files changed, 34 insertions(+), 25 deletions(-)

diff --git a/NEWS b/NEWS
index fc583d5587b..41825a3cd30 100644
--- a/NEWS
+++ b/NEWS
@@ -23,6 +23,7 @@ documents those changes that are of interest to users and administrators.
  -- slurmd - Avoid a crash when starting slurmd version 24.05 with
     SlurmdSpoolDir files that have been upgraded to a newer major version of
     Slurm. Log warnings instead.
+ -- Fix race condition in stepmgr step completion handling.
 
 * Changes in Slurm 24.05.4
 ==========================
diff --git a/src/slurmd/slurmstepd/req.c b/src/slurmd/slurmstepd/req.c
index e7da47c75dd..5c0e21cbfa1 100644
--- a/src/slurmd/slurmstepd/req.c
+++ b/src/slurmd/slurmstepd/req.c
@@ -2163,31 +2163,39 @@ _handle_completion(int fd, stepd_step_rec_t *step, uid_t uid)
 		goto rwfail;
 	FREE_NULL_BUFFER(buffer);
 
-	if (job_step_ptr && do_stepmgr) {
-		int rem = 0;
-		uint32_t max_rc;
-		slurm_step_id_t temp_id = {
-			.job_id = job_step_ptr->job_id,
-			.step_het_comp = NO_VAL,
-			.step_id = step_id
-		};
-
-		step_complete_msg_t req = {
-			.range_first = first,
-			.range_last = last,
-			.step_id = temp_id,
-			.step_rc = step_rc,
-			.jobacct = jobacct
-		};
-
-		step_partial_comp(&req, uid, true, &rem, &max_rc);
-
-		safe_write(fd, &rc, sizeof(int));
-		safe_write(fd, &errnum, sizeof(int));
-
-		jobacctinfo_destroy(jobacct);
-
-		return SLURM_SUCCESS;
+	if (do_stepmgr) {
+		slurm_mutex_lock(&stepmgr_mutex);
+		if (job_step_ptr) {
+			int rem = 0;
+			uint32_t max_rc;
+			slurm_step_id_t temp_id = {
+				.job_id = job_step_ptr->job_id,
+				.step_het_comp = NO_VAL,
+				.step_id = step_id
+			};
+
+			step_complete_msg_t req = {
+				.range_first = first,
+				.range_last = last,
+				.step_id = temp_id,
+				.step_rc = step_rc,
+				.jobacct = jobacct
+			};
+
+			step_partial_comp(&req, uid, true, &rem, &max_rc);
+
+			safe_write(fd, &rc, sizeof(int));
+			safe_write(fd, &errnum, sizeof(int));
+
+			jobacctinfo_destroy(jobacct);
+
+			rc = SLURM_SUCCESS;
+		} else {
+			error("Asked to complete a stepmgr step but we don't have a job_step_ptr. This should never happen.");
+			rc = SLURM_ERROR;
+		}
+		slurm_mutex_unlock(&stepmgr_mutex);
+		return rc;
 	}
 
 	/*

From 6c48f9d89312d24e5789c244807885f35fcbdc86 Mon Sep 17 00:00:00 2001
From: Megan Dahl <megan@schedmd.com>
Date: Tue, 24 Sep 2024 18:41:44 +0200
Subject: [PATCH 62/90] Fix ctld segfault with job arrays and reserved ports

If one has MpiParams or do use resv-ports option, in combination with job
arrays, the controller will segfault. The node_bitmap being referenced has
already been freed. Additionally, the job record was copied from the
previous array task and thus referenced the other tasks resv_port_cnt,
resv_ports, and resv_port_array. Make sure that these copied values get
reset to default values before returning from job_array_split().

Ticket 21008.
---
 NEWS                           |  1 +
 src/common/port_mgr.c          | 48 ++++++++++++++++++++++++++++++++++
 src/slurmctld/job_mgr.c        |  2 ++
 src/slurmctld/node_scheduler.c |  8 ++++--
 4 files changed, 57 insertions(+), 2 deletions(-)

diff --git a/NEWS b/NEWS
index 41825a3cd30..a09a8e5b02b 100644
--- a/NEWS
+++ b/NEWS
@@ -24,6 +24,7 @@ documents those changes that are of interest to users and administrators.
     SlurmdSpoolDir files that have been upgraded to a newer major version of
     Slurm. Log warnings instead.
  -- Fix race condition in stepmgr step completion handling.
+ -- Fix slurmctld segfault with stepmgr and MpiParams when running a job array.
 
 * Changes in Slurm 24.05.4
 ==========================
diff --git a/src/common/port_mgr.c b/src/common/port_mgr.c
index c78abc3956b..dc89241a7c9 100644
--- a/src/common/port_mgr.c
+++ b/src/common/port_mgr.c
@@ -342,6 +342,9 @@ static int _resv_port_alloc(uint16_t resv_port_cnt,
 	static int last_port_alloc = 0;
 	static int dims = -1;
 
+	xassert(!*resv_ports);
+	xassert(!*resv_port_array);
+
 	if (dims == -1)
 		dims = slurmdb_setup_cluster_dims();
 
@@ -389,6 +392,28 @@ extern int resv_port_step_alloc(step_record_t *step_ptr)
 	int rc;
 	int port_inx;
 
+	if (step_ptr->resv_port_array || step_ptr->resv_ports) {
+		/*
+		 * Both resv_ports and resv_port_array need to be NULL.
+		 * If they are not that could lead to resv_ports never being
+		 * freed on nodes, eventually making those nodes unable to
+		 * schedule jobs since their ports could have been allocated
+		 * without being freed. By setting resv_ports and
+		 * resv_port_array to NULL in job_array_split() guarantees that,
+		 * but try to catch this issue if it happens in future.
+		 */
+		error("%pS allocated reserved ports while it already had reserved ports %s",
+		       step_ptr, step_ptr->resv_ports);
+
+		/*
+		 * We can't just call _resv_port_free() because it is not
+		 * guaranteed that the node_bitmap or resv_port_cnt is the same
+		 * from when resv_port_array was allocated.
+		 */
+		xfree(step_ptr->resv_port_array);
+		xfree(step_ptr->resv_ports);
+	}
+
 	rc = _resv_port_alloc(step_ptr->resv_port_cnt,
 			      step_ptr->step_node_bitmap, &step_ptr->resv_ports,
 			      &step_ptr->resv_port_array, &port_inx);
@@ -408,6 +433,29 @@ extern int resv_port_job_alloc(job_record_t *job_ptr)
 	int rc;
 	int port_inx;
 
+	if (job_ptr->resv_port_array || job_ptr->resv_ports) {
+		/*
+		 * Both resv_ports and resv_port_array need to be NULL.
+		 * If they are not that could lead to resv_ports never being
+		 * freed on nodes, eventually making those nodes unable to
+		 * schedule jobs since their ports could have been allocated
+		 * without being freed. By setting resv_ports and
+		 * resv_port_array to NULL in job_array_split() guarantees that,
+		 * but try to catch this issue if it happens in future.
+		 */
+		error("%pJ allocated reserved ports while it already had reserved ports %s. Ports may be lost, which will require a restart of the slurmctld daemon to resolve.",
+		       job_ptr, job_ptr->resv_ports);
+
+		/*
+		 * We can't just call _resv_port_free() because it is not
+		 * guaranteed that the node_bitmap or resv_port_cnt is the same
+		 * from when resv_port_array was allocated. A restart of the
+		 * controller will restore any lost ports.
+		 */
+		xfree(job_ptr->resv_port_array);
+		xfree(job_ptr->resv_ports);
+	}
+
 	rc = _resv_port_alloc(job_ptr->resv_port_cnt,
 			      job_ptr->node_bitmap, &job_ptr->resv_ports,
 			      &job_ptr->resv_port_array, &port_inx);
diff --git a/src/slurmctld/job_mgr.c b/src/slurmctld/job_mgr.c
index d0c02a94447..a3a04aa2562 100644
--- a/src/slurmctld/job_mgr.c
+++ b/src/slurmctld/job_mgr.c
@@ -3477,6 +3477,8 @@ extern job_record_t *job_array_split(job_record_t *job_ptr)
 	job_ptr_pend->resv_name = xstrdup(job_ptr->resv_name);
 	if (job_ptr->resv_list)
 		job_ptr_pend->resv_list = list_shallow_copy(job_ptr->resv_list);
+	job_ptr_pend->resv_ports = NULL;
+	job_ptr_pend->resv_port_array = NULL;
 	job_ptr_pend->resp_host = xstrdup(job_ptr->resp_host);
 	if (job_ptr->select_jobinfo) {
 		job_ptr_pend->select_jobinfo =
diff --git a/src/slurmctld/node_scheduler.c b/src/slurmctld/node_scheduler.c
index 1f1794f1c1f..bda74ea0618 100644
--- a/src/slurmctld/node_scheduler.c
+++ b/src/slurmctld/node_scheduler.c
@@ -2538,6 +2538,7 @@ extern int select_nodes(job_record_t *job_ptr, bool test_only,
 		{ .assoc = READ_LOCK, .qos = WRITE_LOCK, .tres = READ_LOCK };
 	List gres_list_pre = NULL;
 	bool gres_list_pre_set = false;
+	job_record_t *tmp_job;
 
 	xassert(job_ptr);
 	xassert(job_ptr->magic == JOB_MAGIC);
@@ -2942,7 +2943,10 @@ extern int select_nodes(job_record_t *job_ptr, bool test_only,
 
 	job_end_time_reset(job_ptr);
 
-	(void) job_array_post_sched(job_ptr);
+	tmp_job = job_array_post_sched(job_ptr);
+	if (tmp_job && (tmp_job != job_ptr) && (orig_resv_port_cnt == NO_VAL16))
+		tmp_job->resv_port_cnt = orig_resv_port_cnt;
+
 	if (bb_g_job_begin(job_ptr) != SLURM_SUCCESS) {
 		/* Leave job queued, something is hosed */
 		error_code = ESLURM_INVALID_BURST_BUFFER_REQUEST;
@@ -3101,7 +3105,6 @@ extern int select_nodes(job_record_t *job_ptr, bool test_only,
 	}
 
 	if (error_code != SLURM_SUCCESS) {
-		FREE_NULL_BITMAP(job_ptr->node_bitmap);
 		if (gres_list_pre_set &&
 		    (job_ptr->gres_list_req != gres_list_pre)) {
 			FREE_NULL_LIST(job_ptr->gres_list_req);
@@ -3114,6 +3117,7 @@ extern int select_nodes(job_record_t *job_ptr, bool test_only,
 			resv_port_job_free(job_ptr);
 			xfree(job_ptr->resv_ports);
 		}
+		FREE_NULL_BITMAP(job_ptr->node_bitmap);
 	} else
 		FREE_NULL_LIST(gres_list_pre);
 

From c90858ccbedf55832349b7d193324298ed4d64f4 Mon Sep 17 00:00:00 2001
From: Matt Ezell <ezellma@ornl.gov>
Date: Wed, 30 Oct 2024 18:42:42 -0400
Subject: [PATCH 63/90] Reset a job's priority when it is requeued

When a job is requeued it's begin_time is set to now+cred_expire+1, and
the job's priority remains until it's eligible to run again AND until
the decay thread updates the priority. So recalculate the job's priority
right after it has requeued.

Ticket 21312
---
 NEWS                    | 1 +
 src/slurmctld/job_mgr.c | 4 ++++
 2 files changed, 5 insertions(+)

diff --git a/NEWS b/NEWS
index a09a8e5b02b..c248b2ca6a0 100644
--- a/NEWS
+++ b/NEWS
@@ -25,6 +25,7 @@ documents those changes that are of interest to users and administrators.
     Slurm. Log warnings instead.
  -- Fix race condition in stepmgr step completion handling.
  -- Fix slurmctld segfault with stepmgr and MpiParams when running a job array.
+ -- Fix requeued jobs keeping their priority until the decay thread happens.
 
 * Changes in Slurm 24.05.4
 ==========================
diff --git a/src/slurmctld/job_mgr.c b/src/slurmctld/job_mgr.c
index a3a04aa2562..e99e2d58f21 100644
--- a/src/slurmctld/job_mgr.c
+++ b/src/slurmctld/job_mgr.c
@@ -15676,6 +15676,10 @@ void batch_requeue_fini(job_record_t *job_ptr)
 		}
 	}
 
+	/* Reset the priority (begin and accrue times were reset) */
+	if (job_ptr->priority != 0)
+		set_job_prio(job_ptr);
+
 	/*
 	 * If a reservation ended and was a repeated (e.g., daily, weekly)
 	 * reservation, its ID will be different; make sure

From 7eee12524739cabd8c19588f411e0de2d0317517 Mon Sep 17 00:00:00 2001
From: Andy Georges <itkovian@gmail.com>
Date: Wed, 27 Nov 2024 17:28:30 +0100
Subject: [PATCH 64/90] fix: include 24.05.ug branch

---
 .github/workflows/build.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
index 04ee9237cb8..3a69be7cd3c 100644
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@@ -2,9 +2,9 @@ name: slurm C/C++ build
 
 on:
   push:
-    branches: [ 20.11.ug, 22.05.ug ]
+    branches: [ 20.11.ug, 22.05.ug, 24.05.ug ]
   pull_request:
-    branches: [ 20.11.ug, 22.05.ug ]
+    branches: [ 20.11.ug, 22.05.ug, 24.05.ug ]
 
 jobs:
   build:

From 43572f8428ef4b9bf718208b71bf1794f1f704c2 Mon Sep 17 00:00:00 2001
From: Stephen Kendall <stephen@schedmd.com>
Date: Mon, 25 Nov 2024 12:01:24 -0700
Subject: [PATCH 65/90] Docs - move enable_job_state_cache to
 SchedulerParameters

Ticket 21517
---
 doc/man/man5/slurm.conf.5 | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/doc/man/man5/slurm.conf.5 b/doc/man/man5/slurm.conf.5
index 981b790d750..98ba99e476f 100644
--- a/doc/man/man5/slurm.conf.5
+++ b/doc/man/man5/slurm.conf.5
@@ -4155,6 +4155,13 @@ Enable job steps that span heterogeneous job allocations.
 The default value.
 .IP
 
+.TP
+\fBenable_job_state_cache\fR
+Enables an independent cache of job state details within slurmctld. This allows
+processing of `\fBsqueue\fR \-\-only\-job\-state` and replaced RPCs with minimal
+impact on other slurmctld operations.
+.IP
+
 .TP
 \fBenable_user_top\fR
 Enable use of the "scontrol top" command by non\-privileged users.
@@ -4846,13 +4853,6 @@ filenames have no path separators and are located adjacent to slurm.conf.
 Glob patterns (See \fBglob\fR (7)) are not supported.
 .IP
 
-.TP
-\fBenable_job_state_cache\fR
-Enables an independent cache of job state details within slurmctld. This allows
-processing of `\fBsqueue\fR \-\-only\-job\-state` and replaced RPCs with minimal
-impact on other slurmctld operations.
-.IP
-
 .TP
 \fBidle_on_node_suspend\fR
 Mark nodes as idle, regardless of current state, when suspending nodes with

From e13128836cfb3d80e9face53a04cdb27df5efda6 Mon Sep 17 00:00:00 2001
From: Nathan Rini <nate@schedmd.com>
Date: Tue, 26 Nov 2024 19:58:43 -0700
Subject: [PATCH 66/90] Docs - Add docker exceptions to containers guide

Exceptions for 'docker-compose', 'docker swarm',
and docker commands/api from inside containers.

Issue 50171
---
 doc/html/containers.shtml | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/doc/html/containers.shtml b/doc/html/containers.shtml
index e8cea0253bd..17b96110074 100644
--- a/doc/html/containers.shtml
+++ b/doc/html/containers.shtml
@@ -515,10 +515,14 @@ the Slurm controller. The container is run by Slurm on the compute nodes which
 makes having Docker setup a network isolation layer ineffective for the
 container.</li>
 <li><pre>docker exec</pre> command is not supported.</li>
-<li><pre>docker compose</pre> command is not supported.</li>
+<li><pre>docker swarm</pre> command is not supported.</li>
+<li><pre>docker compose</pre>/<pre>docker-compose</pre> command is not supported.</li>
 <li><pre>docker pause</pre> command is not supported.</li>
 <li><pre>docker unpause</pre> command is not supported.</li>
 <li><pre>docker swarm</pre> command is not supported.</li>
+<li>All <pre>docker</pre> commands are not supported inside of containers.</li>
+<li><a href="https://docs.docker.com/reference/api/engine/">Docker API</a> is
+	not supported inside of containers.</li>
 </ol>
 
 <h3>Setup procedure</h3>

From 5502b1bb55e2d37cbf942bf66552d8df6872dbe1 Mon Sep 17 00:00:00 2001
From: Nathan Rini <nate@schedmd.com>
Date: Tue, 26 Nov 2024 20:25:07 -0700
Subject: [PATCH 67/90] Docs - Add podman exceptions to containers guide

Exceptions for 'podman-compose', 'podman farm'
and podman commands/api from inside containers.

Issue 50171
---
 doc/html/containers.shtml | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/doc/html/containers.shtml b/doc/html/containers.shtml
index 17b96110074..7b38b4138f8 100644
--- a/doc/html/containers.shtml
+++ b/doc/html/containers.shtml
@@ -641,8 +641,15 @@ configuration.</li>
 <a href="https://github.com/containers/podman/blob/main/docs/tutorials/basic_networking.md">
 host networking</a></li>
 <li><pre>podman exec</pre> command is not supported.</li>
+<li><pre>podman-compose</pre> command is not supported, due to only being
+	partially implemented. Some compositions may work but each container
+	may be run on different nodes. The network for all containers must be
+	the <code>network_mode: host</code> device.</li>
 <li><pre>podman kube</pre> command is not supported.</li>
 <li><pre>podman pod</pre> command is not supported.</li>
+<li><pre>podman farm</pre> command is not supported.</li>
+<li>All <pre>podman</pre> commands are not supported inside of containers.</li>
+<li>Podman REST API is not supported inside of containers.</li>
 </ol>
 
 <h3>Setup procedure</h3>

From 038eb99a6b8d5be846bc3518faf6a9fca7bf75a6 Mon Sep 17 00:00:00 2001
From: Nathan Rini <nate@schedmd.com>
Date: Tue, 26 Nov 2024 20:32:37 -0700
Subject: [PATCH 68/90] Docs - Swap docker and podman exceptions to use 'code'
 blocks

Issue 50171
---
 doc/html/containers.shtml | 29 +++++++++++++++--------------
 1 file changed, 15 insertions(+), 14 deletions(-)

diff --git a/doc/html/containers.shtml b/doc/html/containers.shtml
index 7b38b4138f8..c1b9e75c4dc 100644
--- a/doc/html/containers.shtml
+++ b/doc/html/containers.shtml
@@ -514,13 +514,14 @@ scrun being isolated from the network and not being able to communicate with
 the Slurm controller. The container is run by Slurm on the compute nodes which
 makes having Docker setup a network isolation layer ineffective for the
 container.</li>
-<li><pre>docker exec</pre> command is not supported.</li>
-<li><pre>docker swarm</pre> command is not supported.</li>
-<li><pre>docker compose</pre>/<pre>docker-compose</pre> command is not supported.</li>
-<li><pre>docker pause</pre> command is not supported.</li>
-<li><pre>docker unpause</pre> command is not supported.</li>
-<li><pre>docker swarm</pre> command is not supported.</li>
-<li>All <pre>docker</pre> commands are not supported inside of containers.</li>
+<li><code>docker exec</code> command is not supported.</li>
+<li><code>docker swarm</code> command is not supported.</li>
+<li><code>docker compose</code>/<code>docker-compose</code> command is not
+	supported.</li>
+<li><code>docker pause</code> command is not supported.</li>
+<li><code>docker unpause</code> command is not supported.</li>
+<li><code>docker swarm</code> command is not supported.</li>
+<li>All <code>docker</code> commands are not supported inside of containers.</li>
 <li><a href="https://docs.docker.com/reference/api/engine/">Docker API</a> is
 	not supported inside of containers.</li>
 </ol>
@@ -640,15 +641,15 @@ configuration.</li>
 <li>All containers must use
 <a href="https://github.com/containers/podman/blob/main/docs/tutorials/basic_networking.md">
 host networking</a></li>
-<li><pre>podman exec</pre> command is not supported.</li>
-<li><pre>podman-compose</pre> command is not supported, due to only being
+<li><code>podman exec</code> command is not supported.</li>
+<li><code>podman-compose</code> command is not supported, due to only being
 	partially implemented. Some compositions may work but each container
 	may be run on different nodes. The network for all containers must be
 	the <code>network_mode: host</code> device.</li>
-<li><pre>podman kube</pre> command is not supported.</li>
-<li><pre>podman pod</pre> command is not supported.</li>
-<li><pre>podman farm</pre> command is not supported.</li>
-<li>All <pre>podman</pre> commands are not supported inside of containers.</li>
+<li><code>podman kube</code> command is not supported.</li>
+<li><code>podman pod</code> command is not supported.</li>
+<li><code>podman farm</code> command is not supported.</li>
+<li>All <code>podman</code> commands are not supported inside of containers.</li>
 <li>Podman REST API is not supported inside of containers.</li>
 </ol>
 
@@ -942,6 +943,6 @@ Overview slides of Sarus are
 
 <hr size=4 width="100%">
 
-<p style="text-align:center;">Last modified 19 November 2024</p>
+<p style="text-align:center;">Last modified 27 November 2024</p>
 
 <!--#include virtual="footer.txt"-->

From cc982bc73909f29c4347112bb895804d311d1334 Mon Sep 17 00:00:00 2001
From: Nathan Rini <nate@schedmd.com>
Date: Tue, 26 Nov 2024 16:28:53 -0700
Subject: [PATCH 69/90] Docs - Add allow userns via apparmor in containers
 guide

23.10+ Ubuntu kernels restrict unprivileged user namespaces which breaks
rootless podman and docker.

Issue 50169
---
 doc/html/containers.shtml | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/doc/html/containers.shtml b/doc/html/containers.shtml
index c1b9e75c4dc..0f5ee7aef6c 100644
--- a/doc/html/containers.shtml
+++ b/doc/html/containers.shtml
@@ -78,7 +78,11 @@ job or any given plugin).</li>
 
 <h2 id="prereq">Prerequisites<a class="slurm_link" href="#prereq"></a></h2>
 <p>The host kernel must be configured to allow user land containers:</p>
-<pre>$ sudo sysctl -w kernel.unprivileged_userns_clone=1</pre>
+<pre>
+sudo sysctl -w kernel.unprivileged_userns_clone=1
+sudo sysctl -w kernel.apparmor_restrict_unprivileged_unconfined=0
+sudo sysctl -w kernel.apparmor_restrict_unprivileged_userns=0
+</pre>
 
 <p>Docker also provides a tool to verify the kernel configuration:
 <pre>$ dockerd-rootless-setuptool.sh check --force

From b5ef918b2cc4db1ffa15a4068556fd53129d3b5c Mon Sep 17 00:00:00 2001
From: Andy Georges <itkovian@gmail.com>
Date: Thu, 5 Dec 2024 15:15:03 +0100
Subject: [PATCH 70/90] fix: add munge developnment to gh workflow build

---
 .github/workflows/build.yml | 25 +++++++++++++------------
 1 file changed, 13 insertions(+), 12 deletions(-)

diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
index 3a69be7cd3c..1bcd969c943 100644
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@@ -2,22 +2,23 @@ name: slurm C/C++ build
 
 on:
   push:
-    branches: [ 20.11.ug, 22.05.ug, 24.05.ug ]
+    branches: [20.11.ug, 22.05.ug, 24.05.ug]
   pull_request:
-    branches: [ 20.11.ug, 22.05.ug, 24.05.ug ]
+    branches: [20.11.ug, 22.05.ug, 24.05.ug]
 
 jobs:
   build:
-
     runs-on: ubuntu-20.04
 
     steps:
-    - uses: actions/checkout@v2
-    - name: configure
-      run: ./configure --enable-multiple-slurmd --prefix=/tmp/slurm/
-    - name: make
-      run: make -j
-    - name: make check
-      run: make -j check
-    - name: make install
-      run: make -j install
+      - uses: actions/checkout@v2
+      - name: Install deps
+        run: sudo apt-get install -y libmunge-dev
+      - name: configure
+        run: ./configure --enable-multiple-slurmd --prefix=/tmp/slurm/
+      - name: make
+        run: make -j
+      - name: make check
+        run: make -j check
+      - name: make install
+        run: make -j install

From 17ec21fb8944d404d7582d7ab0b6c8cfeb689771 Mon Sep 17 00:00:00 2001
From: Nathan Prisbrey <nathan@schedmd.com>
Date: Sat, 30 Nov 2024 23:34:21 +0000
Subject: [PATCH 71/90] Testsuite - Improve logging of slurmdbd start/stop

Ticket 21393

Cherry-picked: eca4a64f53
---
 testsuite/python/lib/atf.py | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/testsuite/python/lib/atf.py b/testsuite/python/lib/atf.py
index 647db89c9c4..304716256ec 100644
--- a/testsuite/python/lib/atf.py
+++ b/testsuite/python/lib/atf.py
@@ -561,6 +561,8 @@ def start_slurmdbd(clean=False, quiet=False):
     if not properties["auto-config"]:
         require_auto_config("wants to start slurmdbd")
 
+    logging.debug("Starting slurmdbd...")
+
     if (
         run_command_exit(
             "sacctmgr show cluster", user=properties["slurm-user"], quiet=quiet
@@ -583,6 +585,8 @@ def start_slurmdbd(clean=False, quiet=False):
             "sacctmgr show cluster", lambda results: results["exit_code"] == 0
         ):
             pytest.fail(f"Slurmdbd is not running")
+        else:
+            logging.debug("Slurmdbd started successfully")
 
 
 def start_slurm(clean=False, quiet=False):
@@ -727,6 +731,8 @@ def stop_slurmdbd(quiet=False):
     if not properties["auto-config"]:
         require_auto_config("wants to stop slurmdbd")
 
+    logging.debug("Stopping slurmdbd...")
+
     # Stop slurmdbd
     results = run_command(
         "sacctmgr shutdown", user=properties["slurm-user"], quiet=quiet
@@ -743,6 +749,8 @@ def stop_slurmdbd(quiet=False):
         timeout=60,
     ):
         failures.append("Slurmdbd is still running")
+    else:
+        logging.debug("Slurmdbd stopped successfully")
 
 
 def stop_slurm(fatal=True, quiet=False):

From abcba2b6120ebaeb1fcd96f11c28e05ed756a688 Mon Sep 17 00:00:00 2001
From: Nathan Prisbrey <nathan@schedmd.com>
Date: Sat, 30 Nov 2024 23:30:45 +0000
Subject: [PATCH 72/90] Testsuite - Fix test_102_5 job race condition

Address race condition where submitted jobs would occasionally start
RUNNING when the test expected PENDING jobs.

Tests running_vs_pending are now not necessary.

Ticket 21393

Cherry-picked: d4ebd3f697
---
 testsuite/python/tests/test_102_5.py | 300 ++++++++++++++-------------
 1 file changed, 152 insertions(+), 148 deletions(-)

diff --git a/testsuite/python/tests/test_102_5.py b/testsuite/python/tests/test_102_5.py
index f6576fe752f..2490fa66cfb 100644
--- a/testsuite/python/tests/test_102_5.py
+++ b/testsuite/python/tests/test_102_5.py
@@ -69,79 +69,127 @@ def submit_job_with(extra_params, xfail=False, fatal=False):
 
 
 def test_qos_removal_single():
-    """Test that removal of a single QOS:
-    1. Marks pending jobs with InvalidQOS
-    2. Rejects new job submissions using removed QOS
+    """Test removing a QOS in use:
+    - Verify that running jobs with that QOS keep running.
+    - Verify that pending jobs are updated to InvalidQOS
+    - Verify that new jobs cannot use the removed QOS.
     """
-
-    # Stop slurmdbd and submit job
+    # Stop slurmdbd to avoid the job info being saved in the DB
     atf.stop_slurmdbd(quiet=True)
-    job_id = submit_job_with(f"--qos={qos1}", fatal=True)
 
-    # Stop slurmctld, start slurmdbd, remove QOS
+    # Submit a blocking job
+    job_id1 = submit_job_with(f"--qos={qos1} --exclusive", fatal=True)
+    assert atf.wait_for_job_state(
+        job_id1, "RUNNING"
+    ), f"Job {job_id1} never started running"
+
+    # Submit another job in the same node to be blocked (due exclusive)
+    node = atf.get_job_parameter(job_id1, "NodeList")
+    job_id2 = submit_job_with(f"--qos={qos1} -w {node}", fatal=True)
+    assert atf.wait_for_job_state(
+        job_id2, "PENDING"
+    ), f"Job {job_id2} should be pending"
+
+    # Stop slurmctld before starting slurmdbd to keep the jobs info out of
+    # the DB, only in slurmctld for the moment.
     atf.stop_slurmctld(quiet=True)
     atf.start_slurmdbd(quiet=True)
+
+    # Remove the QOS from the DB.
+    # Note that slurmdbd won't have the QOS or the jobs using it, while
+    # slurmctld knows the jobs and still thinks that the QOS exists.
     atf.run_command(
         f"sacctmgr -i remove qos {qos1}",
         user=atf.properties["slurm-user"],
         fatal=True,
     )
 
-    # Start slurmctld and verify job state/reason
+    # Start slurmctld and verify job states/reasons are the expected now that
+    # the QOS doesn't exists anymore.
     atf.start_slurmctld(quiet=True)
 
+    # Running job should continue running
     assert atf.wait_for_job_state(
-        job_id, "PENDING", desired_reason="InvalidQOS"
-    ), f"Job {job_id} not in PENDING state with InvalidQOS reason"
+        job_id1, "RUNNING", desired_reason="None"
+    ), f"Previously running job {job_id1} should stay RUNNING with 'None' reason"
+
+    # Pending job should be marked with InvalidQOS
+    assert atf.wait_for_job_state(
+        job_id2, "PENDING", desired_reason="InvalidQOS"
+    ), f"Pending job {job_id2} should be PENDING with InvalidQOS reason"
 
     # Try to submit a new job with removed QOS - should be rejected
     assert (
         submit_job_with(f"--qos={qos1}", xfail=True) == 0
-    ), f"Job submission with removed QOS {qos1} should have failed but got job id {job_id2}"
-
-    # Submit a job with remaining QOS - should succeed
-    assert (
-        submit_job_with(f"--qos={qos2}") != 0
-    ), f"Job submission with valid QOS {qos2} should have succeeded"
+    ), f"Job submission with removed QOS {qos1} should have failed"
 
 
 def test_qos_removal_multiple():
     """Test QOS removal when user has multiple QOS access:
-    1. Verifies jobs with removed QOS get marked InvalidQOS
-    2. Verifies jobs with remaining QOS stay valid
-    3. Verifies new job submissions with removed QOS are rejected
-    4. Verifies new job submissions with remaining QOS succeed
+    - Verify that running jobs with removed QOS keep running.
+    - Verify that pending jobs with removed QOS are updated to InvalidQOS.
+    - Verify that jobs with remaining QOS stay valid.
+    - Verify that new jobs cannot use the removed QOS.
+    - Verify that new job can use the remaining QOS.
     """
 
-    # Stop slurmdbd and submit jobs with different QOSs
+    # Stop slurmdbd to avoid the job info being saved in the DB
     atf.stop_slurmdbd(quiet=True)
-    job_id1 = submit_job_with(f"--qos={qos1}", fatal=True)
-    job_id2 = submit_job_with(f"--qos={qos2}", fatal=True)
 
-    # Stop slurmctld, start slurmdbd, remove first QOS
+    # Submit a blocking job
+    job_id1 = submit_job_with(f"--qos={qos1} --exclusive", fatal=True)
+    assert atf.wait_for_job_state(
+        job_id1, "RUNNING"
+    ), f"Job {job_id1} never started running"
+
+    # Submit two more jobs in the same node to be blocked (due exclusive)
+    node = atf.get_job_parameter(job_id1, "NodeList")
+    job_id2 = submit_job_with(f"--qos={qos1} -w {node}", fatal=True)
+    job_id3 = submit_job_with(f"--qos={qos2} -w {node}", fatal=True)
+
+    # Verify both jobs are pending
+    assert atf.wait_for_job_state(
+        job_id2, "PENDING"
+    ), f"Job {job_id2} should be pending"
+    assert atf.wait_for_job_state(
+        job_id3, "PENDING"
+    ), f"Job {job_id3} should be pending"
+
+    # Stop slurmctld before starting slurmdbd to keep the jobs info out of
+    # the DB, only in slurmctld for the moment.
     atf.stop_slurmctld(quiet=True)
     atf.start_slurmdbd(quiet=True)
+
+    # Remove the QOS from the DB.
+    # Note that slurmdbd won't have the QOS or the jobs using it, while
+    # slurmctld knows the jobs and still thinks that the QOS exists.
     atf.run_command(
         f"sacctmgr -i remove qos {qos1}",
         user=atf.properties["slurm-user"],
         fatal=True,
     )
 
-    # Start slurmctld and verify jobs state/reason
+    # Start slurmctld and verify job states/reasons are the expected now that
+    # the QOS doesn't exists anymore.
     atf.start_slurmctld(quiet=True)
 
-    # First job should be PENDING with InvalidQOS
+    # Running job should continue running
     assert atf.wait_for_job_state(
-        job_id1, "PENDING", desired_reason="InvalidQOS"
-    ), f"Job {job_id1} not in PENDING state and InvalidQOS reason"
+        job_id1, "RUNNING", desired_reason="None"
+    ), f"Previously running job {job_id1} should stay RUNNING with 'None' reason"
 
-    # Second job should stay PENDING with different reason
+    # Pending job with removed QOS should be marked with InvalidQOS
     assert atf.wait_for_job_state(
-        job_id2, "PENDING", timeout=10
-    ), f"Job {job_id2} should be in PENDING state"
+        job_id2, "PENDING", desired_reason="InvalidQOS"
+    ), f"Pending job {job_id2} should be PENDING with InvalidQOS reason"
+
+    # Pending job with remaining QOS should stay valid
+    assert atf.wait_for_job_state(
+        job_id3, "PENDING"
+    ), f"Job {job_id3} should be PENDING"
     assert (
-        atf.get_job_parameter(job_id2, "Reason") != "InvalidQOS"
-    ), "The second job whose QOS was not deleted should not be pending due to 'InvalidQOS'"
+        atf.get_job_parameter(job_id3, "Reason") != "InvalidQOS"
+    ), f"Job {job_id3} using qos2 should not have InvalidQOS reason"
 
     # Try to submit a new job with removed QOS - should be rejected
     assert (
@@ -154,80 +202,56 @@ def test_qos_removal_multiple():
     ), f"Job submission with valid QOS {qos2} should have succeeded"
 
 
-def test_qos_removal_running_vs_pending():
-    """Test QOS removal impact on running vs pending jobs:
-    1. Submit two jobs with same QOS - one running, one pending
-    2. Remove the QOS
-    3. Verify running job continues running
-    4. Verify pending job gets marked with InvalidQOS
+def test_account_removal_single():
+    """Test removing an account in use:
+    - Verify that running jobs with that account keep running.
+    - Verify that pending jobs are updated to InvalidAccount.
+    - Verify that new jobs cannot use the removed account.
     """
 
-    # Stop slurmdbd and submit jobs - use exclusive to ensure only one can run
+    # Stop slurmdbd to avoid the job info being saved in the DB
     atf.stop_slurmdbd(quiet=True)
-    job_id1 = submit_job_with(f"--qos={qos1} --exclusive", fatal=True)
-    job_id2 = submit_job_with(f"--qos={qos1} --exclusive", fatal=True)
 
-    # Wait for first job to start running
+    # Submit a blocking job
+    job_id1 = submit_job_with(f"--account={acct1} --exclusive", fatal=True)
     assert atf.wait_for_job_state(
         job_id1, "RUNNING"
     ), f"Job {job_id1} never started running"
 
-    # Verify second job is pending (due to exclusive)
+    # Submit another job in the same node to be blocked (due exclusive)
+    node = atf.get_job_parameter(job_id1, "NodeList")
+    job_id2 = submit_job_with(f"--account={acct1} -w {node}", fatal=True)
     assert atf.wait_for_job_state(
         job_id2, "PENDING"
     ), f"Job {job_id2} should be pending"
 
-    # Stop slurmctld, start slurmdbd, remove QOS
+    # Stop slurmctld before starting slurmdbd to keep the jobs info out of
+    # the DB, only in slurmctld for the moment.
     atf.stop_slurmctld(quiet=True)
     atf.start_slurmdbd(quiet=True)
+
+    # Remove the account from the DB.
+    # Note that slurmdbd won't have the account or the jobs using it, while
+    # slurmctld knows the jobs and still thinks that the account exists.
     atf.run_command(
-        f"sacctmgr -i remove qos {qos1}",
+        f"sacctmgr -i remove account {acct1}",
         user=atf.properties["slurm-user"],
         fatal=True,
     )
 
-    # Start slurmctld and verify jobs state/reason
+    # Start slurmctld and verify job states/reasons are the expected now that
+    # the account doesn't exists anymore.
     atf.start_slurmctld(quiet=True)
 
     # Running job should continue running
     assert atf.wait_for_job_state(
-        job_id1, "RUNNING"
-    ), f"Previously running job {job_id1} should stay RUNNING"
-    assert (
-        atf.get_job_parameter(job_id1, "Reason") == "None"
-    ), f"Running job {job_id1} should have 'None' as reason"
-
-    # Pending job should be marked with InvalidQOS
-    assert atf.wait_for_job_state(
-        job_id2, "PENDING", desired_reason="InvalidQOS"
-    ), f"Pending job {job_id2} should be PENDING with InvalidQOS reason"
-
-
-def test_account_removal_single():
-    """Test that removal of a single account:
-    1. Marks pending jobs with InvalidAccount
-    2. Rejects new job submissions using removed account
-    """
-
-    # Stop slurmdbd and submit job
-    atf.stop_slurmdbd(quiet=True)
-    job_id = submit_job_with(f"--account={acct1}", fatal=True)
-
-    # Stop slurmctld, start slurmdbd, remove account
-    atf.stop_slurmctld(quiet=True)
-    atf.start_slurmdbd(quiet=True)
-    atf.run_command(
-        f"sacctmgr -i remove account {acct1}",
-        user=atf.properties["slurm-user"],
-        fatal=True,
-    )
-
-    # Start slurmctld and verify job state/reason
-    atf.start_slurmctld(quiet=True)
+        job_id1, "RUNNING", desired_reason="None"
+    ), f"Previously running job {job_id1} should stay RUNNING with 'None' reason"
 
+    # Pending job should be marked with InvalidAccount
     assert atf.wait_for_job_state(
-        job_id, "PENDING", desired_reason="InvalidAccount"
-    ), f"Job {job_id} not in PENDING state and InvalidAccount reason"
+        job_id2, "PENDING", desired_reason="InvalidAccount"
+    ), f"Pending job {job_id2} should be PENDING with InvalidAccount reason"
 
     # Try to submit a new job with removed account - should be rejected
     assert (
@@ -236,98 +260,78 @@ def test_account_removal_single():
 
 
 def test_account_removal_multiple():
-    """Test account removal when user has multiple account access:
-    1. Verifies jobs with removed account get marked InvalidAccount
-    2. Verifies jobs with remaining account stay valid
-    3. Verifies new job submissions with removed account are rejected
-    4. Verifies new job submissions with remaining account succeed
+    """Test removing an account when user has multiple account access:
+    - Verify that running jobs with removed account keep running.
+    - Verify that pending jobs with removed account are updated to InvalidAccount.
+    - Verify that jobs with remaining account stay valid.
+    - Verify that new jobs cannot use the removed account.
+    - Verify that new jobs can use the remaining account.
     """
 
-    # Stop slurmdbd and submit jobs with different accounts
+    # Stop slurmdbd to avoid the job info being saved in the DB
     atf.stop_slurmdbd(quiet=True)
-    job_id1 = submit_job_with(f"--account={acct1}", fatal=True)
-    job_id2 = submit_job_with(f"--account={acct2}", fatal=True)
-
-    # Stop slurmctld, start slurmdbd, remove first account
-    atf.stop_slurmctld(quiet=True)
-    atf.start_slurmdbd(quiet=True)
-    atf.run_command(
-        f"sacctmgr -i remove account {acct1}",
-        user=atf.properties["slurm-user"],
-        fatal=True,
-    )
-
-    # Start slurmctld and verify jobs state/reason
-    atf.start_slurmctld(quiet=True)
-
-    # First job should be PENDING with InvalidAccount
-    assert atf.wait_for_job_state(
-        job_id1, "PENDING", desired_reason="InvalidAccount"
-    ), f"Job {job_id1} not in PENDING state and InvalidAccount reason"
-
-    # Second job should stay PENDING with different reason
-    assert atf.wait_for_job_state(
-        job_id2, "PENDING", timeout=10
-    ), f"Job {job_id2} should be in PENDING state"
-    assert (
-        atf.get_job_parameter(job_id2, "Reason") != "InvalidAccount"
-    ), "The second job whose account was not deleted should not be pending due to 'InvalidAccount'"
-
-    # Try to submit a new job with removed account - should be rejected
-    assert (
-        submit_job_with(f"--account={acct1}", xfail=True) == 0
-    ), f"Job submission with removed account {acct1} should have failed"
-
-    # Submit a job with remaining account - should succeed
-    assert (
-        submit_job_with(f"--account={acct2}") != 0
-    ), f"Job submission with valid account {acct2} should have succeeded"
 
-
-def test_account_removal_running_vs_pending():
-    """Test account removal impact on running vs pending jobs:
-    1. Submit two jobs with same account - one running, one pending
-    2. Remove the account
-    3. Verify running job continues running
-    4. Verify pending job gets marked with InvalidAccount
-    """
-
-    # Stop slurmdbd and submit jobs - use exclusive to ensure only one can run
-    atf.stop_slurmdbd(quiet=True)
+    # Submit a blocking job
     job_id1 = submit_job_with(f"--account={acct1} --exclusive", fatal=True)
-    job_id2 = submit_job_with(f"--account={acct1} --exclusive", fatal=True)
-
-    # Wait for first job to start running
     assert atf.wait_for_job_state(
         job_id1, "RUNNING"
     ), f"Job {job_id1} never started running"
 
-    # Verify second job is pending (due to exclusive)
+    # Submit two more jobs in the same node to be blocked (due exclusive)
+    node = atf.get_job_parameter(job_id1, "NodeList")
+    job_id2 = submit_job_with(f"--account={acct1} -w {node}", fatal=True)
+    job_id3 = submit_job_with(f"--account={acct2} -w {node}", fatal=True)
+
+    # Verify both jobs are pending
     assert atf.wait_for_job_state(
         job_id2, "PENDING"
     ), f"Job {job_id2} should be pending"
+    assert atf.wait_for_job_state(
+        job_id3, "PENDING"
+    ), f"Job {job_id3} should be pending"
 
-    # Stop slurmctld, start slurmdbd, remove account
+    # Stop slurmctld before starting slurmdbd to keep the jobs info out of
+    # the DB, only in slurmctld for the moment.
     atf.stop_slurmctld(quiet=True)
     atf.start_slurmdbd(quiet=True)
+
+    # Remove the account from the DB.
+    # Note that slurmdbd won't have the account or the jobs using it, while
+    # slurmctld knows the jobs and still thinks that the account exists.
     atf.run_command(
         f"sacctmgr -i remove account {acct1}",
         user=atf.properties["slurm-user"],
         fatal=True,
     )
 
-    # Start slurmctld and verify jobs state/reason
+    # Start slurmctld and verify job states/reasons are the expected now that
+    # the account doesn't exists anymore.
     atf.start_slurmctld(quiet=True)
 
     # Running job should continue running
     assert atf.wait_for_job_state(
-        job_id1, "RUNNING"
-    ), f"Previously running job {job_id1} should stay RUNNING"
-    assert (
-        atf.get_job_parameter(job_id1, "Reason") == "None"
-    ), f"Running job {job_id1} should have 'None' as reason"
+        job_id1, "RUNNING", desired_reason="None"
+    ), f"Previously running job {job_id1} should stay RUNNING with 'None' reason"
 
-    # Pending job should be marked with InvalidAccount
+    # Pending job with removed account should be marked with InvalidAccount
     assert atf.wait_for_job_state(
         job_id2, "PENDING", desired_reason="InvalidAccount"
     ), f"Pending job {job_id2} should be PENDING with InvalidAccount reason"
+
+    # Pending job with remaining account should stay valid
+    assert atf.wait_for_job_state(
+        job_id3, "PENDING"
+    ), f"Job {job_id3} should be PENDING"
+    assert (
+        atf.get_job_parameter(job_id3, "Reason") != "InvalidAccount"
+    ), f"Job {job_id3} using acct2 should not have InvalidAccount reason"
+
+    # Try to submit a new job with removed account - should be rejected
+    assert (
+        submit_job_with(f"--account={acct1}", xfail=True) == 0
+    ), f"Job submission with removed account {acct1} should have failed"
+
+    # Submit a job with remaining account - should succeed
+    assert (
+        submit_job_with(f"--account={acct2}") != 0
+    ), f"Job submission with valid account {acct2} should have succeeded"

From aaf75401e188ce611c3654f619e47806abd1e997 Mon Sep 17 00:00:00 2001
From: Tim McMullan <mcmullan@schedmd.com>
Date: Thu, 28 Nov 2024 11:10:10 -0500
Subject: [PATCH 73/90] Fix backup controller crash after reconfiguring.

If the backup controller is in control and scontrol reconfigure is run,
after resuming the primary controller it was possible that the backup
would not honor the request to relinquish control.

In most cases this would result in the backup controller crashing, but
in a quiet enough system both controllers could continue to operate as
the primary.

Changelog: slurmctld - Fix crash and possible split brain issue if the
 backup controller handles an scontrol reconfigure while in control
 before the primary resumes operation.
Ticket: 21532
Cherry-picked: f37371bac8
---
 src/slurmctld/controller.c | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/src/slurmctld/controller.c b/src/slurmctld/controller.c
index 42185e4d99a..ec4f4e8272c 100644
--- a/src/slurmctld/controller.c
+++ b/src/slurmctld/controller.c
@@ -817,6 +817,10 @@ int main(int argc, char **argv)
 		if (slurmctld_config.resume_backup && slurmctld_primary)
 			break;
 
+		/* The backup is now meant to relinquish control */
+		if (slurmctld_config.resume_backup && !slurmctld_primary)
+			backup_has_control = false;
+
 		recover = 2;
 	}
 

From ad37db167796cba755a6a20d2213c7cc964456e4 Mon Sep 17 00:00:00 2001
From: Brian Christiansen <brian@schedmd.com>
Date: Wed, 4 Dec 2024 00:24:38 -0700
Subject: [PATCH 74/90] Fix stepmgr not getting dynamic node addrs from the
 controller

The controller doesn't pack job_record_t's node_addrs, so the stepmgr
wasn't getting them and passing them to the steps. When the steps
completed and tried communicating back to the dynamic stepmgr, it failed to
find its node addr. The job's node_addr's are passed in the cred so we
can just get them from there.

Changelog: Fix stepmgr not getting dynamic node addrs from the controller
Ticket: 21535
Cherry-picked: a4af0fbc1d
---
 src/slurmd/slurmstepd/slurmstepd.c | 21 +++++++++++++++++++++
 1 file changed, 21 insertions(+)

diff --git a/src/slurmd/slurmstepd/slurmstepd.c b/src/slurmd/slurmstepd/slurmstepd.c
index 494fb35dc09..09f20b2eb99 100644
--- a/src/slurmd/slurmstepd/slurmstepd.c
+++ b/src/slurmd/slurmstepd/slurmstepd.c
@@ -881,10 +881,31 @@ _init_from_slurmd(int sock, char **argv, slurm_addr_t **_cli,
 
 		if (task_msg->job_ptr &&
 		    !xstrcmp(conf->node_name, task_msg->job_ptr->batch_host)) {
+			slurm_addr_t *node_addrs;
+
 			/* only allow one stepd to be stepmgr. */
 			job_step_ptr = task_msg->job_ptr;
 			job_step_ptr->part_ptr = task_msg->part_ptr;
 			job_node_array = task_msg->job_node_array;
+
+			/*
+			 * job_record doesn't pack its node_addrs array, so get
+			 * it from the cred.
+			 */
+			if (task_msg->cred &&
+			    (node_addrs = slurm_cred_get(
+				     task_msg->cred,
+				     CRED_DATA_JOB_NODE_ADDRS))) {
+				add_remote_nodes_to_conf_tbls(
+					job_step_ptr->nodes, node_addrs);
+
+				job_step_ptr->node_addrs =
+					xcalloc(job_step_ptr->node_cnt,
+						sizeof(slurm_addr_t));
+				memcpy(job_step_ptr->node_addrs, node_addrs,
+				       job_step_ptr->node_cnt *
+				       sizeof(slurm_addr_t));
+			}
 		}
 
 		break;

From 56a3317b9992d7251edc0f766e1dd659a3a84364 Mon Sep 17 00:00:00 2001
From: Marcin Stolarek <cinek@schedmd.com>
Date: Wed, 27 Nov 2024 17:12:06 +0100
Subject: [PATCH 75/90] stepgmr - avoid sending rc message on success after
 already responding

Changelog: stepmgr - avoid "Unexpected missing socket" errors.
Ticket: 21422
Cherry-picked: 8469d653d1
---
 src/slurmd/slurmstepd/req.c | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/src/slurmd/slurmstepd/req.c b/src/slurmd/slurmstepd/req.c
index 5c0e21cbfa1..d558c6aa4fb 100644
--- a/src/slurmd/slurmstepd/req.c
+++ b/src/slurmd/slurmstepd/req.c
@@ -607,8 +607,6 @@ static int _handle_job_step_get_info(int fd, stepd_step_rec_t *step, uid_t uid)
 		      buffer);
 	slurm_send_node_msg(msg.conn_fd, &response_msg);
 	FREE_NULL_BUFFER(buffer);
-
-	slurm_send_rc_msg(&msg, SLURM_SUCCESS);
 	slurm_free_msg_members(&msg);
 
 done:
@@ -752,9 +750,10 @@ static int _handle_step_layout(int fd, stepd_step_rec_t *step, uid_t uid)
 			      step_layout);
 		slurm_send_node_msg(msg.conn_fd, &response_msg);
 		slurm_step_layout_destroy(step_layout);
+	} else {
+		slurm_send_rc_msg(&msg, rc);
 	}
 
-	slurm_send_rc_msg(&msg, rc);
 	slurm_free_msg_members(&msg);
 
 done:
@@ -789,6 +788,8 @@ static int _handle_job_sbcast_cred(int fd, stepd_step_rec_t *step, uid_t uid)
 	slurm_send_node_msg(msg.conn_fd, &response_msg);
 
 	slurm_free_sbcast_cred_msg(job_info_resp_msg);
+	slurm_free_msg_members(&msg);
+	return rc;
 
 resp:
 	slurm_send_rc_msg(&msg, rc);
@@ -839,6 +840,8 @@ static int _handle_het_job_alloc_info(int fd, stepd_step_rec_t *step, uid_t uid)
 		      resp_list);
 	slurm_send_node_msg(msg.conn_fd, &response_msg);
 	FREE_NULL_LIST(resp_list);
+	slurm_free_msg_members(&msg);
+	return rc;
 
 resp:
 	slurm_send_rc_msg(&msg, rc);

From 990073e59f0f8afe5a10e260c0329772317cac48 Mon Sep 17 00:00:00 2001
From: Brian Christiansen <brian@schedmd.com>
Date: Thu, 5 Dec 2024 23:33:36 -0700
Subject: [PATCH 76/90] Fix scontrol show steps with dynamic stepmgr

Changelog: Fix `scontrol show steps` with dynamic stepmgr
Ticket: 21422
Cherry-picked: 59e69f37ca
---
 src/api/job_step_info.c | 18 ++++++++++++++++--
 1 file changed, 16 insertions(+), 2 deletions(-)

diff --git a/src/api/job_step_info.c b/src/api/job_step_info.c
index 00458feb5b9..d1dd90a501d 100644
--- a/src/api/job_step_info.c
+++ b/src/api/job_step_info.c
@@ -315,8 +315,22 @@ static int _get_stepmgr_steps(void *x, void *arg)
 	slurm_msg_t req_msg;
 	slurm_msg_t_init(&req_msg);
 	slurm_msg_set_r_uid(&req_msg, slurm_conf.slurmd_user_id);
-	slurm_conf_get_addr(sji->stepmgr, &req_msg.address,
-			    req_msg.flags);
+
+	if (slurm_conf_get_addr(sji->stepmgr, &req_msg.address, req_msg.flags))
+	{
+		/*
+		 * The node isn't in the conf, see if the
+		 * controller has an address for it.
+		 */
+		slurm_node_alias_addrs_t *alias_addrs = NULL;
+		if (!slurm_get_node_alias_addrs(sji->stepmgr, &alias_addrs)) {
+			add_remote_nodes_to_conf_tbls(alias_addrs->node_list,
+						      alias_addrs->node_addrs);
+			slurm_free_node_alias_addrs(alias_addrs);
+			slurm_conf_get_addr(sji->stepmgr, &req_msg.address,
+					    req_msg.flags);
+		}
+	}
 
 	job_step_info_request_msg_t req_data = {0};
 	req_data.step_id.job_id = sji->job_id;

From 6a27a08c7919634b75866977be5afafc79d0c0ac Mon Sep 17 00:00:00 2001
From: Albert Gil <albert.gil@schedmd.com>
Date: Wed, 11 Dec 2024 12:19:29 +0100
Subject: [PATCH 77/90] Testsuite - Improve test15.4 adding diagnose info

Cherry-picked: 5b80951ce7
---
 testsuite/expect/test15.4 | 10 +++-------
 1 file changed, 3 insertions(+), 7 deletions(-)

diff --git a/testsuite/expect/test15.4 b/testsuite/expect/test15.4
index 400e8af1bb9..f0862350f5b 100755
--- a/testsuite/expect/test15.4
+++ b/testsuite/expect/test15.4
@@ -53,7 +53,6 @@ set login_grp_info [get_my_id]
 #
 # Submit a slurm job that will execute 'id'
 #
-set timeout $max_job_delay
 spawn $salloc -N1 -t1 $srun $bin_id
 expect {
 	-re "Granted job allocation ($number)" {
@@ -73,15 +72,12 @@ expect {
 	}
 }
 
-if {$got_job_grps == 0} {
-	fail "Did not get user info from slurm job"
-}
+subtest {$got_job_grps != 0} "Verify we were able to get user info from slurm job"
+
 
 #
 # Confirm the user id and group id in the slurm job matches that
 # of the local 'id' execution.
 #
 
-if {$login_grp_info ne $job_grp_info} {
-	fail "Login and slurm user info mismatch"
-}
+subtest {$login_grp_info eq $job_grp_info} "Verify user info from login and job match" "$login_grp_info != $job_grp_info"

From 123076b5208ebc8e4ac02c452f22abae1ce57cd6 Mon Sep 17 00:00:00 2001
From: Tim McMullan <mcmullan@schedmd.com>
Date: Mon, 28 Oct 2024 13:09:08 -0400
Subject: [PATCH 78/90] Make helper function for xgetaddrinfo() that accepts
 custom hints.

Cherry-picked: e47410a62c
Ticket: 20997
---
 src/common/util-net.c | 35 +++++++++++++++++++++--------------
 1 file changed, 21 insertions(+), 14 deletions(-)

diff --git a/src/common/util-net.c b/src/common/util-net.c
index a0fb281e1cb..2bcabe7f1d0 100644
--- a/src/common/util-net.c
+++ b/src/common/util-net.c
@@ -253,6 +253,26 @@ extern char *make_full_path(const char *rpath)
 	return cwd2;
 }
 
+static struct addrinfo *_xgetaddrinfo(const char *hostname, const char *serv,
+				      const struct addrinfo *hints)
+{
+	struct addrinfo *result = NULL;
+	int err;
+
+	err = getaddrinfo(hostname, serv, hints, &result);
+	if (err == EAI_SYSTEM) {
+		error_in_daemon("%s: getaddrinfo(%s:%s) failed: %s: %m",
+				__func__, hostname, serv, gai_strerror(err));
+		return NULL;
+	} else if (err != 0) {
+		error_in_daemon("%s: getaddrinfo(%s:%s) failed: %s",
+				__func__, hostname, serv, gai_strerror(err));
+		return NULL;
+	}
+
+	return result;
+}
+
 extern struct addrinfo *xgetaddrinfo_port(const char *hostname, uint16_t port)
 {
 	char serv[6];
@@ -262,9 +282,7 @@ extern struct addrinfo *xgetaddrinfo_port(const char *hostname, uint16_t port)
 
 extern struct addrinfo *xgetaddrinfo(const char *hostname, const char *serv)
 {
-	struct addrinfo *result = NULL;
 	struct addrinfo hints;
-	int err;
 	bool v4_enabled = slurm_conf.conf_flags & CONF_FLAG_IPV4_ENABLED;
 	bool v6_enabled = slurm_conf.conf_flags & CONF_FLAG_IPV6_ENABLED;
 
@@ -300,18 +318,7 @@ extern struct addrinfo *xgetaddrinfo(const char *hostname, const char *serv)
 		hints.ai_flags |= AI_CANONNAME;
 	hints.ai_socktype = SOCK_STREAM;
 
-	err = getaddrinfo(hostname, serv, &hints, &result);
-	if (err == EAI_SYSTEM) {
-		error_in_daemon("%s: getaddrinfo(%s:%s) failed: %s: %m",
-				__func__, hostname, serv, gai_strerror(err));
-		return NULL;
-	} else if (err != 0) {
-		error_in_daemon("%s: getaddrinfo(%s:%s) failed: %s",
-				__func__, hostname, serv, gai_strerror(err));
-		return NULL;
-	}
-
-	return result;
+	return _xgetaddrinfo(hostname, serv, &hints);
 }
 
 static int _name_cache_find(void *x, void *y)

From 27ceee139f05c484f1ab42719d06f4a688708845 Mon Sep 17 00:00:00 2001
From: Tim McMullan <mcmullan@schedmd.com>
Date: Mon, 28 Oct 2024 13:09:08 -0400
Subject: [PATCH 79/90] Add function to determine what types of addresses a
 host has.

Cherry-picked: 54e3c2ba62
Ticket: 20997
---
 src/common/util-net.c | 32 ++++++++++++++++++++++++++++++++
 src/common/util-net.h |  2 ++
 2 files changed, 34 insertions(+)

diff --git a/src/common/util-net.c b/src/common/util-net.c
index 2bcabe7f1d0..5720e4b0d1f 100644
--- a/src/common/util-net.c
+++ b/src/common/util-net.c
@@ -321,6 +321,38 @@ extern struct addrinfo *xgetaddrinfo(const char *hostname, const char *serv)
 	return _xgetaddrinfo(hostname, serv, &hints);
 }
 
+extern int host_has_addr_family(const char *hostname, const char *srv,
+				bool *ipv4, bool *ipv6)
+{
+	struct addrinfo hints;
+	struct addrinfo *ai_ptr, *ai_start;
+
+	memset(&hints, 0, sizeof(hints));
+
+	hints.ai_family = AF_UNSPEC;
+	hints.ai_flags = AI_ADDRCONFIG | AI_NUMERICSERV | AI_PASSIVE;
+	if (hostname)
+		hints.ai_flags |= AI_CANONNAME;
+	hints.ai_socktype = SOCK_STREAM;
+
+	ai_start = _xgetaddrinfo(hostname, srv, &hints);
+
+	if (!ai_start)
+		return SLURM_ERROR;
+
+	*ipv4 = *ipv6 = false;
+	for (ai_ptr = ai_start; ai_ptr; ai_ptr = ai_ptr->ai_next) {
+		if (ai_ptr->ai_family == AF_INET6)
+			*ipv6 = true;
+		else if (ai_ptr->ai_family == AF_INET)
+			*ipv4 = true;
+	}
+
+	freeaddrinfo(ai_start);
+
+	return SLURM_SUCCESS;
+}
+
 static int _name_cache_find(void *x, void *y)
 {
 	getnameinfo_cache_t *cache_ent = x;
diff --git a/src/common/util-net.h b/src/common/util-net.h
index 7d8c7f9ccfb..c85b9b562d1 100644
--- a/src/common/util-net.h
+++ b/src/common/util-net.h
@@ -87,6 +87,8 @@ extern struct addrinfo *xgetaddrinfo_port(const char *hostname,
 					  uint16_t port);
 extern char *xgetnameinfo(struct sockaddr *addr, socklen_t addrlen);
 
+extern int host_has_addr_family(const char *hostname, const char *srv,
+				bool *ipv4, bool *ipv6);
 /* Functions responsible for cleanup of getnameinfo cache */
 extern void getnameinfo_cache_destroy(void *obj);
 extern void getnameinfo_cache_purge(void);

From c509494958065a6d09fe6be78c4068227f86a27d Mon Sep 17 00:00:00 2001
From: Tim McMullan <mcmullan@schedmd.com>
Date: Mon, 28 Oct 2024 13:09:09 -0400
Subject: [PATCH 80/90] Add has_ipv4 and has_ipv6 address information when
 fetching configs

Cherry-picked: 43a57d9d61
Ticket: 20997
---
 src/common/fetch_config.c | 13 +++++++++++++
 src/common/slurm_resolv.h |  2 ++
 2 files changed, 15 insertions(+)

diff --git a/src/common/fetch_config.c b/src/common/fetch_config.c
index fceccde29a8..24b8f12298e 100644
--- a/src/common/fetch_config.c
+++ b/src/common/fetch_config.c
@@ -47,6 +47,7 @@
 #include "src/common/slurm_protocol_pack.h"
 #include "src/common/slurm_resolv.h"
 #include "src/common/strlcpy.h"
+#include "src/common/util-net.h"
 #include "src/common/xstring.h"
 #include "src/common/xmalloc.h"
 
@@ -150,6 +151,16 @@ static void _fetch_child(List controllers, uint32_t flags)
 	_exit(1);
 }
 
+static int _get_controller_addr_type(void *x, void *arg)
+{
+	ctl_entry_t *ctl = (ctl_entry_t *) x;
+
+	host_has_addr_family(ctl->hostname, NULL, &ctl->has_ipv4,
+			     &ctl->has_ipv6);
+
+	return SLURM_SUCCESS;
+}
+
 extern config_response_msg_t *fetch_config(char *conf_server, uint32_t flags)
 {
 	char *env_conf_server = getenv("SLURM_CONF_SERVER");
@@ -198,6 +209,8 @@ extern config_response_msg_t *fetch_config(char *conf_server, uint32_t flags)
                 }
 	}
 
+	list_for_each(controllers, _get_controller_addr_type, NULL);
+
 	/* If the slurm.key file exists, assume we're using auth/slurm */
 	sack_jwks = get_extra_conf_path("slurm.jwks");
 	sack_key = get_extra_conf_path("slurm.key");
diff --git a/src/common/slurm_resolv.h b/src/common/slurm_resolv.h
index e0fe650daf6..63de8c74ba7 100644
--- a/src/common/slurm_resolv.h
+++ b/src/common/slurm_resolv.h
@@ -40,6 +40,8 @@ typedef struct {
 	uint16_t priority;
 	uint16_t port;
 	char hostname[1024];
+	bool has_ipv4;
+	bool has_ipv6;
 } ctl_entry_t;
 
 /*

From e7b6def6a7b936afca2cc69f78d96a084c1a1fa2 Mon Sep 17 00:00:00 2001
From: Tim McMullan <mcmullan@schedmd.com>
Date: Mon, 28 Oct 2024 13:09:09 -0400
Subject: [PATCH 81/90] Selectively try to fetch configs from controllers over
 IPv6.

Previous default behavior of trying IPv4 remains, however if the
controller appears to be IPv6 only skip the IPv4 attempt.

It is expected that all slurm controllers have the same IP address
families available.

Cherry-picked: bf4a853677
Ticket: 20997
---
 src/common/fetch_config.c | 30 ++++++++++++++++++++++++++----
 1 file changed, 26 insertions(+), 4 deletions(-)

diff --git a/src/common/fetch_config.c b/src/common/fetch_config.c
index 24b8f12298e..dd20d1fd3d6 100644
--- a/src/common/fetch_config.c
+++ b/src/common/fetch_config.c
@@ -64,7 +64,8 @@ static char *client_config_files[] = {
 };
 
 
-static void _init_minimal_conf_server_config(List controllers);
+static void _init_minimal_conf_server_config(List controllers, bool use_v6,
+					     bool reinit);
 
 static int to_parent[2] = {-1, -1};
 
@@ -115,6 +116,7 @@ static config_response_msg_t *_fetch_parent(pid_t pid)
 static void _fetch_child(List controllers, uint32_t flags)
 {
 	config_response_msg_t *config;
+	ctl_entry_t *ctl = NULL;
 	buf_t *buffer = init_buf(1024 * 1024);
 	int len = 0;
 
@@ -129,9 +131,22 @@ static void _fetch_child(List controllers, uint32_t flags)
 	 */
 	slurm_conf_unlock();
 
-	_init_minimal_conf_server_config(controllers);
+	ctl = list_peek(controllers);
+
+	if (ctl->has_ipv6 && !ctl->has_ipv4)
+		_init_minimal_conf_server_config(controllers, true, false);
+	else
+		_init_minimal_conf_server_config(controllers, false, false);
+
 	config = fetch_config_from_controller(flags);
 
+	if (!config && ctl->has_ipv6 && ctl->has_ipv4) {
+		warning("%s: failed to fetch remote configs via IPv4, retrying with IPv6: %m",
+			__func__);
+		_init_minimal_conf_server_config(controllers, true, true);
+		config = fetch_config_from_controller(flags);
+	}
+
 	if (!config) {
 		error("%s: failed to fetch remote configs: %m", __func__);
 		safe_write(to_parent[1], &len, sizeof(int));
@@ -344,7 +359,8 @@ static int _print_controllers(void *x, void *arg)
 	return SLURM_SUCCESS;
 }
 
-static void _init_minimal_conf_server_config(List controllers)
+static void _init_minimal_conf_server_config(List controllers, bool use_v6,
+					     bool reinit)
 {
 	char *conf = NULL, *filename = NULL;
 	int fd;
@@ -356,11 +372,17 @@ static void _init_minimal_conf_server_config(List controllers)
 	if (slurm_conf.authinfo)
 		xstrfmtcat(conf, "AuthInfo=%s\n", slurm_conf.authinfo);
 
+	if (use_v6)
+		xstrcat(conf, "CommunicationParameters=EnableIPv6");
+
 	if ((fd = dump_to_memfd("slurm.conf", conf, &filename)) < 0)
 		fatal("%s: could not write temporary config", __func__);
 	xfree(conf);
 
-	slurm_init(filename);
+	if (reinit)
+		slurm_conf_reinit(filename);
+	else
+		slurm_init(filename);
 
 	close(fd);
 	xfree(filename);

From dd6127502be3c6b9e75988607e9a586e0d845266 Mon Sep 17 00:00:00 2001
From: Tim McMullan <mcmullan@schedmd.com>
Date: Mon, 28 Oct 2024 13:09:09 -0400
Subject: [PATCH 82/90] slurmd - Parse IPv6 addresses provided to --conf-server
 correctly.

When using an IPv6 address directly, wrap the address with [] to
denote what is address vs what is the port.

Cherry-picked: 04babaab63
Ticket: 20997
---
 doc/man/man8/slurmd.8     |  5 ++++-
 src/common/fetch_config.c | 14 +++++++++++++-
 2 files changed, 17 insertions(+), 2 deletions(-)

diff --git a/doc/man/man8/slurmd.8 b/doc/man/man8/slurmd.8
index e35ff45dad3..7f4a4a3f605 100644
--- a/doc/man/man8/slurmd.8
+++ b/doc/man/man8/slurmd.8
@@ -77,10 +77,13 @@ NodeName=node1 CPUs=16 RealMemory=30000 Gres=gpu:2"
 .IP
 
 .TP
-\fB\-\-conf\-server <host>[:<port>]\fR
+\fB\-\-conf\-server <host|ip address>[:<port>]\fR
 Comma\-separated list of controllers, the first being the primary slurmctld. A
 port can (optionally) be specified for each controller. These hosts are where
 the slurmd will fetch the configuration from when running in "configless" mode.
+\fBNOTE\fR: If specifying an IPv6 address, wrap the <ip address> in [] to
+distinguish the address from the port.  This is required even if no port is
+specified.
 .IP
 
 .TP
diff --git a/src/common/fetch_config.c b/src/common/fetch_config.c
index dd20d1fd3d6..0428f3e51bb 100644
--- a/src/common/fetch_config.c
+++ b/src/common/fetch_config.c
@@ -204,9 +204,21 @@ extern config_response_msg_t *fetch_config(char *conf_server, uint32_t flags)
 		server = strtok_r(tmp, ",", &save_ptr);
 		while (server) {
 			ctl_entry_t *ctl = xmalloc(sizeof(*ctl));
+			char *tmp_ptr = NULL;
+
+			if (server[0] == '[')
+				server++;
+
 			strlcpy(ctl->hostname, server, sizeof(ctl->hostname));
 
-			if ((port = xstrchr(ctl->hostname, ':'))) {
+			if ((tmp_ptr = strchr(ctl->hostname, ']'))) {
+				*tmp_ptr = '\0';
+				tmp_ptr++;
+			} else {
+				tmp_ptr = ctl->hostname;
+			}
+
+			if ((port = xstrchr(tmp_ptr, ':'))) {
 				*port = '\0';
 				port++;
 				ctl->port = atoi(port);

From 388e8c3953bfe959cdd92cd586a621e2aa211beb Mon Sep 17 00:00:00 2001
From: Tim McMullan <mcmullan@schedmd.com>
Date: Mon, 28 Oct 2024 13:24:24 -0400
Subject: [PATCH 83/90] NEWS for previous five commits

Changelog: Support IPv6 in configless mode.
Cherry-picked: 050517c526
Ticket: 20997

From 8ab29ce372732a4f446fd36560dd62112841d1bc Mon Sep 17 00:00:00 2001
From: Marshall Garey <marshall@schedmd.com>
Date: Thu, 12 Dec 2024 11:05:59 -0700
Subject: [PATCH 84/90] Docs - Update REST API reference

---
 doc/html/rest_api.shtml | 28 ++++++++++++++--------------
 1 file changed, 14 insertions(+), 14 deletions(-)

diff --git a/doc/html/rest_api.shtml b/doc/html/rest_api.shtml
index 3e29da52719..f540d0c64dd 100644
--- a/doc/html/rest_api.shtml
+++ b/doc/html/rest_api.shtml
@@ -3,7 +3,7 @@
     <div class="app-desc">API to access and control Slurm</div>
     <div class="app-desc">More information: <a href="https://www.schedmd.com/">https://www.schedmd.com/</a></div>
     <div class="app-desc">Contact Info: <a href="sales@schedmd.com">sales@schedmd.com</a></div>
-    <div class="app-desc">Version: Slurm-24.05.4&amp;openapi/slurmdbd&amp;openapi/slurmctld</div>
+    <div class="app-desc">Version: Slurm-24.05.5&amp;openapi/slurmdbd&amp;openapi/slurmctld</div>
     <div class="app-desc">BasePath:</div>
     <div class="license-info">Apache 2.0</div>
     <div class="license-url">https://www.apache.org/licenses/LICENSE-2.0.html</div>
@@ -6169,7 +6169,7 @@
 
       <div class="param-desc"><span class="param-type">Query Parameter</span> — CSV QOS list default: null </div><div class="param">format (optional)</div>
 
-      <div class="param-desc"><span class="param-type">Query Parameter</span> — CSV format list default: null </div><div class="param">id (optional)</div>
+      <div class="param-desc"><span class="param-type">Query Parameter</span> — Ignored; process JSON manually to control output format default: null </div><div class="param">id (optional)</div>
 
       <div class="param-desc"><span class="param-type">Query Parameter</span> — CSV id list default: null </div><div class="param">only_defaults (optional)</div>
 
@@ -6297,7 +6297,7 @@
 
       <div class="param-desc"><span class="param-type">Query Parameter</span> — CSV QOS list default: null </div><div class="param">format (optional)</div>
 
-      <div class="param-desc"><span class="param-type">Query Parameter</span> — CSV format list default: null </div><div class="param">id (optional)</div>
+      <div class="param-desc"><span class="param-type">Query Parameter</span> — Ignored; process JSON manually to control output format default: null </div><div class="param">id (optional)</div>
 
       <div class="param-desc"><span class="param-type">Query Parameter</span> — CSV id list default: null </div><div class="param">only_defaults (optional)</div>
 
@@ -6433,7 +6433,7 @@
 
       <div class="param-desc"><span class="param-type">Query Parameter</span> — Query flags default: null </div><div class="param">format (optional)</div>
 
-      <div class="param-desc"><span class="param-type">Query Parameter</span> — CSV format list default: null </div><div class="param">rpc_version (optional)</div>
+      <div class="param-desc"><span class="param-type">Query Parameter</span> — Ignored; process JSON manually to control output format default: null </div><div class="param">rpc_version (optional)</div>
 
       <div class="param-desc"><span class="param-type">Query Parameter</span> — CSV RPC version list default: null </div><div class="param">usage_end (optional)</div>
 
@@ -7132,7 +7132,7 @@
 
       <div class="param-desc"><span class="param-type">Query Parameter</span> — CSV QOS list default: null </div><div class="param">format (optional)</div>
 
-      <div class="param-desc"><span class="param-type">Query Parameter</span> — CSV format list default: null </div><div class="param">id (optional)</div>
+      <div class="param-desc"><span class="param-type">Query Parameter</span> — Ignored; process JSON manually to control output format default: null </div><div class="param">id (optional)</div>
 
       <div class="param-desc"><span class="param-type">Query Parameter</span> — CSV id list default: null </div><div class="param">only_defaults (optional)</div>
 
@@ -7640,7 +7640,7 @@
 
       <div class="param-desc"><span class="param-type">Query Parameter</span> — CSV QOS list default: null </div><div class="param">format (optional)</div>
 
-      <div class="param-desc"><span class="param-type">Query Parameter</span> — CSV format list default: null </div><div class="param">id (optional)</div>
+      <div class="param-desc"><span class="param-type">Query Parameter</span> — Ignored; process JSON manually to control output format default: null </div><div class="param">id (optional)</div>
 
       <div class="param-desc"><span class="param-type">Query Parameter</span> — CSV id list default: null </div><div class="param">only_defaults (optional)</div>
 
@@ -8156,7 +8156,7 @@
 
       <div class="param-desc"><span class="param-type">Query Parameter</span> — Query flags default: null </div><div class="param">format (optional)</div>
 
-      <div class="param-desc"><span class="param-type">Query Parameter</span> — CSV format list default: null </div><div class="param">rpc_version (optional)</div>
+      <div class="param-desc"><span class="param-type">Query Parameter</span> — Ignored; process JSON manually to control output format default: null </div><div class="param">rpc_version (optional)</div>
 
       <div class="param-desc"><span class="param-type">Query Parameter</span> — CSV RPC version list default: null </div><div class="param">usage_end (optional)</div>
 
@@ -9960,7 +9960,7 @@
 
       <div class="param-desc"><span class="param-type">Query Parameter</span> — CSV extra list default: null </div><div class="param">format (optional)</div>
 
-      <div class="param-desc"><span class="param-type">Query Parameter</span> — CSV format list default: null </div><div class="param">instance_id (optional)</div>
+      <div class="param-desc"><span class="param-type">Query Parameter</span> — Ignored; process JSON manually to control output format default: null </div><div class="param">instance_id (optional)</div>
 
       <div class="param-desc"><span class="param-type">Query Parameter</span> — CSV instance_id list default: null </div><div class="param">instance_type (optional)</div>
 
@@ -10088,7 +10088,7 @@
 
       <div class="param-desc"><span class="param-type">Query Parameter</span> — CSV extra list default: null </div><div class="param">format (optional)</div>
 
-      <div class="param-desc"><span class="param-type">Query Parameter</span> — CSV format list default: null </div><div class="param">instance_id (optional)</div>
+      <div class="param-desc"><span class="param-type">Query Parameter</span> — Ignored; process JSON manually to control output format default: null </div><div class="param">instance_id (optional)</div>
 
       <div class="param-desc"><span class="param-type">Query Parameter</span> — CSV instance_id list default: null </div><div class="param">instance_type (optional)</div>
 
@@ -11488,7 +11488,7 @@
 
       <div class="param-desc"><span class="param-type">Query Parameter</span> — Include job environment default: null </div><div class="param">format (optional)</div>
 
-      <div class="param-desc"><span class="param-type">Query Parameter</span> — CSV format list default: null </div><div class="param">groups (optional)</div>
+      <div class="param-desc"><span class="param-type">Query Parameter</span> — Ignored; process JSON manually to control output format default: null </div><div class="param">groups (optional)</div>
 
       <div class="param-desc"><span class="param-type">Query Parameter</span> — CSV group list default: null </div><div class="param">job_name (optional)</div>
 
@@ -12758,7 +12758,7 @@
 
       <div class="param-desc"><span class="param-type">Query Parameter</span> — CSV QOS id list default: null </div><div class="param">format (optional)</div>
 
-      <div class="param-desc"><span class="param-type">Query Parameter</span> — CSV format list default: null </div><div class="param">name (optional)</div>
+      <div class="param-desc"><span class="param-type">Query Parameter</span> — Ignored; process JSON manually to control output format default: null </div><div class="param">name (optional)</div>
 
       <div class="param-desc"><span class="param-type">Query Parameter</span> — CSV QOS name list default: null </div><div class="param">preempt_mode (optional)</div>
 
@@ -14740,7 +14740,7 @@
 
       <div class="param-desc"><span class="param-type">Query Parameter</span> — CSV cluster name list default: null </div><div class="param">format (optional)</div>
 
-      <div class="param-desc"><span class="param-type">Query Parameter</span> — CSV format name list default: null </div><div class="param">id (optional)</div>
+      <div class="param-desc"><span class="param-type">Query Parameter</span> — Ignored; process JSON manually to control output format default: null </div><div class="param">id (optional)</div>
 
       <div class="param-desc"><span class="param-type">Query Parameter</span> — CSV id list default: null </div><div class="param">name (optional)</div>
 
@@ -15436,7 +15436,7 @@
 
       <div class="param-desc"><span class="param-type">Query Parameter</span> — CSV QOS id list default: null </div><div class="param">format (optional)</div>
 
-      <div class="param-desc"><span class="param-type">Query Parameter</span> — CSV format list default: null </div><div class="param">name (optional)</div>
+      <div class="param-desc"><span class="param-type">Query Parameter</span> — Ignored; process JSON manually to control output format default: null </div><div class="param">name (optional)</div>
 
       <div class="param-desc"><span class="param-type">Query Parameter</span> — CSV QOS name list default: null </div><div class="param">preempt_mode (optional)</div>
 
@@ -15859,7 +15859,7 @@
 
       <div class="param-desc"><span class="param-type">Query Parameter</span> — CSV cluster name list default: null </div><div class="param">format (optional)</div>
 
-      <div class="param-desc"><span class="param-type">Query Parameter</span> — CSV format name list default: null </div><div class="param">id (optional)</div>
+      <div class="param-desc"><span class="param-type">Query Parameter</span> — Ignored; process JSON manually to control output format default: null </div><div class="param">id (optional)</div>
 
       <div class="param-desc"><span class="param-type">Query Parameter</span> — CSV id list default: null </div><div class="param">name (optional)</div>
 

From fae1f13b83cd28c5f66e3f35c7182004298c0fda Mon Sep 17 00:00:00 2001
From: Tim McMullan <mcmullan@schedmd.com>
Date: Thu, 12 Dec 2024 13:10:06 -0700
Subject: [PATCH 85/90] Update NEWS for 24.05.5

---
 NEWS | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/NEWS b/NEWS
index c248b2ca6a0..e84c992e163 100644
--- a/NEWS
+++ b/NEWS
@@ -26,6 +26,13 @@ documents those changes that are of interest to users and administrators.
  -- Fix race condition in stepmgr step completion handling.
  -- Fix slurmctld segfault with stepmgr and MpiParams when running a job array.
  -- Fix requeued jobs keeping their priority until the decay thread happens.
+ -- slurmctld - Fix crash and possible split brain issue if the
+    backup controller handles an scontrol reconfigure while in control
+    before the primary resumes operation.
+ -- Fix stepmgr not getting dynamic node addrs from the controller
+ -- stepmgr - avoid "Unexpected missing socket" errors.
+ -- Fix `scontrol show steps` with dynamic stepmgr
+ -- Support IPv6 in configless mode.
 
 * Changes in Slurm 24.05.4
 ==========================

From 8e8a552a468b7e6c0175dd1e562e8495aa47b4fe Mon Sep 17 00:00:00 2001
From: Tim McMullan <mcmullan@schedmd.com>
Date: Thu, 12 Dec 2024 13:17:17 -0700
Subject: [PATCH 86/90] Update META for 24.05.5.

Update slurm.spec and debian/changelog as well.
---
 META             | 4 ++--
 debian/changelog | 2 +-
 slurm.spec       | 2 +-
 3 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/META b/META
index d83af4e1674..c3441b69275 100644
--- a/META
+++ b/META
@@ -7,8 +7,8 @@
   Name:		slurm
   Major:	24
   Minor:	05
-  Micro:	4
-  Version:	24.05.4
+  Micro:	5
+  Version:	24.05.5
   Release:	1
 
 ##
diff --git a/debian/changelog b/debian/changelog
index 01eb1252b30..8bb6b7e6c24 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,4 +1,4 @@
-slurm-smd (24.05.4-1) UNRELEASED; urgency=medium
+slurm-smd (24.05.5-1) UNRELEASED; urgency=medium
 
   * Initial release.
 
diff --git a/slurm.spec b/slurm.spec
index aab4cdd9c1e..59a13298dbe 100644
--- a/slurm.spec
+++ b/slurm.spec
@@ -1,5 +1,5 @@
 Name:		slurm
-Version:	24.05.4
+Version:	24.05.5
 %define rel	1
 Release:	%{rel}%{?dist}
 Summary:	Slurm Workload Manager

From bd17c8d112330da152cc6d9879801f109d5979dc Mon Sep 17 00:00:00 2001
From: Tim McMullan <mcmullan@schedmd.com>
Date: Thu, 12 Dec 2024 13:21:50 -0700
Subject: [PATCH 87/90] Start NEWS for v24.05.6.

---
 NEWS | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/NEWS b/NEWS
index e84c992e163..4ce1c8bf55e 100644
--- a/NEWS
+++ b/NEWS
@@ -1,6 +1,9 @@
 This file describes changes in recent versions of Slurm. It primarily
 documents those changes that are of interest to users and administrators.
 
+* Changes in Slurm 24.05.6
+==========================
+
 * Changes in Slurm 24.05.5
 ==========================
  -- Fix issue signaling cron jobs resulting in unintended requeues.

From 6443b033d9cb60a5a6cdb226ba115f08fa16bcb9 Mon Sep 17 00:00:00 2001
From: Andy Georges <itkovian@gmail.com>
Date: Fri, 13 Dec 2024 21:35:14 +0100
Subject: [PATCH 88/90] fix: install nvidia-driver-devel for nvml.h

---
 build.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/build.sh b/build.sh
index bc2a9453852..7b0c622085d 100755
--- a/build.sh
+++ b/build.sh
@@ -96,7 +96,7 @@ sudo dnf -y install munge-devel libjwt-devel pam-devel
 sudo dnf -y install http-parser-devel json-c-devel libyaml-devel
 # - features: Nvidia NVML
 sudo dnf -y autoremove cuda-nvml-* nvidia-driver-NVML-* nvidia-driver* libnvidia-ml*
-sudo dnf -y install "$CUDA_NVML_PKG" "$NVDRV_NVML_PKG"
+sudo dnf -y install "$CUDA_NVML_PKG" "$NVDRV_NVML_PKG" "nvidia-driver-devel"
 # - plugins: MPI
 sudo dnf -y install pmix "pmix-devel ${PMIX_VERSION}"  "ucx-devel-${UCX_VERSION}"
 # - plugins: cgroup/v2

From 9984765abb559648d6212996a3904db2b6bb4f77 Mon Sep 17 00:00:00 2001
From: Andy Georges <itkovian@gmail.com>
Date: Fri, 13 Dec 2024 21:39:30 +0100
Subject: [PATCH 89/90] fix: nvidia driver version on el9

---
 build.sh | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/build.sh b/build.sh
index 7b0c622085d..c5b9ea569f0 100755
--- a/build.sh
+++ b/build.sh
@@ -33,9 +33,10 @@ if grep "release 8.8" /etc/redhat-release; then
    CUDA_VERSION=${CUDA_VERSION:-12.6}
    CUDA_NVML_PKG="cuda-nvml-devel-${CUDA_VERSION//./-}"
 elif grep "release 9.4" /etc/redhat-release; then
-    NVDRV_NVML_PKG="libnvidia-ml"
-    CUDA_VERSION=${CUDA_VERSION:-12.6}
-    CUDA_NVML_PKG="cuda-nvml-devel-${CUDA_VERSION//./-}"
+   NVIDIA_DRIVER=${NVIDIA_DRIVER-555.42.06}
+   NVDRV_NVML_PKG="nvidia-driver-NVML${NVIDIA_DRIVER:+-$NVIDIA_DRIVER}"
+   CUDA_VERSION=${CUDA_VERSION:-12.6}
+   CUDA_NVML_PKG="cuda-nvml-devel-${CUDA_VERSION//./-}"
 fi
 
 # Prepare directory structure

From 30a0b7d626c6ee4fa28b489620bc8cb8ae45081d Mon Sep 17 00:00:00 2001
From: Andy Georges <itkovian@gmail.com>
Date: Mon, 16 Dec 2024 10:27:24 +0100
Subject: [PATCH 90/90] fix: nvidia version on el8

---
 build.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/build.sh b/build.sh
index c5b9ea569f0..9e4b715b3c0 100755
--- a/build.sh
+++ b/build.sh
@@ -28,7 +28,7 @@ OUR_RELEASE=${RELEASE:-1}
 # allow _empty_ version, which is used in pipeline
 
 if grep "release 8.8" /etc/redhat-release; then
-   NVIDIA_DRIVER=${NVIDIA_DRIVER-550.90.07}
+   NVIDIA_DRIVER=${NVIDIA_DRIVER-555.42.06}
    NVDRV_NVML_PKG="nvidia-driver-NVML${NVIDIA_DRIVER:+-$NVIDIA_DRIVER}"
    CUDA_VERSION=${CUDA_VERSION:-12.6}
    CUDA_NVML_PKG="cuda-nvml-devel-${CUDA_VERSION//./-}"