FS and slurm clients ok

Author: dtrudg

Diff for: README.md

@@ -4,14 +4,62 @@ This is a Vagrant environment that provides a minimal virtual HPC cluster with
 some commonly encountered filesystems and tools:
 
   - [X] CentOS 7 Base
-  - [ ] 1 master node, 2 compute nodes
-  - [ ] NFS exported from master to compute nodes
-  - [ ] Lustre exported from master to compute nodes
-  - [ ] Slurm 19.05.2 installed and configured
-  - [ ] Singularity 3.4.0 installed on all nodes
-  
+  - [X] 1 master node, 2 compute nodes
+  - [X] NFS exported from master to compute nodes
+  - [X] Lustre exported from master to compute nodes
+  - [X] Slurm 19.05.2 installed and configured
+  - [X] Singularity from EPEL
+  - [X] Dev tools to build Singularity from source
+  - [X] OpenMPI (from repos, 3.0.2-1)
+ 
+ 
+## How to use:
+
+```
+# bring up the master
+#
+# This will fail the first time, as CentOS base image has an IDE controller
+# We need a SATA controller, and there is no easy idempotent way to add one in a
+# Vagrantfile such that initial and re-provisioning will succeeed.
+#
+vagrant up master
+
+# Add the SATA controller we need
+VBoxManage storagectl $(cat .vagrant/machines/master/virtualbox/id) --add sata --name "SATA Controller"
+
+# Bring up the master properly now
+vagrant up master
+
+# bring up the compute nodes
+vagrant up compute01 compute02
+
+```
+
+In this minimal cluster we can be naughty and use the master node as a login
+node. For your experiments, note that currently:
+
+ - The cluster has a common `vagrant` user across all nodes - this is the user
+   you should use for Slurm jobs
+ - $HOME is not mounted over NFS (yet)
+ - Put anything you need across the cluster into `/nfs` or `/lustre`
+ - OpenMPI is available as a module via `module load mpi/openmpi3-x86_64`
+
+
+```
+# login to master
+$ vagrant ssh
+
+# Run hostname across the cluster (2 nodes)
+$ srun -N 2 hostname
+
+# /nfs & /lustre are mounted across master and nodes
+# Run an interactive session on a node
+# Look in the shared filesystems
+$ srun --pty /bin/bash
+$ ls /nfs /lustre
+```
 
-Caveats:
+## Caveats
 
   * The lustre setup is very simple. There are 1 each of MGS/MDT/OST and they
     are all running on the master VM.

Diff for: Vagrantfile

@@ -54,11 +54,12 @@ Vagrant.configure(2) do |config|
     # Provisioning scripts
     master.vm.provision "shell", inline: "hostnamectl set-hostname master"
     master.vm.provision "shell", path: "common-1-initial.sh"
-    master.vm.provision "shell", path: "common-2-lustre_kernel.sh"
+    master.vm.provision "shell", path: "common-2-software.sh"
+    master.vm.provision "shell", path: "master-1-lustre_kernel.sh"
     master.vm.provision :reload
-    master.vm.provision "shell", path: "master-1-lustre_fs.sh"
-    master.vm.provision "shell", path: "master-2-nfs_fs.sh"
-    master.vm.provision "shell", path: "master-3-slurm_master.sh"
+    master.vm.provision "shell", path: "master-2-lustre_fs.sh"
+    master.vm.provision "shell", path: "master-3-nfs_fs.sh"
+    master.vm.provision "shell", path: "master-4-slurm_master.sh"
   end
 
   # Compute node 1
@@ -73,11 +74,28 @@ Vagrant.configure(2) do |config|
     end
    compute01.vm.provision "shell", inline: "hostnamectl set-hostname compute01"
    compute01.vm.provision "shell", path: "common-1-initial.sh"
-   compute01.vm.provision "shell", path: "common-2-lustre_kernel.sh"
-   compute01.vm.provision :reload
+   compute01.vm.provision "shell", path: "common-2-software.sh"
    compute01.vm.provision "shell", path: "compute-1-lustre_client.sh"
-#   compute01.vm.provision "shell", path: "compute-2-nfs_client.sh"
-#   compute01.vm.provision "shell", path: "compute-3-slurm_client.sh"
+   compute01.vm.provision "shell", path: "compute-2-nfs_client.sh"
+   compute01.vm.provision "shell", path: "compute-3-slurm_client.sh"
+  end
+
+  # Compute node 2
+  config.vm.define "compute02" do |compute02|
+    compute02.vm.box = "centos/7"
+    compute02.vm.synced_folder ".", "/vagrant", disabled: true
+    compute02.vm.network "private_network", ip: "10.0.4.102", nic_type: "virtio"
+    compute02.vm.provider "virtualbox" do |v|
+      v.memory = 1024
+      v.cpus = 1
+      v.customize ['modifyvm', :id, '--nictype1', 'virtio']
+    end
+    compute02.vm.provision "shell", inline: "hostnamectl set-hostname compute01"
+    compute02.vm.provision "shell", path: "common-1-initial.sh"
+    compute02.vm.provision "shell", path: "common-2-software.sh"
+    compute02.vm.provision "shell", path: "compute-1-lustre_client.sh"
+    compute02.vm.provision "shell", path: "compute-2-nfs_client.sh"
+    compute02.vm.provision "shell", path: "compute-3-slurm_client.sh"
   end
 
 end

Diff for: common-2-software.sh

@@ -0,0 +1,19 @@
+#!/bin/bash
+
+#########################################
+# Useful software to have on the cluster
+#########################################
+
+# Utilities
+yum -y install vim-enhanced htop iftop
+
+# Development
+yum -y groupinstall "Development Tools"
+sudo yum install -y git golang openssl-devel libuuid-devel \
+     libseccomp-devel squashfs-tools cryptsetup
+
+# EPEL Singularity
+yum -y install singularity
+
+# OpenMPI
+yum -y install openmpi3

Diff for: compute-1-lustre_client.sh

@@ -4,11 +4,35 @@
 # Install Lustre Kernel Modules and Setup client
 ################################################
 
-# Install modules into the now running lustre kernel
-yum --nogpgcheck --enablerepo=lustre-client-2.10.0 install \
+# Setup repo
+cat >/etc/yum.repos.d/lustre-repo.repo <<\__EOF
+[lustre-server]
+name=lustre-server
+baseurl=https://downloads.whamcloud.com/public/lustre/latest-release/el7/server
+# exclude=*debuginfo*
+gpgcheck=0
+
+[lustre-client]
+name=lustre-client
+baseurl=https://downloads.whamcloud.com/public/lustre/latest-release/el7/client
+# exclude=*debuginfo*
+gpgcheck=0
+
+[e2fsprogs-wc]
+name=e2fsprogs-wc
+baseurl=https://downloads.whamcloud.com/public/e2fsprogs/latest/el7
+# exclude=*debuginfo*
+gpgcheck=0
+__EOF
+
+# Install client modules
+yum -y --nogpgcheck --enablerepo=lustre-client install \
     kmod-lustre-client \
     lustre-client
 
+modprobe lustre
+modprobe lnet
+
 # Setup our lnet network
 if ! lnetctl ping master; then
     lnetctl net add --net tcp0 --if eth1
@@ -18,7 +42,7 @@ fi
 # Setup mounts & mount them
 mkdir -p /lustre
 if ! grep -q "lustre" /etc/fstab; then
-    echo "master@tcp01:/lustre1        /lustre         lustre  defaults_netdev 0 0" >> /etc/fstab
+    echo "master@tcp0:/lustre1        /lustre         lustre  defaults,_netdev 0 0" >> /etc/fstab
 fi
 
 if ! mountpoint -q /lustre; then

Diff for: compute-2-nfs_client.sh

@@ -0,0 +1,16 @@
+#!/bin/bash
+
+################################################
+# Mount NFS share from master
+################################################
+
+# Setup mounts & mount them
+mkdir -p /nfs
+if ! grep -q "nfs" /etc/fstab; then
+    echo "master:/nfsdata        /nfs         nfs4  defaults,_netdev 0 0" >> /etc/fstab
+fi
+
+if ! mountpoint -q /nfs; then
+    mount /nfs
+fi
+

Diff for: compute-3-slurm_client.sh

@@ -0,0 +1,71 @@
+#!/bin/bash
+
+##############################################################
+# Setup slurm client
+##############################################################
+
+# Setup users
+export MUNGEUSER=991
+groupadd -g $MUNGEUSER munge
+useradd  -m -c "MUNGE Uid 'N' Gid Emporium" -d /var/lib/munge -u $MUNGEUSER -g munge  -s /sbin/nologin munge
+export SLURMUSER=992
+groupadd -g $SLURMUSER slurm
+useradd  -m -c "SLURM workload manager" -d /var/lib/slurm -u $SLURMUSER -g slurm  -s /bin/bash slurm
+
+# Install munge
+yum -y install epel-release
+yum -y install munge munge-libs munge-devel
+
+# Create a munge key
+echo "THIS_IS_NOT_A_VERY_SECURE_MUNGE_KEY_BUT_IT_WILL_DO" > /etc/munge/munge.key
+chown munge.munge /etc/munge/munge.key
+chmod 400 /etc/munge/munge.key
+
+# Correct perms & enable service
+chown -R munge: /etc/munge/ /var/log/munge/
+chmod 0700 /etc/munge/ /var/log/munge/
+systemctl enable munge
+systemctl start munge
+
+# Install slurm RPMs (built on master)
+yum -y install /nfs/slurm-rpms/slurm-*.rpm
+
+# Minimal slurm config
+cat <<EOF >> /etc/slurm/slurm.conf
+ControlMachine=master
+ControlAddr=10.0.4.100
+MpiDefault=none
+ProctrackType=proctrack/pgid
+ReturnToService=1
+SlurmctldPidFile=/var/run/slurmctld.pid
+SlurmdPidFile=/var/run/slurmd.pid
+SlurmdSpoolDir=/var/spool/slurmd
+SlurmUser=slurm
+StateSaveLocation=/var/spool/slurmctld
+SwitchType=switch/none
+TaskPlugin=task/none
+FastSchedule=1
+SchedulerType=sched/backfill
+SelectType=select/linear
+AccountingStorageType=accounting_storage/none
+ClusterName=dev
+JobAcctGatherType=jobacct_gather/none
+SlurmctldLogFile=/var/log/slurmctld.log
+SlurmdLogFile=/var/log/slurmd.log
+#
+# COMPUTE NODES
+NodeName=compute01 NodeAddr=10.0.4.101 CPUs=1 State=UNKNOWN
+NodeName=compute02 NodeAddr=10.0.4.102 CPUs=1 State=UNKNOWN
+PartitionName=super Nodes=compute0[1-2] Default=YES MaxTime=INFINITE State=UP
+EOF
+
+# Start slurm
+systemctl enable slurmd
+systemctl start slurmd
+
+# Check sinfo works
+sinfo
+
+# Mark node ready to rock
+scontrol update nodename=$(hostname) State=resume
+

Diff for: common-2-lustre_kernel.sh renamed to master-1-lustre_kernel.sh

@@ -56,3 +56,17 @@ fi
 if ! mountpoint -q /ost; then
     mount /ost
 fi
+
+# Mount the filesystem to our own host
+mkdir -p /lustre
+if ! grep -q "/lustre" /etc/fstab; then
+    echo "master@tcp0:/lustre1        /lustre         lustre  defaults_netdev 0 0" >> /etc/fstab
+fi
+
+if ! mountpoint -q /lustre; then
+    mount /lustre
+fi
+
+# Make perms wide open on it
+chmod 777 /lustre
+

Diff for: master-1-lustre_fs.sh renamed to master-2-lustre_fs.sh

@@ -31,3 +31,12 @@ systemctl enable nfs-server rpcbind
 echo "/nfsdata 10.0.4.0/24(rw,sync,no_root_squash)" > /etc/exports
 exportfs -r
 
+# Mount the filesystem to our own host
+mkdir -p /nfs
+if ! grep -q "nfs" /etc/fstab; then
+    echo "master:/nfsdata        /nfs         nfs4  defaults,_netdev 0 0" >> /etc/fstab
+fi
+
+if ! mountpoint -q /nfs; then
+    mount /nfs
+fi

Diff for: master-2-nfs_fs.sh renamed to master-3-nfs_fs.sh

@@ -41,7 +41,7 @@ mkdir -p /nfsdata/slurm-rpms
 cp /root/rpmbuild/RPMS/x86_64/slurm*.rpm /nfsdata/slurm-rpms/
 
 # Correct perms
-mkdir /var/spool/slurmctld
+mkdir -p /var/spool/slurmctld
 chown slurm: /var/spool/slurmctld
 chmod 755 /var/spool/slurmctld
 touch /var/log/slurmctld.log
@@ -52,7 +52,7 @@ chown slurm: /var/log/slurm_jobacct.log /var/log/slurm_jobcomp.log
 # Minimal slurm config
 cat <<EOF >> /etc/slurm/slurm.conf
 ControlMachine=master
-ControlAddr=10.0.4.1
+ControlAddr=10.0.4.100
 MpiDefault=none
 ProctrackType=proctrack/pgid
 ReturnToService=1
@@ -73,8 +73,8 @@ SlurmctldLogFile=/var/log/slurmctld.log
 SlurmdLogFile=/var/log/slurmd.log
 #
 # COMPUTE NODES
-NodeName=compute01 NodeAddr=10.0.4.2 CPUs=1 State=UNKNOWN
-NodeName=compute02 NodeAddr=10.0.4.3 CPUs=1 State=UNKNOWN
+NodeName=compute01 NodeAddr=10.0.4.101 CPUs=1 State=UNKNOWN
+NodeName=compute02 NodeAddr=10.0.4.102 CPUs=1 State=UNKNOWN
 PartitionName=super Nodes=compute0[1-2] Default=YES MaxTime=INFINITE State=UP
 EOF