Skip to content

Commit f5efdf5

Browse files
jbuschejbusche
committed
Counter fix and combined operator check
Signed-off-by: James Busche <[email protected]>
1 parent 8fb3be4 commit f5efdf5

File tree

5 files changed

+86
-80
lines changed

5 files changed

+86
-80
lines changed

‎test/perf-test/kwokmcadperf.sh

+30-26
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,7 @@ function help() {
1010
echo
1111
echo "Preconditions: "
1212
echo " - The script assumes you've logged into your cluster already. If not, it will tell you to login."
13-
echo " - The script checks that you have the mcad-controller installed, otherwise it'll tell you to install it first."
13+
echo " - The script checks that you have the mcad-controller installed (or the newer CodeFlare Operator), otherwise it'll tell you to install it first."
1414
echo " - The script checks that you have the kwok-controller installed, otherwise it'll tell you to install it first."
1515
echo
1616
echo "Options:"
@@ -40,12 +40,15 @@ function check_mcad_installed_status() {
4040
res2="$?"
4141
kubectl get crd |grep appwrapper &> /dev/null
4242
res3="$?"
43+
kubectl get pod -A |grep codeflare-operator-manager &> /dev/null
44+
res4="$?"
4345
set -e
4446
MCAD="$res2"
4547
CRD="$res3"
46-
if [[ $MCAD == 1 ]] || [[ $CRD == 1 ]]
48+
CODEFLARE="$res4"
49+
if [[ $MCAD == 1 ]] && [[ $CODEFLARE == 1 ]] || [[ $CRD == 1 ]]
4750
then
48-
echo "You need Install MCAD Controller first before running this script"
51+
echo "You need Install the MCAD Controller or the latest CodeFlare Operator first before running this script"
4952
exit 1
5053
else
5154
echo "Nice, MCAD Controller is installed"
@@ -95,7 +98,7 @@ echo
9598
check_kwok_installed_status
9699

97100
echo
98-
read -p "How many fake KWOK appwrapper jobs do you want?" jobs
101+
read -p "How many fake KWOK appwrapper jobs do you want?" JOBS
99102

100103
# Start the timer now
101104
SECONDS=0
@@ -106,54 +109,55 @@ echo " "
106109
echo "Jobs started at: $STARTTIME" |tee fake-job-$STARTTIME.log
107110
echo " "
108111

109-
# This fixes the number of jobs to be one less so the for loop gets the right amount
110-
((realjobs=$jobs-1))
111-
112-
for num in $(eval echo "{0.."$realjobs"}")
112+
COUNTER=1
113+
while [ $COUNTER -le $JOBS ]
113114
do
114-
next_num=$(($num + 1))
115-
echo "Submitting job $next_num"
115+
ORIG_COUNTER=$(($COUNTER - 1))
116+
echo "Submitting job $COUNTER"
117+
116118
# Had to do this OSTYPE because sed acts differently on Linux versus Mac
117119
case "$OSTYPE" in
118120
linux-gnu*)
119-
sed -i "s/fake-defaultaw-schd-spec-with-timeout-$num/fake-defaultaw-schd-spec-with-timeout-$next_num/g" ${SCRIPT_DIR}/preempt-exp-kwok.yaml ;;
120-
darwin*)
121-
sed -i '' "s/fake-defaultaw-schd-spec-with-timeout-$num/fake-defaultaw-schd-spec-with-timeout-$next_num/g" ${SCRIPT_DIR}/preempt-exp-kwok.yaml ;;
122-
*)
123-
sed -i "s/fake-defaultaw-schd-spec-with-timeout-$num/fake-defaultaw-schd-spec-with-timeout-$next_num/g" ${SCRIPT_DIR}/preempt-exp-kwok.yaml ;;
121+
sed -i "s/fake-defaultaw-schd-spec-with-timeout-$ORIG_COUNTER/fake-defaultaw-schd-spec-with-timeout-$COUNTER/g" ${SCRIPT_DIR}/preempt-exp-kwok.yaml ;;
122+
darwin*)
123+
sed -i '' "s/fake-defaultaw-schd-spec-with-timeout-$ORIG_COUNTER/fake-defaultaw-schd-spec-with-timeout-$COUNTER/g" ${SCRIPT_DIR}/preempt-exp-kwok.yaml ;;
124+
*)
125+
sed -i "s/fake-defaultaw-schd-spec-with-timeout-$ORIG_COUNTER/fake-defaultaw-schd-spec-with-timeout-$COUNTER/g" ${SCRIPT_DIR}/preempt-exp-kwok.yaml ;;
124126
esac
125127
kubectl apply -f ${SCRIPT_DIR}/preempt-exp-kwok.yaml
128+
COUNTER=$[$COUNTER +1]
126129
done
127130

131+
128132
# Let's reset the original preempt-exp-kwok.yaml file back to original value
129133
case "$OSTYPE" in
130134
linux-gnu*)
131-
sed -i "s/fake-defaultaw-schd-spec-with-timeout-$next_num/fake-defaultaw-schd-spec-with-timeout-1/g" ${SCRIPT_DIR}/preempt-exp-kwok.yaml ;;
135+
sed -i "s/fake-defaultaw-schd-spec-with-timeout-$JOBS/fake-defaultaw-schd-spec-with-timeout-0/g" ${SCRIPT_DIR}/preempt-exp-kwok.yaml ;;
132136
darwin*)
133-
sed -i '' "s/fake-defaultaw-schd-spec-with-timeout-$next_num/fake-defaultaw-schd-spec-with-timeout-1/g" ${SCRIPT_DIR}/preempt-exp-kwok.yaml ;;
137+
sed -i '' "s/fake-defaultaw-schd-spec-with-timeout-$JOBS/fake-defaultaw-schd-spec-with-timeout-0/g" ${SCRIPT_DIR}/preempt-exp-kwok.yaml ;;
134138
*)
135-
sed -i "s/fake-defaultaw-schd-spec-with-timeout-$next_num/fake-defaultaw-schd-spec-with-timeout-1/g" ${SCRIPT_DIR}/preempt-exp-kwok.yaml ;;
139+
sed -i "s/fake-defaultaw-schd-spec-with-timeout-$JOBS/fake-defaultaw-schd-spec-with-timeout-0/g" ${SCRIPT_DIR}/preempt-exp-kwok.yaml ;;
136140
esac
137141

138142
# Check for all jobs to report complete
139-
jobstatus=`kubectl get jobs -n default --no-headers --field-selector status.successful=1 |wc -l`
143+
JOBSTATUS=`kubectl get jobs -n default --no-headers --field-selector status.successful=1 |wc -l`
140144

141-
while [ $jobstatus -lt $jobs ]
145+
while [ $JOBSTATUS -lt $JOBS ]
142146
do
143-
echo "Number of completed jobs is: " $jobstatus " and the goal is: " $jobs
147+
echo "Number of completed jobs is: " $JOBSTATUS " and the goal is: " $JOBS
144148
sleep 10
145-
jobstatus=`kubectl get jobs -n default --no-headers --field-selector status.successful=1 |wc -l`
149+
JOBSTATUS=`kubectl get jobs -n default --no-headers --field-selector status.successful=1 |wc -l`
146150
done
147151

148152
echo " "
149153
export FINISHTIME=`date +"%T"`
150-
echo "All $jobstatus jobs finished: $FINISHTIME" |tee -a fake-job-$STARTTIME.log
151-
echo "Total amount of time for $jobs appwrappers is: $SECONDS seconds" |tee -a ${SCRIPT_DIR}/fake-job-$STARTTIME.log
154+
echo "All $JOBSTATUS jobs finished: $FINISHTIME" |tee -a fake-job-$STARTTIME.log
155+
echo "Total amount of time for $JOBS appwrappers is: $SECONDS seconds" |tee -a ${SCRIPT_DIR}/fake-job-$STARTTIME.log
152156
echo " "
153-
echo "Test results are stored in this file: ${SCRIPT_DIR}/fake-job-$next_num-$STARTTIME.log"
157+
echo "Test results are stored in this file: ${SCRIPT_DIR}/fake-job-$JOBS-$STARTTIME.log"
154158

155159
# Rename the log to show the number of jobs used
156-
mv ${SCRIPT_DIR}/fake-job-$STARTTIME.log ${SCRIPT_DIR}/fake-job-$next_num-$STARTTIME.log
160+
mv ${SCRIPT_DIR}/fake-job-$STARTTIME.log ${SCRIPT_DIR}/fake-job-$JOBS-$STARTTIME.log
157161

158162
#Ask if you want to auto-cleanup the appwrapper jobs
159163
echo "Do you want to cleanup the most recently created appwrappers? [Y/n]"

‎test/perf-test/nodes.sh

+18-19
Original file line numberDiff line numberDiff line change
@@ -65,51 +65,50 @@ echo "Checking whether we have a valid cluster login or not..."
6565
check_kubectl_login_status
6666

6767
# Track whether you have the KWOK controller installed
68-
echo "Checking MCAD Controller installation status"
68+
echo "Checking KWOK Controller installation status"
6969
echo
7070
check_kwok_installed_status
7171

7272
echo
73-
read -p "How many simulated KWOK nodes do you want?" nodes
73+
read -p "How many simulated KWOK nodes do you want?" NODES
7474

75-
echo "Nodes number is $nodes"
75+
echo "Nodes number is $NODES"
7676
echo " "
7777

78-
# This fixes the number of jobs to be one less so the for loop gets the right amount
79-
((realnodes=$nodes-1))
80-
echo "The real number of nodes is $realnodes"
81-
82-
for num in $(eval echo "{0.."$realnodes"}")
78+
COUNTER=1
79+
while [ $COUNTER -le $NODES ]
8380
do
84-
next_num=$(($num + 1))
85-
echo "Submitting node $next_num"
81+
ORIG_COUNTER=$(($COUNTER - 1))
82+
echo "Submitting node $COUNTER"
8683
# Had to do this OSTYPE because sed acts differently on Linux versus Mac
8784
case "$OSTYPE" in
8885
linux-gnu*)
89-
sed -i "s/kwok-node-$num/kwok-node-$next_num/g" ${SCRIPT_DIR}/node.yaml ;;
90-
darwin*)
91-
sed -i '' "s/kwok-node-$num/kwok-node-$next_num/g" ${SCRIPT_DIR}/node.yaml ${SCRIPT_DIR}/node.yaml ;;
92-
*)
93-
sed -i "/kwok-node-$num/kwok-node-$next_num/g" ${SCRIPT_DIR}/node.yaml ;;
86+
sed -i "s/kwok-node-$ORIG_COUNTER/kwok-node-$COUNTER/g" ${SCRIPT_DIR}/node.yaml ;;
87+
darwin*)
88+
sed -i '' "s/kwok-node-$ORIG_COUNTER/kwok-node-$COUNTER/g" ${SCRIPT_DIR}/node.yaml ${SCRIPT_DIR}/node.yaml ;;
89+
*)
90+
sed -i "/kwok-node-$ORIG_COUNTER/kwok-node-$COUNTER/g" ${SCRIPT_DIR}/node.yaml ;;
9491
esac
9592
kubectl apply -f ${SCRIPT_DIR}/node.yaml
93+
COUNTER=$[$COUNTER +1]
9694
done
9795

96+
9897
# Let's reset the original node.yaml file back to original value
9998
case "$OSTYPE" in
10099
linux-gnu*)
101-
sed -i "s/kwok-node-$next_num/kwok-node-0/g" ${SCRIPT_DIR}/node.yaml ;;
100+
sed -i "s/kwok-node-$NODES/kwok-node-0/g" ${SCRIPT_DIR}/node.yaml ;;
102101
darwin*)
103-
sed -i '' "s/kwok-node-$next_num/kwok-node-0/g" ${SCRIPT_DIR}/node.yaml ;;
102+
sed -i '' "s/kwok-node-$NODES/kwok-node-0/g" ${SCRIPT_DIR}/node.yaml ;;
104103
*)
105-
sed -i "s/kwok-node-$next_num/kwok-node-0/g" ${SCRIPT_DIR}/node.yaml ;;
104+
sed -i "s/kwok-node-$NODES/kwok-node-0/g" ${SCRIPT_DIR}/node.yaml ;;
106105
esac
107106

108107
# Check for all nodes to report complete
109108
echo "Waiting until all the simualted pods become ready:"
110109
kubectl wait --for=condition=Ready nodes --selector type=kwok --timeout=600s
111110
echo " "
112-
echo "Total amount of simulated nodes requested is: $nodes"
111+
echo "Total amount of simulated nodes requested is: $NODES"
113112
echo "Total number of created nodes is: "`kubectl get nodes --selector type=kwok -o name |wc -l`
114113
kubectl get nodes --selector type=kwok
115114

‎test/perf-test/perf.sh

+29-26
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,7 @@ function help() {
99
echo
1010
echo "Preconditions: "
1111
echo " - The script assumes you've logged into your cluster already. If not, it will tell you to login."
12-
echo " - The script checks that you have the mcad-controller installed, otherwise it'll tell you to install it first."
12+
echo " - The script checks that you have the mcad-controller installed (or the newer CodeFlare Operator), otherwise it'll tell you to install it first."
1313
echo
1414
echo "Options:"
1515
echo " -h Print this help message"
@@ -38,12 +38,15 @@ function check_mcad_installed_status() {
3838
res2="$?"
3939
kubectl get crd |grep appwrapper &> /dev/null
4040
res3="$?"
41+
kubectl get pod -A |grep codeflare-operator-manager &> /dev/null
42+
res4="$?"
4143
set -e
4244
MCAD="$res2"
4345
CRD="$res3"
44-
if [[ $MCAD == 1 ]] || [[ $CRD == 1 ]]
46+
CODEFLARE="$res4"
47+
if [[ $MCAD == 1 ]] && [[ $CODEFLARE == 1 ]] || [[ $CRD == 1 ]]
4548
then
46-
echo "You need Install MCAD Controller first before running this script"
49+
echo "You need Install the MCAD Controller or the latest CodeFlare Operator first before running this script"
4750
exit 1
4851
else
4952
echo "Nice, MCAD Controller is installed"
@@ -73,65 +76,65 @@ echo
7376
check_mcad_installed_status
7477

7578
echo
76-
read -p "How many appwrapper jobs do you want?" jobs
79+
read -p "How many appwrapper jobs do you want?" JOBS
7780

7881
# Start the timer now
7982
SECONDS=0
8083

81-
echo "jobs number is $jobs"
84+
echo "jobs number is $JOBS"
8285
export STARTTIME=`date +"%T"`
8386
echo " "
8487
echo "Jobs started at: $STARTTIME" |tee job-$STARTTIME.log
8588
echo " "
8689

87-
# This fixes the number of jobs to be one less so the for loop gets the right amount
88-
((realjobs=$jobs-1))
89-
90-
for num in $(eval echo "{0.."$realjobs"}")
90+
COUNTER=1
91+
while [ $COUNTER -le $JOBS ]
9192
do
92-
next_num=$(($num + 1))
93-
echo "Submitting job $next_num"
93+
ORIG_COUNTER=$(($COUNTER - 1))
94+
echo "Submitting job $COUNTER"
95+
9496
# Had to do this OSTYPE because sed acts differently on Linux versus Mac
9597
case "$OSTYPE" in
9698
linux-gnu*)
97-
sed -i "s/defaultaw-schd-spec-with-timeout-$num/defaultaw-schd-spec-with-timeout-$next_num/g" ${SCRIPT_DIR}/preempt-exp.yaml ;;
98-
darwin*)
99-
sed -i '' "s/defaultaw-schd-spec-with-timeout-$num/defaultaw-schd-spec-with-timeout-$next_num/g" ${SCRIPT_DIR}/preempt-exp.yaml ;;
100-
*)
101-
sed -i "s/defaultaw-schd-spec-with-timeout-$num/defaultaw-schd-spec-with-timeout-$next_num/g" ${SCRIPT_DIR}/preempt-exp.yaml ;;
99+
sed -i "s/defaultaw-schd-spec-with-timeout-$ORIG_COUNTER/defaultaw-schd-spec-with-timeout-$COUNTER/g" ${SCRIPT_DIR}/preempt-exp.yaml ;;
100+
darwin*)
101+
sed -i '' "s/defaultaw-schd-spec-with-timeout-$ORIG_COUNTER/defaultaw-schd-spec-with-timeout-$COUNTER/g" ${SCRIPT_DIR}/preempt-exp.yaml ;;
102+
*)
103+
sed -i "s/defaultaw-schd-spec-with-timeout-$ORIG_COUNTER/defaultaw-schd-spec-with-timeout-$COUNTER/g" ${SCRIPT_DIR}/preempt-exp.yaml ;;
102104
esac
103105
kubectl apply -f ${SCRIPT_DIR}/preempt-exp.yaml
106+
COUNTER=$[$COUNTER +1]
104107
done
105108

106109
# Let's reset the original preempt-exp.yaml file back to original value
107110
case "$OSTYPE" in
108111
linux-gnu*)
109-
sed -i "s/defaultaw-schd-spec-with-timeout-$next_num/defaultaw-schd-spec-with-timeout-1/g" ${SCRIPT_DIR}/preempt-exp.yaml ;;
112+
sed -i "s/defaultaw-schd-spec-with-timeout-$JOBS/defaultaw-schd-spec-with-timeout-0/g" ${SCRIPT_DIR}/preempt-exp.yaml ;;
110113
darwin*)
111-
sed -i '' "s/defaultaw-schd-spec-with-timeout-$next_num/defaultaw-schd-spec-with-timeout-1/g" ${SCRIPT_DIR}/preempt-exp.yaml ;;
114+
sed -i '' "s/defaultaw-schd-spec-with-timeout-$JOBS/defaultaw-schd-spec-with-timeout-0/g" ${SCRIPT_DIR}/preempt-exp.yaml ;;
112115
*)
113-
sed -i "s/defaultaw-schd-spec-with-timeout-$next_num/defaultaw-schd-spec-with-timeout-1/g" ${SCRIPT_DIR}/preempt-exp.yaml ;;
116+
sed -i "s/defaultaw-schd-spec-with-timeout-$JOBS/defaultaw-schd-spec-with-timeout-0/g" ${SCRIPT_DIR}/preempt-exp.yaml ;;
114117
esac
115118

116119
# Check for all jobs to report complete
117-
jobstatus=`kubectl get jobs -n default --no-headers --field-selector status.successful=1 |wc -l`
120+
JOBSTATUS=`kubectl get jobs -n default --no-headers --field-selector status.successful=1 |wc -l`
118121

119-
while [ $jobstatus -lt $jobs ]
122+
while [ $JOBSTATUS -lt $JOBS ]
120123
do
121-
echo "Number of completed jobs is: " $jobstatus " and the goal is: " $jobs
124+
echo "Number of completed jobs is: " $JOBSTATUS " and the goal is: " $JOBS
122125
sleep 10
123-
jobstatus=`kubectl get jobs -n default --no-headers --field-selector status.successful=1 |wc -l`
126+
JOBSTATUS=`kubectl get jobs -n default --no-headers --field-selector status.successful=1 |wc -l`
124127
done
125128

126129
echo " "
127130
export FINISHTIME=`date +"%T"`
128131
echo "All $jobstatus jobs finished: $FINISHTIME" |tee -a job-$STARTTIME.log
129-
echo "Total amount of time for $jobs appwrappers is: $SECONDS seconds" |tee -a ${SCRIPT_DIR}/job-$STARTTIME.log
132+
echo "Total amount of time for $JOBS appwrappers is: $SECONDS seconds" |tee -a ${SCRIPT_DIR}/job-$STARTTIME.log
130133
echo " "
131-
echo "Test results are stored in this file: ${SCRIPT_DIR}/job-$next_num-$STARTTIME.log"
134+
echo "Test results are stored in this file: ${SCRIPT_DIR}/job-$JOBS-$STARTTIME.log"
132135

133136
# Rename the log to show the number of jobs used
134-
mv ${SCRIPT_DIR}/job-$STARTTIME.log ${SCRIPT_DIR}/job-$next_num-$STARTTIME.log
137+
mv ${SCRIPT_DIR}/job-$STARTTIME.log ${SCRIPT_DIR}/job-$JOBS-$STARTTIME.log
135138

136139
#Ask if you want to auto-cleanup the appwrapper jobs
137140
echo "Do you want to cleanup the most recently created appwrappers? [Y/n]"

‎test/perf-test/preempt-exp-kwok.yaml

+4-4
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
apiVersion: workload.codeflare.dev/v1beta1
22
kind: AppWrapper
33
metadata:
4-
name: fake-defaultaw-schd-spec-with-timeout-1
4+
name: fake-defaultaw-schd-spec-with-timeout-0
55
namespace: default
66
spec:
77
schedulingSpec:
@@ -29,15 +29,15 @@ spec:
2929
kind: Job
3030
metadata:
3131
namespace: default
32-
name: fake-defaultaw-schd-spec-with-timeout-1
32+
name: fake-defaultaw-schd-spec-with-timeout-0
3333
spec:
3434
parallelism: 1
3535
completions: 1
3636
template:
3737
metadata:
3838
namespace: default
3939
labels:
40-
appwrappers.workload.codeflare.dev: "fake-defaultaw-schd-spec-with-timeout-1"
40+
appwrapper.mcad.ibm.com: "fake-defaultaw-schd-spec-with-timeout-0"
4141
spec:
4242
affinity:
4343
nodeAffinity:
@@ -55,6 +55,6 @@ spec:
5555
operator: "Exists"
5656
effect: "NoSchedule"
5757
containers:
58-
- name: fake-defaultaw-schd-spec-with-timeout-1
58+
- name: fake-defaultaw-schd-spec-with-timeout-0
5959
image: fake-image
6060
restartPolicy: Never

‎test/perf-test/preempt-exp.yaml

+5-5
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
apiVersion: workload.codeflare.dev/v1beta1
22
kind: AppWrapper
33
metadata:
4-
name: defaultaw-schd-spec-with-timeout-1
4+
name: defaultaw-schd-spec-with-timeout-0
55
namespace: default
66
spec:
77
schedulingSpec:
@@ -29,20 +29,20 @@ spec:
2929
kind: Job
3030
metadata:
3131
namespace: default
32-
name: defaultaw-schd-spec-with-timeout-1
32+
name: defaultaw-schd-spec-with-timeout-0
3333
# labels:
34-
# appwrappers.workload.codeflare.dev: defaultaw-schd-spec-with-timeout-1
34+
# appwrapper.mcad.ibm.com: defaultaw-schd-spec-with-timeout-0
3535
spec:
3636
parallelism: 1
3737
completions: 1
3838
template:
3939
metadata:
4040
namespace: default
4141
labels:
42-
appwrappers.workload.codeflare.dev: "defaultaw-schd-spec-with-timeout-1"
42+
appwrapper.mcad.ibm.com: "defaultaw-schd-spec-with-timeout-0"
4343
spec:
4444
containers:
45-
- name: defaultaw-schd-spec-with-timeout-1
45+
- name: defaultaw-schd-spec-with-timeout-0
4646
image: ubi8-minimal:latest
4747
command: [ "/bin/bash", "-c", "--" ]
4848
args: [ "sleep 10" ]

0 commit comments

Comments
 (0)