Skip to content

Commit b04c13c

Browse files
leftwoleftwo
authoredJun 21, 2024··
Parameterize more tests (#1364)
This updates more of the tests in the tools directory to honor both `REGION_ROOT ` and `WORK_ROOT` to help avoid conflicts and give tests a place to store region files.
1 parent 49be926 commit b04c13c

5 files changed

+62
-45
lines changed
 

‎tools/test_live_repair.sh

+19-9
Original file line numberDiff line numberDiff line change
@@ -20,8 +20,15 @@ function ctrl_c() {
2020
${dsc} cmd shutdown
2121
}
2222

23-
loop_log=/tmp/test_live_repair_summary.log
24-
test_log=/tmp/test_live_repair.log
23+
REGION_ROOT=${REGION_ROOT:-/var/tmp/test_live_repair}
24+
mkdir -p "$REGION_ROOT"
25+
26+
# Location of logs and working files
27+
WORK_ROOT=${WORK_ROOT:-/tmp}
28+
mkdir -p "$WORK_ROOT"
29+
30+
loop_log="$WORK_ROOT"/test_live_repair_summary.log
31+
test_log="$WORK_ROOT"/test_live_repair.log
2532

2633
ROOT=$(cd "$(dirname "$0")/.." && pwd)
2734
export BINDIR=${BINDIR:-$ROOT/target/debug}
@@ -52,12 +59,13 @@ while getopts 'l:' opt; do
5259
esac
5360
done
5461

55-
echo "" > ${loop_log}
56-
echo "" > ${test_log}
57-
echo "starting $(date)" | tee ${loop_log}
62+
echo "" > "$loop_log"
63+
echo "" > "$test_log"
64+
echo "starting $(date)" | tee "$loop_log"
5865
echo "Tail $test_log for test output"
5966

6067
if ! ${dsc} create --cleanup \
68+
--region-dir "$REGION_ROOT" \
6169
--region-count 4 \
6270
--ds-bin "$downstairs" \
6371
--extent-size 4000 \
@@ -66,6 +74,7 @@ if ! ${dsc} create --cleanup \
6674
exit 1
6775
fi
6876
${dsc} start --ds-bin "$downstairs" \
77+
--region-dir "$REGION_ROOT" \
6978
--region-count 4 >> "$test_log" 2>&1 &
7079
dsc_pid=$!
7180
sleep 5
@@ -85,6 +94,7 @@ if ! "$crucible_test" fill "${args[@]}" -q -g "$gen"\
8594
--verify-out alan --retry-activate >> "$test_log" 2>&1 ; then
8695
echo Failed on initial verify seed, check "$test_log"
8796
${dsc} cmd shutdown
97+
exit 1
8898
fi
8999
(( gen += 1 ))
90100

@@ -106,7 +116,7 @@ while [[ $count -le $loops ]]; do
106116
(( err += 1 ))
107117
duration=$SECONDS
108118
printf "[%03d] Error $result after %d:%02d\n" "$count" \
109-
$((duration / 60)) $((duration % 60)) | tee -a ${loop_log}
119+
$((duration / 60)) $((duration % 60)) | tee -a "$loop_log"
110120
mv "$test_log" "$test_log".lastfail
111121
break
112122
fi
@@ -121,20 +131,20 @@ last_run_seconds:%d\n" \
121131
$((duration / 60)) $((duration % 60)) \
122132
$((ave / 60)) $((ave % 60)) \
123133
$((total / 60)) $((total % 60)) \
124-
"$err" $duration | tee -a ${loop_log}
134+
"$err" $duration | tee -a "$loop_log"
125135
(( count += 1 ))
126136

127137
done
128138
${dsc} cmd shutdown
129139
wait "$dsc_pid"
130140

131141
sleep 4
132-
echo "Final results:" | tee -a ${loop_log}
142+
echo "Final results:" | tee -a "$loop_log"
133143
printf "[%03d] %d:%02d ave:%d:%02d total:%d:%02d errors:%d last_run_seconds:%d\n" \
134144
"$count" \
135145
$((duration / 60)) $((duration % 60)) \
136146
$((ave / 60)) $((ave % 60)) \
137147
$((total / 60)) $((total % 60)) \
138-
"$err" $duration | tee -a ${loop_log}
148+
"$err" $duration | tee -a "$loop_log"
139149
echo "$(date) Test ends with $err" >> "$test_log" 2>&1
140150
exit "$err"

‎tools/test_mem.sh

+4-1
Original file line numberDiff line numberDiff line change
@@ -80,7 +80,10 @@ function show_mem_summary() {
8080

8181
}
8282

83-
test_mem_log="/tmp/test_mem_log.txt"
83+
# Location of logs and working files
84+
WORK_ROOT=${WORK_ROOT:-/tmp}
85+
mkdir -p "$WORK_ROOT"
86+
test_mem_log="$WORK_ROOT/test_mem_log.txt"
8487
# Create a region with the given extent_size ($1) and extent_count ($2)
8588
# Once created, write to every block in the region, then display memory usage.
8689
function mem_test() {

‎tools/test_perf.sh

+1
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,7 @@ count=30000
3131
prefill=0
3232
read_loops=2
3333
write_loops=2
34+
REGION_ROOT=${REGION_ROOT:-/var/tmp/test_perf}
3435
region_dir="/var/tmp/dsc"
3536

3637
while getopts 'b:c:g:hfr:w:' opt; do

‎tools/test_replay.sh

+8-5
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,10 @@ function ctrl_c() {
1414
${dsc} cmd shutdown
1515
}
1616

17-
test_log=/tmp/test_replay.log
17+
WORK_ROOT=${WORK_ROOT:-/tmp}
18+
mkdir -p "$WORK_ROOT"
19+
20+
test_log="$WORK_ROOT/test_replay.log"
1821

1922
ROOT=$(cd "$(dirname "$0")/.." && pwd)
2023
export BINDIR=${BINDIR:-$ROOT/target/debug}
@@ -45,8 +48,8 @@ while getopts 'l:' opt; do
4548
esac
4649
done
4750

48-
echo "" > ${test_log}
49-
echo "starting $(date)" | tee ${test_log}
51+
echo "" > "$test_log"
52+
echo "starting $(date)" | tee "$test_log"
5053
echo "Tail $test_log for test output"
5154

5255
echo "Creating downstairs regions" | tee -a "$test_log"
@@ -90,11 +93,11 @@ result=$?
9093
duration=$SECONDS
9194
if [[ $result -ne 0 ]]; then
9295
printf "Error $result after %d:%02d\n" \
93-
$((duration / 60)) $((duration % 60)) | tee -a ${test_log}
96+
$((duration / 60)) $((duration % 60)) | tee -a "$test_log"
9497
else
9598
(( gen += 1 ))
9699
printf "Replays:%d time: %d:%02d\n" \
97-
"$loops" $((duration / 60)) $((duration % 60)) | tee -a ${test_log}
100+
"$loops" $((duration / 60)) $((duration % 60)) | tee -a "$test_log"
98101

99102
echo "Do final verify" | tee -a "$test_log"
100103
if ! "$crucible_test" verify "${args[@]}" -q -g "$gen"\

‎tools/test_restart_repair.sh

+30-30
Original file line numberDiff line numberDiff line change
@@ -90,27 +90,28 @@ while getopts 'l:' opt; do
9090
esac
9191
done
9292

93-
export loop_log=/tmp/repair_restart.log
94-
export test_log=/tmp/repair_restart_test.log
95-
export dsc_log=/tmp/repair_restart_dsc.log
96-
export region_dir="./var"
97-
98-
echo "" > ${loop_log}
99-
echo "starting $(date)" | tee ${loop_log}
100-
echo "" > ${test_log}
93+
WORK_ROOT=${WORK_ROOT:-/tmp}
94+
export loop_log="$WORK_ROOT/repair_restart.log"
95+
export test_log="$WORK_ROOT/repair_restart_test.log"
96+
export dsc_log="$WORK_ROOT/repair_restart_dsc.log"
97+
REGION_ROOT=${REGION_ROOT:-/var/tmp/test_restart_repair}
98+
99+
echo "" > "$loop_log"
100+
echo "starting $(date)" | tee "$loop_log"
101+
echo "" > "$test_log"
101102
echo "Tail $test_log for test output"
102103
echo "Tail $loop_log for summary output"
103104
echo "Tail $dsc_log for dsc outout"
104105

105106
echo "Create a new region to test" | tee -a "${loop_log}"
106107
ulimit -n 65536
107-
if ! "$dsc" create --cleanup --ds-bin "$cds" --extent-count 61 --extent-size 5120 --region-dir "$region_dir"; then
108-
echo "Failed to create region at $region_dir"
108+
if ! "$dsc" create --cleanup --ds-bin "$cds" --extent-count 61 --extent-size 5120 --region-dir "$REGION_ROOT"; then
109+
echo "Failed to create region at $REGION_ROOT"
109110
exit 1
110111
fi
111112

112113
echo "Starting the downstairs" | tee -a "${loop_log}"
113-
"$dsc" start --ds-bin "$cds" --region-dir "$region_dir" >> ${dsc_log} 2>&1 &
114+
"$dsc" start --ds-bin "$cds" --region-dir "$REGION_ROOT" >> "$dsc_log" 2>&1 &
114115
dsc_pid=$!
115116
# Sleep 5 to give the downstairs time to get going.
116117
sleep 5
@@ -158,10 +159,10 @@ stop_all_downstairs
158159
sleep 7
159160

160161
# Create the "old" region files
161-
rm -rf "$region_dir"/8810.old "$region_dir"/8820.old "$region_dir"/8830.old
162-
cp -R "$region_dir"/8810 "$region_dir"/8810.old || ctrl_c
163-
cp -R "$region_dir"/8820 "$region_dir"/8820.old || ctrl_c
164-
cp -R "$region_dir"/8830 "$region_dir"/8830.old || ctrl_c
162+
rm -rf "$REGION_ROOT"/8810.old "$REGION_ROOT"/8820.old "$REGION_ROOT"/8830.old
163+
cp -R "$REGION_ROOT"/8810 "$REGION_ROOT"/8810.old || ctrl_c
164+
cp -R "$REGION_ROOT"/8820 "$REGION_ROOT"/8820.old || ctrl_c
165+
cp -R "$REGION_ROOT"/8830 "$REGION_ROOT"/8830.old || ctrl_c
165166

166167
# Bring the downstairs back online.
167168
echo "$(date) Bring downstairs back online" | tee -a "$loop_log"
@@ -182,7 +183,7 @@ fi
182183
echo "Fill completed, wait for downstairs to start restarting" >> "$test_log"
183184
duration=$SECONDS
184185
printf "Initial fill and verify took: %d:%02d \n" \
185-
$((duration / 60)) $((duration % 60)) | tee -a ${loop_log}
186+
$((duration / 60)) $((duration % 60)) | tee -a "$loop_log"
186187

187188
# Now run the repair loop
188189
count=1
@@ -202,19 +203,19 @@ while [[ $count -le $loops ]]; do
202203
echo "$(date) move regions" >> "$test_log"
203204
choice=$((RANDOM % 3))
204205
if [[ $choice -eq 0 ]]; then
205-
rm -rf "$region_dir"/8810
206-
cp -R "$region_dir"/8810.old "$region_dir"/8810
206+
rm -rf "$REGION_ROOT"/8810
207+
cp -R "$REGION_ROOT"/8810.old "$REGION_ROOT"/8810
207208
elif [[ $choice -eq 1 ]]; then
208-
rm -rf "$region_dir"/8820
209-
cp -R "$region_dir"/8820.old "$region_dir"/8820
209+
rm -rf "$REGION_ROOT"/8820
210+
cp -R "$REGION_ROOT"/8820.old "$REGION_ROOT"/8820
210211
else
211-
rm -rf "$region_dir"/8830
212-
cp -R "$region_dir"/8830.old "$region_dir"/8830
212+
rm -rf "$REGION_ROOT"/8830
213+
cp -R "$REGION_ROOT"/8830.old "$REGION_ROOT"/8830
213214
fi
214215
echo "$(date) regions moved, current dump outputs:" >> "$test_log"
215-
$cds dump --no-color -d "$region_dir"/8810 \
216-
-d "$region_dir"/8820 \
217-
-d "$region_dir"/8830 >> "$test_log" 2>&1
216+
$cds dump --no-color -d "$REGION_ROOT"/8810 \
217+
-d "$REGION_ROOT"/8820 \
218+
-d "$REGION_ROOT"/8830 >> "$test_log" 2>&1
218219

219220
echo "$(date) resume downstairs" >> "$test_log"
220221
bring_all_downstairs_online
@@ -236,7 +237,7 @@ while [[ $count -le $loops ]]; do
236237
(( err += 1 ))
237238
duration=$SECONDS
238239
printf "[%03d] Error $result in one test after %d:%02d\n" "$count" \
239-
$((duration / 60)) $((duration % 60)) | tee -a ${loop_log}
240+
$((duration / 60)) $((duration % 60)) | tee -a "$loop_log"
240241
mv "$test_log" "$test_log".lastfail
241242
break
242243
fi
@@ -249,7 +250,7 @@ while [[ $count -le $loops ]]; do
249250
printf "[%03d/%03d] %d:%02d ave:%d:%02d total:%d:%02d errors:%d \
250251
last_run_seconds:%d\n" "$count" "$loops" $((duration / 60)) $((duration % 60)) \
251252
$((ave / 60)) $((ave % 60)) $((total / 60)) $((total % 60)) \
252-
"$err" $duration | tee -a ${loop_log}
253+
"$err" $duration | tee -a "$loop_log"
253254
(( count += 1 ))
254255

255256
done
@@ -258,11 +259,10 @@ if [[ -n "$dsc_pid" ]]; then
258259
wait "$dsc_pid"
259260
fi
260261

261-
echo "Final results $(date):" | tee -a ${loop_log}
262+
echo "Final results $(date):" | tee -a "$loop_log"
262263
printf "[%03d] %d:%02d ave:%d:%02d total:%d:%02d errors:%d last_run_seconds:%d\n" \
263264
"$count" $((duration / 60)) $((duration % 60)) \
264265
$((ave / 60)) $((ave % 60)) \
265266
$((total / 60)) $((total % 60)) \
266-
"$err" $duration | tee -a ${loop_log}
267+
"$err" $duration | tee -a "$loop_log"
267268
exit "$err"
268-

0 commit comments

Comments
 (0)
Please sign in to comment.