Skip to content

Commit 46bc98f

Browse files
BigAndiniBigAndini
authored
Merge pull request #53 from patrickdk77/master
Add Zstandard support, Fix 2 xz bugs
2 parents 7c32e42 + 6d63c38 commit 46bc98f

File tree

1 file changed

+95
-3
lines changed

1 file changed

+95
-3
lines changed

‎automysqlbackup

Lines changed: 95 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,7 @@ let "filename_flag_gz=0x02"
3131
let "filename_flag_bz2=0x04"
3232
let "filename_flag_diff=0x08"
3333
let "filename_flag_xz=0x10"
34+
let "filename_flag_zst=0x20"
3435
# END _flags
3536

3637
# BEGIN _errors_notifications
@@ -274,6 +275,22 @@ gzip_compression() {
274275
fi
275276
}
276277

278+
# @info: zstd compression
279+
zstd_compression() {
280+
var=("$@")
281+
re='^[0-9]*$'
282+
if [[ "x$CONFIG_multicore" = 'xyes' ]]; then
283+
if [[ "x$CONFIG_multicore_threads" != 'xauto' ]] && [[ "x$CONFIG_multicore_threads" =~ $re ]]; then
284+
var=( "-T${CONFIG_multicore_threads}" "${var[@]}" )
285+
else
286+
var=( "-T0" "${var[@]}" )
287+
fi
288+
zstd "${var[@]}"
289+
else
290+
zstd "${var[@]}"
291+
fi
292+
}
293+
277294
# @info: xz compression
278295
xz_compression() {
279296
var=("$@")
@@ -455,6 +472,9 @@ backup_local_files () {
455472
'xz')
456473
echo "tar -cJvf ${1}${suffix} ${CONFIG_backup_local_files[@]}";
457474
;;
475+
'zstd')
476+
echo "tar -I zstd -cvf ${1}${suffix} ${CONFIG_backup_local_files[@]}";
477+
;;
458478
*)
459479
echo "tar -cvf ${1}${suffix} ${CONFIG_backup_local_files[@]}";
460480
;;
@@ -475,6 +495,10 @@ backup_local_files () {
475495
tar -cJvf "${1}${suffix}" "${CONFIG_backup_local_files[@]}";
476496
return $?
477497
;;
498+
'zstd')
499+
tar -I zstd -cvf "${1}${suffix}" "${CONFIG_backup_local_files[@]}";
500+
return $?
501+
;;
478502
*)
479503
tar -cvf "${1}${suffix}" "${CONFIG_backup_local_files[@]}";
480504
return $?
@@ -571,6 +595,7 @@ parse_configuration () {
571595
'gzip') suffix='.gz';;
572596
'bzip2') suffix='.bz2';;
573597
'xz') suffix='.xz';;
598+
'zstd') suffix='.zst';;
574599
*) suffix='';;
575600
esac
576601
# <- determine suffix
@@ -612,6 +637,9 @@ dbstatus() {
612637
'xz')
613638
echo "dry-running: $MYSQLSHOW ${opt_dbstatus[@]} | xz_compression > ${1}${suffix}";
614639
;;
640+
'zstd')
641+
echo "dry-running: $MYSQLSHOW ${opt_dbstatus[@]} | zstd_compression > ${1}${suffix}";
642+
;;
615643
*)
616644
echo "dry-running: $MYSQLSHOW ${opt_dbstatus[@]} > ${1}${suffix}";
617645
;;
@@ -631,6 +659,10 @@ dbstatus() {
631659
$MYSQLSHOW "${opt_dbstatus[@]}" | xz_compression > "${1}${suffix}";
632660
return $?
633661
;;
662+
'zstd')
663+
$MYSQLSHOW "${opt_dbstatus[@]}" | zstd_compression > "${1}${suffix}";
664+
return $?
665+
;;
634666
*)
635667
$MYSQLSHOW "${opt_dbstatus[@]}" > "${1}${suffix}";
636668
return $?
@@ -655,6 +687,9 @@ fullschema () {
655687
'xz')
656688
echo "dry-running: $MYSQLDUMP ${opt_fullschema[@]} | xz_compression > ${1}${suffix}";
657689
;;
690+
'zstd')
691+
echo "dry-running: $MYSQLDUMP ${opt_fullschema[@]} | zstd_compression > ${1}${suffix}";
692+
;;
658693
*)
659694
echo "dry-running: $MYSQLDUMP ${opt_fullschema[@]} > ${1}${suffix}";
660695
;;
@@ -674,6 +709,10 @@ fullschema () {
674709
$MYSQLDUMP "${opt_fullschema[@]}" | xz_compression > "${1}${suffix}";
675710
return $?
676711
;;
712+
'zstd')
713+
$MYSQLDUMP "${opt_fullschema[@]}" | zstd_compression > "${1}${suffix}";
714+
return $?
715+
;;
677716
*)
678717
$MYSQLDUMP "${opt_fullschema[@]}" > "${1}${suffix}";
679718
return $?
@@ -750,13 +789,14 @@ process_dbs() {
750789
# ## -> get latest differential manifest entry for specified db
751790
# if get_latest_manifest_entry_for_db "$db" 1; then
752791
# pid="${manifest_entry[2]}"
753-
# # filename format: prefix_db_YYYY-MM-DD_HHhMMm_[A-Za-z0-9]{8}(.sql|.diff)(.gz|.bz2|.xz)(.enc)
792+
# # filename format: prefix_db_YYYY-MM-DD_HHhMMm_[A-Za-z0-9]{8}(.sql|.diff)(.gz|.bz2|.xz|.zst)(.enc)
754793
# FileStub=${manifest_entry[0]%.@(sql|diff)*}
755794
# FileExt=${manifest_entry[0]#"$FileStub"}
756795
# re=".*\.enc.*"; [[ "$FileExt" =~ $re ]] && let "filename_flags|=$filename_flag_encrypted"
757796
# re=".*\.gz.*"; [[ "$FileExt" =~ $re ]] && let "filename_flags|=$filename_flag_gz"
758797
# re=".*\.bz2.*"; [[ "$FileExt" =~ $re ]] && let "filename_flags|=$filename_flag_bz2"
759798
# re=".*\.xz.*"; [[ "$FileExt" =~ $re ]] && let "filename_flags|=$filename_flag_xz"
799+
# re=".*\.zst.*"; [[ "$FileExt" =~ $re ]] && let "filename_flags|=$filename_flag_zst"
760800
# re=".*\.diff.*"; [[ "$FileExt" =~ $re ]] && let "filename_flags|=$filename_flag_diff"
761801
# manifest_latest_diff_entry=("${manifest_entry[@]}")
762802
# else # no entries in manifest
@@ -768,13 +808,14 @@ process_dbs() {
768808
# Create a differential backup if a master entry in the manifest exists, it isn't the day we do weekly master backups or the master file we fetched is already from today.
769809
if get_latest_manifest_entry_for_db "$db" 0 && ( (( ${date_dayno_of_week} != ${CONFIG_do_weekly} )) || [[ "${manifest_entry[0]}" = *_$(date +%Y-%m-%d)_* ]] ); then
770810
pid="${manifest_entry[2]}"
771-
# filename format: prefix_db_YYYY-MM-DD_HHhMMm_[A-Za-z0-9]{8}(.sql|.diff)(.gz|.bz2|xz)(.enc)
811+
# filename format: prefix_db_YYYY-MM-DD_HHhMMm_[A-Za-z0-9]{8}(.sql|.diff)(.gz|.bz2|,xz|.zst)(.enc)
772812
FileStub="${manifest_entry[0]%.@(sql|diff)*}"
773813
FileExt="${manifest_entry[0]#"$FileStub"}"
774814
re=".*\.enc.*"; [[ "$FileExt" =~ $re ]] && let "filename_flags|=$filename_flag_encrypted"
775815
re=".*\.gz.*"; [[ "$FileExt" =~ $re ]] && let "filename_flags|=$filename_flag_gz"
776816
re=".*\.bz2.*"; [[ "$FileExt" =~ $re ]] && let "filename_flags|=$filename_flag_bz2"
777817
re=".*\.xz.*"; [[ "$FileExt" =~ $re ]] && let "filename_flags|=$filename_flag_xz"
818+
re=".*\.zst.*"; [[ "$FileExt" =~ $re ]] && let "filename_flags|=$filename_flag_zst"
778819
re=".*\.diff.*"; [[ "$FileExt" =~ $re ]] && let "filename_flags|=$filename_flag_diff"
779820
manifest_latest_master_entry=("${manifest_entry[@]}")
780821
else # no entries in manifest
@@ -803,6 +844,9 @@ process_dbs() {
803844
'xz')
804845
$MYSQLDUMP "${opt[@]}" "$@" | xz_compression > "$cfname";
805846
;;
847+
'zstd')
848+
$MYSQLDUMP "${opt[@]}" "$@" | zstd_compression > "$cfname";
849+
;;
806850
*)
807851
$MYSQLDUMP "${opt[@]}" "$@" > "$cfname";
808852
;;
@@ -848,13 +892,26 @@ process_dbs() {
848892
diff "${manifest_latest_master_entry[0]}" <($MYSQLDUMP "${opt[@]}" "$@") | xz_compression > "$cfname";
849893
fi
850894
;;
895+
'zstd')
896+
if (( $filename_flags & $filename_flag_zst )); then
897+
diff <(zstd_compression -dc "${manifest_latest_master_entry[0]}") <($MYSQLDUMP "${opt[@]}" "$@") | xz_compression > "$cfname";
898+
elif (( $filename_flags & $filename_flag_bz2 )); then
899+
diff <(bzip2_compression -dc "${manifest_latest_master_entry[0]}") <($MYSQLDUMP "${opt[@]}" "$@") | xz_compression > "$cfname";
900+
elif (( $filename_flags & $filename_flag_xz )); then
901+
diff <(xz_compression -dc "${manifest_latest_master_entry[0]}") <($MYSQLDUMP "${opt[@]}" "$@") | xz_compression > "$cfname";
902+
else
903+
diff "${manifest_latest_master_entry[0]}" <($MYSQLDUMP "${opt[@]}" "$@") | xz_compression > "$cfname";
904+
fi
905+
;;
851906
*)
852907
if (( $filename_flags & $filename_flag_gz )); then
853908
diff <(gzip_compression -dc "${manifest_latest_master_entry[0]}") <($MYSQLDUMP "${opt[@]}" "$@") > "$cfname";
854909
elif (( $filename_flags & $filename_flag_bz2 )); then
855910
diff <(bzip2_compression -dc "${manifest_latest_master_entry[0]}") <($MYSQLDUMP "${opt[@]}" "$@") > "$cfname";
856911
elif (( $filename_flags & $filename_flag_xz )); then
857912
diff <(xz_compression -dc "${manifest_latest_master_entry[0]}") <($MYSQLDUMP "${opt[@]}" "$@") > "$cfname";
913+
elif (( $filename_flags & $filename_flag_zst )); then
914+
diff <(zstd_compression -dc "${manifest_latest_master_entry[0]}") <($MYSQLDUMP "${opt[@]}" "$@") > "$cfname";
858915
else
859916
diff "${manifest_latest_master_entry[0]}" <($MYSQLDUMP "${opt[@]}" "$@") > "$cfname";
860917
fi
@@ -878,6 +935,9 @@ process_dbs() {
878935
'xz')
879936
echo "dry-running: $MYSQLDUMP ${opt[@]} $@ | xz_compression > ${cfname}"
880937
;;
938+
'zstd')
939+
echo "dry-running: $MYSQLDUMP ${opt[@]} $@ | zstd_compression > ${cfname}"
940+
;;
881941
*)
882942
echo "dry-running: $MYSQLDUMP ${opt[@]} $@ > ${cfname}"
883943
;;
@@ -903,6 +963,10 @@ process_dbs() {
903963
$MYSQLDUMP "${opt[@]}" "$@" | xz_compression > "${cfname}"
904964
ret=$?
905965
;;
966+
'zstd')
967+
$MYSQLDUMP "${opt[@]}" "$@" | zstd_compression > "${cfname}"
968+
ret=$?
969+
;;
906970
*)
907971
$MYSQLDUMP "${opt[@]}" "$@" > "${cfname}"
908972
ret=$?
@@ -1138,12 +1202,20 @@ check_dependencies () {
11381202
CONFIG_multicore='no' # turn off multicore support, since the program isn't there
11391203
echo "WARNING: Turning off multicore support, since xz isn't there."
11401204
fi
1205+
elif [[ "x$CONFIG_mysql_dump_compression" = 'xzstd' ]]; then
1206+
if type zstd &>/dev/null; then
1207+
echo "zstd ... found."
1208+
else
1209+
CONFIG_multicore='no' # turn off multicore support, since the program isn't there
1210+
echo "WARNING: Turning off multicore support, since zstd isn't there."
1211+
fi
11411212
fi
11421213

11431214
else
11441215
[[ "x$CONFIG_mysql_dump_compression" = 'xbzip2' ]] && dependencies=("${dependencies[@]}" 'bzip2' )
11451216
[[ "x$CONFIG_mysql_dump_compression" = 'xgzip' ]] && dependencies=("${dependencies[@]}" 'gzip' )
11461217
[[ "x$CONFIG_mysql_dump_compression" = 'xxz' ]] && dependencies=("${dependencies[@]}" 'xz' )
1218+
[[ "x$CONFIG_mysql_dump_compression" = 'xzstd' ]] && dependencies=("${dependencies[@]}" 'zstd' )
11471219
fi
11481220

11491221
if [[ "x$CONFIG_mailcontent" = 'xlog' || "x$CONFIG_mailcontent" = 'xquiet' ]]; then
@@ -2137,6 +2209,8 @@ method_list_manifest_entries () {
21372209
let "filename_flag_gz=0x02"
21382210
let "filename_flag_bz2=0x04"
21392211
let "filename_flag_diff=0x08"
2212+
let "filename_flag_xz=0x10"
2213+
let "filename_flag_zst=0x20"
21402214

21412215
##############################################################################################################
21422216
# BEGIN __STARTUP
@@ -2235,6 +2309,7 @@ method_list_manifest_entries () {
22352309
re=".*\.gz.*"; [[ "$FileExt" =~ $re ]] && let "master_flags|=$filename_flag_gz"
22362310
re=".*\.bz2.*"; [[ "$FileExt" =~ $re ]] && let "master_flags|=$filename_flag_bz2"
22372311
re=".*\.xz.*"; [[ "$FileExt" =~ $re ]] && let "master_flags|=$filename_flag_xz"
2312+
re=".*\.zst.*"; [[ "$FileExt" =~ $re ]] && let "master_flags|=$filename_flag_zst"
22382313
re=".*\.diff.*"; [[ "$FileExt" =~ $re ]] && let "master_flags|=$filename_flag_diff"
22392314
if (( $master_flags & $filename_flag_gz )); then
22402315
declare -a testarray=("${to_rm[@]}")
@@ -2257,6 +2332,13 @@ method_list_manifest_entries () {
22572332
to_rm=("${to_rm[@]}" "${master%.xz}")
22582333
fi
22592334
master="${master%.xz}"
2335+
elif (( $master_flags & $filename_flag_zst )); then
2336+
declare -a testarray=("${to_rm[@]}")
2337+
if ! in_array "${master%.zst}"; then
2338+
zstd_compression -dc "$master" > "${master%.zst}"
2339+
to_rm=("${to_rm[@]}" "${master%.zst}")
2340+
fi
2341+
master="${master%.zst}"
22602342
else
22612343
:
22622344
fi
@@ -2298,13 +2380,15 @@ method_diff_to_full() {
22982380
re=".*\.gz.*"; [[ "$FileExt" =~ $re ]] && let "diff_flags|=$filename_flag_gz"
22992381
re=".*\.bz2.*"; [[ "$FileExt" =~ $re ]] && let "diff_flags|=$filename_flag_bz2"
23002382
re=".*\.xz.*"; [[ "$FileExt" =~ $re ]] && let "diff_flags|=$filename_flag_xz"
2383+
re=".*\.zst.*"; [[ "$FileExt" =~ $re ]] && let "diff_flags|=$filename_flag_zst"
23012384
re=".*\.diff.*"; [[ "$FileExt" =~ $re ]] && let "diff_flags|=$filename_flag_diff"
23022385
FileStub="${master%.@(sql|master)*}"
23032386
FileExt="${master#"$FileStub"}"
23042387
re=".*\.enc.*"; [[ "$FileExt" =~ $re ]] && let "master_flags|=$filename_flag_encrypted"
23052388
re=".*\.gz.*"; [[ "$FileExt" =~ $re ]] && let "master_flags|=$filename_flag_gz"
23062389
re=".*\.bz2.*"; [[ "$FileExt" =~ $re ]] && let "master_flags|=$filename_flag_bz2"
23072390
re=".*\.xz.*"; [[ "$FileExt" =~ $re ]] && let "master_flags|=$filename_flag_xz"
2391+
re=".*\.zst.*"; [[ "$FileExt" =~ $re ]] && let "master_flags|=$filename_flag_zst"
23082392
re=".*\.diff.*"; [[ "$FileExt" =~ $re ]] && let "master_flags|=$filename_flag_diff"
23092393

23102394
# TODO: Differential backup with encryption is not yet implemented!
@@ -2328,6 +2412,10 @@ method_diff_to_full() {
23282412
xz_compression -dc "$diff" > "${diff%.xz}"
23292413
to_rm=("${to_rm[@]}" "${diff%.xz}")
23302414
diff="${diff%.xz}"
2415+
elif (( $diff_flags & $filename_flag_zst )); then
2416+
zstd_compression -dc "$diff" > "${diff%.zst}"
2417+
to_rm=("${to_rm[@]}" "${diff%.zst}")
2418+
diff="${diff%.zst}"
23312419
else
23322420
:
23332421
fi
@@ -2340,10 +2428,14 @@ method_diff_to_full() {
23402428
bzip2_compression -dc "$master" > "${master%.bz2}"
23412429
to_rm=("${to_rm[@]}" "${master%.bz2}")
23422430
master="${master%.bz2}"
2343-
elif (( $master_flags & $filename_flag_bz2 )); then
2431+
elif (( $master_flags & $filename_flag_xz )); then
23442432
xz_compression -dc "$master" > "${master%.xz}"
23452433
to_rm=("${to_rm[@]}" "${master%.xz}")
23462434
master="${master%.xz}"
2435+
elif (( $master_flags & $filename_flag_zst )); then
2436+
zstd_compression -dc "$master" > "${master%.zst}"
2437+
to_rm=("${to_rm[@]}" "${master%.zst}")
2438+
master="${master%.zst}"
23472439
else
23482440
:
23492441
fi

0 commit comments

Comments
 (0)