commit e6a9c973dbb7be6e46ed9a7fe34df0635635fed6 Author: Johannes Meixner Date: Tue Jul 12 13:59:28 2022 +0200 Merge pull request #2831 from pcahyna/rsync-url-fix-refactor Refactor rsync URL support, fixes rsync OUTPUT_URL: The code to parse rsync:// URLs was BACKUP_URL specific. If one specified BACKUP=RSYNC and an OUTPUT_URL different from BACKUP_URL, the OUTPUT_URL was ignored and the output files went to BACKUP_URL. Fix by introducing generic functions for rsync URL parsing and use them for both BACKUP_URL and OUTPUT_URL, as appropriate. Replace all uses of global RSYNC_* variables derived from BACKUP_URL by those functions. There also was inconsistent special handling for OUTPUT=PXE which is now removed: An rsync OUTPUT_URL with OUTPUT=PXE now creates the RSYNC_PREFIX directory at the destination and the URL is interpreted as in all other cases. See https://github.com/rear/rear/pull/2831 and https://github.com/rear/rear/issues/2781 diff --git a/usr/share/rear/backup/NETFS/default/200_check_rsync_relative_option.sh b/usr/share/rear/backup/NETFS/default/200_check_rsync_relative_option.sh deleted file mode 120000 index 336b83f5..00000000 --- a/usr/share/rear/backup/NETFS/default/200_check_rsync_relative_option.sh +++ /dev/null @@ -1 +0,0 @@ -../../RSYNC/default/200_check_rsync_relative_option.sh \ No newline at end of file diff --git a/usr/share/rear/backup/NETFS/default/210_check_rsync_relative_option.sh b/usr/share/rear/backup/NETFS/default/210_check_rsync_relative_option.sh new file mode 120000 index 00000000..0570eb44 --- /dev/null +++ b/usr/share/rear/backup/NETFS/default/210_check_rsync_relative_option.sh @@ -0,0 +1 @@ +../../RSYNC/default/210_check_rsync_relative_option.sh \ No newline at end of file diff --git a/usr/share/rear/backup/RSYNC/GNU/Linux/610_start_selinux.sh b/usr/share/rear/backup/RSYNC/GNU/Linux/610_start_selinux.sh index 1692ba4c..dd198ede 100644 --- a/usr/share/rear/backup/RSYNC/GNU/Linux/610_start_selinux.sh +++ b/usr/share/rear/backup/RSYNC/GNU/Linux/610_start_selinux.sh @@ -6,29 +6,29 @@ local backup_prog_rc touch "${TMP_DIR}/selinux.autorelabel" cat $TMP_DIR/selinux.mode > $SELINUX_ENFORCE Log "Restored original SELinux mode" - case $RSYNC_PROTO in + case $(rsync_proto "$BACKUP_URL") in (ssh) # for some reason rsync changes the mode of backup after each run to 666 # FIXME: Add an explanatory comment why "2>/dev/null" is useful here # or remove it according to https://github.com/rear/rear/issues/1395 - ssh $RSYNC_USER@$RSYNC_HOST "chmod $v 755 ${RSYNC_PATH}/${RSYNC_PREFIX}/backup" 2>/dev/null + ssh $(rsync_remote_ssh "$BACKUP_URL") "chmod $v 755 $(rsync_path_full "$BACKUP_URL")/backup" 2>/dev/null $BACKUP_PROG -a "${TMP_DIR}/selinux.autorelabel" \ - "$RSYNC_USER@$RSYNC_HOST:${RSYNC_PATH}/${RSYNC_PREFIX}/backup/.autorelabel" 2>/dev/null + "$(rsync_remote_full "$BACKUP_URL")/backup/.autorelabel" 2>/dev/null backup_prog_rc=$? if [ $backup_prog_rc -ne 0 ]; then - LogPrint "Failed to create .autorelabel on ${RSYNC_PATH}/${RSYNC_PREFIX}/backup [${rsync_err_msg[$backup_prog_rc]}]" - #StopIfError "Failed to create .autorelabel on ${RSYNC_PATH}/${RSYNC_PREFIX}/backup" + LogPrint "Failed to create .autorelabel on $(rsync_path_full "$BACKUP_URL")/backup [${rsync_err_msg[$backup_prog_rc]}]" + #StopIfError "Failed to create .autorelabel on $(rsync_path_full "$BACKUP_URL")/backup" fi ;; (rsync) $BACKUP_PROG -a "${TMP_DIR}/selinux.autorelabel" "${BACKUP_RSYNC_OPTIONS[@]}" \ - "${RSYNC_PROTO}://${RSYNC_USER}@${RSYNC_HOST}:${RSYNC_PORT}/${RSYNC_PATH}/${RSYNC_PREFIX}/backup/.autorelabel" + "$(rsync_remote_full "$BACKUP_URL")/backup/.autorelabel" backup_prog_rc=$? if [ $backup_prog_rc -ne 0 ]; then - LogPrint "Failed to create .autorelabel on ${RSYNC_PATH}/${RSYNC_PREFIX}/backup [${rsync_err_msg[$backup_prog_rc]}]" - #StopIfError "Failed to create .autorelabel on ${RSYNC_PATH}/${RSYNC_PREFIX}/backup" + LogPrint "Failed to create .autorelabel on $(rsync_path_full "$BACKUP_URL")/backup [${rsync_err_msg[$backup_prog_rc]}]" + #StopIfError "Failed to create .autorelabel on $(rsync_path_full "$BACKUP_URL")/backup" fi ;; diff --git a/usr/share/rear/backup/RSYNC/GNU/Linux/620_force_autorelabel.sh b/usr/share/rear/backup/RSYNC/GNU/Linux/620_force_autorelabel.sh index 9a17d6bb..de57d571 100644 --- a/usr/share/rear/backup/RSYNC/GNU/Linux/620_force_autorelabel.sh +++ b/usr/share/rear/backup/RSYNC/GNU/Linux/620_force_autorelabel.sh @@ -4,29 +4,29 @@ local backup_prog_rc > "${TMP_DIR}/selinux.autorelabel" - case $RSYNC_PROTO in + case $(rsync_proto "$BACKUP_URL") in (ssh) # for some reason rsync changes the mode of backup after each run to 666 # FIXME: Add an explanatory comment why "2>/dev/null" is useful here # or remove it according to https://github.com/rear/rear/issues/1395 - ssh $RSYNC_USER@$RSYNC_HOST "chmod $v 755 ${RSYNC_PATH}/${RSYNC_PREFIX}/backup" 2>/dev/null + ssh $(rsync_remote_ssh "$BACKUP_URL") "chmod $v 755 $(rsync_path_full "$BACKUP_URL")/backup" 2>/dev/null $BACKUP_PROG -a "${TMP_DIR}/selinux.autorelabel" \ - "$RSYNC_USER@$RSYNC_HOST:${RSYNC_PATH}/${RSYNC_PREFIX}/backup/.autorelabel" 2>/dev/null + "$(rsync_remote_full "$BACKUP_URL")/backup/.autorelabel" 2>/dev/null backup_prog_rc=$? if [ $backup_prog_rc -ne 0 ]; then - LogPrint "Failed to create .autorelabel on ${RSYNC_PATH}/${RSYNC_PREFIX}/backup [${rsync_err_msg[$backup_prog_rc]}]" - #StopIfError "Failed to create .autorelabel on ${RSYNC_PATH}/${RSYNC_PREFIX}/backup" + LogPrint "Failed to create .autorelabel on $(rsync_path_full "$BACKUP_URL")/backup [${rsync_err_msg[$backup_prog_rc]}]" + #StopIfError "Failed to create .autorelabel on $(rsync_path_full "$BACKUP_URL")/backup" fi ;; (rsync) $BACKUP_PROG -a "${TMP_DIR}/selinux.autorelabel" "${BACKUP_RSYNC_OPTIONS[@]}" \ - "${RSYNC_PROTO}://${RSYNC_USER}@${RSYNC_HOST}:${RSYNC_PORT}/${RSYNC_PATH}/${RSYNC_PREFIX}/backup/.autorelabel" + "$(rsync_remote_full "$BACKUP_URL")/backup/.autorelabel" backup_prog_rc=$? if [ $backup_prog_rc -ne 0 ]; then - LogPrint "Failed to create .autorelabel on ${RSYNC_PATH}/${RSYNC_PREFIX}/backup [${rsync_err_msg[$backup_prog_rc]}]" - #StopIfError "Failed to create .autorelabel on ${RSYNC_PATH}/${RSYNC_PREFIX}/backup" + LogPrint "Failed to create .autorelabel on $(rsync_path_full "$BACKUP_URL")/backup [${rsync_err_msg[$backup_prog_rc]}]" + #StopIfError "Failed to create .autorelabel on $(rsync_path_full "$BACKUP_URL")/backup" fi ;; diff --git a/usr/share/rear/backup/RSYNC/default/200_make_prefix_dir.sh b/usr/share/rear/backup/RSYNC/default/200_make_prefix_dir.sh new file mode 100644 index 00000000..81aa6879 --- /dev/null +++ b/usr/share/rear/backup/RSYNC/default/200_make_prefix_dir.sh @@ -0,0 +1,28 @@ +# Create RSYNC_PREFIX/backup on remote rsync server +# RSYNC_PREFIX=$HOSTNAME as set in default.conf + +local proto host + +proto="$(rsync_proto "$BACKUP_URL")" +host="$(rsync_host "$BACKUP_URL")" + +mkdir -p $v -m0750 "${TMP_DIR}/rsync/${RSYNC_PREFIX}" >&2 || Error "Could not mkdir '${TMP_DIR}/rsync/${RSYNC_PREFIX}'" +mkdir -p $v -m0755 "${TMP_DIR}/rsync/${RSYNC_PREFIX}/backup" >&2 || Error "Could not mkdir '${TMP_DIR}/rsync/${RSYNC_PREFIX}/backup'" + +case $proto in + + (ssh) + $BACKUP_PROG -a $v -r "${TMP_DIR}/rsync/${RSYNC_PREFIX}" "$(rsync_remote "$BACKUP_URL")" >/dev/null 2>&1 \ + || Error "Could not create '$(rsync_path_full "$BACKUP_URL")' on remote ${host}" + ;; + + (rsync) + $BACKUP_PROG -a $v -r "${TMP_DIR}/rsync/${RSYNC_PREFIX}" "${BACKUP_RSYNC_OPTIONS[@]}" "$(rsync_remote "$BACKUP_URL")/" >/dev/null \ + || Error "Could not create '$(rsync_path_full "$BACKUP_URL")' on remote ${host}" + ;; + +esac + +# We don't need it anymore, from now we operate on the remote copy +rmdir $v "${TMP_DIR}/rsync/${RSYNC_PREFIX}/backup" +rmdir $v "${TMP_DIR}/rsync/${RSYNC_PREFIX}" diff --git a/usr/share/rear/backup/RSYNC/default/200_check_rsync_relative_option.sh b/usr/share/rear/backup/RSYNC/default/210_check_rsync_relative_option.sh similarity index 91% rename from usr/share/rear/backup/RSYNC/default/200_check_rsync_relative_option.sh rename to usr/share/rear/backup/RSYNC/default/210_check_rsync_relative_option.sh index cedee9ce..692616b7 100644 --- a/usr/share/rear/backup/RSYNC/default/200_check_rsync_relative_option.sh +++ b/usr/share/rear/backup/RSYNC/default/210_check_rsync_relative_option.sh @@ -1,4 +1,4 @@ -# 200_check_rsync_relative_option.sh +# 210_check_rsync_relative_option.sh # See issue #871 for details # check for the --relative option in BACKUP_RSYNC_OPTIONS array diff --git a/usr/share/rear/backup/RSYNC/default/450_calculate_req_space.sh b/usr/share/rear/backup/RSYNC/default/450_calculate_req_space.sh index eb99dbf6..037e49c0 100644 --- a/usr/share/rear/backup/RSYNC/default/450_calculate_req_space.sh +++ b/usr/share/rear/backup/RSYNC/default/450_calculate_req_space.sh @@ -1,6 +1,12 @@ # here we will calculate the space required to hold the backup archive on the remote rsync system # This file is part of Relax-and-Recover, licensed under the GNU General # Public License. Refer to the included COPYING for full text of license. +local proto host path + +proto="$(rsync_proto "$BACKUP_URL")" +host="$(rsync_host "$BACKUP_URL")" +path="$(rsync_path "$BACKUP_URL")" + _local_size=0 _remote_size=0 while read -r ; do @@ -13,17 +19,17 @@ while read -r ; do done < $TMP_DIR/backup-include.txt LogPrint "Estimated size of local file systems is $(( _local_size / 1024 )) MB" -case $RSYNC_PROTO in +case $proto in (ssh) - LogPrint "Calculating size of $RSYNC_HOST:$RSYNC_PATH" - ssh -l $RSYNC_USER $RSYNC_HOST "df -P $RSYNC_PATH" >$TMP_DIR/rs_size - StopIfError "Failed to determine size of $RSYNC_PATH" + LogPrint "Calculating size of ${host}:${path}" + ssh $(rsync_remote_ssh "$BACKUP_URL") "df -P ${path}" >$TMP_DIR/rs_size + StopIfError "Failed to determine size of ${path}" _div=1 # 1024-blocks grep -q "512-blocks" $TMP_DIR/rs_size && _div=2 # HPUX: divide with 2 to get kB size _remote_size=$( tail -n 1 $TMP_DIR/rs_size | awk '{print $2}' ) _remote_size=$(( _remote_size / _div )) [[ $_remote_size -gt $_local_size ]] - StopIfError "Not enough disk space available on $RSYNC_HOST:$RSYNC_PATH ($_remote_size < $_local_size)" + StopIfError "Not enough disk space available on ${host}:${path} ($_remote_size < $_local_size)" ;; (rsync) # TODO: how can we calculate the free size on remote system via rsync protocol?? diff --git a/usr/share/rear/backup/RSYNC/default/500_make_rsync_backup.sh b/usr/share/rear/backup/RSYNC/default/500_make_rsync_backup.sh index 750a04ca..aa8192c0 100644 --- a/usr/share/rear/backup/RSYNC/default/500_make_rsync_backup.sh +++ b/usr/share/rear/backup/RSYNC/default/500_make_rsync_backup.sh @@ -5,6 +5,11 @@ local backup_prog_rc local backup_log_message +local host path + +host="$(rsync_host "$BACKUP_URL")" +path="$(rsync_path "$BACKUP_URL")" + Log "Include list:" while read -r ; do Log " $REPLY" @@ -14,26 +19,27 @@ while read -r ; do Log " $REPLY" done < $TMP_DIR/backup-exclude.txt -LogPrint "Creating $BACKUP_PROG backup on '${RSYNC_HOST}:${RSYNC_PATH}'" +LogPrint "Creating $BACKUP_PROG backup on '${host}:${path}'" ProgressStart "Running backup operation" ( case "$(basename $BACKUP_PROG)" in (rsync) + # We are in a subshell, so this change will not propagate to later scripts BACKUP_RSYNC_OPTIONS+=( --one-file-system --delete --exclude-from=$TMP_DIR/backup-exclude.txt --delete-excluded ) - case $RSYNC_PROTO in + case $(rsync_proto "$BACKUP_URL") in (ssh) - Log $BACKUP_PROG "${BACKUP_RSYNC_OPTIONS[@]}" $(cat $TMP_DIR/backup-include.txt) "${RSYNC_USER}@${RSYNC_HOST}:${RSYNC_PATH}/${RSYNC_PREFIX}/backup" + Log $BACKUP_PROG "${BACKUP_RSYNC_OPTIONS[@]}" $(cat $TMP_DIR/backup-include.txt) "$(rsync_remote_full "$BACKUP_URL")/backup" $BACKUP_PROG "${BACKUP_RSYNC_OPTIONS[@]}" $(cat $TMP_DIR/backup-include.txt) \ - "${RSYNC_USER}@${RSYNC_HOST}:${RSYNC_PATH}/${RSYNC_PREFIX}/backup" + "$(rsync_remote_full "$BACKUP_URL")/backup" ;; (rsync) $BACKUP_PROG "${BACKUP_RSYNC_OPTIONS[@]}" $(cat $TMP_DIR/backup-include.txt) \ - "${RSYNC_PROTO}://${RSYNC_USER}@${RSYNC_HOST}:${RSYNC_PORT}/${RSYNC_PATH}/${RSYNC_PREFIX}/backup" + "$(rsync_remote_full "$BACKUP_URL")/backup" ;; esac @@ -57,11 +63,11 @@ get_size() { } check_remote_df() { - echo $(ssh ${RSYNC_USER}@${RSYNC_HOST} df -P ${RSYNC_PATH} 2>/dev/null | tail -1 | awk '{print $5}' | sed -e 's/%//') + echo $(ssh $(rsync_remote_ssh "$BACKUP_URL") df -P ${path} 2>/dev/null | tail -1 | awk '{print $5}' | sed -e 's/%//') } check_remote_du() { - x=$(ssh ${RSYNC_USER}@${RSYNC_HOST} du -sb ${RSYNC_PATH}/${RSYNC_PREFIX}/backup 2>/dev/null | awk '{print $1}') + x=$(ssh $(rsync_remote_ssh "$BACKUP_URL") du -sb $(rsync_path_full "$BACKUP_URL")/backup 2>/dev/null | awk '{print $1}') [[ -z "${x}" ]] && x=0 echo $x } @@ -81,7 +87,7 @@ case "$(basename $BACKUP_PROG)" in case $i in 300) - [[ $(check_remote_df) -eq 100 ]] && Error "Disk is full on system ${RSYNC_HOST}" + [[ $(check_remote_df) -eq 100 ]] && Error "Disk is full on system ${host}" ;; 15|30|45|60|75|90|105|120|135|150|165|180|195|210|225|240|255|270|285) diff --git a/usr/share/rear/backup/RSYNC/default/700_copy_backup_log.sh b/usr/share/rear/backup/RSYNC/default/700_copy_backup_log.sh index b90d459b..76b9f971 100644 --- a/usr/share/rear/backup/RSYNC/default/700_copy_backup_log.sh +++ b/usr/share/rear/backup/RSYNC/default/700_copy_backup_log.sh @@ -1,26 +1,27 @@ # copy the backup.log & rear.log file to remote destination with timestamp added -local timestamp +local timestamp proto timestamp=$( date +%Y%m%d.%H%M ) +proto="$(rsync_proto "$BACKUP_URL")" # compress the log file first gzip "$TMP_DIR/$BACKUP_PROG_ARCHIVE.log" || Error "Failed to 'gzip $TMP_DIR/$BACKUP_PROG_ARCHIVE.log'" -case $RSYNC_PROTO in +case $proto in (ssh) # FIXME: Add an explanatory comment why "2>/dev/null" is useful here # or remove it according to https://github.com/rear/rear/issues/1395 $BACKUP_PROG -a "${TMP_DIR}/${BACKUP_PROG_ARCHIVE}.log.gz" \ - "${RSYNC_USER}@${RSYNC_HOST}:${RSYNC_PATH}/${RSYNC_PREFIX}/${BACKUP_PROG_ARCHIVE}-${timestamp}.log.gz" 2>/dev/null + "$(rsync_remote_full "$BACKUP_URL")/${BACKUP_PROG_ARCHIVE}-${timestamp}.log.gz" 2>/dev/null - $BACKUP_PROG -a "$RUNTIME_LOGFILE" "${RSYNC_USER}@${RSYNC_HOST}:${RSYNC_PATH}/${RSYNC_PREFIX}/rear-${timestamp}.log" 2>/dev/null + $BACKUP_PROG -a "$RUNTIME_LOGFILE" "$(rsync_remote_full "$BACKUP_URL")/rear-${timestamp}.log" 2>/dev/null ;; (rsync) $BACKUP_PROG -a "${TMP_DIR}/${BACKUP_PROG_ARCHIVE}.log.gz" "${BACKUP_RSYNC_OPTIONS[@]}" \ - "${RSYNC_PROTO}://${RSYNC_USER}@${RSYNC_HOST}:${RSYNC_PORT}/${RSYNC_PATH}/${RSYNC_PREFIX}/${BACKUP_PROG_ARCHIVE}-${timestamp}.log.gz" + "$(rsync_remote_full "$BACKUP_URL")/${BACKUP_PROG_ARCHIVE}-${timestamp}.log.gz" - $BACKUP_PROG -a "$RUNTIME_LOGFILE" "${BACKUP_RSYNC_OPTIONS[@]}" "${RSYNC_PROTO}://${RSYNC_USER}@${RSYNC_HOST}:${RSYNC_PORT}/${RSYNC_PATH}/${RSYNC_PREFIX}//rear-${timestamp}.log" + $BACKUP_PROG -a "$RUNTIME_LOGFILE" "${BACKUP_RSYNC_OPTIONS[@]}" "$(rsync_remote_full "$BACKUP_URL")//rear-${timestamp}.log" ;; esac diff --git a/usr/share/rear/lib/global-functions.sh b/usr/share/rear/lib/global-functions.sh index 32aeb8ca..2edb64a6 100644 --- a/usr/share/rear/lib/global-functions.sh +++ b/usr/share/rear/lib/global-functions.sh @@ -259,7 +259,7 @@ function url_scheme() { # the scheme is the leading part up to '://' local scheme=${url%%://*} # rsync scheme does not have to start with rsync:// it can also be scp style - # see the comments in usr/share/rear/prep/RSYNC/default/100_check_rsync.sh + # see the comments in usr/share/rear/lib/rsync-functions.sh echo $scheme | grep -q ":" && echo rsync || echo $scheme } diff --git a/usr/share/rear/lib/rsync-functions.sh b/usr/share/rear/lib/rsync-functions.sh new file mode 100644 index 00000000..443a9625 --- /dev/null +++ b/usr/share/rear/lib/rsync-functions.sh @@ -0,0 +1,178 @@ +# Functions for manipulation of rsync URLs (both OUTPUT_URL and BACKUP_URL) + +#### OLD STYLE: +# BACKUP_URL=[USER@]HOST:PATH # using ssh (no rsh) +# +# with rsync protocol PATH is a MODULE name defined in remote /etc/rsyncd.conf file +# BACKUP_URL=[USER@]HOST::PATH # using rsync +# BACKUP_URL=rsync://[USER@]HOST[:PORT]/PATH # using rsync (is not compatible with new style!!!) + +#### NEW STYLE: +# BACKUP_URL=rsync://[USER@]HOST[:PORT]/PATH # using ssh +# BACKUP_URL=rsync://[USER@]HOST[:PORT]::/PATH # using rsync + +function rsync_validate () { + local url="$1" + + if [[ "$(url_scheme "$url")" != "rsync" ]]; then # url_scheme still recognizes old style + BugError "Non-rsync URL $url !" + fi +} + +# Determine whether the URL specifies the use of the rsync protocol (rsyncd) or ssh +# Do not call on non-rsync URLs (use url_scheme first) +function rsync_proto () { + local url="$1" + + rsync_validate "$url" + if egrep -q '(::)' <<< $url ; then # new style '::' means rsync protocol + echo rsync + else + echo ssh + fi +} + +# Functions to parse the URL into its components: +# USER, HOST, PORT, PATH + +function rsync_user () { + local url="$1" + local host + + host=$(url_host "$url") + + if grep -q '@' <<< $host ; then + echo "${host%%@*}" # grab user name + else + echo root + fi +} + +function rsync_host () { + local url="$1" + local host + local path + + host=$(url_host "$url") + path=$(url_path "$url") + # remove USER@ if present + local tmp2="${host#*@}" + + case "$(rsync_proto "$url")" in + (rsync) + # tmp2=witsbebelnx02::backup or tmp2=witsbebelnx02:: + echo "${tmp2%%::*}" + ;; + (ssh) + # tmp2=host or tmp2=host: + echo "${tmp2%%:*}" + ;; + esac +} + +function rsync_path () { + local url="$1" + local host + local path + local url_without_scheme + local url_without_scheme_user + + host=$(url_host "$url") + path=$(url_path "$url") + local tmp2="${host#*@}" + + url_without_scheme="${url#*//}" + url_without_scheme_user="${url_without_scheme#$(rsync_user "$url")@}" + + case "$(rsync_proto "$url")" in + + (rsync) + if grep -q '::' <<< $url_without_scheme_user ; then + # we can not use url_path here, it uses / as separator, not :: + local url_after_separator="${url_without_scheme_user##*::}" + # remove leading / - this is a module name + echo "${url_after_separator#/}" + else + echo "${path#*/}" + fi + ;; + (ssh) + if [ "$url_without_scheme" == "$url" ]; then + # no scheme - old-style URL + if grep -q ':' <<< $url_without_scheme_user ; then + echo "${url_without_scheme_user##*:}" + else + BugError "Old-style rsync URL $url without : !" + fi + else + echo "$path" + fi + ;; + + esac +} + +function rsync_port () { + # XXX changing port not implemented yet + echo 873 +} + +# Full path to the destination directory on the remote server, +# includes RSYNC_PREFIX. RSYNC_PREFIX is not given by the URL, +# it is a global parameter (by default derived from hostname). +function rsync_path_full () { + local url="$1" + + echo "$(rsync_path "$url")/${RSYNC_PREFIX}" +} + +# Argument for the ssh command to log in to the remote host ("user@host") +function rsync_remote_ssh () { + local url="$1" + + local user host + + user="$(rsync_user "$url")" + host="$(rsync_host "$url")" + + echo "${user}@${host}" +} + +# Argument for the rsync command to reach the remote host, without path. +function rsync_remote_base () { + local url="$1" + + local user host port + + user="$(rsync_user "$url")" + host="$(rsync_host "$url")" + port="$(rsync_port "$url")" + + case "$(rsync_proto "$url")" in + + (rsync) + echo "rsync://${user}@${host}:${port}/" + ;; + (ssh) + echo "$(rsync_remote_ssh "$url"):" + ;; + + esac +} + +# Complete argument to rsync to reach the remote location identified by URL, +# but without the added RSYNC_PREFIX. +# This essentially converts our rsync:// URLs into a form accepted by the rsync command. +function rsync_remote () { + local url="$1" + + echo "$(rsync_remote_base "$url")$(rsync_path "$url")" +} + +# Complete argument to rsync including even RSYNC_PREFIX. +# Determined from the URL and RSYNC_PREFIX. +function rsync_remote_full () { + local url="$1" + + echo "$(rsync_remote_base "$url")$(rsync_path_full "$url")" +} diff --git a/usr/share/rear/output/RSYNC/default/200_make_prefix_dir.sh b/usr/share/rear/output/RSYNC/default/200_make_prefix_dir.sh index 519febf5..d00d15e4 100644 --- a/usr/share/rear/output/RSYNC/default/200_make_prefix_dir.sh +++ b/usr/share/rear/output/RSYNC/default/200_make_prefix_dir.sh @@ -1,20 +1,32 @@ # Create RSYNC_PREFIX under the local TMP_DIR and also on remote rsync server # RSYNC_PREFIX=$HOSTNAME as set in default.conf -# create temporary local work-spaces to collect files (we already make the remote backup dir with the correct mode!!) +local proto host scheme + +scheme="$(url_scheme "$OUTPUT_URL")" + +# we handle only rsync:// output schemes. +# ToDo: why does handling of the output URL scheme belong under RSYNC (which is a backup method)? +# OUTPUT_URL is independent on the chosen backup method, so this code should be moved to be backup-independent. +test "rsync" = "$scheme" || return 0 + +proto="$(rsync_proto "$OUTPUT_URL")" +host="$(rsync_host "$OUTPUT_URL")" + +# create temporary local work-spaces to collect files mkdir -p $v -m0750 "${TMP_DIR}/rsync/${RSYNC_PREFIX}" >&2 || Error "Could not mkdir '${TMP_DIR}/rsync/${RSYNC_PREFIX}'" -mkdir -p $v -m0755 "${TMP_DIR}/rsync/${RSYNC_PREFIX}/backup" >&2 || Error "Could not mkdir '${TMP_DIR}/rsync/${RSYNC_PREFIX}/backup'" -case $RSYNC_PROTO in +case $proto in (ssh) - $BACKUP_PROG -a $v -r "${TMP_DIR}/rsync/${RSYNC_PREFIX}" "${RSYNC_USER}@${RSYNC_HOST}:${RSYNC_PATH}" >/dev/null 2>&1 \ - || Error "Could not create '${RSYNC_PATH}/${RSYNC_PREFIX}' on remote ${RSYNC_HOST}" + $BACKUP_PROG -a $v -r "${TMP_DIR}/rsync/${RSYNC_PREFIX}" "$(rsync_remote "$OUTPUT_URL")" >/dev/null 2>&1 \ + || Error "Could not create '$(rsync_path_full "$OUTPUT_URL")' on remote ${host}" ;; (rsync) - $BACKUP_PROG -a $v -r "${TMP_DIR}/rsync/${RSYNC_PREFIX}" "${BACKUP_RSYNC_OPTIONS[@]}" "${RSYNC_PROTO}://${RSYNC_USER}@${RSYNC_HOST}:${RSYNC_PORT}/${RSYNC_PATH}/" >/dev/null \ - || Error "Could not create '${RSYNC_PATH}/${RSYNC_PREFIX}' on remote ${RSYNC_HOST}" + # This must run before the backup stage. Otherwise --relative gets added to BACKUP_RSYNC_OPTIONS + $BACKUP_PROG -a $v -r "${TMP_DIR}/rsync/${RSYNC_PREFIX}" "${BACKUP_RSYNC_OPTIONS[@]}" "$(rsync_remote "$OUTPUT_URL")/" >/dev/null \ + || Error "Could not create '$(rsync_path_full "$OUTPUT_URL")' on remote ${host}" ;; esac diff --git a/usr/share/rear/output/RSYNC/default/900_copy_result_files.sh b/usr/share/rear/output/RSYNC/default/900_copy_result_files.sh index 96b62da1..4ddf3cb4 100644 --- a/usr/share/rear/output/RSYNC/default/900_copy_result_files.sh +++ b/usr/share/rear/output/RSYNC/default/900_copy_result_files.sh @@ -1,6 +1,17 @@ # # copy resulting files to remote network (backup) location +local proto scheme + +scheme="$(url_scheme "$OUTPUT_URL")" + +# we handle only rsync:// output schemes. +# ToDo: why does handling of the output URL scheme belong under RSYNC (which is a backup method)? +# OUTPUT_URL is independent on the chosen backup method, so this code should be moved to be backup-independent. +test "rsync" = "$scheme" || return 0 + +proto="$(rsync_proto "$OUTPUT_URL")" + LogPrint "Copying resulting files to $OUTPUT_URL location" # if called as mkbackuponly then we just don't have any result files. @@ -19,21 +30,21 @@ cp $v $(get_template "RESULT_usage_$OUTPUT.txt") "${TMP_DIR}/rsync/${RSYNC_PREFI cat "$RUNTIME_LOGFILE" >"${TMP_DIR}/rsync/${RSYNC_PREFIX}/rear.log" \ || Error "Could not copy $RUNTIME_LOGFILE to local rsync location" -case $RSYNC_PROTO in +case $proto in (ssh) - Log "$BACKUP_PROG -a ${TMP_DIR}/rsync/${RSYNC_PREFIX}/ ${RSYNC_USER}@${RSYNC_HOST}:${RSYNC_PATH}/${RSYNC_PREFIX}/" + Log "$BACKUP_PROG -a ${TMP_DIR}/rsync/${RSYNC_PREFIX}/ $(rsync_remote_full "$OUTPUT_URL")/" # FIXME: Add an explanatory comment why "2>/dev/null" is useful here # or remove it according to https://github.com/rear/rear/issues/1395 - $BACKUP_PROG -a "${TMP_DIR}/rsync/${RSYNC_PREFIX}/" "${RSYNC_USER}@${RSYNC_HOST}:${RSYNC_PATH}/${RSYNC_PREFIX}/" 2>/dev/null \ + $BACKUP_PROG -a "${TMP_DIR}/rsync/${RSYNC_PREFIX}/" "$(rsync_remote_full "$OUTPUT_URL")/" 2>/dev/null \ || Error "Could not copy '${RESULT_FILES[*]}' to $OUTPUT_URL location" ;; (rsync) - Log "$BACKUP_PROG -a ${TMP_DIR}/rsync/${RSYNC_PREFIX}/ ${BACKUP_RSYNC_OPTIONS[*]} ${RSYNC_PROTO}://${RSYNC_USER}@${RSYNC_HOST}:${RSYNC_PORT}/${RSYNC_PATH}/${RSYNC_PREFIX}/" + Log "$BACKUP_PROG -a ${TMP_DIR}/rsync/${RSYNC_PREFIX}/ ${BACKUP_RSYNC_OPTIONS[*]} $(rsync_remote_full "$OUTPUT_URL")/" # FIXME: Add an explanatory comment why "2>/dev/null" is useful here # or remove it according to https://github.com/rear/rear/issues/1395 - $BACKUP_PROG -a "${TMP_DIR}/rsync/${RSYNC_PREFIX}/" "${BACKUP_RSYNC_OPTIONS[@]}" "${RSYNC_PROTO}://${RSYNC_USER}@${RSYNC_HOST}:${RSYNC_PORT}/${RSYNC_PATH}/${RSYNC_PREFIX}/" 2>/dev/null \ + $BACKUP_PROG -a "${TMP_DIR}/rsync/${RSYNC_PREFIX}/" "${BACKUP_RSYNC_OPTIONS[@]}" "$(rsync_remote_full "$OUTPUT_URL")/" 2>/dev/null \ || Error "Could not copy '${RESULT_FILES[*]}' to $OUTPUT_URL location" ;; diff --git a/usr/share/rear/prep/RSYNC/GNU/Linux/200_selinux_in_use.sh b/usr/share/rear/prep/RSYNC/GNU/Linux/200_selinux_in_use.sh index eb7df29e..84500039 100644 --- a/usr/share/rear/prep/RSYNC/GNU/Linux/200_selinux_in_use.sh +++ b/usr/share/rear/prep/RSYNC/GNU/Linux/200_selinux_in_use.sh @@ -25,8 +25,10 @@ case $(basename $BACKUP_PROG) in (rsync) if grep -q "no xattrs" "$TMP_DIR/rsync_protocol"; then + local host + host="$(rsync_host "$BACKUP_URL")" # no xattrs compiled in remote rsync, so saving SELinux attributes are not possible - Log "WARNING: --xattrs not possible on system ($RSYNC_HOST) (no xattrs compiled in rsync)" + Log "WARNING: --xattrs not possible on system ($host) (no xattrs compiled in rsync)" # $TMP_DIR/selinux.mode is a trigger during backup to disable SELinux cat $SELINUX_ENFORCE > $TMP_DIR/selinux.mode RSYNC_SELINUX= # internal variable used in recover mode (empty means disable SELinux) diff --git a/usr/share/rear/prep/RSYNC/default/100_check_rsync.sh b/usr/share/rear/prep/RSYNC/default/100_check_rsync.sh index c964a148..448a1b1a 100644 --- a/usr/share/rear/prep/RSYNC/default/100_check_rsync.sh +++ b/usr/share/rear/prep/RSYNC/default/100_check_rsync.sh @@ -3,97 +3,40 @@ # This file is part of Relax-and-Recover, licensed under the GNU General # Public License. Refer to the included COPYING for full text of license. -#### OLD STYLE: -# BACKUP_URL=[USER@]HOST:PATH # using ssh (no rsh) -# -# with rsync protocol PATH is a MODULE name defined in remote /etc/rsyncd.conf file -# BACKUP_URL=[USER@]HOST::PATH # using rsync -# BACKUP_URL=rsync://[USER@]HOST[:PORT]/PATH # using rsync (is not compatible with new style!!!) - -#### NEW STYLE: -# BACKUP_URL=rsync://[USER@]HOST[:PORT]/PATH # using ssh -# BACKUP_URL=rsync://[USER@]HOST[:PORT]::/PATH # using rsync - if test -z "$BACKUP_URL" ; then Error "Missing BACKUP_URL=rsync://[USER@]HOST[:PORT][::]/PATH !" fi -local host=$(url_host $BACKUP_URL) local scheme=$(url_scheme $BACKUP_URL) # url_scheme still recognizes old style -local path=$(url_path $BACKUP_URL) if [[ "$scheme" != "rsync" ]]; then Error "Missing BACKUP_URL=rsync://[USER@]HOST[:PORT][::]/PATH !" fi -RSYNC_PROTO= # ssh or rsync -RSYNC_USER= -RSYNC_HOST= -RSYNC_PORT=873 # default port (of rsync server) -RSYNC_PATH= - - -if egrep -q '(::)' <<< $BACKUP_URL ; then # new style '::' means rsync protocol - RSYNC_PROTO=rsync -else - RSYNC_PROTO=ssh -fi - -if grep -q '@' <<< $host ; then - RSYNC_USER="${host%%@*}" # grab user name -else - RSYNC_USER=root -fi - -# remove USER@ if present (we don't need it anymore) -local tmp2="${host#*@}" - -case "$RSYNC_PROTO" in - - (rsync) - # tmp2=witsbebelnx02::backup or tmp2=witsbebelnx02:: - RSYNC_HOST="${tmp2%%::*}" - # path=/gdhaese1@witsbebelnx02::backup or path=/backup - if grep -q '::' <<< $path ; then - RSYNC_PATH="${path##*::}" - else - RSYNC_PATH="${path##*/}" - fi - ;; - (ssh) - # tmp2=host or tmp2=host: - RSYNC_HOST="${tmp2%%:*}" - RSYNC_PATH=$path - ;; - -esac - -#echo RSYNC_PROTO=$RSYNC_PROTO -#echo RSYNC_USER=$RSYNC_USER -#echo RSYNC_HOST=$RSYNC_HOST -#echo RSYNC_PORT=$RSYNC_PORT -#echo RSYNC_PATH=$RSYNC_PATH +local host proto +host="$(rsync_host "$BACKUP_URL")" +proto="$(rsync_proto "$BACKUP_URL")" # check if host is reachable if test "$PING" ; then - ping -c 2 "$RSYNC_HOST" >/dev/null || Error "Backup host [$RSYNC_HOST] not reachable." + ping -c 2 "$host" >/dev/null || Error "Backup host [$host] not reachable." else Log "Skipping ping test" fi # check protocol connectivity -case "$RSYNC_PROTO" in +case "$proto" in (rsync) - Log "Test: $BACKUP_PROG ${BACKUP_RSYNC_OPTIONS[*]} ${RSYNC_PROTO}://${RSYNC_USER}@${RSYNC_HOST}:${RSYNC_PORT}/" - $BACKUP_PROG "${BACKUP_RSYNC_OPTIONS[@]}" ${RSYNC_PROTO}://${RSYNC_USER}@${RSYNC_HOST}:${RSYNC_PORT}/ >/dev/null \ - || Error "Rsync daemon not running on $RSYNC_HOST" + Log "Test: $BACKUP_PROG ${BACKUP_RSYNC_OPTIONS[*]} $(rsync_remote_base "$BACKUP_URL")" + $BACKUP_PROG "${BACKUP_RSYNC_OPTIONS[@]}" $(rsync_remote_base "$BACKUP_URL") >/dev/null \ + || Error "Rsync daemon not running on $host" ;; (ssh) - Log "Test: ssh ${RSYNC_USER}@${RSYNC_HOST} /bin/true" - ssh ${RSYNC_USER}@${RSYNC_HOST} /bin/true >/dev/null 2>&1 \ - || Error "Secure shell connection not setup properly [$RSYNC_USER@$RSYNC_HOST]" + Log "Test: ssh $(rsync_remote_ssh "$BACKUP_URL") /bin/true" + ssh $(rsync_remote_ssh "$BACKUP_URL") /bin/true >/dev/null 2>&1 \ + || Error "Secure shell connection not setup properly [$(rsync_remote_ssh "$BACKUP_URL")]" ;; esac diff --git a/usr/share/rear/prep/RSYNC/default/150_check_rsync_protocol_version.sh b/usr/share/rear/prep/RSYNC/default/150_check_rsync_protocol_version.sh index e9103531..becf35a0 100644 --- a/usr/share/rear/prep/RSYNC/default/150_check_rsync_protocol_version.sh +++ b/usr/share/rear/prep/RSYNC/default/150_check_rsync_protocol_version.sh @@ -3,15 +3,18 @@ # Public License. Refer to the included COPYING for full text of license. # try to grab the rsync protocol version of rsync on the remote server -local remote_mountpoint +local remote_mountpoint host path proto +host="$(rsync_host "$BACKUP_URL")" +path="$(rsync_path "$BACKUP_URL")" +proto="$(rsync_proto "$BACKUP_URL")" if [ -z "$RSYNC_PROTOCOL_VERSION" ]; then - case $RSYNC_PROTO in + case $proto in (ssh) - ssh ${RSYNC_USER}@${RSYNC_HOST} rsync --version >"$TMP_DIR/rsync_protocol" 2>&1 \ - || Error "Secure shell connection not setup properly [$RSYNC_USER@$RSYNC_HOST]" + ssh $(rsync_remote_ssh "$BACKUP_URL") rsync --version >"$TMP_DIR/rsync_protocol" 2>&1 \ + || Error "Secure shell connection not setup properly [$(rsync_remote_ssh "$BACKUP_URL")]" if grep -q "protocol version" "$TMP_DIR/rsync_protocol" ; then RSYNC_PROTOCOL_VERSION=$(grep 'protocol version' "$TMP_DIR/rsync_protocol" | awk '{print $6}') else @@ -24,29 +27,29 @@ if [ -z "$RSYNC_PROTOCOL_VERSION" ]; then RSYNC_PROTOCOL_VERSION=29 # being conservative (old rsync) ;; esac - Log "Remote rsync system ($RSYNC_HOST) uses rsync protocol version $RSYNC_PROTOCOL_VERSION" + Log "Remote rsync system ($host) uses rsync protocol version $RSYNC_PROTOCOL_VERSION" else - Log "Remote rsync system ($RSYNC_HOST) uses rsync protocol version $RSYNC_PROTOCOL_VERSION (overruled by user)" + Log "Remote rsync system ($host) uses rsync protocol version $RSYNC_PROTOCOL_VERSION (overruled by user)" fi -if [ "${RSYNC_USER}" != "root" -a $RSYNC_PROTO = "ssh" ]; then +if [ "$(rsync_user "$BACKUP_URL")" != "root" -a $proto = "ssh" ]; then if [ $RSYNC_PROTOCOL_VERSION -gt 29 ]; then if grep -q "no xattrs" "$TMP_DIR/rsync_protocol"; then # no xattrs available in remote rsync, so --fake-super is not possible - Error "rsync --fake-super not possible on system ($RSYNC_HOST) (no xattrs compiled in rsync)" + Error "rsync --fake-super not possible on system ($host) (no xattrs compiled in rsync)" else # when using --fake-super we must have user_xattr mount options on the remote mntpt - remote_mountpoint=$(ssh ${RSYNC_USER}@${RSYNC_HOST} 'cd ${RSYNC_PATH}; df -P .' 2>/dev/null | tail -1 | awk '{print $6}') - ssh ${RSYNC_USER}@${RSYNC_HOST} "cd ${RSYNC_PATH} && touch .is_xattr_supported && setfattr -n user.comment -v 'File created by ReaR to test if this filesystems supports extended attributes.' .is_xattr_supported && getfattr -n user.comment .is_xattr_supported 1>/dev/null; find .is_xattr_supported -empty -delete" \ + remote_mountpoint=$(ssh $(rsync_remote_ssh "$BACKUP_URL") 'cd ${path}; df -P .' 2>/dev/null | tail -1 | awk '{print $6}') + ssh $(rsync_remote_ssh "$BACKUP_URL") "cd ${path} && touch .is_xattr_supported && setfattr -n user.comment -v 'File created by ReaR to test if this filesystems supports extended attributes.' .is_xattr_supported && getfattr -n user.comment .is_xattr_supported 1>/dev/null; find .is_xattr_supported -empty -delete" \ || Error "Remote file system $remote_mountpoint does not have user_xattr mount option set!" #BACKUP_RSYNC_OPTIONS+=( --xattrs --rsync-path="rsync --fake-super" ) # see issue #366 for explanation of removing --xattrs BACKUP_RSYNC_OPTIONS+=( --rsync-path="rsync --fake-super" ) fi else - Error "rsync --fake-super not possible on system ($RSYNC_HOST) (please upgrade rsync to 3.x)" + Error "rsync --fake-super not possible on system ($host) (please upgrade rsync to 3.x)" fi fi diff --git a/usr/share/rear/restore/RSYNC/default/400_restore_rsync_backup.sh b/usr/share/rear/restore/RSYNC/default/400_restore_rsync_backup.sh index 993088be..0fa08587 100644 --- a/usr/share/rear/restore/RSYNC/default/400_restore_rsync_backup.sh +++ b/usr/share/rear/restore/RSYNC/default/400_restore_rsync_backup.sh @@ -7,7 +7,11 @@ get_size() { local backup_prog_rc local restore_log_message -LogPrint "Restoring $BACKUP_PROG backup from '${RSYNC_HOST}:${RSYNC_PATH}'" +local host path +host="$(rsync_host "$BACKUP_URL")" +path="$(rsync_path "$BACKUP_URL")" + +LogPrint "Restoring $BACKUP_PROG backup from '${host}:${path}'" ProgressStart "Restore operation" ( @@ -15,18 +19,18 @@ ProgressStart "Restore operation" (rsync) - case $RSYNC_PROTO in + case $(rsync_proto "$BACKUP_URL") in (ssh) - Log $BACKUP_PROG "${BACKUP_RSYNC_OPTIONS[@]}" "${RSYNC_USER}@${RSYNC_HOST}:${RSYNC_PATH}/${RSYNC_PREFIX}/backup"/ $TARGET_FS_ROOT/ + Log $BACKUP_PROG "${BACKUP_RSYNC_OPTIONS[@]}" "$(rsync_remote_full "$BACKUP_URL")/backup"/ $TARGET_FS_ROOT/ $BACKUP_PROG "${BACKUP_RSYNC_OPTIONS[@]}" \ - "${RSYNC_USER}@${RSYNC_HOST}:${RSYNC_PATH}/${RSYNC_PREFIX}/backup"/ \ + "$(rsync_remote_full "$BACKUP_URL")/backup"/ \ $TARGET_FS_ROOT/ ;; (rsync) $BACKUP_PROG "${BACKUP_RSYNC_OPTIONS[@]}" \ - "${RSYNC_PROTO}://${RSYNC_USER}@${RSYNC_HOST}:${RSYNC_PORT}/${RSYNC_PATH}/${RSYNC_PREFIX}/backup"/ $TARGET_FS_ROOT/ + "$(rsync_remote_full "$BACKUP_URL")/backup"/ $TARGET_FS_ROOT/ ;; esac diff --git a/usr/share/rear/verify/RSYNC/default/550_check_remote_backup_archive.sh b/usr/share/rear/verify/RSYNC/default/550_check_remote_backup_archive.sh index b2fb72f5..76132794 100644 --- a/usr/share/rear/verify/RSYNC/default/550_check_remote_backup_archive.sh +++ b/usr/share/rear/verify/RSYNC/default/550_check_remote_backup_archive.sh @@ -1,14 +1,14 @@ # check the backup archive on remote rsync server -case $RSYNC_PROTO in +case $(rsync_proto "$BACKUP_URL") in (ssh) - ssh ${RSYNC_USER}@${RSYNC_HOST} "ls -ld ${RSYNC_PATH}/${RSYNC_PREFIX}/backup" >/dev/null 2>&1 \ - || Error "Archive not found on [$RSYNC_USER@$RSYNC_HOST:${RSYNC_PATH}/${RSYNC_PREFIX}]" + ssh $(rsync_remote_ssh "$BACKUP_URL") "ls -ld $(rsync_path_full "$BACKUP_URL")/backup" >/dev/null 2>&1 \ + || Error "Archive not found on [$(rsync_remote_full "$BACKUP_URL")]" ;; (rsync) - $BACKUP_PROG "${RSYNC_PROTO}://${RSYNC_USER}@${RSYNC_HOST}:${RSYNC_PORT}/${RSYNC_PATH}/${RSYNC_PREFIX}/backup" >/dev/null 2>&1 \ - || Error "Archive not found on [$RSYNC_USER@$RSYNC_HOST:${RSYNC_PATH}/${RSYNC_PREFIX}]" + $BACKUP_PROG "$(rsync_remote_full "$BACKUP_URL")/backup" >/dev/null 2>&1 \ + || Error "Archive not found on [$(rsync_remote_full "$BACKUP_URL")]" ;; esac