The expected behavior of the script is to download a file from a URL if the file is not present in the current directory and compare the md5 checksum of the file against the md5 checksum from the server. If the file is present then perform the verification again and download the file if the verification fails. As I have a poor internet connection I attempt the download and verification a second time if the initial verification fails. If verification fails on the second attempt the script exits and logs a fatal error.
The script is called with two inputs. The first input is a file that contains the full URL with filename to download. One file per line. The second input is a file that contains the expected md5checksum and the filename. Expected format of checksum file list "checksum" "filename" without the quotes.
Currently the script works as expected. As I'm in the process of teaching myself bash/sh scripting. Is there a cleaner way I can implement the expected behavior. I am aware of the continue option for wget. I choose not to use it as it felt "too easy" and not in the spirit of learning sh scripting.
The two sourced files can be found at my forks of the original repos https://github.com/CJ-Systems/log4sh and https://github.com/CJ-Systems/shlib/blob/master/functions/shlib_ansi
#!/bin/sh
#
# load log4sh
if [ -r ./lib/log4sh ]; then
. ./lib/log4sh
else
echo "ERROR: could not load (log4sh)" >&2
exit 1
fi
# load shlib_ansi
if [ -r ./lib/shlib_ansi ]; then
. ./lib/shlib_ansi
else
echo "ERROR: could not load (shlib_ansi)" >&2
exit 1
fi
# Download single file
downloadfile() {
local fn=$1
log DEBUG "${shlib_ansi_red}Downloading $bn ${shlib_ansi_none}"
wget --quiet --show-progress --no-use-server-timestamps $fn -O $bn
}
# Compare md5 checksums
verifyfile() {
local check_sum=$1
exp_md5=$(cat $check_sum | grep -i $bn | cut -d' ' -f1)
md5_local=$(md5sum $bn | cut -d' ' -f1)
if [ $md5_local = $exp_md5 ]
then
log DEBUG "Checksum valid for $bn"
return 0
else
log DEBUG "Checksum invalid for $bn"
return 1
fi
}
# Download all files in file_list
download() {
local file_list=$1
local check_sums=$2
for f in $(cat $file_list)
do
# Get name file of file to download
bn=$(basename $f)
# If file not present attempt to download
if [ ! -f $bn ]
then
log INFO "Downloading $bn"
downloadfile $f
if verifyfile $check_sums
then
log DEBUG "Verification of $bn successful"
else
log WARN "Verification of $bn failed. Retrying download"
downloadfile $f
if ! verifyfile $check_sums
then
log FATAL "Failed to verify $bn download unsuccessful"
exit
else
log DEBUG "Verification successful"
fi
fi
# Local copy of file exists verify checksum redownload if necessary
else
log INFO "File $bn already downloaded. Verifying checksum"
if ! verifyfile $check_sums
then
log WARN "Verification of $bn failed. Redownloading"
downloadfile $f
if verifyfile $check_sums
then
log INFO "Redownload and verification of $bn successful"
else
log WARN "Verification of $bn failed. Retrying download"
downloadfile $f
if ! verifyfile $check_sums
then
log FATAL "Failed to verify $bn redownload unsuccessful"
exit
fi
fi
else log INFO "Verification of $bn successful"
fi
fi
done;
}
shlib_ansi_init auto
download $1 $2
