#!/bin/tcsh -f
# JLdL 01Nov11.
#
# This is a Linux kernel mirroring script using rsync.
#
# Possible upstream sites:
#
# This site is an official Linux kernel mirror, and the
# _ following fixed names are associated to it:
# _ ftp.if-usp.lkams.kernel.org
# _ www.if-usp.lkams.kernel.org
#
# This is the internal kernel.org site; filehub.kernel.org is
# _ a DNS round-robin address, and filehub2.kernel.org is the
# _ less-loaded machine according to the kernel.org news.
# rsync://mirrors@filehub.kernel.org/pub/
# rsync://mirrors@filehub1.kernel.org/pub/
#
# This script has the following technical characteristics:
# _ 1) It has a local locking mechanism, so as to prevent
# _    two instances of the program from running at the
# _    same time on the local server.
# _ 2) It does the standard universal lock within the root
# _    directory of the mirror, so as to inform downstream
# _    mirrors of an update in progress.
# _ 3) It verifies that same lock in the upstream server,
# _    and waits for the update to finish there before
# _    starting the update of the local mirror.
# _ 4) It uses the rsync options "--delete-after" and
# _    "--delay-updates" in order to minimize the mirror
# _    downtime for users during the update.
# _ 5) It has an interactive mode and a batch mode, this
# _    last one meant to be run from a cron job, by means
# _    of a symlink <file>.batch pointing to this file.
# _ 6) It creates the correct local timestamp file, using
# _    the symbolic name of this site, within the
# _    subdirectory project/trace, if it exists.
#
# PART 1: preparation: setting all the necessary variables.
#
# Section 1A: variables that may need customization.
#
# Define the root of the mirror filesystem.
set root = /sft
#
# Define the directory containing the mirror.
set mdir = $root/pub
#
# Define the symbolic name of this site.
set site = sft.if.usp.br
#
# Define the name of the remote host.
#set rhst = mirrors@filehub2.kernel.org
#set rhst = mirrors@filehub.kernel.org
set rhst = rsync.kernel.org
#
# Define the remote rsync URL address.
set rurl = "rsync://$rhst/pub/"
#
# Section 1B: variables that should need no customization.
#
# Define a filename for control purposes.
set file = `basename $0 | cut -d. -f1`
#
# Set the batch-mode flag.
if ( `basename $0 | cut -d. -f2` == batch ) then
    set btch = 1
else
    set btch = 0
endif
#
# Define the lock directory.
set lckd = /var/lock$root
#
# Define the lock file.
set lock = $lckd/$file
#
# Define the log directory.
set logd = $root/log
#
# Define the error file.
set errf = $logd/$file.err
#
# Define the log file.
set logf = $logd/$file.log
#
# Define the maximum number of files to delete.
set maxd = 10000
#
# Define the temporary directory.
set tmpd = $root/tmp
#
# Define the name of the standard flag for the update.
set flag = Archive-Update-in-Progress
#
# Use the symbolic name of the site for the local flag.
set lflg = $flag-$site
#
# Use the name of the remote host for remote flag.
set rflg = $flag-$rhst
#
# Build an exclusion list for files and directories.
set excl = ""
#
# Exclude the directories which are not required
# _ for an official kernel.org mirror.
set excl = "$excl --exclude scm"
set excl = "$excl --exclude site"
set excl = "$excl --exclude dist"
#
# Exclude the default temporary directories used by rsync
# _ when the "--delay-updates" option is in effect.
set excl = "$excl --exclude \.\~tmp\~"
#
# TEMPORARY EMERGENCY DEVICE: breakage at the original site.
#set excl = "$excl --exclude linux/kernel/people/arnd/patches/2.6.19-rc6-arnd1/broken-out/ps3-support/"
#
# Build a list of options for rsync, so that it will
# _ work well in mirror mode.
set opts = ""
set opts = "$opts --recursive"
set opts = "$opts --times"
set opts = "$opts --links"
set opts = "$opts --hard-links"
set opts = "$opts --delete"
set opts = "$opts --delete-after"
set opts = "$opts --max-delete=$maxd"
set opts = "$opts --delay-updates"
#set opts = "$opts --compress"
set opts = "$opts --verbose"
set opts = "$opts --partial"
set opts = "$opts --temp-dir=$tmpd"
set opts = "$opts --stats"
set opts = "$opts --human-readable"
#
# Since we are an official mirror, we have
# _ privileged password access; this is
# _ where that password is located.
#set pswd = "--password-file=/root/etc/passwd"
set pswd = ""
#
# PART 2: the action within the mirror filesystem.
#
# Make sure that the mirror directory exists.
if ( ! -d $mdir ) then
    mkdir $mdir
endif
#
# Make sure that the lock directory exists.
if ( ! -d $lckd ) then
    mkdir $lckd
endif
#
# Make sure that the log directory exists.
if ( ! -d $logd ) then
    mkdir $logd
endif
#
# Make sure that the temporary directory exists.
if ( ! -d $tmpd ) then
    mkdir $tmpd
endif
#
# Go to the appropriate directory in the mirror filesystem.
cd $mdir
#
# Check for a lock file.
if ( -f $lock ) then
    echo "${file}: this script is already running" | tee $errf
    exit 1
endif
#
# Make a lock file.
touch $lock
#
# On interruption, go to the clean exit label; note that this
# _ does not work for unexpected errors within the commands
# _ called from this script.
onintr cleanexit
#
# Verify whether the upstream archive is being updated at this time;
# _ if it is, then wait and try again until that update is finished;
# _ however, only do this if running in batch mode.
if ( $btch ) then
    #
    # Start the log file.
    cat /dev/null >&! $logf
    #
    # Start a trial counter.
    set trl = 1
    #
    # A label for the waiting loop.
    testagain:
    #
    # Try to simply list the flag file at the upstream site.
    #rsync $pswd $rurl$flag-\* >& /dev/null
    rsync $pswd $rurl$rflg >& /dev/null
    #
    # Keep the exit status in a separate variable.
    set stat = $status
    #
    # If the transfer was successful, the update archive is being
    # _ updated at this time, so wait for that update to finish;
    # _ however, the transfer may have failed for some other
    # _ reason, besides the flag file being absent, so there
    # _ is more then one alternative to be tested; in any
    # _ case retry every five minutes until it succeeds,
    # _ up to a certain maximum number of times.
    #
    # Error code 23 means that the server was successfully
    # _ connected but the transfer failed, meaning that
    # _ the file requested was not found; this is the
    # _ sole situation in which we proceed.
    if ( $stat != 23 ) then
	#
	# Error code 0 means that the transfer was successful.
	if ( $stat == 0 ) then
	    #
	    # Write a status message in the log file.
	    ( echo -n "Upstream archive being updated on " ; date ) >>& $logf
	#
	# Other error codes probably mean that the connection
	# _ to the server failed for some reason.
	else
	    #
	    # Write a status message in the log file.
	    ( echo -n "Error accessing upstream server $rhst on " ; \
							    date ) >>& $logf
	    #
	endif
	#
	# Go to the next trial.
	@ trl = $trl + 1
	#
	# Try up to 60 times (5 hours).
	if ( $trl <= 60 ) then
	    #
	    # Wait for 5 minutes.
	    sleep 300
	    #
	    # Go back and try again.
	    goto testagain
	    #
	else
	    #
	    # Exit cleanly.
	    goto cleanexit
	    #
	endif
	#
    endif
    #
    # Clean up the log file.
    cat /dev/null >&! $logf
#
# Run in interactive mode.
else
    #
    # Try to simply list the flag file at the upstream site.
    #rsync $pswd $rurl$flag-\* >& /dev/null
    rsync $pswd $rurl$rflg >& /dev/null
    #
    # Keep the exit status in a separate variable.
    set stat = $status
    #
    # If the transfer was successful, the update archive is being
    # _ updated at this time, therefore simply quit.
    if ( $stat == 0 ) then
	#
	# Write out an error message and quit.
	echo "${file}: ERROR: upstream archive being updated"
	#
	# Exit cleanly.
	goto cleanexit
	#
    endif
    #
endif
#
# Raise the standard universal flag during the update.
touch $lflg
#
# Disable shell expansion of the "*" character which
# _ may be contained in the excl variable.
set noglob
#
# Use rsync with all options and exclusions; if it fails, then
# _ try again, but only do this if running in batch mode.
if ( $btch ) then
    #
    # Start a trial counter.
    set trl = 1
    #
    # A label for error handling.
    again:
    #
    # Start the log file with the date.
    ( echo -n "Starting trial $trl on " ; date ) >&! $logf
    #
    # Run the mirror, logging to the log file.
    rsync $opts $pswd $excl $rurl . >>& $logf
    #
    # Keep the exit status in a separate variable.
    set stat = $status
    #
    # Retry every five minutes until it succeeds,
    # _ up to a certain maximum number of times.
    if ( $stat != 0 ) then
	#
	# Go to the next trial.
	@ trl = $trl + 1
	#
	# Try up to 60 times (5 hours).
	if ( $trl <= 60 ) then
	    #
	    # Wait for 5 minutes.
	    sleep 300
	    #
	    # Go back and try again.
	    goto again
	    #
	endif
	#
    endif
    #
    # End the log file with the date.
    ( echo -n "Finished on " ; date ) >>& $logf
    #
    # Fix the log file.
    /sft/sbin/doit-fix-log-file $logf
#
# Run in interactive mode.
else
    rsync $opts $pswd --progress $excl $rurl .
endif
#
# Restore shell expansion of special characters.
unset noglob
#
# Lower the standard universal flag for the update.
if ( -f $lflg ) then
    rm -f $lflg
endif
#
# Update the local timestamp file in the trace directory.
if ( -d project/trace ) then
    date -u >! project/trace/$site
endif
#
# Remove the lock file.
cleanexit:
rm -f $lock