#!/bin/bash
# vim: set tabstop=2:softtabstop=0:shiftwidth=2:syntax=sh
# backup-lxc-dc-container-v15
#
#V15-009
#|=============================================================================|
#|                *** PROPRIETARY RIGHTS NOTICE ***                            |
#|   This material contains proprietary information of MULI Holdings P/L,      |
#|   Wahroonga, NSW, Australia and/or its licensor, embodying                  |
#|   "CONFIDENTIAL INFORMATION" ideas and expressions, no part of which may    |
#|   be reproduced or transmitted without permission of MULI Management P/L.   |
#|      ***  COPYRIGHT  1982 - 2022  ***  MULI Holdings P/L  ***               |
#|                          All rights reserved.                               |
#|=============================================================================|

###
###
# This script should be installed in /virt/backup-V15/backup-scripts
# There should be another directory  /virt/backup-V15/control
# This directory should hold files corresponding to the container name to be backed up from the opposing server
# i.e. ds5:/virt/backup-V15/control/c10010 for container c10010 on ds4.
#   These files should contain the data store on which it is to be backed up to ie /doe for c10010 and /dee for c10000
#		Remembering on the even server ds4 all data stores should end in e /doe and /dee for the even server.
#
#
# This script requires: mutt and remote access between ds4 and ds5 to be configured
#
###
###

# create abort function
send_failure_email ()
{
	mutt -s "`hostname`: FAILURE ${Container} backup log" ${Email_address_failure} <  ${Log_File}
}
send_failure_email_arg ()
{
	mutt -s "`hostname`: FAILURE ${Container} backup" ${Email_address_failure} <<!
${1}
!
}

Email_address_muli=backups@muli.com.au
Email_address_failure=support@muli.com.au
Email_address_client=${Email_address_muli}


# Check on Backup folders
Backup_prefix=/virt/backup-V15
if [ ! -d "${Backup_prefix}" ];then
	mess="$0: failed: no Backup_prefix=${Backup_prefix} on system"
	echo "${mess}"
	send_failure_email_arg "${mess}"
	exit 1
fi


#check on $Log_File
if [ ! -d ${Backup_prefix}/backup-logs ];then
	mkdir ${Backup_prefix}/backup-logs
	res=$?
	if [ ${res} -ne 0 ]; then
		mess="$0: failed to make ${Backup_prefix}/backup-logs"
		echo "${mess}"
		send_failure_email_arg "${mess}"
		exit 1
	fi
fi

#check on $Control File
if [ ! -d ${Backup_prefix}/control ];then
	mkdir ${Backup_prefix}/control
	res=$?
	if [ ${res} -ne 0 ]; then
		mess="$0: failed to make ${Backup_prefix}/control"
		echo "${mess}"
		send_failure_email_arg "${mess}"
		exit 1
	fi
fi
Control_dir="${Backup_prefix}/control"
exists=`ls ${Control_dir}`
if [ -z "${exists}" ];then
	mess="$0: missing container files in ${Control_dir}"
	echo "${mess}"
	send_failure_email_arg "${mess}"
	exit 1
fi

myhost=`hostname`
if [ ${myhost} = "ds4" ];then
	Backup_host=ds5
elif [ ${myhost} = "ds5" ];then
	Backup_host=ds4
else
	mess="$0: myhost = ${myhost} cannot determine backup host"
	echo "${mess}"
	send_failure_email_arg "${mess}"
	exit 1
fi;

Log_Dir=${Backup_prefix}/backup-logs
if [ ! -d ${Log_Dir} ];then
	mkdir ${Log_Dir}
	res=$?
	if [ ${res} -ne 0 ]; then
		mess="$0: failed to make ${Log_Dir}"
		echo "${mess}"
		send_failure_email_arg "${mess}"
		exit 1
	fi
fi


#check for rsync
rsync_prog=`which rsync`
if [ -z "${rsync_prog}" ]; then
	"Aborting 	rsync is missing"
	"FAIL Aborting rsync is missing `date`" >> ${Status_File}
	send_failure_email
	exit 1
fi

Daynum=`date +%j`

Status_File=${Log_Dir}/BackupStatus-${Backup_host}_${Daynum}
echo "Status_File = ${Status_File}"

## Obtain container name and data starage location from control/container for backup loop
cd ${Control_dir}
for Container in `ls *`
do
	echo "Container = ${Container}"
	Backup_store=`cat ${Control_dir}/${Container}`
	echo "Backup store = ${Backup_store}"
	if [ ! -d ${Backup_store} ];then
		mess="$0: incorrect backup store: ${Backup_store} in ${Control_dir} container = ${Container}"
		echo "${mess}"
		send_failure_email_arg "${mess}"
		exit 1
	fi

	Log_File=${Log_Dir}/backup-${Container}_${Daynum}
	echo "Log file = ${Log_File}"
	exec > ${Log_File} 2>&1

	echo " " >> ${Status_File}
	echo "#### Backup of ${Backup_host}:${Container}" >> ${Status_File}
	echo "${Container} Backup" >> ${Status_File}
	echo "START `date`" >> ${Status_File}
	echo "#### Backup of ${Container} ###"

	## Extract profile to obtain storage pool e.g location from profile name
	## and confirm container
	Partition=`lxc info ${Backup_host}:${Container} | grep Profiles | cut -f2 -d":" | cut -f2 -d"_"`
	res=$?
	if [ ${res} -ne 0 ];then
		mess="$0:Failed Command:  Partition=`lxc info ${Backup_host}:${Container} | grep Profiles | cut -f2 -d: | cut -f2 -d_`"
		echo "${mess}"
		echo "${mess}" >> ${Status_File}
		echo "exited with ${res}"
		send_failure_email_arg "${mess}"
		exit 1
	fi
	if [ "/${Partition}" != "${Backup_store}" ]; then
		mess="$0: Failed Partition=${Partition} not equal Backup_store=${Backup_store}"
		echo "${mess}"
		echo "${mess}" >> ${Status_File}
		send_failure_email_arg "${mess}"
		exit 1
	fi

	Backup_folder="${Backup_store}/backups"
	if [ ! -d ${Backup_folder} ];then
		mkdir ${Backup_folder}
		res=$?
		if [ ${res} -ne 0 ]; then
			mess="$0: failed to make ${Backup_folder}"
			echo "${mess}"
			echo "${mess}" >> ${Status_File}
			send_failure_email_arg "${mess}"
			exit 1
		fi
	fi
	Backup_location="${Backup_folder}/${Container}"
	echo "Backup_location = ${Backup_folder}"
	if [ ! -d ${Backup_location} ];then
		mkdir ${Backup_location}
		res=$?
		if [ ${res} -ne 0 ]; then
			mess="$0: failed to make ${Backup_location}"
			echo "${mess}"
			echo "${mess}" >> ${Status_File}
			send_failure_email_arg "${mess}"
			exit 1
		fi
	fi
	if [ ! -d ${Backup_location}/mulidata ];then
		mkdir ${Backup_location}/mulidata
		res=$?
		if [ ${res} -ne 0 ]; then
			mess="$0: failed to make ${Backup_location}/mulidata"
			echo "${mess}"
			echo "${mess}" >> ${Status_File}
			send_failure_email_arg "${mess}"
			exit 1
		fi
	fi
	if [ ! -d ${Backup_location}/etc ];then
		mkdir ${Backup_location}/etc
		res=$?
		if [ ${res} -ne 0 ]; then
			mess="$0: failed to make ${Backup_location}/etc"
			echo "${mess}"
			echo "${mess}" >> ${Status_File}
			send_failure_email_arg "${mess}"
			exit 1
		fi
	fi

	declare -i active=0
	active=`lxc list ${Backup_host}:${Container} | grep -c "RUNNING"`
	if [ ${active} -ne 1 ];then
		mess="$0: failed to find running ${Container} on ${Backup_host}"
		echo "${mess}"
		echo "${mess}" >> ${Status_File}
		send_failure_email_arg "${mess}"
		exit 1
	fi

	#We have the container running on Backup_host and the local backup location
	# run the script to do the internal container database dumps and flat file transfer
	#shutdown the container to enable rsync of mulidata
	# do the rsync of /usr/local/mulidata
	#restart the container
	
	echo "lxc exec ${Backup_host}:${Container} -- /bin/systemctl stop Mbbag-web `date`" >> ${Status_File}
	lxc exec ${Backup_host}:${Container} -- /bin/systemctl stop Mbbag-web
	echo "lxc exec ${Backup_host}:${Container} -- /bin/systemctl stop Mbbag-nginx `date`" >> ${Status_File}
	lxc exec ${Backup_host}:${Container} -- /bin/systemctl stop Mbbag-nginx
	echo
	echo "running mulipg14_backup_all.sh for ${Container}"
	echo
	echo "START lxc exec ${Backup_host}:${Container} --  /usr/local/muli/scripts/mulipg14_backup_all.sh `date`" >> ${Status_File}
	lxc exec ${Backup_host}:${Container} -- /usr/local/muli/scripts/mulipg14_backup_all.sh

	echo "lxc stop ${Backup_host}:${Container} `date`"
	echo "lxc stop ${Backup_host}:${Container} `date`" >> ${Status_File}
	echo "lxc stop ${Backup_host}:${Container} `date`" >> ${Log_File}
	lxc stop ${Backup_host}:${Container}
	counter=0
	while [ $counter -le 5 ]
	do
		echo " loop counter=${counter} Sleep 5 ">> ${Status_File}
		((counter++))
		sleep 5
		active=`lxc list ${Backup_host}: | grep ${Container} | grep -c "RUNNING"`
		if [ ${active} -eq 0 ];then
			echo "${Backup_host}:${Container} STopped  `date`" >> ${Status_File}
			counter=6
		fi
	done
	if [ ${active} -eq 1 ];then
		mess="$0: failed to stop running ${Container} on ${Backup_host}"
		echo "send failure email with mess = ${mess} `date`" >> ${Status_File}
		send_failure_email_arg "${mess}"
		echo ">>>> Continuing to next container <<<< `date`" >> ${Status_File}
		continue
	fi

	echo "#### Transfer ${Backup_host}:${Container} #####"
	echo "#### Transfer ${Backup_host}:${Container} #####" >> ${Status_File}
	echo "#### Transfer ${Backup_host}:${Container} #####" >> ${Log_File}
	echo "${rsync_prog} -avz --delete ${Backup_host}:${Backup_store}/lxd-storage-pool/containers/${Container}/rootfs/usr/local/mulidata/ ${Backup_location}/mulidata/ `date`"
	echo "${rsync_prog} -avz --delete ${Backup_host}:${Backup_store}/lxd-storage-pool/containers/${Container}/rootfs/usr/local/mulidata/ ${Backup_location}/mulidata/ `date`" >> ${Status_File}
	echo "${rsync_prog} -avz --delete ${Backup_host}:${Backup_store}/lxd-storage-pool/containers/${Container}/rootfs/usr/local/mulidata/ ${Backup_location}/mulidata/ `date`" >> ${Log_File}
	${rsync_prog} -avz --delete ${Backup_host}:${Backup_store}/lxd-storage-pool/containers/${Container}/rootfs/usr/local/mulidata/ ${Backup_location}/mulidata/

	echo "${rsync_prog} -avz --delete ${Backup_host}:${Backup_store}/lxd-storage-pool/containers/${Container}/rootfs/etc/  ${Backup_location}/etc/ `date`"
	echo "${rsync_prog} -avz --delete ${Backup_host}:${Backup_store}/lxd-storage-pool/containers/${Container}/rootfs/etc/  ${Backup_location}/etc/ `date`" >> ${Status_File}
	echo "${rsync_prog} -avz --delete ${Backup_host}:${Backup_store}/lxd-storage-pool/containers/${Container}/rootfs/etc/  ${Backup_location}/etc/ `date`" >> ${Log_File}
	${rsync_prog} -avz --delete ${Backup_host}:${Backup_store}/lxd-storage-pool/containers/${Container}/rootfs/etc/  ${Backup_location}/etc/

	echo " ##### Restart Container ####"
	echo " ##### Restart Container ####" >> ${Status_File}
	echo " ##### Restart Container ####" >> ${Log_File}
	echo "lxc start ${Backup_host}:${Container} `date`" >> ${Status_File}
	lxc start ${Backup_host}:${Container}
	if [ $? -eq 0 ];then
		echo " Start status successfull " >> ${Status_File}
		echo " Start status successfull " >> ${Log_File}
	fi
	counter=0
	while [ $counter -le 5 ]
	do
		echo " looping counter=${counter} Sleep 5 ">> ${Status_File}
		((counter++))
		sleep 5
		active=`lxc list ${Backup_host}:${Container} | grep -c "RUNNING"`
		if [ ${active} -eq 1 ];then
			echo "${Backup_host}:${Container} started   `date`" >> ${Status_File}
			counter=6
		fi
		if [ $counter -eq 2 ];then
			echo "counter = 2 issuing start again `date`" >> ${Status_File}
			lxc start ${Backup_host}:${Container}
		fi
	done
	if [ ${active} -ne 1 ];then
		mess="$0: failed to find running ${Container} on ${Backup_host}"
		echo "${mess}"
		echo "${mess}" >> ${Status_File}
		echo "${mess}" >> ${Log_File}
		send_failure_email_arg "${mess}"
		echo ">>>> Continuing to next container <<<< `date`" >> ${Status_File}
		continue
	fi

	echo "%%%%%%%%% Completed backup for ${Container} %%%%%%%%% `date`" >> ${Status_File}
	mutt -s "`hostname`: ${Container} backup log" ${Email_address_muli} <  ${Log_File}

done

echo "O.K. Backups Finished `date`" >> ${Status_File}

if [ -f /var/run/reboot-required ]; then
	echo "%%%% reboot required %%%%"
	echo "%%%% reboot required %%%%" >> ${Status_file}
	echo " /usr/sbin/shutdown --reboot +1 Issued @ `date`" >> ${Status_file}
	/usr/sbin/shutdown --reboot 00:30
	mutt -s "`hostname`: Backup initiated reboot for 00:30 " support@muli.com.au
else
	echo " no reboot required"
fi;

mutt -s "`hostname`: Backup Status for ${Backup_host}" ${Email_address_muli} -a ${Status_File} <<!
Completed backup of ${Backup_host}
!

exit 0



###############################################################################
#=>Vn			Who			When			Risk			What
#=.
#=>V15-001	BrowCh	19.01.21	R00000834	1st pass V15 dc backup control script
#=>V15-002	BrowCh	09.02.21	R00000834	corrected rsync scripting
#=>V15-003	BrowCh	26.04.22	R00000000	updated for  pg14
#=>V15-004	BrowCh	29.11.22	R00000000	updated to not truncate between containers send status file to support
#=>V15-005	BrowCh	30.11.22	R00000000	added reboot at end of script if /var/run/reboot-required exists
#=>V15-006	BrowCh	01.12.22	R00000000	corrected failure_email_arg usage and testing for lxc start success
#=>V15-007	BrowCh	02.12.22	R00000000	added sleep 5 before checking for stopped container
#=>V15-008	BrowCh	05.12.22	R00000000	added looping to container status checks
#=>V15-009	BrowCh	06.12.22	R00000000	Add additional checks on start
#=..
###############################################################################
