dbackup

#!/usr/bin/perl -w
# Copyright (c) 2010-2014, AllWorldIT
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.
#
#
# Author: Nigel Kukard <nkukard@lbsd.net>


# TODO:
# - USE incremental backups!!!! fucking awesome
#
# - Read in defaultSystemExcl extras from /proc/mounts
# - proc, nfs, nfs4, fusectl fuse.glusterfs, sysfs, tmpfs, devtmpfs, devpts
#


use strict;
use Compress::Zlib;
use Config::IniFiles;
use Cwd;
use Fcntl qw (:mode);
use File::Find;
use File::Path qw( rmtree mkpath );
use File::Spec;
use Getopt::Long;
use POSIX qw (strftime lchown);
use Data::Dumper;
use MIME::Base64;



my $VERSION = "0.0.x";
# System dirs we don't care about
my @defaultSystemExcl = ("/dev","/run","/proc","/sys","/tmp","/var/tmp","/misc","/media","/mnt");
# These should be backed up separately
my @defaultDataExcl = (
		# We cannot backup the DB reliably by just copying files
		"/var/lib/mysql","/var/lib/pgsql",

		# Exclude amavis working files
		"/var/amavis/tmp/",
		"/var/lib/amavis/tmp/",
		"/var/spool/amavis/virusmails/",

		"/var/spool/mailman/retry",
		# Exclude postfix queues
		"/var/spool/postfix/active",
		"/var/spool/postfix/defer",
		"/var/spool/postfix/deferred",
		"/var/spool/postfix/public",
		"/var/spool/postfix/private",
		# Exclude squid cache
		"/var/spool/squid",

		# Yum cache
		"/var/cache/yum",
		# Apt cache
		"/var/cache/apt/archives",

		# cPanel
		"/home/virtfs"
);


# Backup constants
use constant {
	LOG_DEBUG => 5,
	LOG_INFO => 4,
	LOG_NOTICE => 3,
	LOG_WARNING => 2,
	LOG_ERROR => 1
};

use constant {
	ST_FILE_CHANGED => 1,
	ST_DIR_NEW => 2,
	ST_DIR_SYS => 4,
	ST_DIR_CHANGED => 8,
	ST_FILE_ATTR_CHANGED => 16,
	ST_FILE_NEW => 32
};


# Main config
my %config = (
	'log-level' => LOG_NOTICE,
	'tar' => 'tar',
	'exclude-system' => 0,
	'backup-upgrade' => 0,
	'compress' => "bzip2"
);
# Choose default compression method
if (checkPATH("xz")) {
	$config{'compress'} = "xz";
}



print(STDERR "AWIT-DBackup v$VERSION, Copyright (c) 2010-2014, AllWorldIT\n\n");


# Grab options
my %optctl = ();
GetOptions(\%optctl,
	"help",

	"config=s",
	"log-level=i",
	"tar=s",
	"manifest-format=s",

	"backup",
	"backup-upgrade",
	"compress:s",
	"exclude-system",
	"system-base=s@",
	"system-dir=s@",

	"tar-ignore-failed-read=s@",

	"exclude-data",
	"data-dir=s@",

	"exclude-file=s@",
	"exclude-path=s@",

	"exclude-fs-type=s@",

	"restore",
	"tar-keep-newer",
	"tar-keep-old-files",
);

# Check for help
if (defined($optctl{'help'})) {
	displayHelp();
	exit 0;
}

# Check for invalid combination
if (defined($optctl{'backup'}) && defined($optctl{'restore'})) {
	print(STDERR "ERROR: Cannot backup AND restore at the same time\n\n");
	displayHelp();
	exit 1;
}

# Make sure we only have 2 additional args
if (@ARGV > 2 || @ARGV < 2) {
		print(STDERR "ERROR: Invalid number of arguments\n\n");
		displayHelp();
		exit 1;
}

# Check both dirs exist...
if (! -d $ARGV[0]) {
	print(STDERR "ERROR: Source directory '".$ARGV[0]."' does not exist!\n\n");
	exit 1;
}
if (! -d $ARGV[1]) {
	print(STDERR "ERROR: Destination directory '".$ARGV[1]."' does not exist!\n\n");
	exit 1;
}

# If backup file is specified and it does not exist, throw an error
if (defined($optctl{'config'}) && ! -f $optctl{'config'}) {
	print(STDERR "ERROR: Configuration file '".$optctl{'config'}."' NOT found!\n");
	exit 1;
}
# Check config file was specified, if not set default
my $configFile;
if (!defined($configFile = $optctl{'config'})) {
	$configFile = "/etc/dbackup.conf";
}
# If we have a config file, read it in
if (-f $configFile) {
	# Use config file, ignore case
	tie my %inifile, 'Config::IniFiles', (
			-file => $optctl{'config'},
			-nocase => 1
	) or die "Failed to open config file '".$optctl{'config'}."': ".join("\n",@Config::IniFiles::errors);

	# Check if we actually have something...
	if (defined($inifile{'backup'})) {
		# Loop with config items
		foreach my $item (keys %{$inifile{'backup'}}) {
			$config{$item} = $inifile{'backup'}{$item};
		}
	}
}

#
# Process config
#
if (!defined($config{'system-base'})) {
	# Quick hack to see if we have a cmdline option
	if (!defined($optctl{'system-base'})) {
		$config{'system-base'} = ["/"];
	} else {
		$config{'system-base'} = [];
	}
} else {
	$config{'system-base'} = toArray($config{'system-base'});
}

if (!defined($config{'system-dir'})) {
	$config{'system-dir'} = [];
} else {
	$config{'system-dir'} = toArray($config{'system-dir'});
}

if (!defined($config{'tar-ignore-failed-read'})) {
	$config{'tar-ignore-failed-read'} = [];
} else {
	$config{'tar-ignore-failed-read'} = toArray($config{'tar-ignore-failed-read'});
}

if (!defined($config{'data-dir'})) {
	$config{'data-dir'} = [];
} else {
	$config{'data-dir'} = toArray($config{'data-dir'});
}

if (!defined($config{'exclude-file'})) {
	$config{'exclude-file'} = [];
} else {
	$config{'exclude-file'} = toArray($config{'exclude-file'});
}

if (!defined($config{'exclude-path'})) {
	$config{'exclude-path'} = [];
} else {
	$config{'exclude-path'} = toArray($config{'exclude-path'});
}

if (!defined($config{'exclude-fs-type'})) {
	$config{'exclude-fs-type'} = [];
} else {
	$config{'exclude-fs-type'} = toArray($config{'exclude-fs-type'});
}


#
# Check commandline options
#

if (defined($optctl{'backup-upgrade'})) {
	$config{'backup-upgrade'} = 1;
}

if (defined($optctl{'compress'})) {
	# Why use --compress with --restore?
	if (defined($optctl{'restore'})) {
		print(STDERR "WARNING: The use of --compress with --restore does not make sense\n");
	}

	# Make sure its valid
	if (
				$optctl{'compress'} ne "xz" &&
				$optctl{'compress'} ne "bzip2" &&
				$optctl{'compress'} ne "gzip" &&
				$optctl{'compress'} ne "lz" &&
				$optctl{'compress'} ne "none"
	) {
		print(STDERR "ERROR: Compression is invalid, valid values are: xz bzip2 gzip lz none\n\n");
		displayHelp();
		exit 1;
	}
	$config{'compress'} = $optctl{'compress'};
}

if (defined($optctl{'tar'})) {
	# Check if tar is executable
	if ( -x $optctl{'tar'}) {
		$config{'tar'} = $optctl{'tar'};
	} else {
		print(STDERR "ERROR: tar '".$optctl{'tar'}."' does not exist or is not executable\n");
	}
}
my $tarVer = getTarVer($config{'tar'});

if (defined($optctl{'log-level'})) {
	$config{'log-level'} = $optctl{'log-level'};
}

if (defined($optctl{'manifest-format'})) {
	if (
				$optctl{'manifest-format'} ne "null" &&
				$optctl{'manifest-format'} ne "newline"
	) {
		print(STDERR "ERROR: Manifest format is invalid, valid values are: null newline\n\n");
		displayHelp();
		exit 1;
	}
	$config{'manifest-format'} = $optctl{'manifest-format'};
}

if (defined($optctl{'exclude-system'})) {
	$config{'exclude-system'} = 1;
}

if (defined($optctl{'system-base'})) {
	push(@{$config{'system-base'}},@{$optctl{'system-base'}});
}

if (defined($optctl{'system-dir'})) {
	push(@{$config{'system-dir'}},@{$optctl{'system-dir'}});
}

if (defined($optctl{'tar-ignore-failed-read'})) {
	push(@{$config{'tar-ignore-failed-read'}},@{$optctl{'tar-ignore-failed-read'}});
}

if (defined($optctl{'data-dir'})) {
	push(@{$config{'data-dir'}},@{$optctl{'data-dir'}});
}

if (defined($optctl{'exclude-file'})) {
	push(@{$config{'exclude-file'}},@{$optctl{'exclude-file'}});
}

if (defined($optctl{'exclude-path'})) {
	push(@{$config{'exclude-path'}},@{$optctl{'exclude-path'}});
}

if (defined($optctl{'exclude-fs-type'})) {
	push(@{$config{'exclude-fs-type'}},@{$optctl{'exclude-fs-type'}});
}

if (@{$config{'exclude-fs-type'}} > 0) {
	# Pull in mounts
	open(my $fh,"<","/proc/mounts")
			or die "ERROR: Failed to open '/proc/mounts': $!";
	while (my $line = <$fh>) {
		chomp($line);
		# Split off items we need
		my (undef,$path,$type) = split(/\s+/,$line);
		# Loop with types and check
		foreach my $item (@{$config{'exclude-fs-type'}}) {
			if ($item eq $type) {
				push(@{$config{'exclude-path'}},$path);
			}
		}
	}
	close($fh);
}

# Look for parallel versions of the compression program we going to use
$config{'compress-gzip'} = "gzip";
if (checkPATH("pigz")) {
	$config{'compress-gzip'} = "pigz";
}
$config{'compress-bzip2'} = "bzip2";
if (checkPATH("pbzip2")) {
	$config{'compress-bzip2'} = "pbzip2";
} elsif (checkPATH("lbzip2")) {
	$config{'compress-lbzip2'} = "lbzip2";
}
$config{'compress-xz'} = "xz";
if (checkPATH("pxz")) {
	$config{'compress-xz'} = "pxz";
} elsif (checkPATH("pixz")) {
	$config{'compress-xz'} = "pixz";
}
$config{'compress-lz'} = "clzip";
if (checkPATH("plzip")) {
	$config{'compress-lz'} = "plzip";
} elsif (checkPATH("pdlzip")) {
	$config{'compress-lz'} = "pdlzip";
}


# Sanitize the source and dest
my $sourceDir = File::Spec->rel2abs($ARGV[0]);
my $destDir = File::Spec->rel2abs($ARGV[1]);


# Backup function
sub backup
{
	our ($source,$dest) = @_;


	# State infor for current dir
	our %doBackup;
	our %srcFileList;
	our %srcDirList;
	# Original .state file read in during backup
	our %origFileList;
	our %origDirList;
	our %origPathAttribs;
	# New attributes
	our %newPathAttribs;


	# Preprocess dir
	sub backup_preprocess {
		my @list = @_;

		# Strip first part of the path
		(my $path = $File::Find::dir) =~ s#^$source/?##;

		# Check if this dir should be backed up or not....
		if ( -f "$source/$path/.nodbackup" ) {
			printLog(LOG_INFO,"S: Path '[$source]/($path)' EXCLUDED from backup with .nodbackup\n");
			return ();
		}
		# Exclude trash folders
		if ( "$source/$path" =~ /(\.Trash-[0-9]{4})$/) {
			printLog(LOG_INFO,"S: Path '[$source]/($path)' EXCLUDED from backup, Linux trash folder\n");
			return ();
		}

		# Make sure even if we processing empty dirs that we actually record it
		if (!defined($srcFileList{$path})) {
			# Just set to empty hash ref
			$srcFileList{$path} = {};
		}

		# Exclude system directories
		if ($config{'exclude-system'}) {
			# Loop through system dirs
			foreach my $sysdir (@{$config{'system-base'}}) {
				# Loop with system paths
				foreach my $excl (@defaultSystemExcl,@{$config{'system-dir'}}) {
					# Sanitize path
					my $testPath = File::Spec->rel2abs("$sysdir/$excl");
					# Check...
					if ("/$path" eq $testPath) {
						printLog(LOG_INFO,"S: Path '[$source]/($path)' is a system directory, ignoring files\n");
						$doBackup{$path} |= ST_DIR_SYS;
						return ();
					}
				}
				# Check if we excluding data dirs
				if (defined($config{'exclude-data'})) {
					# Loop with data dirs
					foreach my $excl (@defaultDataExcl,@{$config{'data-dir'}}) {
						# Sanitize path
						my $testPath = File::Spec->rel2abs("$sysdir/$excl");
						# Check...
						if ("/$path" eq $testPath) {
							printLog(LOG_INFO,"S: Path '[$source]/($path)' is a data directory, ignoring files\n");
							$doBackup{$path} |= ST_DIR_SYS;
							return ();
						}
					}
				}
			}
		}
		# Exclude paths
		foreach my $item (@{$config{'exclude-path'}}) {
			# Check...
			if ("/$path" =~ $item) {
				printLog(LOG_INFO,"S: Path '[$source]/($path)' is an excluded path, ignoring files\n");
				$doBackup{$path} |= ST_DIR_SYS;
				return ();
			}
		}

		# Check if this dir exists on the backup, if not we need to back it up obviously...
		if ( ! -d "$dest/$path") {
			$doBackup{$path} |= ST_DIR_NEW;
		# This can probably only be because a backup was continued
		} elsif ( ! -f "$dest/$path/.dbackup-state" ) {
			# First backup will never have the main dir .dbackup-state file existing
			if ($path ne "") {
				printLog(LOG_WARNING,"State file '[$dest]/($path)/.dbackup-state' does not exist\n");
			}
			$doBackup{$path} |= ST_DIR_NEW;
		}

		# Cannot read in for new dirs can we...
		if (($doBackup{$path} & ST_DIR_NEW) != ST_DIR_NEW) {
			# Load state file
			loadStateFile("$dest/$path/.dbackup-state",$path,\%origFileList,\%origDirList,\%origPathAttribs);

			# Check we have at least the version attribute
			if (!defined($origPathAttribs{$path}) || !defined($origPathAttribs{$path}->{'dbackup.version'})) {
				printLog(LOG_ERROR,"No dbackup version information found in '[$dest]/($path)/.dbackup-state', IGNORING\n");
				delete($origDirList{$path});
				delete($origFileList{$path});

			# NK:
			# Sanity check, some versions created empty'ish .dbackup-state files with no sequence  < 2012-01-21
			# If we have a manifest, we should have a backup, therefore a format and sequence
			} elsif (
					-f "$dest/$path/dbackup0.manifest" && (
						!defined($origPathAttribs{$path}->{'format'}) ||
						!defined($origPathAttribs{$path}->{'sequence'})
					)
			) {
				printLog(LOG_ERROR,"No dbackup format/sequence information found in ".
						"'[$dest]/($path)/.dbackup-state', IGNORING\n");
				delete($origDirList{$path});
				delete($origFileList{$path});

			# Check if we going to upgrade...
			} else {
				# Are we doing backup upgrades?
				if ($config{'backup-upgrade'}) {
					# Pull in versions
					my $a = getNumericVer($origPathAttribs{$path}->{'dbackup.version'});
					my $b = getNumericVer($VERSION);
					# Compare
					if (
							$a < 16		# Things before 0.0.16 may have .dbackup-state issues and missing
										# required info.
					) {
						# Check if end in .x
						if ($VERSION =~ /\.x$/) {
							printLog(LOG_WARNING,"S: Path '[$source]/($path)' ".
									"cannot be upgraded, dbackup is TRUNK version\n");
						} else {
							printLog(LOG_INFO,"S: Path '[$source]/($path)' will be backed up, upgrading\n");
							delete($origDirList{$path});
							delete($origFileList{$path});
							# Force state file update if dir is empty
							$doBackup{$path} |= ST_DIR_CHANGED;
						}
					}
				}
			}
		}

		# Apply filter...
		if ((@{$config{'exclude-file'}}) > 0) {
			my @newList;
			foreach my $item (@list) {
				my $match = 0;
				# Apply only to files
				if ( -f "$source/$path/$item") {
					# Loop with exclude filters
					foreach my $filter (@{$config{'exclude-file'}}) {
						# Check for match
						if ($item =~ $filter) {
							printLog(LOG_INFO,"S: Path '[$source]/($path)/$item' matches file exclude filter, ignoring\n");
							# XXX: NK - If this ws backed up before and now not, dir has changed?
							$match = 1;
							last;
						}
					}
				}
				# If not matched, add to new list
				if (!$match) {
					push(@newList,$item);
				}
			}
			# Replace the list now
			@list = @newList;
		}

		return @list;
	}

	# Process files
	sub backup_process {
		my $name = $File::Find::name;
		# Strip first part of the path
		(my $path = $File::Find::dir) =~ s#^$source/?##;

		# Store current filename
		my $item = $_;

		# We use lstat so we don't follow symlinks that don't exist
		my @stat = lstat($name);
		# Split it up
		my ($dev,$ino,$mode,$nlink,$uid,$gid,$rdev,$size,$atime,$mtime,
				$ctime,$blksize,$blocks) = @stat;

		# Process directory
		if (S_ISDIR($mode)) {
			# Skip over main dir
			if ($item eq ".") {
				return;
			}

			# Make sure sub dirs have a santized name
			my $ppath = ($path ne "") ? "$path/$item" : $item;

			# Set default flags to nothing...
			$doBackup{$ppath} = 0;

			# Ok, lets see whats going on with the dir
			if (!defined($origDirList{$path}->{$item})) {
				printLog(LOG_DEBUG,"S: Directory '[$source]/($path)/$item' is new\n");
				$doBackup{$path} |= ST_DIR_CHANGED;

			# Check if mtime matches
			} elsif (
					!defined($origDirList{$path}->{$item}->{'mtime'}) ||
					$origDirList{$path}->{$item}->{'mtime'} ne $mtime
			) {
				printLog(LOG_DEBUG,"S: Directory '[$source]/($path)/$item' was modified\n");
				$doBackup{$path} |= ST_DIR_CHANGED;
			# Check if uid matches
			} elsif (
					!defined($origDirList{$path}->{$item}->{'uid'}) ||
					$origDirList{$path}->{$item}->{'uid'} ne $uid
			) {
				printLog(LOG_DEBUG,"S: Directory '[$source]/($path)/$item' UID was changed\n");
				$doBackup{$path} |= ST_DIR_CHANGED;
			# Check if gid matches
			} elsif (
					!defined($origDirList{$path}->{$item}->{'gid'}) ||
					$origDirList{$path}->{$item}->{'gid'} ne $gid
			) {
				printLog(LOG_DEBUG,"S: Directory '[$source]/($path)/$item' GID was changed\n");
				$doBackup{$path} |= ST_DIR_CHANGED;
			# Check if mode matches
			} elsif (
					!defined($origDirList{$path}->{$item}->{'mode'}) ||
					$origDirList{$path}->{$item}->{'mode'} ne $mode
			) {
				printLog(LOG_DEBUG,"S: Directory '[$source]/($path)/$item' MODE was changed\n");
				$doBackup{$path} |= ST_DIR_CHANGED;
			}

			# Record details...
			$srcDirList{$path}->{$item}->{'size'} = $size;
			$srcDirList{$path}->{$item}->{'ctime'} = $ctime;
			$srcDirList{$path}->{$item}->{'atime'} = $atime;
			$srcDirList{$path}->{$item}->{'mtime'} = $mtime;
			$srcDirList{$path}->{$item}->{'uid'} = $uid;
			$srcDirList{$path}->{$item}->{'gid'} = $gid;
			$srcDirList{$path}->{$item}->{'mode'} = $mode;

			return;

		# Check if we're a socket
		} elsif (S_ISSOCK($mode)) {
			# Blatently ignore...
			return;
		}

		# Initialize file state
		my $state = 0;

		# Check if file does not exists on backup
		if (!defined($origFileList{$path}->{$item})) {
			if (($doBackup{$path} & ST_DIR_NEW) != ST_DIR_NEW) {
				printLog(LOG_DEBUG,"S: File '[$source]/($path)/$item' is new\n");
			}
			$state |= ST_FILE_NEW;

		# Check if size matches
		} elsif (
				!defined($origFileList{$path}->{$item}->{'size'}) ||
				$origFileList{$path}->{$item}->{'size'} ne $size
		) {
			printLog(LOG_DEBUG,"S: File '[$source]/($path)/$item' has different size\n");
			$state |= ST_FILE_CHANGED;

		#
		# NK: Does not like chmod changes to files???
		#
		# Check if ctime matches
#		} elsif (
#				!defined($origFileList{$path}->{$item}->{'ctime'}) ||
#				$origFileList{$path}->{$item}->{'ctime'} ne $ctime
#		) {
#			printLog(LOG_DEBUG,"S: File '[$source]/($path)/$item' was recreated\n");
#			$state |= ST_FILE_CHANGED;

		# Check if mtime matches
		} elsif (
				!defined($origFileList{$path}->{$item}->{'mtime'}) ||
				$origFileList{$path}->{$item}->{'mtime'} ne $mtime
		) {
			printLog(LOG_DEBUG,"S: File '[$source]/($path)/$item' was modified\n");
			$state |= ST_FILE_CHANGED;

		# Check if uid matches
		} elsif (
				!defined($origFileList{$path}->{$item}->{'uid'}) ||
				$origFileList{$path}->{$item}->{'uid'} ne $uid
		) {
			printLog(LOG_DEBUG,"S: File '[$source]/($path)/$item' UID was changed\n");
			$state |= ST_FILE_ATTR_CHANGED

		# Check if gid matches
		} elsif (
				!defined($origFileList{$path}->{$item}->{'gid'}) ||
				$origFileList{$path}->{$item}->{'gid'} ne $gid
		) {
			printLog(LOG_DEBUG,"S: File '[$source]/($path)/$item' GID was changed\n");
			$state |= ST_FILE_ATTR_CHANGED;

		# Check if mode matches
		} elsif (
				!defined($origFileList{$path}->{$item}->{'mode'}) ||
				$origFileList{$path}->{$item}->{'mode'} ne $mode
		) {
			printLog(LOG_DEBUG,"S: File '[$source]/($path)/$item' MODE was changed\n");
			$state |= ST_FILE_ATTR_CHANGED;
		}

		# Update state of path, this is used below, the $state is used to track WHAT changed
		if ($state) {
			$doBackup{$path} |= ST_FILE_CHANGED;
		}

		# Record details...
		$srcFileList{$path}->{$item}->{'size'} = $size;
		$srcFileList{$path}->{$item}->{'ctime'} = $ctime;
		$srcFileList{$path}->{$item}->{'atime'} = $atime;
		$srcFileList{$path}->{$item}->{'mtime'} = $mtime;
		$srcFileList{$path}->{$item}->{'uid'} = $uid;
		$srcFileList{$path}->{$item}->{'gid'} = $gid;
		$srcFileList{$path}->{$item}->{'mode'} = $mode;
		$srcFileList{$path}->{$item}->{'_state'} = $state;
	}


	# Time to do stuff
	sub backup_postprocess {
		# Strip first part of the path
		(my $path = $File::Find::dir) =~ s#^$source/?##;
		# Get current lists
		my $curDirList = $srcDirList{$path};
		my $curFileList = $srcFileList{$path};

		# Compare the two file lists, we know what was added, lets see if anything was removed
		foreach my $fname (keys %{$origFileList{$path}}) {
			if (!defined($srcFileList{$path}->{$fname})) {
				printLog(LOG_DEBUG,"File '[$source]/($path)/$fname' removed\n");
				$doBackup{$path} |= ST_FILE_CHANGED;
			}
		}

		# Create backup dir
		if (($doBackup{$path} & ST_DIR_NEW) == ST_DIR_NEW && ! -d "$dest/$path") {
			mkpath("$dest/$path");
		}

		# NK:
		# Before we do anything, we copy our previous attributes.
		# If we do not write an archive below, we still have our stuff.
		# If there is something special, we still have it. Best idea? right?
		foreach my $item (keys %{$origPathAttribs{$path}}) {
			$newPathAttribs{$path}->{$item} = $origPathAttribs{$path}->{$item};
		}

		# Check if we need to backup this dir
		if (($doBackup{$path} & ST_FILE_CHANGED) == ST_FILE_CHANGED) {
			printLog(LOG_INFO,"B: Path '[$source]/$path' backing up\n");

			# If we not backing up a system dir, we create a manifest and backup the files
			if (($doBackup{$path} & ST_DIR_SYS) != ST_DIR_SYS) {
				# Did file contents change?
				my $changedFiles = 0;
				my $changedAttrs = 0;
				my @newFiles = ();
				my @manifestList;

				# Take a look quick...
				foreach my $item (sort keys %{$curFileList}) {
					# Lets see whats happened...
					if (($curFileList->{$item}->{'_state'} & ST_FILE_NEW) == ST_FILE_NEW) {
						push(@newFiles,$item);
					}
					if (($curFileList->{$item}->{'_state'} & ST_FILE_CHANGED) == ST_FILE_CHANGED) {
						$changedFiles = 1;
					}
					if (($curFileList->{$item}->{'_state'} & ST_FILE_ATTR_CHANGED) == ST_FILE_ATTR_CHANGED) {
						$changedAttrs = 1;
					}
				}

				# If we don't have a sequence, this is a new backup?
				if (!defined($newPathAttribs{$path}->{'sequence'} = $origPathAttribs{$path}->{'sequence'})) {
					$newPathAttribs{$path}->{'sequence'} = 0;
					@manifestList = keys %{$curFileList};
				} else {
					# If this is new files, then just inc the counter
					if (@newFiles > 0 && !$changedFiles && !$changedAttrs) {
						$newPathAttribs{$path}->{'sequence'}++;
						@manifestList = @newFiles;
					# If files changed we blow everything away
					} else {
						$newPathAttribs{$path}->{'sequence'} = 0;
						# Remove old files
						removeBackups(
								"$dest/$path",
								$origPathAttribs{$path}->{'format'},
								$origPathAttribs{$path}->{'compression'},
								$origPathAttribs{$path}->{'sequence'}
						);
						@manifestList = keys %{$curFileList};
					}
				}


				# Get current sequence
				my $seq = $newPathAttribs{$path}->{'sequence'};

				# Setup tar args
				my @tarArgs = ();
				my $tarExt;
				if ($config{'compress'} eq "xz") {
					push(@tarArgs,"--use-compress-program",$config{'compress-xz'});
					$tarExt = ".xz";
				} elsif ($config{'compress'} eq "bzip2") {
					push(@tarArgs,"--use-compress-program",$config{'compress-bzip2'});
					$tarExt = ".bz2";
				} elsif ($config{'compress'} eq "gzip") {
					push(@tarArgs,"--use-compress-program",$config{'compress-gzip'});
					$tarExt = ".gz";
				} elsif ($config{'compress'} eq "lz") {
					push(@tarArgs,"--use-compress-program",$config{'compress-lz'});
					$tarExt = ".lz";
				} elsif ($config{'compress'} eq "none") {
					$tarExt = "";
				}

				# Save format and compression
				$newPathAttribs{$path}->{'format'} = "tar";
				$newPathAttribs{$path}->{'compression'} = $config{'compress'};

				# Sort out manifest format
				my $manifestDelim;
				# Are we overriding the manifest format?
				if (defined($config{'manifest-format'})) {
					$newPathAttribs{$path}->{'manifest.format'} = $config{'manifest-format'};
				} else {
					# If not choose best
					if (defined($origPathAttribs{$path}->{'manifest.format'})) {
						$newPathAttribs{$path}->{'manifest.format'} = $origPathAttribs{$path}->{'manifest.format'};
					} else {
						# Tar pre-1.22 is buggered with \0 manifest files
						if ($tarVer > 122) {
							$newPathAttribs{$path}->{'manifest.format'} = "null";
						} else {
							$newPathAttribs{$path}->{'manifest.format'} = "newline";
						}
					}
				}

				# Setup manifest delim & tar options
				if ($newPathAttribs{$path}->{'manifest.format'} eq "null") {
					push(@tarArgs,"--null");
					push(@tarArgs,"--files-from", "$dest/$path/dbackup$seq.manifest");
					push(@tarArgs,"--no-unquote");
					$manifestDelim = "\0";
				} elsif ($newPathAttribs{$path}->{'manifest.format'} eq "newline") {
					push(@tarArgs,"--files-from", "$dest/$path/dbackup$seq.manifest");
					$manifestDelim = "\n";
				} else {
					printLog(LOG_ERROR,"Invalid manifest.format '".$newPathAttribs{$path}->{'manifest.format'}."'\n");
					exit 1;
				}

				# File list to backup
				open(my $fh,">","$dest/$path/dbackup$seq.manifest")
						or die "Failed to open '[$dest]/($path)/dbackup$seq.manifest': $!";
				foreach my $item (@manifestList) {
					# Sanity check to see if this filename is going to work
					if ($manifestDelim eq "\n" && $item =~ /[\n"'\\]/) {
						printLog(LOG_ERROR,"ERROR: Your version of tar cannot backup => path:'$path', file '$item'\n");
						next;
					}
					# If this is not the top dir, use /
					if ($path ne "") {
						print($fh "$path/$item$manifestDelim");
					# If it is, don't use /
					} else {
						print($fh "$item$manifestDelim");
					}
				}
				close($fh);

				# tar specifics...
				if ($newPathAttribs{$path}->{'format'} eq "tar") {
					# Exclude paths for tar-ignore-failed-read
					foreach my $item (@{$config{'tar-ignore-failed-read'}}) {
						# Check...
						if ("/$path" eq $item) {
							printLog(LOG_NOTICE,"B: Path '[$source]/$path' ignore-failed-read passed to tar\n");
							push(@tarArgs,"--ignore-failed-read",$path);
						}
					}
				}

				# Change dir and start backup
				system(
					"tar",
					"--create",
					"--verbose",
					# Output file
					"--file", "$dest/$path/dbackup$seq.tar$tarExt",
					# cd into here first...
					"--directory", "$source",
					# Do not recurse
					"--no-recursion",
					# Output to this file
					"--index-file", "$dest/$path/dbackup$seq.index",
					# Incremental options
#					"--listed-incremental", "$dest/$path/dbackup.snar",
#					"--no-check-device",
					@tarArgs
				);
				if ($? == -1) {
					printLog(LOG_ERROR,"Failed to execute: $!\n");
					exit 1;
				} elsif ($? & 127) {
					printLog(LOG_ERROR,"Child died with signal ".($? & 127)."\n");
					exit 1;
				} else {
					my $retcode = $? >> 8;
					# If tar died, lets die too
					if ($retcode >= 2) {
						printLog(LOG_ERROR,"tar died with error code $retcode\n");
#						exit 1;
					}
				}
			} else { # if (($doBackup{$path} & ST_DIR_SYS) != ST_DIR_SYS) {
					# Remove old files
					removeBackups(
							"$dest/$path",
							$origPathAttribs{$path}->{'format'},
							$origPathAttribs{$path}->{'compression'},
							$origPathAttribs{$path}->{'sequence'}
					);
			}
		} # if (($doBackup{$path} & ST_FILE_CHANGED) == ST_FILE_CHANGED) {


		# Check if we need to record directory changes...
		if (
					($doBackup{$path} & ST_FILE_CHANGED) == ST_FILE_CHANGED ||
					($doBackup{$path} & ST_DIR_CHANGED) == ST_DIR_CHANGED ||
					($doBackup{$path} & ST_DIR_NEW) == ST_DIR_NEW
		) {
			# If we have a revision, bump it
			if (defined($newPathAttribs{$path}->{'revision'} = $origPathAttribs{$path}->{'revision'})) {
				$newPathAttribs{$path}->{'revision'} = $origPathAttribs{$path}->{'revision'} + 1;
			} else {
				$newPathAttribs{$path}->{'revision'} = 0;
			}

			# Set this as the dbackup that created this file
			$newPathAttribs{$path}->{'dbackup.version'} = $VERSION;

			printLog(LOG_DEBUG,"B: Writing state '[$source]/($path)' - revision ".$newPathAttribs{$path}->{'revision'}."\n");

			# Write out state, we do this AFTER we have backed up, so if we stopped, we can just  continue
			my $gz = gzopen("$dest/$path/.dbackup-state","wb")
					or die "Failed to open '[$dest]/($path)/.dbackup-state': $!";
			# Loop with attributes
			foreach my $item (keys %{$newPathAttribs{$path}}) {
				$gz->gzwrite(join("\0",
						"a",
						$item,
						$newPathAttribs{$path}->{$item}
					)."\n"
				);
			}
			# Loop with directory list
			foreach my $item (sort keys %{$curDirList}) {
				my $ename = encode_base64($item);
				$ename =~ s/\n//g;
				# Save state of source files
				$gz->gzwrite(join("\0",
						"d",
						$ename,
						$curDirList->{$item}->{'ctime'},
						$curDirList->{$item}->{'atime'},
						$curDirList->{$item}->{'mtime'},
						$curDirList->{$item}->{'uid'},
						$curDirList->{$item}->{'gid'},
						$curDirList->{$item}->{'mode'}
					)."\n"
				);
			}
			# Loop with file list
			foreach my $item (sort keys %{$curFileList}) {
				my $ename = encode_base64($item);
				$ename =~ s/\n//g;
				# Save state of source files
				$gz->gzwrite(join("\0",
						"f",
						$ename,
						$curFileList->{$item}->{'size'},
						$curFileList->{$item}->{'ctime'},
						$curFileList->{$item}->{'atime'},
						$curFileList->{$item}->{'mtime'},
						$curFileList->{$item}->{'uid'},
						$curFileList->{$item}->{'gid'},
						$curFileList->{$item}->{'mode'}