#!/usr/bin/perl -w # Copyright (c) 2010-2014, AllWorldIT # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see <http://www.gnu.org/licenses/>. # # # Author: Nigel Kukard <nkukard@lbsd.net> # TODO: # - USE incremental backups!!!! fucking awesome # # - Read in defaultSystemExcl extras from /proc/mounts # - proc, nfs, nfs4, fusectl fuse.glusterfs, sysfs, tmpfs, devtmpfs, devpts # use strict; use Compress::Zlib; use Config::IniFiles; use Cwd; use Fcntl qw (:mode); use File::Find; use File::Path qw( rmtree mkpath ); use File::Spec; use Getopt::Long; use POSIX qw (strftime lchown); use Data::Dumper; use MIME::Base64; my $VERSION = "0.0.21"; # System dirs we don't care about my @defaultSystemExcl = ("/dev","/run","/proc","/sys","/tmp","/var/tmp","/misc","/media","/mnt"); # These should be backed up separately my @defaultDataExcl = ( # We cannot backup the DB reliably by just copying files "/var/lib/mysql","/var/lib/pgsql", # Exclude amavis working files "/var/amavis/tmp/", "/var/lib/amavis/tmp/", "/var/spool/amavis/virusmails/", "/var/spool/mailman/retry", # Exclude postfix queues "/var/spool/postfix/active", "/var/spool/postfix/defer", "/var/spool/postfix/deferred", "/var/spool/postfix/public", "/var/spool/postfix/private", # Exclude squid cache "/var/spool/squid", # Yum cache "/var/cache/yum", # Apt cache "/var/cache/apt/archives", # cPanel "/home/virtfs" ); # Backup constants use constant { LOG_DEBUG => 5, LOG_INFO => 4, LOG_NOTICE => 3, LOG_WARNING => 2, LOG_ERROR => 1 }; use constant { ST_FILE_CHANGED => 1, ST_DIR_NEW => 2, ST_DIR_SYS => 4, ST_DIR_CHANGED => 8, ST_FILE_ATTR_CHANGED => 16, ST_FILE_NEW => 32, }; # Main config my %config = ( 'log-level' => LOG_NOTICE, 'tar' => 'tar', 'exclude-system' => 0, 'backup-upgrade' => 0, ); # Choose default compression method if (-x "/bin/xz" || -x "/usr/bin/xz") { $config{'compress'} = "xz"; } else { $config{'compress'} = "bzip2"; } print(STDERR "DBackup v$VERSION, Copyright (c) 2010-2014, AllWorldIT\n\n"); # Grab options my %optctl = (); GetOptions(\%optctl, "help", "config=s", "log-level=i", "tar=s", "manifest-format=s", "backup", "backup-upgrade", "compress:s", "exclude-system", "system-base=s@", "system-dir=s@", "tar-ignore-failed-read=s@", "exclude-data", "data-dir=s@", "exclude-file=s@", "exclude-path=s@", "exclude-fs-type=s@", "restore", "tar-keep-newer", "tar-keep-old-files", ); # Check for help if (defined($optctl{'help'})) { displayHelp(); exit 0; } # Check for invalid combination if (defined($optctl{'backup'}) && defined($optctl{'restore'})) { print(STDERR "ERROR: Cannot backup AND restore at the same time\n\n"); displayHelp(); exit 1; } # Check config file exists if (!defined($optctl{'config'})) { $optctl{'config'} = "/etc/dbackupc.conf"; } if (! -f $optctl{'config'}) { print(STDERR "ERROR: Configuration file '".$optctl{'config'}."' NOT found!\n"); exit 1; } # Make sure we only have 2 additional args if (@ARGV > 2 || @ARGV < 2) { print(STDERR "ERROR: Invalid number of arguments\n\n"); displayHelp(); exit 1; } # Check both dirs exist... if (! -d $ARGV[0]) { print(STDERR "ERROR: Source directory '".$ARGV[0]."' does not exist!\n\n"); exit 1; } if (! -d $ARGV[1]) { print(STDERR "ERROR: Destination directory '".$ARGV[1]."' does not exist!\n\n"); exit 1; } # Use config file, ignore case tie my %inifile, 'Config::IniFiles', ( -file => $optctl{'config'}, -nocase => 1 ) or die "Failed to open config file '".$optctl{'config'}."': ".join("\n",@Config::IniFiles::errors); # Check if we actually have something... if (defined($inifile{'backup'})) { # Loop with config items foreach my $item (keys %{$inifile{'backup'}}) { $config{$item} = $inifile{'backup'}{$item}; } } # # Process config # if (!defined($config{'system-base'})) { # Quick hack to see if we have a cmdline option if (!defined($optctl{'system-base'})) { $config{'system-base'} = ["/"]; } else { $config{'system-base'} = []; } } else { $config{'system-base'} = toArray($config{'system-base'}); } if (!defined($config{'system-dir'})) { $config{'system-dir'} = []; } else { $config{'system-dir'} = toArray($config{'system-dir'}); } if (!defined($config{'tar-ignore-failed-read'})) { $config{'tar-ignore-failed-read'} = []; } else { $config{'tar-ignore-failed-read'} = toArray($config{'tar-ignore-failed-read'}); } if (!defined($config{'data-dir'})) { $config{'data-dir'} = []; } else { $config{'data-dir'} = toArray($config{'data-dir'}); } if (!defined($config{'exclude-file'})) { $config{'exclude-file'} = []; } else { $config{'exclude-file'} = toArray($config{'exclude-file'}); } if (!defined($config{'exclude-path'})) { $config{'exclude-path'} = []; } else { $config{'exclude-path'} = toArray($config{'exclude-path'}); } if (!defined($config{'exclude-fs-type'})) { $config{'exclude-fs-type'} = []; } else { $config{'exclude-fs-type'} = toArray($config{'exclude-fs-type'}); } # # Check commandline options # if (defined($optctl{'backup-upgrade'})) { $config{'backup-upgrade'} = 1; } if (defined($optctl{'compress'})) { # Why use --compress with --restore? if (defined($optctl{'restore'})) { print(STDERR "WARNING: The use of --compress with --restore does not make sense\n"); } # Make sure its valid if ( $optctl{'compress'} ne "xz" && $optctl{'compress'} ne "bzip2" && $optctl{'compress'} ne "gzip" && $optctl{'compress'} ne "none" ) { print(STDERR "ERROR: Compression is invalid, valid values are: xz bzip2 gzip none\n\n"); displayHelp(); exit 1; } $config{'compress'} = $optctl{'compress'}; } if (defined($optctl{'tar'})) { # Check if tar is executable if ( -x $optctl{'tar'}) { $config{'tar'} = $optctl{'tar'}; } else { print(STDERR "ERROR: tar '".$optctl{'tar'}."' does not exist or is not executable\n"); } } my $tarVer = getTarVer($config{'tar'}); if (defined($optctl{'log-level'})) { $config{'log-level'} = $optctl{'log-level'}; } if (defined($optctl{'manifest-format'})) { if ( $optctl{'manifest-format'} ne "null" && $optctl{'manifest-format'} ne "newline" ) { print(STDERR "ERROR: Manifest format is invalid, valid values are: null newline\n\n"); displayHelp(); exit 1; } $config{'manifest-format'} = $optctl{'manifest-format'}; } if (defined($optctl{'exclude-system'})) { $config{'exclude-system'} = 1; } if (defined($optctl{'system-base'})) { push(@{$config{'system-base'}},@{$optctl{'system-base'}}); } if (defined($optctl{'system-dir'})) { push(@{$config{'system-dir'}},@{$optctl{'system-dir'}}); } if (defined($optctl{'tar-ignore-failed-read'})) { push(@{$config{'tar-ignore-failed-read'}},@{$optctl{'tar-ignore-failed-read'}}); } if (defined($optctl{'data-dir'})) { push(@{$config{'data-dir'}},@{$optctl{'data-dir'}}); } if (defined($optctl{'exclude-file'})) { push(@{$config{'exclude-file'}},@{$optctl{'exclude-file'}}); } if (defined($optctl{'exclude-path'})) { push(@{$config{'exclude-path'}},@{$optctl{'exclude-path'}}); } if (defined($optctl{'exclude-fs-type'})) { push(@{$config{'exclude-fs-type'}},@{$optctl{'exclude-fs-type'}}); } if (@{$config{'exclude-fs-type'}} > 0) { # Pull in mounts open(MOUNTS, "< /proc/mounts") or die "ERROR: Failed to open '/proc/mounts': $!"; while (my $line = <MOUNTS>) { chomp($line); # Split off items we need my (undef,$path,$type) = split(/\s+/,$line); # Loop with types and check foreach my $item (@{$config{'exclude-fs-type'}}) { if ($item eq $type) { push(@{$config{'exclude-path'}},$path); } } } close(MOUNTS); } # Sanitize the source and dest my $sourceDir = File::Spec->rel2abs($ARGV[0]); my $destDir = File::Spec->rel2abs($ARGV[1]); # Backup function sub backup { our ($source,$dest) = @_; # State infor for current dir our %doBackup; our %srcFileList; our %srcDirList; # Original .state file read in during backup our %origFileList; our %origDirList; our %origPathAttribs; # New attributes our %newPathAttribs; # Preprocess dir sub backup_preprocess { my @list = @_; # Strip first part of the path (my $path = $File::Find::dir) =~ s#^$source/?##; # Check if this dir should be backed up or not.... if ( -f "$source/$path/.nodbackup" ) { printLog(LOG_INFO,"S: Path '[$source]/($path)' EXCLUDED from backup with .nodbackup\n"); return (); } # Exclude trash folders if ( "$source/$path" =~ /(\.Trash-[0-9]{4})$/) { printLog(LOG_INFO,"S: Path '[$source]/($path)' EXCLUDED from backup, Linux trash folder\n"); return (); } # Make sure even if we processing empty dirs that we actually record it if (!defined($srcFileList{$path})) { # Just set to empty hash ref $srcFileList{$path} = {}; } # Exclude system directories if ($config{'exclude-system'}) { # Loop through system dirs foreach my $sysdir (@{$config{'system-base'}}) { # Loop with system paths foreach my $excl (@defaultSystemExcl,@{$config{'system-dir'}}) { # Sanitize path my $testPath = File::Spec->rel2abs("$sysdir/$excl"); # Check... if ("/$path" eq $testPath) { printLog(LOG_INFO,"S: Path '[$source]/($path)' is a system directory, ignoring files\n"); $doBackup{$path} |= ST_DIR_SYS; return (); } } # Check if we excluding data dirs if (defined($config{'exclude-data'})) { # Loop with data dirs foreach my $excl (@defaultDataExcl,@{$config{'data-dir'}}) { # Sanitize path my $testPath = File::Spec->rel2abs("$sysdir/$excl"); # Check... if ("/$path" eq $testPath) { printLog(LOG_INFO,"S: Path '[$source]/($path)' is a data directory, ignoring files\n"); $doBackup{$path} |= ST_DIR_SYS; return (); } } } } } # Exclude paths foreach my $item (@{$config{'exclude-path'}}) { # Check... if ("/$path" =~ $item) { printLog(LOG_INFO,"S: Path '[$source]/($path)' is an excluded path, ignoring files\n"); $doBackup{$path} |= ST_DIR_SYS; return (); } } # Check if this dir exists on the backup, if not we need to back it up obviously... if ( ! -d "$dest/$path") { $doBackup{$path} |= ST_DIR_NEW; # This can probably only be because a backup was continued } elsif ( ! -f "$dest/$path/.dbackup-state" ) { # First backup will never have the main dir .dbackup-state file existing if ($path ne "") { printLog(LOG_WARNING,"State file '[$dest]/($path)/.dbackup-state' does not exist\n"); } $doBackup{$path} |= ST_DIR_NEW; } # Cannot read in for new dirs can we... if (($doBackup{$path} & ST_DIR_NEW) != ST_DIR_NEW) { # Load state file loadStateFile("$dest/$path/.dbackup-state",$path,\%origFileList,\%origDirList,\%origPathAttribs); # Check we have at least the version attribute if (!defined($origPathAttribs{$path}) || !defined($origPathAttribs{$path}->{'dbackup.version'})) { printLog(LOG_ERROR,"No dbackup version information found in '[$dest]/($path)/.dbackup-state', IGNORING\n"); delete($origDirList{$path}); delete($origFileList{$path}); # NK: # Sanity check, some versions created empty'ish .dbackup-state files with no sequence < 2012-01-21 # If we have a manifest, we should have a backup, therefore a format and sequence } elsif ( -f "$dest/$path/dbackup0.manifest" && ( !defined($origPathAttribs{$path}->{'format'}) || !defined($origPathAttribs{$path}->{'sequence'}) ) ) { printLog(LOG_ERROR,"No dbackup format/sequence information found in ". "'[$dest]/($path)/.dbackup-state', IGNORING\n"); delete($origDirList{$path}); delete($origFileList{$path}); # Check if we going to upgrade... } else { # Are we doing backup upgrades? if ($config{'backup-upgrade'}) { # Pull in versions my $a = getNumericVer($origPathAttribs{$path}->{'dbackup.version'}); my $b = getNumericVer($VERSION); # Compare if ( $a < 16 # Things before 0.0.16 may have .dbackup-state issues and missing # required info. ) { # Check if end in .x if ($VERSION =~ /\.x$/) { printLog(LOG_WARNING,"S: Path '[$source]/($path)' ". "cannot be upgraded, dbackup is TRUNK version\n"); } else { printLog(LOG_INFO,"S: Path '[$source]/($path)' will be backed up, upgrading\n"); delete($origDirList{$path}); delete($origFileList{$path}); # Force state file update if dir is empty $doBackup{$path} |= ST_DIR_CHANGED; } } } } } # Apply filter... if ((@{$config{'exclude-file'}}) > 0) { my @newList; foreach my $item (@list) { my $match = 0; # Apply only to files if ( -f "$source/$path/$item") { # Loop with exclude filters foreach my $filter (@{$config{'exclude-file'}}) { # Check for match if ($item =~ $filter) { printLog(LOG_INFO,"S: Path '[$source]/($path)/$item' matches file exclude filter, ignoring\n"); # XXX: NK - If this ws backed up before and now not, dir has changed? $match = 1; last; } } } # If not matched, add to new list if (!$match) { push(@newList,$item); } } # Replace the list now @list = @newList; } return @list; } # Process files sub backup_process { my $name = $File::Find::name; # Strip first part of the path (my $path = $File::Find::dir) =~ s#^$source/?##; # Store current filename my $item = $_; # We use lstat so we don't follow symlinks that don't exist my @stat = lstat($name); # Split it up my ($dev,$ino,$mode,$nlink,$uid,$gid,$rdev,$size,$atime,$mtime, $ctime,$blksize,$blocks) = @stat; # Process directory if (S_ISDIR($mode)) { # Skip over main dir if ($item eq ".") { return; } # Make sure sub dirs have a santized name my $ppath = ($path ne "") ? "$path/$item" : $item; # Set default flags to nothing... $doBackup{$ppath} = 0; # Ok, lets see whats going on with the dir if (!defined($origDirList{$path}->{$item})) { printLog(LOG_DEBUG,"S: Directory '[$source]/($path)/$item' is new\n"); $doBackup{$path} |= ST_DIR_CHANGED; # Check if mtime matches } elsif ( !defined($origDirList{$path}->{$item}->{'mtime'}) || $origDirList{$path}->{$item}->{'mtime'} ne $mtime ) { printLog(LOG_DEBUG,"S: Directory '[$source]/($path)/$item' was modified\n"); $doBackup{$path} |= ST_DIR_CHANGED; # Check if uid matches } elsif ( !defined($origDirList{$path}->{$item}->{'uid'}) || $origDirList{$path}->{$item}->{'uid'} ne $uid ) { printLog(LOG_DEBUG,"S: Directory '[$source]/($path)/$item' UID was changed\n"); $doBackup{$path} |= ST_DIR_CHANGED; # Check if gid matches } elsif ( !defined($origDirList{$path}->{$item}->{'gid'}) || $origDirList{$path}->{$item}->{'gid'} ne $gid ) { printLog(LOG_DEBUG,"S: Directory '[$source]/($path)/$item' GID was changed\n"); $doBackup{$path} |= ST_DIR_CHANGED; # Check if mode matches } elsif ( !defined($origDirList{$path}->{$item}->{'mode'}) || $origDirList{$path}->{$item}->{'mode'} ne $mode ) { printLog(LOG_DEBUG,"S: Directory '[$source]/($path)/$item' MODE was changed\n"); $doBackup{$path} |= ST_DIR_CHANGED; } # Record details... $srcDirList{$path}->{$item}->{'size'} = $size; $srcDirList{$path}->{$item}->{'ctime'} = $ctime; $srcDirList{$path}->{$item}->{'atime'} = $atime; $srcDirList{$path}->{$item}->{'mtime'} = $mtime; $srcDirList{$path}->{$item}->{'uid'} = $uid; $srcDirList{$path}->{$item}->{'gid'} = $gid; $srcDirList{$path}->{$item}->{'mode'} = $mode; return; # Check if we're a socket } elsif (S_ISSOCK($mode)) { # Blatently ignore... return; } # Initialize file state my $state = 0; # Check if file does not exists on backup if (!defined($origFileList{$path}->{$item})) { if (($doBackup{$path} & ST_DIR_NEW) != ST_DIR_NEW) { printLog(LOG_DEBUG,"S: File '[$source]/($path)/$item' is new\n"); } $state |= ST_FILE_NEW; # Check if size matches } elsif ( !defined($origFileList{$path}->{$item}->{'size'}) || $origFileList{$path}->{$item}->{'size'} ne $size ) { printLog(LOG_DEBUG,"S: File '[$source]/($path)/$item' has different size\n"); $state |= ST_FILE_CHANGED; # # NK: Does not like chmod changes to files??? # # Check if ctime matches # } elsif ( # !defined($origFileList{$path}->{$item}->{'ctime'}) || # $origFileList{$path}->{$item}->{'ctime'} ne $ctime # ) { # printLog(LOG_DEBUG,"S: File '[$source]/($path)/$item' was recreated\n"); # $state |= ST_FILE_CHANGED; # Check if mtime matches } elsif ( !defined($origFileList{$path}->{$item}->{'mtime'}) || $origFileList{$path}->{$item}->{'mtime'} ne $mtime ) { printLog(LOG_DEBUG,"S: File '[$source]/($path)/$item' was modified\n"); $state |= ST_FILE_CHANGED; # Check if uid matches } elsif ( !defined($origFileList{$path}->{$item}->{'uid'}) || $origFileList{$path}->{$item}->{'uid'} ne $uid ) { printLog(LOG_DEBUG,"S: File '[$source]/($path)/$item' UID was changed\n"); $state |= ST_FILE_ATTR_CHANGED # Check if gid matches } elsif ( !defined($origFileList{$path}->{$item}->{'gid'}) || $origFileList{$path}->{$item}->{'gid'} ne $gid ) { printLog(LOG_DEBUG,"S: File '[$source]/($path)/$item' GID was changed\n"); $state |= ST_FILE_ATTR_CHANGED; # Check if mode matches } elsif ( !defined($origFileList{$path}->{$item}->{'mode'}) || $origFileList{$path}->{$item}->{'mode'} ne $mode ) { printLog(LOG_DEBUG,"S: File '[$source]/($path)/$item' MODE was changed\n"); $state |= ST_FILE_ATTR_CHANGED; } # Update state of path, this is used below, the $state is used to track WHAT changed if ($state) { $doBackup{$path} |= ST_FILE_CHANGED; } # Record details... $srcFileList{$path}->{$item}->{'size'} = $size; $srcFileList{$path}->{$item}->{'ctime'} = $ctime; $srcFileList{$path}->{$item}->{'atime'} = $atime; $srcFileList{$path}->{$item}->{'mtime'} = $mtime; $srcFileList{$path}->{$item}->{'uid'} = $uid; $srcFileList{$path}->{$item}->{'gid'} = $gid; $srcFileList{$path}->{$item}->{'mode'} = $mode; $srcFileList{$path}->{$item}->{'_state'} = $state; } # Time to do stuff sub backup_postprocess { # Strip first part of the path (my $path = $File::Find::dir) =~ s#^$source/?##; # Get current lists my $curDirList = $srcDirList{$path}; my $curFileList = $srcFileList{$path}; # Compare the two file lists, we know what was added, lets see if anything was removed foreach my $fname (keys %{$origFileList{$path}}) { if (!defined($srcFileList{$path}->{$fname})) { printLog(LOG_DEBUG,"File '[$source]/($path)/$fname' removed\n"); $doBackup{$path} |= ST_FILE_CHANGED; } } # Create backup dir if (($doBackup{$path} & ST_DIR_NEW) == ST_DIR_NEW && ! -d "$dest/$path") { mkpath("$dest/$path"); } # NK: # Before we do anything, we copy our previous attributes. # If we do not write an archive below, we still have our stuff. # If there is something special, we still have it. Best idea? right? foreach my $item (keys %{$origPathAttribs{$path}}) { $newPathAttribs{$path}->{$item} = $origPathAttribs{$path}->{$item}; } # Check if we need to backup this dir if (($doBackup{$path} & ST_FILE_CHANGED) == ST_FILE_CHANGED) { printLog(LOG_INFO,"B: Path '[$source]/$path' backing up\n"); # If we not backing up a system dir, we create a manifest and backup the files if (($doBackup{$path} & ST_DIR_SYS) != ST_DIR_SYS) { # Did file contents change? my $changedFiles = 0; my $changedAttrs = 0; my @newFiles = (); my @manifestList; # Take a look quick... foreach my $item (sort keys %{$curFileList}) { # Lets see whats happened... if (($curFileList->{$item}->{'_state'} & ST_FILE_NEW) == ST_FILE_NEW) { push(@newFiles,$item); } if (($curFileList->{$item}->{'_state'} & ST_FILE_CHANGED) == ST_FILE_CHANGED) { $changedFiles = 1; } if (($curFileList->{$item}->{'_state'} & ST_FILE_ATTR_CHANGED) == ST_FILE_ATTR_CHANGED) { $changedAttrs = 1; } } # If we don't have a sequence, this is a new backup? if (!defined($newPathAttribs{$path}->{'sequence'} = $origPathAttribs{$path}->{'sequence'})) { $newPathAttribs{$path}->{'sequence'} = 0; @manifestList = keys %{$curFileList}; } else { # If this is new files, then just inc the counter if (@newFiles > 0 && !$changedFiles && !$changedAttrs) { $newPathAttribs{$path}->{'sequence'}++; @manifestList = @newFiles; # If files changed we blow everything away } else { $newPathAttribs{$path}->{'sequence'} = 0; # Remove old files removeBackups( "$dest/$path", $origPathAttribs{$path}->{'format'}, $origPathAttribs{$path}->{'compression'}, $origPathAttribs{$path}->{'sequence'} ); @manifestList = keys %{$curFileList}; } } # Get current sequence my $seq = $newPathAttribs{$path}->{'sequence'}; # Setup tar args my @tarArgs = (); my $tarExt; if ($config{'compress'} eq "xz") { push(@tarArgs,"--xz"); $tarExt = ".xz"; } elsif ($config{'compress'} eq "bzip2") { push(@tarArgs,"--bzip2"); $tarExt = ".bz2"; } elsif ($config{'compress'} eq "gzip") { push(@tarArgs,"--gzip"); $tarExt = ".gz"; } elsif ($config{'compress'} eq "none") { $tarExt = ""; } # Save format and compression $newPathAttribs{$path}->{'format'} = "tar"; $newPathAttribs{$path}->{'compression'} = $config{'compress'}; # Sort out manifest format my $manifestDelim; # Are we overriding the manifest format? if (defined($config{'manifest-format'})) { $newPathAttribs{$path}->{'manifest.format'} = $config{'manifest-format'}; } else { # If not choose best if (defined($origPathAttribs{$path}->{'manifest.format'})) { $newPathAttribs{$path}->{'manifest.format'} = $origPathAttribs{$path}->{'manifest.format'}; } else { # Tar pre-1.22 is buggered with \0 manifest files if ($tarVer > 122) { $newPathAttribs{$path}->{'manifest.format'} = "null"; } else { $newPathAttribs{$path}->{'manifest.format'} = "newline"; } } } # Setup manifest delim & tar options if ($newPathAttribs{$path}->{'manifest.format'} eq "null") { push(@tarArgs,"--null"); push(@tarArgs,"--files-from", "$dest/$path/dbackup$seq.manifest"); push(@tarArgs,"--no-unquote"); $manifestDelim = "\0"; } elsif ($newPathAttribs{$path}->{'manifest.format'} eq "newline") { push(@tarArgs,"--files-from", "$dest/$path/dbackup$seq.manifest"); $manifestDelim = "\n"; } else { printLog(LOG_ERROR,"Invalid manifest.format '".$newPathAttribs{$path}->{'manifest.format'}."'\n"); exit 1; } # File list to backup open(MANIFEST,"> $dest/$path/dbackup$seq.manifest") or die "Failed to open '[$dest]/($path)/dbackup$seq.manifest': $!"; foreach my $item (@manifestList) { # Sanity check to see if this filename is going to work if ($manifestDelim eq "\n" && $item =~ /[\n"'\\]/) { printLog(LOG_ERROR,"ERROR: Your version of tar cannot backup => path:'$path', file '$item'\n"); next; } # If this is not the top dir, use / if ($path ne "") { print(MANIFEST "$path/$item$manifestDelim"); # If it is, don't use / } else { print(MANIFEST "$item$manifestDelim"); } } close(MANIFEST); # tar specifics... if ($newPathAttribs{$path}->{'format'} eq "tar") { # Exclude paths for tar-ignore-failed-read foreach my $item (@{$config{'tar-ignore-failed-read'}}) { # Check... if ("/$path" eq $item) { printLog(LOG_NOTICE,"B: Path '[$source]/$path' ignore-failed-read passed to tar\n"); push(@tarArgs,"--ignore-failed-read",$path); } } } # Change dir and start backup system( "tar", "--create", "--verbose", # Output file "--file", "$dest/$path/dbackup$seq.tar$tarExt", # cd into here first... "--directory", "$source", # Do not recurse "--no-recursion", # Output to this file "--index-file", "$dest/$path/dbackup$seq.index", # Incremental options # "--listed-incremental", "$dest/$path/dbackup.snar", # "--no-check-device", @tarArgs ); if ($? == -1) { printLog(LOG_ERROR,"Failed to execute: $!\n"); exit 1; } elsif ($? & 127) { printLog(LOG_ERROR,"Child died with signal ".($? & 127)."\n"); exit 1; } else { my $retcode = $? >> 8; # If tar died, lets die too if ($retcode >= 2) { printLog(LOG_ERROR,"tar died with error code $retcode\n"); # exit 1; } } # if (($doBackup{$path} & ST_DIR_SYS) != ST_DIR_SYS) { } else { # Remove old files removeBackups( "$dest/$path", $origPathAttribs{$path}->{'format'}, $origPathAttribs{$path}->{'compression'}, $origPathAttribs{$path}->{'sequence'} ); } } # if (($doBackup{$path} & ST_FILE_CHANGED) == ST_FILE_CHANGED) { # Check if we need to record directory changes... if ( ($doBackup{$path} & ST_FILE_CHANGED) == ST_FILE_CHANGED || ($doBackup{$path} & ST_DIR_CHANGED) == ST_DIR_CHANGED || ($doBackup{$path} & ST_DIR_NEW) == ST_DIR_NEW ) { # If we have a revision, bump it if (defined($newPathAttribs{$path}->{'revision'} = $origPathAttribs{$path}->{'revision'})) { $newPathAttribs{$path}->{'revision'} = $origPathAttribs{$path}->{'revision'} + 1; } else { $newPathAttribs{$path}->{'revision'} = 0; } # Set this as the dbackup that created this file $newPathAttribs{$path}->{'dbackup.version'} = $VERSION; printLog(LOG_DEBUG,"B: Writing state '[$source]/($path)' - revision ".$newPathAttribs{$path}->{'revision'}."\n"); # Write out state, we do this AFTER we have backed up, so if we stopped, we can just continue my $gz = gzopen("$dest/$path/.dbackup-state","wb") or die "Failed to open '[$dest]/($path)/.dbackup-state': $!"; # Loop with attributes foreach my $item (keys %{$newPathAttribs{$path}}) { $gz->gzwrite(join("\0", "a", $item, $newPathAttribs{$path}->{$item} )."\n" ); } # Loop with directory list foreach my $item (sort keys %{$curDirList}) { my $ename = encode_base64($item); $ename =~ s/\n//g; # Save state of source files $gz->gzwrite(join("\0", "d", $ename, $curDirList->{$item}->{'ctime'}, $curDirList->{$item}->{'atime'}, $curDirList->{$item}->{'mtime'}, $curDirList->{$item}->{'uid'}, $curDirList->{$item}->{'gid'}, $curDirList->{$item}->{'mode'} )."\n" ); } # Loop with file list foreach my $item (sort keys %{$curFileList}) { my $ename = encode_base64($item); $ename =~ s/\n//g; # Save state of source files $gz->gzwrite(join("\0", "f", $ename, $curFileList->{$item}->{'size'}, $curFileList->{$item}->{'ctime'}, $curFileList->{$item}->{'atime'}, $curFileList->{$item}->{'mtime'}, $curFileList->{$item}->{'uid'}, $curFileList->{$item}->{'gid'}, $curFileList->{$item}->{'mode'} )."\n" ); } # Finally close $gz->gzclose(); } # State clear delete($origFileList{$path}); } printLog(LOG_NOTICE,"BACKUP START: $source => $dest\n"); printLog(LOG_INFO,"Compression: ".$config{'compress'}."\n"); # Check if we excluding system files if ($config{'exclude-system'}) { printLog(LOG_NOTICE,"Exclude System Base: ".join(", ",@{$config{'system-base'}})."\n"); printLog(LOG_NOTICE,"Exclude System Dirs: ".join(", ",@defaultSystemExcl,@{$config{'system-dir'}})."\n"); # If we excluding data too... if ($config{'exclude-data'}) { printLog(LOG_NOTICE,"Exclude Data Dirs: ".join(", ",@defaultDataExcl,@{$config{'data-dir'}})."\n"); } } printLog(LOG_NOTICE,"Exclude Paths: ".join(", ",@{$config{'exclude-path'}})."\n"); printLog(LOG_NOTICE,"Exclude Files: ".join(", ",@{$config{'exclude-file'}})."\n"); # We need an entry for our main dir before we start $doBackup{""} = 0; # This basically does our backup for us... find ( { preprocess => \&backup_preprocess, wanted => \&backup_process, postprocess => \&backup_postprocess }, $source ); printLog(LOG_INFO,"Processing stale directories...\n"); my @rmlist; # Total item by item find ( { wanted => sub { }, # We just need post processing ... postprocess => sub { # Strip first part of the path (my $path = $File::Find::dir) =~ s#^$dest/?##; # Does this dir exist on the backup, but not on the fileserver? if (!defined($srcFileList{$path})) { push(@rmlist,$path); } }, }, $dest ); foreach my $rmitem ( @rmlist ) { printLog(LOG_DEBUG,"Remove path '[$dest]/($rmitem)'\n"); rmtree("$dest/$rmitem"); } printLog(LOG_NOTICE,"BACKUP END\n"); } # Do backup if (defined($optctl{'backup'})) { backup($sourceDir,$destDir); } elsif (defined($optctl{'restore'})) { restore($sourceDir,$destDir); } else { print(STDERR "ERROR: No command given\n\n"); displayHelp(); exit 1; } # Restore function sub restore { our ($source,$dest) = @_; # State infor for current dir # our %doBackup; # our %srcFileList; # our %srcDirList; our %origFileList; our %origDirList; our %origPathAttribs; # Preprocess dir sub restore_preprocess { my @list = @_; # Strip first part of the path (my $path = $File::Find::dir) =~ s#^$source/?##; # Check if we have all our source files if ( ! -f "$source/$path/.dbackup-state" ) { printLog(LOG_WARNING,"S: Path '[$source]/($path)' is MISSING .dbackup-state\n"); return @list; } # Load state file loadStateFile("$source/$path/.dbackup-state",$path,\%origFileList,\%origDirList,\%origPathAttribs); # Check we have at least the version attribute if (!defined($origPathAttribs{$path}) || !defined($origPathAttribs{$path}->{'dbackup.version'})) { printLog(LOG_ERROR, "No dbackup version information found in '[$source]/($path)/.dbackup-state', IGNORING\n"); delete($origDirList{$path}); delete($origFileList{$path}); return @list; } # During our check, we need to create the dir, we may not have a backup file # so we do it here. if ( ! -d "$dest/$path") { printLog(LOG_INFO,"R: Path '[$dest]/($path)' creating directory\n"); mkpath("$dest/$path"); } # The lack of a sequence number probably means we didn't output any tar files during backup if (!defined($origPathAttribs{$path}->{'sequence'})) { return @list; } # Loop through backup sequences for (my $seq = 0; $seq <= $origPathAttribs{$path}->{'sequence'}; $seq++) { printLog(LOG_DEBUG,"R: Path '[$dest]/($path)' restoring data ".($seq+1)." of ". ($origPathAttribs{$path}->{'sequence'}+1)."\n"); # Args for tar my @tarArgs = (); my $tarExt; if ($origPathAttribs{$path}->{'compression'} eq "xz") { push(@tarArgs,"--xz"); $tarExt = ".xz"; } elsif ($origPathAttribs{$path}->{'compression'} eq "bzip2") { push(@tarArgs,"--bzip2"); $tarExt = ".bz2"; } elsif ($origPathAttribs{$path}->{'compression'} eq "gzip") { push(@tarArgs,"--gzip"); $tarExt = ".gz"; } elsif ($origPathAttribs{$path}->{'compression'} eq "none") { $tarExt = ""; } # Check what type of backup we did if ( ! -f "$source/$path/dbackup$seq.tar$tarExt" ) { # If we have any other part of the backup, show a message if ( ( -f "$source/$path/dbackup$seq.index" || -f "$source/$path/dbackup$seq.manifest" || -f "$source/$path/dbackup$seq.snar" ) && $seq > 0 ) { printLog(LOG_WARNING,"Backup file '[$source]/($path)/dbackup$seq.tar$tarExt' not found, IGNORING\n"); } return @list; } # Check we have files we need/created if ( ! -f "$source/$path/dbackup$seq.index" ) { printLog(LOG_WARNING,"Manifest file '[$source]/($path)/dbackup$seq.index' not found\n"); return @list; } if ( ! -f "$source/$path/dbackup$seq.manifest" ) { printLog(LOG_WARNING,"Manifest file '[$source]/($path)/dbackup$seq.manifest' not found\n"); return @list; } # Check if manifest.format is undefined, if it is, use the default if (!defined($origPathAttribs{$path}->{'manifest.format'})) { # Check if we have a commandline override if (defined($optctl{'manifest-format'})) { $origPathAttribs{$path}->{'manifest.format'} = $optctl{'manifest-format'}; } else { # NK: # Default to null # This saves us from having to regenerate manifest.format attrs and backups for older # versions of dbackup. $origPathAttribs{$path}->{'manifest.format'} = "null"; } } # Setup manifest delim & tar options if ($origPathAttribs{$path}->{'manifest.format'} eq "null") { push(@tarArgs,"--null"); push(@tarArgs,"--files-from", "$source/$path/dbackup$seq.manifest"); push(@tarArgs,"--no-unquote"); } elsif ($origPathAttribs{$path}->{'manifest.format'} eq "newline") { push(@tarArgs,"--files-from", "$source/$path/dbackup$seq.manifest"); } else { printLog(LOG_ERROR,"Invalid manifest.format '".$origPathAttribs{$path}->{'manifest.format'}. "' for '[$source]/($path)', try setting the default with --manifest-format=\n"); exit 1; } # Sanity check if ($tarVer < 122 && $origPathAttribs{$path}->{'manifest.format'} eq "null") { printLog(LOG_ERROR,"Your version of tar does NOT support manifest.format of 'null'\n"); exit 1; } # Other tar options if (defined($optctl{'tar-keep-newer'})) { push(@tarArgs,"--keep-newer-files"); } if (defined($optctl{'tar-keep-old-files'})) { push(@tarArgs,"--keep-old-files"); } # Change dir and start backup system( "tar", "--extract", # Output file "--file", "$source/$path/dbackup$seq.tar$tarExt", # cd into here first... "--directory", "$dest", # Do not recurse "--no-recursion", # Incremental options # "--incremental", @tarArgs ); if ($? == -1) { printLog(LOG_ERROR,"Failed to execute: $!\n"); exit 1; } elsif ($? & 127) { printLog(LOG_ERROR,"Child died with signal ".($? & 127)."\n"); exit 1; } else { my $retcode = $? >> 8; # If tar died, lets die too if ($retcode >= 2) { printLog(LOG_ERROR,"tar died with error code $retcode\n"); # exit 1; } } } return @list; } # Process files sub restore_process { my $name = $File::Find::name; # Strip first part of the path (my $path = $File::Find::dir) =~ s#^$source/?##; # Store current filename my $item = $_; # print(STDERR "PROCESS: Path = $path, Name = $name, Item = $item)\n"); } # Time to do stuff sub restore_postprocess { # Strip first part of the path (my $path = $File::Find::dir) =~ s#^$source/?##; printLog(LOG_DEBUG,"R: Path '[$dest]/($path)' restoring meta-data\n"); # Loop with directories in this path foreach my $dname (keys %{$origDirList{$path}}) { # Set dir my $dir = $origDirList{$path}->{$dname}; # Full dirname my $fdirname = "$dest/$path/$dname"; # Could be a dir with no contents of one which was ignored, in which case we must create it if (! -d $fdirname) { printLog(LOG_INFO,"R: Path '[$dest]/($path)/$dname' creating missing directory\n"); mkpath($fdirname); } # Restore attribs if (!chown($dir->{'uid'},$dir->{'gid'},$fdirname)) { printLog(LOG_ERROR,"Failed to chown(".$dir->{'uid'}.",".$dir->{'gid'}.") '$fdirname': $!\n"); } if (!chmod($dir->{'mode'},$fdirname)) { printLog(LOG_ERROR,"Failed to chmod(".$dir->{'mode'}.") '$fdirname': $!\n"); } if (!utime($dir->{'atime'},$dir->{'mtime'},$fdirname)) { printLog(LOG_ERROR,"Failed to utime(".$dir->{'atime'}.",".$dir->{'mtime'}.") '$fdirname': $!\n"); } } # Loop with files in this path foreach my $fname (keys %{$origFileList{$path}}) { # Set dir my $file = $origFileList{$path}->{$fname}; # Full dirname my $ffilename = "$dest/$path/$fname"; # Restore for links and files if (!lchown($file->{'uid'},$file->{'gid'},$ffilename)) { printLog(LOG_ERROR,"Failed to lchown(".$file->{'uid'}.",".$file->{'gid'}.") '$ffilename': $!\n"); } # Ignore links for the rest... next if (S_ISLNK($file->{'mode'})); # Restore mode & utime only for files if (!chmod($file->{'mode'},$ffilename)) { printLog(LOG_ERROR,"Failed to chmod(".$file->{'mode'}.") '$ffilename': $!\n"); } if (!utime($file->{'atime'},$file->{'mtime'},$ffilename)) { printLog(LOG_ERROR,"Failed to utime(".$file->{'atime'}.",".$file->{'mtime'}.") '$ffilename': $!\n"); } } } printLog(LOG_NOTICE,"RESTORE START: $source => $dest\n"); # This basically does our backup for us... find ( { preprocess => \&restore_preprocess, wanted => \&restore_process, postprocess => \&restore_postprocess }, $source ); printLog(LOG_NOTICE,"RESTORE END\n"); } # Display log line sub printLog { my ($level,$msg,@args) = @_; # Work out level txt all nicely my $levelTxt = "UNKNOWN"; if ($level == LOG_DEBUG) { $levelTxt = "DEBUG"; } elsif ($level == LOG_INFO) { $levelTxt = "INFO"; } elsif ($level == LOG_NOTICE) { $levelTxt = "NOTICE"; } elsif ($level == LOG_WARNING) { $levelTxt = "WARNING"; } elsif ($level == LOG_ERROR) { $levelTxt = "ERROR"; } # Check log level if ($level <= $config{'log-level'}) { printf(STDERR "%s/$levelTxt: %s", strftime('%F %T',localtime()), $msg, @args); } } # Display usage sub displayHelp { # Build some strings we need my $systemDirStr = join (", ",@defaultSystemExcl); my $dataDirStr = join (", ",@defaultDataExcl); print(STDERR<<EOF); Usage: $0 [args] <src> <dst> General Options: --help What you seeing now. --config=file Config file to use. --log-level 5 = debug, 4 = info, 3 = notice 2 = warning, 1 = error --tar Path to tar binary. Backing Options: --backup Backup src to dst. --backup-upgrade Upgrade backup to new dbackup ver. --compress=<xz|bz2|gzip|none> Compression method to use defaults to using xz, or bzip2 if xz unavail. --exclude-data Exclude all data dirs listed below. --exclude-system Exclude system dirs listed below. --exclude-path=pcre PCRE to exclude paths from backup. --exclude-file=pcre PCRE to exclude files from backup. --exclude-fs-type=fstype Filesystem type to exclude. --data-dir=dir Add an additional data directory. --system-dir=dir Add an additional system directory. --system-base=path Add a system base. This defaults to / and this option will override that. --tar-ignore-failed-read=path This is passed to tar only. It will not cause errors of files that cannot be read for the path matched. Restore Options: --restore Restore src to dst. --tar-keep-newer Do not overwrite newer files. --tar-keep-old-files Don't replace existing files. Builtins: --------- system-dirs: $systemDirStr data-dirs: $dataDirStr EOF } # # LIBRARY # sub loadStateFile { my ($statefile,$key,$fileList,$dirList,$pathAttribs) = @_; # Read in backup state file, we do it here because we may not have a backup file # to restore my $gz = gzopen($statefile,"rb") or die "FATAL ERROR: Failed to open '$statefile': $!"; # Read in file list my $line; while ($gz->gzreadline($line) > 0) { chomp($line); # Pull out array of items my @aline = split(/\0/,$line); my $type = shift(@aline); # Check if its defined if (defined($type)) { # If its a file... if ($type eq "f") { my ($ename,$size,$ctime,$atime,$mtime,$uid,$gid,$mode) = @aline; my $name = decode_base64($ename); # Setup our hash $fileList->{$key}->{$name}->{'size'} = $size; $fileList->{$key}->{$name}->{'ctime'} = $ctime; $fileList->{$key}->{$name}->{'atime'} = $atime; $fileList->{$key}->{$name}->{'mtime'} = $mtime; $fileList->{$key}->{$name}->{'uid'} = $uid; $fileList->{$key}->{$name}->{'gid'} = $gid; $fileList->{$key}->{$name}->{'mode'} = $mode; # If its a dir } elsif ($type eq "d") { my ($ename,$ctime,$atime,$mtime,$uid,$gid,$mode) = @aline; my $name = decode_base64($ename); # Setup our hash $dirList->{$key}->{$name}->{'ctime'} = $ctime; $dirList->{$key}->{$name}->{'atime'} = $atime; $dirList->{$key}->{$name}->{'mtime'} = $mtime; $dirList->{$key}->{$name}->{'uid'} = $uid; $dirList->{$key}->{$name}->{'gid'} = $gid; $dirList->{$key}->{$name}->{'mode'} = $mode; # Attribute } elsif ($type eq "a") { my ($name,$value) = @aline; $pathAttribs->{$key}->{$name} = $value; # Unknown } else { print(STDERR "ERROR: Invalid type '$type' in '$statefile'\n"); exit 1; } } } # Close gzip file $gz->gzclose(); } sub removeBackups { my ($path,$format,$compression,$seqs) = @_; # Loop through backup sequences for (my $i = 0; $i <= $seqs; $i++) { my $tarExt; if ($compression eq "xz") { $tarExt = ".xz"; } elsif ($compression eq "bzip2") { $tarExt = ".bz2"; } elsif ($compression eq "gzip") { $tarExt = ".gz"; } elsif ($compression eq "none") { $tarExt = ""; } # Unlink backup files if (-f "$path/dbackup$i.tar$tarExt") { unlink("$path/dbackup$i.tar$tarExt") or die "ERROR: Failed to remove file '$path/dbackup$i.tar$tarExt': $!"; } # Unlink manifest if (-f "$path/dbackup$i.manifest") { unlink("$path/dbackup$i.manifest") or die "ERROR: Failed to remove file '$path/dbackup$i.manifest': $!"; } # Unlink index if (-f "$path/dbackup$i.index") { unlink("$path/dbackup$i.index") or die "ERROR: Failed to remove file '$path/dbackup$i.index': $!"; } # Unlink snar if (-f "$path/dbackup$i.snar") { unlink("$path/dbackup$i.snar") or die "ERROR: Failed to remove file '$path/dbackup$i.snar': $!"; } } } sub toArray { my $param = shift; if (ref $param eq "ARRAY") { return $param; } else { return [ split(/\n/,$param) ]; } } sub getTarVer { my $tar = shift; open(TAR, $config{'tar'} . " --version |") or die "FAILED to execute '".$config{'tar'}."': $!"; if (!($tarVer = <TAR>)) { print(STDERR "ERROR: Failed to read tar version\n"); exit 1; } ($tarVer) = ($tarVer =~ /([0-9]+\.[0-9]+)/); if (!defined($tarVer) || $tarVer eq "") { print(STDERR "ERROR: Failed to parse tar version\n"); exit 1; } $tarVer =~ s/\.//g; close(TAR); if ($tarVer < 100) { print(STDERR "ERROR: Failed to read tar version or your version is simply too old\n"); exit 1; } return $tarVer; } sub getNumericVer { my $a = shift; # 000 000 000 my @a = split(/\./,$a); # Quick fix for devel versions $a[2] = 0 if ($a[2] eq "x"); $a[2] =~ s/[a-z]$//; return ( ($a[0]*1000000) + ($a[1]*1000) + $a[2] ); } # vim: ts=4