|
| 1 | +#!/usr/bin/python |
| 2 | + |
| 3 | +import sys |
| 4 | +import string |
| 5 | +import shutil |
| 6 | +import getopt |
| 7 | +import os |
| 8 | +import os.path |
| 9 | +import syslog |
| 10 | +import errno |
| 11 | +import logging |
| 12 | +import tempfile |
| 13 | +import datetime |
| 14 | +import subprocess |
| 15 | +import json |
| 16 | +import paramiko |
| 17 | + |
| 18 | +from operator import itemgetter |
| 19 | + |
| 20 | +""" |
| 21 | +----------------------------------------------------------------------------- |
| 22 | +An incremental backup system that pushes backups to a remote server. Useful |
| 23 | +for remote systems that aren't always on (laptops). Backups use rsync and hard |
| 24 | +links to keep multiple full copies while using minimal space. It is assumed |
| 25 | +that the rotatebackups.py script exists on the remote backup server and that |
| 26 | +the proper ssh keys have been setup from the pushing server to the backup |
| 27 | +server. |
| 28 | +
|
| 29 | +A pid file is placed into the system temp directory to prevent concurrent |
| 30 | +backups from running at once. The script provides options for the number of |
| 31 | +backups to keep. After the max number of backups is reached, backups are |
| 32 | +deleted starting with the oldest backup first. |
| 33 | +
|
| 34 | +Backup paths can be either local or remote. The backup root directory where |
| 35 | +the backups are stored must be local and must already exist. If a users isn't |
| 36 | +specified then the remote user used by ssh for rsync is considered to be backup. |
| 37 | +
|
| 38 | +Use the -h or the --help flag to get a listing of options. |
| 39 | +
|
| 40 | +Program: Push Backups |
| 41 | +Author: Dennis E. Kubes |
| 42 | +Date: May 01, 2013 |
| 43 | +Revision: 1.0 |
| 44 | +
|
| 45 | +Revision | Author | Comment |
| 46 | +----------------------------------------------------------------------------- |
| 47 | +20131430-1.0 Dennis E. Kubes Initial creation of script. |
| 48 | +----------------------------------------------------------------------------- |
| 49 | +""" |
| 50 | +class PushBackup: |
| 51 | + |
| 52 | + def __init__(self, name="backup", server=None, keep=90, store=None, |
| 53 | + config_file=None, user="root", ssh_key=None, rotate_script=None): |
| 54 | + self.name = name |
| 55 | + self.server = server |
| 56 | + self.keep = keep |
| 57 | + self.config_file = config_file |
| 58 | + self.store = store |
| 59 | + self.user = user |
| 60 | + self.ssh_key = ssh_key |
| 61 | + self.rotate_script = rotate_script |
| 62 | + |
| 63 | + def run_command(self, command=None, shell=False, ignore_errors=False, |
| 64 | + ignore_codes=None): |
| 65 | + result = subprocess.call(command, shell=False) |
| 66 | + if result and not ignore_errors and (not ignore_codes or result in set(ignore_codes)): |
| 67 | + raise BaseException(str(command) + " " + str(result)) |
| 68 | + |
| 69 | + def backup(self): |
| 70 | + |
| 71 | + # create the ssh client to run the remote rotate script |
| 72 | + client = paramiko.SSHClient() |
| 73 | + client.set_missing_host_key_policy(paramiko.AutoAddPolicy()) |
| 74 | + client.load_system_host_keys() |
| 75 | + client.connect(self.server, username=self.user, key_filename=self.ssh_key) |
| 76 | + |
| 77 | + # rotate the backups remotely by running the rotatebackups.py script on the |
| 78 | + # remote backup server |
| 79 | + rotate_cmd = [self.rotate_script, "-k", str(self.keep), "-t", self.store] |
| 80 | + stdin, stdout, stderr = client.exec_command(" ".join(rotate_cmd)) |
| 81 | + rotated_names = stdout.readlines() |
| 82 | + client.close() |
| 83 | + |
| 84 | + rsync_to = None |
| 85 | + if not rotated_names: |
| 86 | + # get the current date and timestamp and the zero backup name |
| 87 | + now = datetime.datetime.now() |
| 88 | + padding = len(str(self.keep)) |
| 89 | + tstamp = now.strftime("%Y%m%d%H%M%S") |
| 90 | + zbackup_name = string.join(["".zfill(padding), tstamp, self.name], ".") |
| 91 | + rsync_to = self.store + os.sep + zbackup_name |
| 92 | + else: |
| 93 | + rsync_to = rotated_names[0] |
| 94 | + |
| 95 | + # create the base rsync command with excludes |
| 96 | + rsync_base = ["rsync", "-avR", "--ignore-errors", "--delete", "--delete-excluded"] |
| 97 | + |
| 98 | + # get the paths to backup either from the command line or from a paths file |
| 99 | + bpaths = [] |
| 100 | + expaths = [] |
| 101 | + if self.config_file: |
| 102 | + |
| 103 | + pf = open(self.config_file, "r") |
| 104 | + config = json.load(pf) |
| 105 | + pf.close() |
| 106 | + |
| 107 | + # add the paths to backup |
| 108 | + bpaths.extend(config["backup"]) |
| 109 | + |
| 110 | + # add and filter/exclude options |
| 111 | + if "exclude" in config: |
| 112 | + for exclude in config["exclude"]: |
| 113 | + rsync_base.extend(["--exclude", exclude]) |
| 114 | + |
| 115 | + # one rsync command per path, ignore files vanished errors |
| 116 | + for bpath in bpaths: |
| 117 | + bpath = bpath.strip() |
| 118 | + rsync_cmd = rsync_base[:] |
| 119 | + rsync_cmd.append(bpath) |
| 120 | + rsync_cmd.append(self.user + "@" + self.server + ":" + rsync_to) |
| 121 | + logging.debug(rsync_cmd) |
| 122 | + self.run_command(command=rsync_cmd, ignore_errors=True) |
| 123 | + |
| 124 | +""" |
| 125 | +Prints out the usage for the command line. |
| 126 | +""" |
| 127 | +def usage(): |
| 128 | + usage = ["pushbackup.py [-hnksctuxr]\n"] |
| 129 | + usage.append(" [-h | --help] prints this help and usage message\n") |
| 130 | + usage.append(" [-n | --name] backup namespace\n") |
| 131 | + usage.append(" [-k | --keep] number of backups to keep before deleting\n") |
| 132 | + usage.append(" [-s | --server] the server to push to backup to\n") |
| 133 | + usage.append(" [-c | --config] configuration file with backup paths\n") |
| 134 | + usage.append(" [-t | --store] directory locally to store the backups\n") |
| 135 | + usage.append(" [-u | --user] the remote username used to ssh for backups\n") |
| 136 | + usage.append(" [-x | --ssh-key] the ssh key used to connect to the backup\n") |
| 137 | + usage.append(" [-r | --rotate-script] the rotatebackups script remote location\n") |
| 138 | + message = string.join(usage) |
| 139 | + print message |
| 140 | + |
| 141 | +""" |
| 142 | +Main method that starts up the backup. |
| 143 | +""" |
| 144 | +def main(argv): |
| 145 | + |
| 146 | + # set the default values |
| 147 | + pid_file = tempfile.gettempdir() + os.sep + "pushbackup.pid" |
| 148 | + name = "backup" |
| 149 | + keep = 90 |
| 150 | + server = None |
| 151 | + config_file = None |
| 152 | + store = None |
| 153 | + user = "backup" |
| 154 | + ssh_key = os.path.expanduser("~/.ssh/id_rsa") |
| 155 | + rotate_script = "rotatebackups.py" |
| 156 | + |
| 157 | + try: |
| 158 | + |
| 159 | + # process the command line options |
| 160 | + opts, args = getopt.getopt(argv, "hn:k:s:c:t:u:x:r:", ["help", "name=", |
| 161 | + "keep=", "server=", "config=", "store=", "user=", "ssh-key=", |
| 162 | + "rotate-script="]) |
| 163 | + |
| 164 | + # if no arguments print usage |
| 165 | + if len(argv) == 0: |
| 166 | + usage() |
| 167 | + sys.exit() |
| 168 | + |
| 169 | + # loop through all of the command line options and set the appropriate |
| 170 | + # values, overriding defaults |
| 171 | + for opt, arg in opts: |
| 172 | + if opt in ("-h", "--help"): |
| 173 | + usage() |
| 174 | + sys.exit() |
| 175 | + elif opt in ("-n", "--name"): |
| 176 | + name = arg |
| 177 | + elif opt in ("-k", "--keep"): |
| 178 | + keep = int(arg) |
| 179 | + elif opt in ("-s", "--server"): |
| 180 | + server = arg |
| 181 | + elif opt in ("-c", "--config"): |
| 182 | + config_file = arg |
| 183 | + elif opt in ("-t", "--store"): |
| 184 | + store = arg |
| 185 | + elif opt in ("-u", "--user"): |
| 186 | + user = arg |
| 187 | + elif opt in ("-x", "--ssh-key"): |
| 188 | + ssh_key = arg |
| 189 | + elif opt in ("-r", "--rotate-script"): |
| 190 | + rotate_script = arg |
| 191 | + |
| 192 | + except getopt.GetoptError, msg: |
| 193 | + # if an error happens print the usage and exit with an error |
| 194 | + usage() |
| 195 | + sys.exit(errno.EIO) |
| 196 | + |
| 197 | + # check options are set correctly |
| 198 | + if config_file == None or store == None or server == None: |
| 199 | + usage() |
| 200 | + sys.exit(errno.EPERM) |
| 201 | + |
| 202 | + # process backup, catch any errors, and perform cleanup |
| 203 | + try: |
| 204 | + |
| 205 | + # another backup can't already be running, if pid file doesn't exist, then |
| 206 | + # create it |
| 207 | + if os.path.exists(pid_file): |
| 208 | + logging.warning("Backup running, %s pid exists, exiting." % pid_file) |
| 209 | + sys.exit(errno.EBUSY) |
| 210 | + else: |
| 211 | + pid = str(os.getpid()) |
| 212 | + f = open(pid_file, "w") |
| 213 | + f.write("%s\n" % pid) |
| 214 | + f.close() |
| 215 | + |
| 216 | + # create the backup object and call its backup method |
| 217 | + pbackup = PushBackup(name, server, keep, store, config_file, user, |
| 218 | + ssh_key, rotate_script) |
| 219 | + pbackup.backup() |
| 220 | + |
| 221 | + except(Exception): |
| 222 | + logging.exception("Incremental backup failed.") |
| 223 | + finally: |
| 224 | + os.remove(pid_file) |
| 225 | + |
| 226 | +# if we are running the script from the command line, run the main function |
| 227 | +if __name__ == "__main__": |
| 228 | + main(sys.argv[1:]) |
| 229 | + |
| 230 | + |
0 commit comments