#!/usr/bin/env python # # Amazon S3 Automated and Encrypted Rsync Backup # # Philippe Niquille # June 15, 2008 - v0.1 - philippe at niquille dot com # import os, sys, subprocess # Rsync dirs are listed in tuples: # the first element, [0], represents the parent directory which will be rsynced into a particular bucket # --> !! ALWAYS use trailing slashes here # the susequent elements [1+] are subdirectories of parent-dir which will be encryped and rsynced # --> NEVER use trailing slashes here PATHS = ( ('/Users/fluppel/Desktop/Inbox/s3rsync/', ('/Users/fluppel/Desktop/Inbox/s3rsync/encrypt_me',)), ('/Users/fluppel/Desktop/Inbox/test_due/',), ) # Remark: try using folder names without whitespaces, since --exclude in rsync does not work !?! MURK_KEY_FILE = "/Users/fluppel/.murk" MURK_KEYWORD = "" SSH_PRIV_KEY="/Users/fluppel/.ssh/s3rsync_dsa.priv" # Use the service at http://www.s3rsync.com/index.php/Rsync_to_S3 S3_RSYNC_USER = "" S3_BUCKETS = ('your_bucket_one','your_bucket_two') # Bucket names need to be global, remember S3_ID = "" S3_SECRET = "" ##### ***** ##### ***** if not os.path.isfile(MURK_KEY_FILE): print "ERROR: murk key file %s does not exist, exiting." % MURK_KEY_FILE sys.exit() if not os.path.isfile(SSH_PRIV_KEY): print "ERROR: SSH private key file %s does not exist, exiting." % SSH_PRIV_KEY sys.exit() c = 0 for l in PATHS: # Check if directory first in list (the rsync dir) exists if os.path.exists(l[0]): print "working on %s:" % l[0] # By default exclude these: exclude = "--exclude='*.Trash' --exclude='*.DS_Store' " # Check for additional dirs which need to be encrypted before rsyncing try: if os.path.exists(l[1][0]): crypt_rm = [] for crypt_dir in l[1]: # do nothing with the first (rsync) dir if crypt_dir != l[0]: if os.path.exists(crypt_dir): print " |_ encrypting %s .." % crypt_dir crypt_dir_escaped = crypt_dir.replace(" ","\ ") exclude += '--exclude="%s" ' % crypt_dir.replace(l[0],"") # pack the crypt_dir in a tar and encrypt it with 256 bit AES, strip whitespaces from path for tar to work # tmp file is needed since piping the tar somehow renders the file unusable cmd = "tar -cf /tmp/tmp_no_cipher.tar %s 2> /dev/null && murk -n -c aes-256 -k %s -o '%s.tar.zm' %s /tmp/tmp_no_cipher.tar && rm /tmp/tmp_no_cipher.tar" % (crypt_dir_escaped, MURK_KEY_FILE, os.path.join(os.path.dirname(crypt_dir), os.path.basename(crypt_dir)), MURK_KEYWORD) subprocess.call(cmd, shell=True) # prepare list of *.tar.zm to be deleted crypt_rm.append('%s.tar.zm' % crypt_dir) except IndexError: pass print " |_ rsyncing %s .." % l[0] print exclude und = '%%' # save every rsync dir to a new bucket such as listed in S3_BUCKETS cmd = "rsync --stats -v -v %s-e 'ssh -i %s' -az '%s' %s@farm.s3rsync.com:%s%s%s%s%s%s%s | tail -n 2 | growlnotify -t 'S3 Rsync'" % (exclude, SSH_PRIV_KEY, l[0], S3_RSYNC_USER, os.path.basename(l[0]), und, S3_BUCKETS[c], und, S3_ID, und, S3_SECRET) subprocess.call(cmd, shell=True) # again loop through encrypted dir list to delete now rsynced *.tar.zm files for f in crypt_rm: if os.path.isfile(f): cmd = "rm %s" % f.replace(" ","\ ") print cmd subprocess.call(cmd, shell=True) c += 1