commit

tm2018 · tm2018 · commit a1ab2ecb8ff5 · 2017-09-25T15:38:15.000+08:00
diff --git a/123 b/123
diff --git a/del_files.py b/del_files.py
@@ -0,0 +1,63 @@
+# -*- coding: UTF-8 -*-
+__author__ = 'bobo'
+
+'''
+note:the scripts will delete dir which like "path/IP/2017-xx-xx",for example:/home/backup/10.1.1.148/2017-09-19
+class：
+    host_path:match path/ip from path which has set
+    check_dir:check son dir and judge whether matching your rules
+    del_dir:del dir
+
+use：
+1.set path your environment
+2.set time for normal files and log file to del
+3.set host list
+'''
+import os
+import re
+import shutil
+from datetime import datetime
+
+
+time_now = datetime.now()
+path = '/home/backup'
+#set time for delete
+common_day = 30
+log_day = 180
+log_host = ['10.1.1.145','10.1.1.155']
+
+class clean:
+    def __init__(self,base_path):
+        self.base_path = base_path
+
+    def host_path(self):
+        host_path = list(os.listdir(self.base_path))
+        for i in range(len(host_path)):
+            #re to match path:base_path/IP
+            host_rep = re.compile(r'(?<![\.\d])(?:\d{1,3}\.){3}\d{1,3}(?![\.\d])')
+            if host_rep.match(host_path[i]):
+                host_dir = host_path[i]
+                self.check_dir(host_dir)
+
+    def check_dir(self,dir):
+        host_dir = os.path.join(self.base_path,dir)
+        back_dir = os.listdir(host_dir)
+        for i in range(len(back_dir)):
+            back_time = datetime.strptime(back_dir[i],"%Y-%m-%d")
+            day_value = time_now - back_time
+            day_sub = day_value.days
+            if dir in log_host:   
+                if day_sub > log_day:
+                    del_dir = os.path.join(host_dir,back_dir[i])
+                    self.del_dir(del_dir)
+                else:
+                    print "dir %s/%s backup at %s,it's under %s天，system will skip it" %(host_dir,back_dir[i],back_time,log_day)
+            else:
+                if day_sub > common_day:
+                    del_dir = os.path.join(host_dir,back_dir[i])
+                    self.del_dir(del_dir)
+                else:
+                    print "dir %s/%s backup at %s,it's under %s天，system will skip it" %(host_dir,back_dir[i],back_time,common_day)
+    def del_dir(self,dir):
+        shutil.rmtree(dir)
+clean(path).host_path()
diff --git a/functions.py b/functions.py
@@ -0,0 +1,180 @@
+# -*- coding:utf-8 -*-
+__author__ = 'bobo'
+
+import subprocess,os
+import time
+import re
+import paramiko
+import logging
+
+time_now = time.strftime('%Y-%m-%d',time.localtime(time.time()))
+#####################################################################################################################
+#split to get your server's info for connection for your config file
+def hp(server):
+    servers = re.split(':',server)
+    server = servers[0]
+    try:
+        port = servers[1]
+    except:
+        port = '22'
+    try:
+        user = servers[2]
+    except:
+        user = 'root'
+    try:
+        passwd = servers[3]
+    except:
+        passwd = 0
+    return server,port,user,passwd
+
+#check dest dir , if not exist,it will create
+def dir(base,server,time_now):
+    path = '%s%s/%s' %(base,server,time_now)
+    if os.path.isdir(path):
+        pass
+    else:
+        os.makedirs(path)
+    return path
+
+#connect ssh
+def ssh(server,port,cmd):
+    try:
+        #set ssh-keygen to connect remote server
+        pkeyfile = '/root/.ssh/id_rsa'
+        key = paramiko.RSAKey.from_private_key_file(pkeyfile)
+        ssh = paramiko.SSHClient()
+        ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy)
+        ssh.connect(server,port,timeout=5,pkey=key)
+        cmd_status = 1
+        stdin,stdout,stderr = ssh.exec_command(cmd)
+        print '%s: %scmd has sent sucessfuly\n' %(server,cmd)
+        err = stderr.readlines()
+        for msg in err:
+            #tak out blanks and line break
+            msg = msg.strip()
+            log(msg,server)
+            cmd_status = 0
+        ssh.close()
+    except BaseException,e:
+        print '%s\tssh login failed!! error:%s\n' %(server,e)
+        log(e,server)
+        cmd_status = 0
+    return cmd_status   
+#write error log
+def log(msg,host=0):
+    logging.basicConfig(level=logging.WARNING,
+                format = '%(asctime)s %(filename)s[line %(lineno)d]: %(message)s',
+                datefmt = '%b %d %Y %H:%M:%S',
+                filename = '/home/backup/logs/backup_error.log',
+                filemode = 'a')
+    message = '%s %s %s' %(msg,'targethost:',host)
+    logging.error(message)
+
+#####################################################################################################################
+#get file from remote server cycles
+def get_file(servers,items,base_path):
+    for i in range(len(servers)):
+        servers_port = hp(servers[i])
+        servers[i] = servers_port[0]
+        port = servers_port[1]
+        for j in range(len(items)):
+             dir_dest = dir(base_path,servers[i],time_now)
+             back_files(servers[i], items[j],port,dir_dest)
+             time.sleep(0.5)
+
+#function for get_file
+def back_files(server,item,port,dest):
+    if item[-2:]=='gz':
+        print ".gz file ,it will be remove after geting "
+        cmd = '%s%s:%s %s %s' % ('/usr/bin/rsync -aucvvLzP --remove-source-files --progress root@', server, item, dest,'-e \'ssh -p '+port+'\'')
+    else:
+        cmd = '%s%s:%s %s %s' % ('/usr/bin/rsync -aucvvLzP --progress root@', server, item, dest,'-e \'ssh -p '+port+'\'')
+    try:
+        result = subprocess.Popen(cmd,shell=True,stderr=subprocess.PIPE)
+        stderr = result.stderr.readlines()
+        for msg in stderr:
+            msg = msg.strip()
+            log(msg,server)
+        result.wait()
+    except Exception,e:
+        print '%s:%s' %('backup failed',e)
+        log(e,server)
+#########################################################
+#get sql from remote server cycles
+def get_sql(servers,items,base_path,user=0,passwd=0):
+    for i in range(len(servers)):
+        host = hp(servers[i])
+        servers[i] = host[0]
+        port = host[1]
+        user = host[2]
+        passwd = host[3]
+        #get server and database from dict
+        items1 = items[servers[i]]
+        #get database from databases
+        item_server = re.split(':',items1)
+        for j in range(len(item_server)):
+             dir_dest = dir(base_path,servers[i],time_now)
+             sql_sucess = backup_sqls(servers[i],port,item_server[j],user,passwd)
+             #not sucess to backup,it will skip this database
+             if sql_sucess == 0:
+                print '%s:%s backup failed,system will skip it' %(servers[i],item_server[j])
+                continue
+             #get dir of remote server
+             sql_src = '%s%s' %(sql_path,item_server[j])
+             #get sql file from remote server
+             get_back_files(servers[i], sql_src,port,dir_dest)
+             time.sleep(0.5)
+#exec remote sql cmd
+#you must insure /usr/bin/mysqldump exists and it's avaliable
+def backup_sqls(server,port,database,user_sql,passwd_sql=0):
+    port = int(port)
+    #passwd_sql==0 if sql passwd is null
+    mysqldump = '/usr/bin/mysqldump -u'
+    global sql_path
+    sql_path = '/home/'
+    if passwd_sql == 0:
+        cmd = '%s %s %s %s %s%s' %(mysqldump,user_sql,database,'>',sql_path,database)
+    else:
+        cmd = '%s %s %s%s %s %s %s%s' %(mysqldump,user_sql, '-p',passwd_sql,database,'>',sql_path,database)
+    sql_sucess = ssh(server,port,cmd)
+    return sql_sucess
+#########################################################
+##get svn file from remote server cycles
+def get_svn(servers,items,base_path):
+    for server in servers:
+        host = hp(server)
+        server = host[0]
+        port = host[1]
+        #get file you want to backup from dict
+        items1 = items[server]
+        items = re.split(':',items1)
+        for item in items:
+             dir_dest = dir(base_path,server,time_now)
+             svn_sucess,svn_path = backup_svn(server,item,port,base_path)
+             #if not sucess to backup,system will skip it
+             if svn_sucess == 0:
+                print '%s:%s backup failed,system will skip it' %(servers,svn_path)
+                continue
+             #get svn dir from remote server
+             svn_src = '%s%s' %(svn_path,'.svn')
+             print svn_src
+             #get svn_file
+             get_back_files(servers,svn_src,port,dir_dest)
+             time.sleep(0.5)
+
+#function to backup svn file
+def backup_svn(server,item,port,base_path):
+    port = int(port)
+    #get son dir
+    p = re.compile(r'^/[A-Za-z0-9]*/')
+    path = p.sub(r'',item)
+    svn_path = "%s%s" %(base_path,path)
+    mkdir = "mkdir -p %s" %(svn_path)
+    ssh(server,port,mkdir)
+    #exec remote cmd to backup svn
+    cmd = "/usr/bin/svnadmin dump %s > %s.svn" %(item,svn_path)
+    svn_sucess = ssh(server,port,cmd)
+    svn_sucess = 1
+    return svn_sucess,svn_path
+#########################################################
+#####################################################################################################################
diff --git a/hosts.py b/hosts.py
@@ -0,0 +1,43 @@
+# -*- coding:utf-8 -*-
+__author__ = 'bobo'
+
+#the file is config file
+
+
+#base_path in your dest server
+base_path = '/home/backup/'
+
+#it can be null if your server's ssh port is 22
+#################################################################################################################
+#nginx
+nginx_servers = ['10.1.1.184:9022','10.1.1.185:9022']
+nginx_items = ['/etc/nginx/conf.d','/etc/nginx/nginx.conf','/home/config','/var/log/nginx/*.gz']
+
+#jenkins
+jenkins_servers = ['172.16.20.18']
+jenkins_items = ['/home/jenkins']
+
+#phpredis
+phpredis_servers = ['172.16.20.49']
+phpredis_items = ['/etc/nginx/nginx.conf','/etc/nginx/nginx.d','/var/www']
+
+#keepalived
+keepalived_servers = ['10.1.1.186:9022','10.1.1.176:9022']
+keepalived_items = ['/etc/keepalived/keepalived.conf']
+
+#nexus
+nexus_servers = ['10.1.1.40:22']
+nexus_items = ['/home/nexus-2.14.0-01','/home/sonatype-work']
+
+#################################################################################################################
+###sqlinfo,sql_databases must be dict
+#sql_servers = ['host:ssh_port:mysql_user:mysql_passwd']
+#sql_databases = {'host':'database1:database2:databasen'}
+
+sql_servers = ['192.168.110.4:22:root','192.168.110.5:22:root:123456']
+sql_databases = {'192.168.110.4':'zentao:test1:test2','192.168.110.5':'mysql:test1:test2'}
+#################################################################################################################
+#svn,items must be dict
+svn_servers = ['10.1.1.231']
+svn_items = {'10.1.1.231':'/home/Repositories/abcd;/home/Repositories/main:/home/Repositories/manager'}
+#################################################################################################################
diff --git a/main.py b/main.py
@@ -0,0 +1,11 @@
+# -*- coding:utf-8 -*-
+__author__ = 'bobo'
+
+from functions import  *
+from hosts import *
+
+
+if __name__ == '__main__':
+        get_sql(sql_servers,sql_databases,base_path)
+        get_svn(svn_servers,svn_items,base_path)
+
