From 0a4f55a811c5ede4e4c1b09dcc4381fa196b851f Mon Sep 17 00:00:00 2001 From: mahmoodnasr Date: Mon, 5 Oct 2020 09:03:25 +0200 Subject: [PATCH] CI #Initial --- client/autodeploy_client/Client.py | 8 ++ client/autodeploy_client/ClientJob.py | 18 ++++ server/Request.py | 19 +++++ server/autodeploy-server.py | 10 +++ server/integrator/autointegrator.py | 115 ++++++++++++++++++++++++++ webapp/autoDeploy/autoDeploy/api.py | 82 +++++++++++++++++- webapp/autoDeploy/deployment/views.py | 3 - 7 files changed, 251 insertions(+), 4 deletions(-) create mode 100644 server/integrator/autointegrator.py diff --git a/client/autodeploy_client/Client.py b/client/autodeploy_client/Client.py index 19d285f..8979372 100644 --- a/client/autodeploy_client/Client.py +++ b/client/autodeploy_client/Client.py @@ -82,6 +82,14 @@ def Deploy(self, workdir, configFile, owner=''): result = self._send(msg) return result + def Integrate(self, workdir, configFile, owner=''): + global msg + if owner == '': + owner = Config.Owner + msg = Job.createIntegrateMessage(workdir=workdir, configFile=configFile, scm=self.scm, owner=owner) + result = self._send(msg) + return result + def CheckUp(self): return Connect.connect(self.server, self.port, 5) diff --git a/client/autodeploy_client/ClientJob.py b/client/autodeploy_client/ClientJob.py index fbc95c0..17ee72a 100644 --- a/client/autodeploy_client/ClientJob.py +++ b/client/autodeploy_client/ClientJob.py @@ -111,6 +111,24 @@ def createDeployMessage(owner, workdir, scm, configFile, options=None): f += '' return f +def createIntegrateMessage(owner, workdir, scm, configFile, options=None): + sec=base64.encodestring(importKey().encrypt(owner+scm+"INTEGRATE","")[0]) + f = '\n'%(owner,"INTEGRATE",sec,scm) + f += '%s'%workdir + f += '%s'%configFile + print(configFile) + conf=open(str(configFile)).read() + f += '%s'%(base64.encodestring(conf)) + + if options: + f += '' + for option in list(options.keys()): + f += "" % (option, options[option]) + + f += "" + f += '' + return f + def createListCommitsMessage(owner, workdir, key, scm, options=None): sec=base64.encodestring(importKey().encrypt(owner+scm+"LIST-COMMITS","")[0]) f = '\n'%( owner,"LIST-COMMITS",sec,scm) diff --git a/server/Request.py b/server/Request.py index 081ec03..9689e46 100755 --- a/server/Request.py +++ b/server/Request.py @@ -133,6 +133,25 @@ def parseDeployJob(message): "scm":scm,"options": optionsDict} return params +def parseIntegrateJob(message): + params = {} + optionsDict = {} + doc = xml.dom.minidom.parseString(message) + Job = doc.getElementsByTagName('job')[0] + scm=Job.getAttribute("scm") + workdir= getValue(Job, 'workdir') + configFile=getValue(Job,"configFile") + requestType = Job.getAttribute('type') + owner = Job.getAttribute('owner') + fileBase64=getValue(Job,"file") + if not os.path.exists(os.path.dirname(configFile)): + os.makedirs(os.path.dirname(configFile)) + open(configFile,"w").write(base64.decodestring(fileBase64)) + print('Recieved New Job from ' + owner + '.....') + params = {"workdir": workdir,"owner": owner, "requestType": requestType,"configFile":configFile, + "scm":scm,"options": optionsDict} + return params + def parseGetCommitsJob(message): optionsDict={} doc = xml.dom.minidom.parseString(message) diff --git a/server/autodeploy-server.py b/server/autodeploy-server.py index 94c7e28..e929684 100755 --- a/server/autodeploy-server.py +++ b/server/autodeploy-server.py @@ -9,6 +9,7 @@ from . import Common from server.scm import Git as git from .deployer import autodeployer +from .integrator import autointegrator import traceback import yaml JOBS = {} @@ -175,6 +176,15 @@ def HandleClient(clientsock): res="Done" except Exception as e: res="ERR:"+traceback.format_exc() + elif req["requestType"]=="INTEGRATE": + print(msg) + job = Request.parseIntegrateJob(msg) + try: + config=yaml.safe_load(open(job["configFile"])) + autointegrator.runTest(config,job["workdir"]) + res="Done" + except Exception as e: + res="ERR:"+traceback.format_exc() if cmd!="": print(cmd) res=Common.run(cmd) diff --git a/server/integrator/autointegrator.py b/server/integrator/autointegrator.py new file mode 100644 index 0000000..0836c2f --- /dev/null +++ b/server/integrator/autointegrator.py @@ -0,0 +1,115 @@ +__author__ = 'mohamed' + +import sys +import os +import yaml +import subprocess + + +EOM = "\n\n###" +debug = False +slient = False + + +def run(executer, raiseError=True,exitcode=False, id=None, wait=True): + PIPE = subprocess.PIPE + p = subprocess.Popen(executer, stdout=PIPE, stderr=PIPE, shell=True) + (stdout, stderr) = p.communicate() + st = stderr + if id: + f = open("/tmp/" + id + ".err", 'w') + f.write(st) + f.flush() + f.close() + f = open("/tmp/" + id + ".out", 'w') + f.write(stdout) + f.flush() + f.close() + + if len(stderr) > 0 and "fatal" in str(stderr): + return "ERR:" + str(stderr) + if stdout == "": return "Done" + if exitcode: + return p.returncode + return stdout + + +def printNotication(message): + if slient: return + print(message) + print("=" * len(message)) + + +def runEvents(config, workdir, event, raiseErrorOnStdErr=True): + if event in config["events"].keys(): + for script in config["events"][event]: + wait = True + if not script["location"].startswith("/"): + cmd = workdir + script["location"] + else: + cmd = script["location"] + if "interpreter" in script.keys(): + cmd = "%s %s" % (script["interpreter"], cmd) + if "run-as" in script.keys(): + if not script["run-as"] == "root": + cmd = "su %s -c %s" % (script["run-as"], cmd) + if not slient: print("Running:", cmd) + if "wait" in script.keys(): + wait = script["wait"] + if "ignore-stderr" in script.keys(): + if script["ignore-stderr"] in ("yes", "True", "true", "y", "True", True): + raiseErrorOnStdErr = False + run(cmd, raiseErrorOnStdErr, wait=wait) + + +def handleRuns(tasks, workdir): + for task in tasks: + cmd = "%s %s" % (task['interpreter'], task["location"]) + task_reult = run(cmd,exitcode=True) + if task_reult not in [0,'0']: + print("Task Failed") + break + else: + print("Task Success") + + +def runTest(config, workdir=".", raiseErrorOnStdErr=True): + printNotication("Running Before Run scripts:") + runEvents(config, workdir, "beforeRun", raiseErrorOnStdErr) + + printNotication("Starting Test Scripts") + + if not "tasks" in config.keys(): + if not slient: print(" No tasks to run ... skipping") + else: + if not slient: print(" Running tasks") + handleRuns(config['tasks'], workdir) + if not slient: print(" Tasks done") + printNotication("Test Scripts Done.......") + + printNotication("Starting After Install Scripts") + runEvents(config, workdir, "afterRun", raiseErrorOnStdErr) + return "Done" + + +if __name__ == "__main__": + config = None + workdir = None + stdErr = True + for arg in sys.argv[1:]: + if "--config" in arg: + yamlFile = arg.split("=")[1] + config = yaml.safe_load(open(yamlFile)) + print(config) + elif "--workdir" in arg: + workdir = arg.split("=")[1] + elif "--no-stderr" in arg: + stdErr = False + elif "--slient" in arg: + slient = True + elif "--debug" in arg: + debug = True + if not config: + print("--config should be set") + exit() + runTest(config, workdir, stdErr) \ No newline at end of file diff --git a/webapp/autoDeploy/autoDeploy/api.py b/webapp/autoDeploy/autoDeploy/api.py index 9dff51e..21f21c9 100644 --- a/webapp/autoDeploy/autoDeploy/api.py +++ b/webapp/autoDeploy/autoDeploy/api.py @@ -31,7 +31,7 @@ def cloneCD(request): project = CDModels.Project.objects.get(name=request.GET["project_name"]) c = Client(scm, ip, port, project.sshKey.key) res = c.Clone(project.repo, project.working_dir) - return HttpResponse(res)\ + return HttpResponse(res) @csrf_protect def cloneCI(request): @@ -121,5 +121,85 @@ def deploy(request): "Dear User,
This is an automated notification that a new version of %s has been deployed at: %s.
%s" % ( project.name, link, changes_text), fromUser=None, cc="", bcc="", ) return HttpResponse(res + ",," + link) + else: + return HttpResponse(res) + +@csrf_protect +def integrate(request): + from webapp.autoDeploy.autoDeploy import Common + server = CDModels.Server.objects.get(name=request.session["integrate_server"]) + project = CIModels.Project.objects.get(name=request.session["integrate_project"]) + last_Integration = None + try: + last_Integration = CIModels.Integration_server.objects.filter(server=server, project=project).latest() + except: + pass + D = CIModels.Integration_server() + c = Client(str(project.repo_type), server.ip, server.port) + D.project = project + D.server = server + if "tag" in request.GET: + res = c.SwitchTag(project.working_dir, request.GET["tag"]) + D.update_type = "tag" + D.update_version = request.GET["tag"] + project.lastTag = request.GET["tag"] + elif "commit" in request.GET: + if request.GET["commit"] != "HEAD": + res = c.SwitchCommit(project.working_dir, request.GET["commit"]) + D.update_type = "commit" + D.update_version = request.GET["commit"] + project.lastCommit = request.GET["commit"] + res = c.Integrate(project.working_dir, project.configFile) + if not "ERR:" in res: + D.datetime = timezone.now() + D.has_new_version = False + D.save() + project.lastUpdate = timezone.now() + project.newVersion = False + project.save() + print(project.integration_link) + if not "http://" in project.integration_link: + print("in if") + link = "http://" + server.DNS + project.integration_link + print(link) + if project.emailUsers != "" or project.emailUsers != " " and last_Integration != None: + changes = c.getChangeLog(project.working_dir, since=last_Integration.update_version, + to=request.GET["commit"]) + changes_text = "

Changes

" + else: + changes_text = "" + Common.send(project.emailUsers.replace(",", ";"), "New version of %s integrated" % project.name, + "Dear User,
This is an automated notification that a new version of %s has been integrated at: %s
%s" % ( + project.name, link, changes_text), fromUser=None, cc="", bcc="", ) + + return HttpResponse(res + ",," + link) + else: + print("in else") + link = project.integration_link + if project.emailUsers != "" or project.emailUsers != " ": + changes = c.getChangeLog(project.working_dir, since=last_Integration.update_version, + to=request.GET["commit"]) + changes_text = "

Changes

" + else: + changes_text = "" + + Common.send(project.emailUsers.replace(",", ";"), "New version of %s integrated" % project.name, + "Dear User,
This is an automated notification that a new version of %s has been integrated at: %s.
%s" % ( + project.name, link, changes_text), fromUser=None, cc="", bcc="", ) + return HttpResponse(res + ",," + link) else: return HttpResponse(res) \ No newline at end of file diff --git a/webapp/autoDeploy/deployment/views.py b/webapp/autoDeploy/deployment/views.py index b1de4d2..14d981c 100644 --- a/webapp/autoDeploy/deployment/views.py +++ b/webapp/autoDeploy/deployment/views.py @@ -4,10 +4,7 @@ from .tables import * from django_tables2.export.export import TableExport from django.views.decorators.csrf import csrf_protect -from django_tables2_reports.config import RequestConfigReport from django_tables2.config import RequestConfig -import sys -sys.path.append("../../../client") from autodeploy_client import Client from django.shortcuts import redirect from django.template.context_processors import csrf