CI #Initial

client/autodeploy_client/Client.py

@@ -82,6 +82,14 @@ def Deploy(self, workdir, configFile, owner=''):
         result = self._send(msg)
         return result
 
+    def Integrate(self, workdir, configFile, owner=''):
+        global msg
+        if owner == '':
+            owner = Config.Owner
+        msg = Job.createIntegrateMessage(workdir=workdir, configFile=configFile, scm=self.scm, owner=owner)
+        result = self._send(msg)
+        return result
+
     def CheckUp(self):
         return Connect.connect(self.server, self.port, 5)
 

client/autodeploy_client/ClientJob.py

@@ -111,6 +111,24 @@ def createDeployMessage(owner, workdir, scm, configFile, options=None):
     f += '</job>'
     return f
 
+def createIntegrateMessage(owner, workdir, scm, configFile, options=None):
+    sec=base64.encodestring(importKey().encrypt(owner+scm+"INTEGRATE","")[0])
+    f = '<job owner="%s" type="%s" sec="%s" scm="%s">\n'%(owner,"INTEGRATE",sec,scm)
+    f += '<workdir>%s</workdir>'%workdir
+    f += '<configFile>%s</configFile>'%configFile
+    print(configFile)
+    conf=open(str(configFile)).read()
+    f += '<file>%s</file>'%(base64.encodestring(conf))
+
+    if options:
+        f += '<options>'
+        for option in list(options.keys()):
+            f += "<option name='%s'>%s</option>" % (option, options[option])
+
+        f += "</options>"
+    f += '</job>'
+    return f
+
 def createListCommitsMessage(owner, workdir, key, scm, options=None):
     sec=base64.encodestring(importKey().encrypt(owner+scm+"LIST-COMMITS","")[0])
     f = '<job  owner="%s" type="%s" sec="%s" scm="%s">\n'%( owner,"LIST-COMMITS",sec,scm)

server/Request.py

@@ -133,6 +133,25 @@ def parseDeployJob(message):
               "scm":scm,"options": optionsDict}
     return params
 
+def parseIntegrateJob(message):
+    params = {}
+    optionsDict = {}
+    doc = xml.dom.minidom.parseString(message)
+    Job = doc.getElementsByTagName('job')[0]
+    scm=Job.getAttribute("scm")
+    workdir= getValue(Job, 'workdir')
+    configFile=getValue(Job,"configFile")
+    requestType = Job.getAttribute('type')
+    owner = Job.getAttribute('owner')
+    fileBase64=getValue(Job,"file")
+    if not os.path.exists(os.path.dirname(configFile)):
+        os.makedirs(os.path.dirname(configFile))
+    open(configFile,"w").write(base64.decodestring(fileBase64))
+    print('Recieved New Job from  ' + owner + '.....')
+    params = {"workdir": workdir,"owner": owner,  "requestType": requestType,"configFile":configFile,
+              "scm":scm,"options": optionsDict}
+    return params
+
 def parseGetCommitsJob(message):
     optionsDict={}
     doc = xml.dom.minidom.parseString(message)

server/autodeploy-server.py

@@ -9,6 +9,7 @@
 from . import Common
 from server.scm import Git as git
 from .deployer import autodeployer
+from .integrator import autointegrator
 import  traceback
 import yaml
 JOBS = {}
@@ -175,6 +176,15 @@ def HandleClient(clientsock):
                     res="Done"
                 except Exception as e:
                     res="ERR:"+traceback.format_exc()
+            elif req["requestType"]=="INTEGRATE":
+                print(msg)
+                job = Request.parseIntegrateJob(msg)
+                try:
+                    config=yaml.safe_load(open(job["configFile"]))
+                    autointegrator.runTest(config,job["workdir"])
+                    res="Done"
+                except Exception as e:
+                    res="ERR:"+traceback.format_exc()
             if cmd!="":
                 print(cmd)
                 res=Common.run(cmd)

server/integrator/autointegrator.py

@@ -0,0 +1,115 @@
+__author__ = 'mohamed'
+
+import sys
+import os
+import yaml
+import subprocess
+
+
+EOM = "\n\n###"
+debug = False
+slient = False
+
+
+def run(executer, raiseError=True,exitcode=False, id=None, wait=True):
+    PIPE = subprocess.PIPE
+    p = subprocess.Popen(executer, stdout=PIPE, stderr=PIPE, shell=True)
+    (stdout, stderr) = p.communicate()
+    st = stderr
+    if id:
+        f = open("/tmp/" + id + ".err", 'w')
+        f.write(st)
+        f.flush()
+        f.close()
+        f = open("/tmp/" + id + ".out", 'w')
+        f.write(stdout)
+        f.flush()
+        f.close()
+
+    if len(stderr) > 0 and "fatal" in str(stderr):
+        return "ERR:" + str(stderr)
+    if stdout == "": return "Done"
+    if exitcode:
+        return p.returncode
+    return stdout
+
+
+def printNotication(message):
+    if slient: return
+    print(message)
+    print("=" * len(message))
+
+
+def runEvents(config, workdir, event, raiseErrorOnStdErr=True):
+    if event in config["events"].keys():
+        for script in config["events"][event]:
+            wait = True
+            if not script["location"].startswith("/"):
+                cmd = workdir + script["location"]
+            else:
+                cmd = script["location"]
+            if "interpreter" in script.keys():
+                cmd = "%s %s" % (script["interpreter"], cmd)
+            if "run-as" in script.keys():
+                if not script["run-as"] == "root":
+                    cmd = "su %s -c %s" % (script["run-as"], cmd)
+            if not slient: print("Running:", cmd)
+            if "wait" in script.keys():
+                wait = script["wait"]
+            if "ignore-stderr" in script.keys():
+                if script["ignore-stderr"] in ("yes", "True", "true", "y", "True", True):
+                    raiseErrorOnStdErr = False
+            run(cmd, raiseErrorOnStdErr, wait=wait)
+
+
+def handleRuns(tasks, workdir):
+    for task in tasks:
+        cmd = "%s %s" % (task['interpreter'], task["location"])
+        task_reult = run(cmd,exitcode=True)
+        if task_reult not in [0,'0']:
+            print("Task Failed")
+            break
+        else:
+            print("Task Success")
+
+
+def runTest(config, workdir=".", raiseErrorOnStdErr=True):
+    printNotication("Running Before Run scripts:")
+    runEvents(config, workdir, "beforeRun", raiseErrorOnStdErr)
+
+    printNotication("Starting Test Scripts")
+
+    if not "tasks" in config.keys():
+        if not slient: print("  No tasks to run ... skipping")
+    else:
+        if not slient: print("     Running tasks")
+        handleRuns(config['tasks'], workdir)
+        if not slient: print("     Tasks done")
+    printNotication("Test Scripts Done.......")
+
+    printNotication("Starting After Install Scripts")
+    runEvents(config, workdir, "afterRun", raiseErrorOnStdErr)
+    return "Done"
+
+
+if __name__ == "__main__":
+    config = None
+    workdir = None
+    stdErr = True
+    for arg in sys.argv[1:]:
+        if "--config" in arg:
+            yamlFile = arg.split("=")[1]
+            config = yaml.safe_load(open(yamlFile))
+            print(config)
+        elif "--workdir" in arg:
+            workdir = arg.split("=")[1]
+        elif "--no-stderr" in arg:
+            stdErr = False
+        elif "--slient" in arg:
+            slient = True
+        elif "--debug" in arg:
+            debug = True
+    if not config:
+        print("--config  should be set")
+        exit()
+    runTest(config, workdir, stdErr)

webapp/autoDeploy/autoDeploy/api.py

@@ -31,7 +31,7 @@ def cloneCD(request):
     project = CDModels.Project.objects.get(name=request.GET["project_name"])
     c = Client(scm, ip, port, project.sshKey.key)
     res = c.Clone(project.repo, project.working_dir)
-    return HttpResponse(res)\
+    return HttpResponse(res)
 
 @csrf_protect
 def cloneCI(request):
@@ -121,5 +121,85 @@ def deploy(request):
                             "Dear User,<br/> This is an automated notification that a new version of %s has been deployed at: %s.<br>%s" % (
                             project.name, link, changes_text), fromUser=None, cc="", bcc="", )
             return HttpResponse(res + ",," + link)
+    else:
+        return HttpResponse(res)
+
+@csrf_protect
+def integrate(request):
+    from webapp.autoDeploy.autoDeploy import Common
+    server = CDModels.Server.objects.get(name=request.session["integrate_server"])
+    project = CIModels.Project.objects.get(name=request.session["integrate_project"])
+    last_Integration = None
+    try:
+        last_Integration = CIModels.Integration_server.objects.filter(server=server, project=project).latest()
+    except:
+        pass
+    D = CIModels.Integration_server()
+    c = Client(str(project.repo_type), server.ip, server.port)
+    D.project = project
+    D.server = server
+    if "tag" in request.GET:
+        res = c.SwitchTag(project.working_dir, request.GET["tag"])
+        D.update_type = "tag"
+        D.update_version = request.GET["tag"]
+        project.lastTag = request.GET["tag"]
+    elif "commit" in request.GET:
+        if request.GET["commit"] != "HEAD":
+            res = c.SwitchCommit(project.working_dir, request.GET["commit"])
+        D.update_type = "commit"
+        D.update_version = request.GET["commit"]
+        project.lastCommit = request.GET["commit"]
+    res = c.Integrate(project.working_dir, project.configFile)
+    if not "ERR:" in res:
+        D.datetime = timezone.now()
+        D.has_new_version = False
+        D.save()
+        project.lastUpdate = timezone.now()
+        project.newVersion = False
+        project.save()
+        print(project.integration_link)
+        if not "http://" in project.integration_link:
+            print("in if")
+            link = "http://" + server.DNS + project.integration_link
+            print(link)
+            if project.emailUsers != "" or project.emailUsers != " " and last_Integration != None:
+                changes = c.getChangeLog(project.working_dir, since=last_Integration.update_version,
+                                         to=request.GET["commit"])
+                changes_text = "<h3>Changes</h3><ul>"
+                found = False
+                for change in changes:
+                    if change.endswith(":"): continue
+                    changes_text += "<li>%s</li>" % change
+                    found = True
+                if found:
+                    changes_text += "</ul>"
+                else:
+                    changes_text = ""
+                Common.send(project.emailUsers.replace(",", ";"), "New version of %s integrated" % project.name,
+                            "Dear User,<br/> This is an automated notification that a new version of %s has been integrated at: %s<br/>%s" % (
+                            project.name, link, changes_text), fromUser=None, cc="", bcc="", )
+
+            return HttpResponse(res + ",," + link)
+        else:
+            print("in else")
+            link = project.integration_link
+            if project.emailUsers != "" or project.emailUsers != " ":
+                changes = c.getChangeLog(project.working_dir, since=last_Integration.update_version,
+                                         to=request.GET["commit"])
+                changes_text = "<h3>Changes</h3><ul>"
+                found = False
+                for change in changes:
+                    if change.endswith(":"): continue
+                    changes_text += "<li>%s</li>" % change
+                    found = True
+                if found:
+                    changes_text += "</ul>"
+                else:
+                    changes_text = ""
+
+                Common.send(project.emailUsers.replace(",", ";"), "New version of %s integrated" % project.name,
+                            "Dear User,<br/> This is an automated notification that a new version of %s has been integrated at: %s.<br>%s" % (
+                            project.name, link, changes_text), fromUser=None, cc="", bcc="", )
+            return HttpResponse(res + ",," + link)
     else:
         return HttpResponse(res)

webapp/autoDeploy/deployment/views.py

@@ -4,10 +4,7 @@
 from .tables import *
 from django_tables2.export.export import TableExport
 from django.views.decorators.csrf import csrf_protect
-from django_tables2_reports.config import RequestConfigReport
 from django_tables2.config import RequestConfig
-import sys
-sys.path.append("../../../client")
 from autodeploy_client import Client
 from django.shortcuts import redirect
 from django.template.context_processors import csrf