Skip to content

Commit b0acdbc

Browse files
author
Manuel Amador (Rudd-O)
committed
Split out build responsibility into per-project wscript_build files. Installation of generic directories like bindir, and creation of systemvms, remain in toplevel wscript_build. Make some waf code useful and reusable in the form of tools.
1 parent 11fb89a commit b0acdbc

File tree

16 files changed

+342
-246
lines changed

16 files changed

+342
-246
lines changed

‎agent/wscript_build

+7
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
import Options
2+
3+
bld.install_files("${AGENTLIBDIR}",
4+
bld.path.ant_glob("storagepatch/**",src=True,bld=False,dir=False,flat=True),
5+
cwd=bld.path,relative_trick=True)
6+
if not Options.options.PRESERVECONFIG:
7+
bld.install_files_filtered("${AGENTSYSCONFDIR}","conf/*")

‎client/wscript_build

+11
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
import Options
2+
3+
start_path = bld.path.find_dir("WEB-INF")
4+
bld.install_files('${MSENVIRON}/webapps/client/WEB-INF',
5+
start_path.ant_glob("**",src=True,bld=False,dir=False,flat=True),
6+
cwd=start_path,relative_trick=True)
7+
8+
if not Options.options.PRESERVECONFIG:
9+
bld.install_files_filtered("${MSCONF}","tomcatconf/*")
10+
bld.install_files("${MSCONF}",'tomcatconf/db.properties',chmod=0640)
11+
bld.setownership("${MSCONF}/db.properties","root",bld.env.MSUSER)

‎console-proxy/wscript_build

+10
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
import Options
2+
3+
# binary unsubstitutable files:
4+
bld.install_files("${CPLIBDIR}",bld.path.ant_glob("images/**",src=True,bld=False,dir=False,flat=True),cwd=bld.path,relative_trick=True)
5+
6+
# text substitutable files (substitute with tokens from the environment bld.env):
7+
bld.substitute('css/** js/** ui/** scripts/**',install_to="${CPLIBDIR}")
8+
9+
# config files (do not replace them if preserve config option is true)
10+
if not Options.options.PRESERVECONFIG: bld.install_files_filtered("${CPSYSCONFDIR}","conf.dom0/*")

‎daemonize/wscript_build

+7
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
if bld.env.DISTRO not in ['Windows','Mac']:
2+
# build / install declarations of the daemonization utility - except for Windows
3+
bld(
4+
name='daemonize',
5+
features='cc cprogram',
6+
source='daemonize.c',
7+
target='cloud-daemonize')

‎deps/wscript_build

+1
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
bld.install_files('${JAVADIR}','*.jar')

‎patches/wscript_build

+29
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,29 @@
1+
import os, Utils, glob, re
2+
3+
bld.substitute("*/**",name="patchsubst")
4+
5+
for virttech in Utils.to_list(bld.path.ant_glob("*",dir=True)):
6+
if virttech in ["shared","wscript_build"]: continue
7+
patchfiles = bld.path.ant_glob('%s/** shared/**'%virttech,src=True,bld=True,dir=False,flat=True)
8+
tgen = bld(
9+
features = 'tar',#Utils.tar_up,
10+
source = patchfiles,
11+
target = '%s-patch.tgz'%virttech,
12+
name = '%s-patch_tgz'%virttech,
13+
root = "patches/%s"%virttech,
14+
rename = lambda x: re.sub(".subst$","",x),
15+
after = 'patchsubst',
16+
)
17+
bld.process_after(tgen)
18+
if virttech != "xenserver":
19+
# xenserver uses the patch.tgz file later to make an ISO, so we do not need to install it
20+
bld.install_as("${AGENTLIBDIR}/scripts/vm/hypervisor/%s/patch.tgz"%virttech, "%s-patch.tgz"%virttech)
21+
22+
tgen = bld(
23+
rule = 'cp ${SRC} ${TGT}',
24+
source = 'xenserver-patch.tgz',
25+
target = 'patch.tgz',
26+
after = 'xenserver-patch_tgz',
27+
name = 'patch_tgz'
28+
)
29+
bld.process_after(tgen)

‎python/wscript_build

+3
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
if bld.env.DISTRO not in ['Windows','Mac']:
2+
obj = bld(features = 'py',name='pythonmodules')
3+
obj.find_sources_in_dirs('lib', exts=['.py'])

‎scripts/wscript_build

+1
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
bld.substitute('**',"${AGENTLIBDIR}/scripts",chmod=0755)

‎server/wscript_build

+4
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
import Options
2+
3+
if not Options.options.PRESERVECONFIG:
4+
bld.install_files_filtered("${SERVERSYSCONFDIR}","conf/*")

‎tools/waf/mkisofs.py

+65
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,65 @@
1+
import Utils
2+
from TaskGen import feature, before
3+
import Task
4+
import os
5+
6+
# fixme: this seems to hang waf with 100% CPU
7+
8+
def detect(conf):
9+
conf.find_program("mkisofs",mandatory=True,var='MKISOFS')
10+
11+
def iso_up(task):
12+
tgt = task.outputs[0].bldpath(task.env)
13+
if os.path.exists(tgt): os.unlink(tgt)
14+
inps = []
15+
for inp in task.inputs:
16+
if inp.id&3==Node.BUILD:
17+
src = inp.bldpath(task.env)
18+
srcname = src
19+
srcname = "/".join(srcname.split("/")[1:]) # chop off default/
20+
else:
21+
src = inp.srcpath(task.env)
22+
srcname = src
23+
srcname = "/".join(srcname.split("/")[1:]) # chop off ../
24+
inps.append(src)
25+
ret = Utils.exec_command(
26+
[
27+
task.generator.env.MKISOFS,
28+
"-quiet",
29+
"-r",
30+
"-o",tgt,
31+
] + inps, shell=False)
32+
if ret != 0: return ret
33+
if task.chmod: os.chmod(tgt,task.chmod)
34+
35+
def apply_iso(self):
36+
Utils.def_attrs(self,fun=iso_up)
37+
self.default_install_path=0
38+
lst=self.to_list(self.source)
39+
self.meths.remove('apply_core')
40+
self.dict=getattr(self,'dict',{})
41+
out = self.path.find_or_declare(self.target)
42+
ins = []
43+
for x in Utils.to_list(self.source):
44+
node = self.path.find_resource(x)
45+
if not node:raise Utils.WafError('cannot find input file %s for processing'%x)
46+
ins.append(node)
47+
if self.dict and not self.env['DICT_HASH']:
48+
self.env=self.env.copy()
49+
keys=list(self.dict.keys())
50+
keys.sort()
51+
lst=[self.dict[x]for x in keys]
52+
self.env['DICT_HASH']=str(Utils.h_list(lst))
53+
tsk=self.create_task('iso',ins,out)
54+
tsk.fun=self.fun
55+
tsk.dict=self.dict
56+
tsk.dep_vars=['DICT_HASH']
57+
tsk.install_path=self.install_path
58+
tsk.chmod=self.chmod
59+
if not tsk.env:
60+
tsk.debug()
61+
raise Utils.WafError('task without an environment')
62+
63+
Task.task_type_from_func('iso',func=iso_up)
64+
feature('iso')(apply_iso)
65+
before('apply_core')(apply_iso)

‎tools/waf/tar.py

+74
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,74 @@
1+
import Utils
2+
import tarfile
3+
from TaskGen import feature, before
4+
import Task
5+
import os
6+
7+
# this is a clever little thing
8+
# given a list of nodes, build or source
9+
# construct a tar file containing them
10+
# rooted in the parameter root =, specified in the task generator
11+
# and renaming the names of the files according to a rename(x) function passed to the task generator as well
12+
# if a build node's result of rename() has the same name as a source node, the build node will take precedence
13+
# for as long as the build node appears later than the source node (this is an implementation detail of waf we are relying on)
14+
def tar_up(task):
15+
tgt = task.outputs[0].bldpath(task.env)
16+
if os.path.exists(tgt): os.unlink(tgt)
17+
if tgt.lower().endswith(".bz2"): z = tarfile.open(tgt,"w:bz2")
18+
elif tgt.lower().endswith(".gz"): z = tarfile.open(tgt,"w:gz")
19+
elif tgt.lower().endswith(".tgz"): z = tarfile.open(tgt,"w:gz")
20+
else: z = tarfile.open(tgt,"w")
21+
fileset = {}
22+
for inp in task.inputs:
23+
src = inp.srcpath(task.env)
24+
if src.startswith(".."):
25+
srcname = Utils.relpath(src,os.path.join("..",".")) # file in source dir
26+
else:
27+
srcname = Utils.relpath(src,os.path.join(task.env.variant(),".")) # file in artifacts dir
28+
if task.generator.rename: srcname = task.generator.rename(srcname)
29+
for dummy in task.generator.root.split("/"):
30+
splittedname = srcname.split("/")
31+
srcname = "/".join(splittedname[1:])
32+
fileset[srcname] = src
33+
for srcname,src in fileset.items():
34+
ti = tarfile.TarInfo(srcname)
35+
ti.mode = 0755
36+
ti.size = os.path.getsize(src)
37+
f = file(src)
38+
z.addfile(ti,fileobj=f)
39+
f.close()
40+
z.close()
41+
if task.chmod: os.chmod(tgt,task.chmod)
42+
return 0
43+
44+
def apply_tar(self):
45+
Utils.def_attrs(self,fun=tar_up)
46+
self.default_install_path=0
47+
lst=self.to_list(self.source)
48+
self.meths.remove('apply_core')
49+
self.dict=getattr(self,'dict',{})
50+
out = self.path.find_or_declare(self.target)
51+
ins = []
52+
for x in Utils.to_list(self.source):
53+
node = self.path.find_resource(x)
54+
if not node:raise Utils.WafError('cannot find input file %s for processing'%x)
55+
ins.append(node)
56+
if self.dict and not self.env['DICT_HASH']:
57+
self.env=self.env.copy()
58+
keys=list(self.dict.keys())
59+
keys.sort()
60+
lst=[self.dict[x]for x in keys]
61+
self.env['DICT_HASH']=str(Utils.h_list(lst))
62+
tsk=self.create_task('tar',ins,out)
63+
tsk.fun=self.fun
64+
tsk.dict=self.dict
65+
tsk.dep_vars=['DICT_HASH']
66+
tsk.install_path=self.install_path
67+
tsk.chmod=self.chmod
68+
if not tsk.env:
69+
tsk.debug()
70+
raise Utils.WafError('task without an environment')
71+
72+
Task.task_type_from_func('tar',func=tar_up)
73+
feature('tar')(apply_tar)
74+
before('apply_core')(apply_tar)

‎ui/wscript_build

+41
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,41 @@
1+
import Utils, os
2+
3+
# binary unsubstitutable files:
4+
bld.install_files("${MSENVIRON}/webapps/client",bld.path.ant_glob("*.ico **/*png **/*jpg **/*gif",src=True,bld=False,dir=False,flat=True),cwd=bld.path,relative_trick=True)
5+
6+
# text substitutable files (substitute with tokens from the environment bld.env):
7+
bld.substitute('*html **/*html **/*js **/*css **/*properties **/*jsp *jsp',install_to="${MSENVIRON}/webapps/client")
8+
9+
# -> minification of UI files
10+
def minifyjs(task):
11+
tgt = task.outputs[0].bldpath(task.env)
12+
inputfiles = []
13+
outputfile = ['--js_output_file',tgt]
14+
for inp in task.inputs:
15+
src = inp.srcpath(task.env)
16+
inputfiles.append(src)
17+
newinputfiles = []
18+
for inputfile in inputfiles:
19+
if inputfile not in newinputfiles:
20+
newinputfiles.append('--js')
21+
newinputfiles.append(inputfile)
22+
compilerjar = os.path.join(bld.srcnode.abspath(),'tools','gcc','compiler.jar')
23+
return Utils.exec_command(["java",'-jar',compilerjar] + newinputfiles + outputfile,log=True)
24+
25+
javascripts = [
26+
['scripts/jquery-1.4.2.min.js','scripts/date.js'],
27+
Utils.to_list(bld.path.ant_glob('scripts/jquery*js')),
28+
['scripts/cloud.core.js','scripts/cloud.core.callbacks.js'],
29+
Utils.to_list(bld.path.ant_glob('scripts/cloud*js')),
30+
]
31+
sourcefiles = []
32+
for lst in javascripts:
33+
for x in lst:
34+
if x not in sourcefiles: sourcefiles.append(x)
35+
tgen = bld(
36+
rule = minifyjs,
37+
source = sourcefiles,
38+
target = 'scripts/cloud.min.js',
39+
name = 'minifyjs',
40+
)
41+
bld.install_files("${MSENVIRON}/webapps/client/scripts", "scripts/cloud.min.js")

‎vnet/wscript_build

+21
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,21 @@
1+
import Utils
2+
3+
if bld.env.DISTRO not in ['Windows','Mac']:
4+
# build / install declarations of vnet
5+
files = """vnetd/connection.c vnetd/select.c vnetd/timer.c vnetd/spinlock.c vnetd/skbuff.c
6+
vnetd/vnetd.c vnet-module/skb_util.c vnet-module/sxpr_util.c vnet-module/timer_util.c
7+
vnet-module/etherip.c vnet-module/vnet.c vnet-module/vnet_eval.c vnet-module/vnet_forward.c
8+
vnet-module/vif.c vnet-module/tunnel.c vnet-module/sa.c vnet-module/varp.c
9+
libxutil/allocate.c libxutil/enum.c libxutil/file_stream.c libxutil/hash_table.c
10+
libxutil/iostream.c libxutil/lexis.c libxutil/socket_stream.c libxutil/string_stream.c
11+
libxutil/sxpr.c libxutil/sxpr_parser.c libxutil/sys_net.c libxutil/sys_string.c libxutil/util.c"""
12+
files = [ "src/%s"%s for s in Utils.to_list(files) ]
13+
bld(
14+
name='vnetd',
15+
features='cc cprogram',
16+
source= files,
17+
includes="src/libxutil src/vnet-module src/vnetd",
18+
lib='dl pthread'.split(),
19+
target='%s-vnetd'%bld.env.PACKAGE,
20+
install_path="${SBINDIR}"
21+
)

‎wscript

+32
Original file line numberDiff line numberDiff line change
@@ -342,6 +342,24 @@ def _install_files_filtered(self,destdir,listoffiles,**kwargs):
342342
return ret
343343
Build.BuildContext.install_files_filtered = _install_files_filtered
344344

345+
def _substitute(self,listoffiles,install_to=None,cwd=None,dict=None,name=None,**kwargs):
346+
if cwd is None: cwd = self.path
347+
tgenkwargs = {}
348+
if name is not None: tgenkwargs["name"] = name
349+
if isinstance(listoffiles,str) and '**' in listoffiles:
350+
listoffiles = cwd.ant_glob(listoffiles,flat=True)
351+
elif isinstance(listoffiles,str) and '*' in listoffiles:
352+
listoffiles = [ n for x in listoffiles.split() for n in _glob(cwd.abspath() + os.sep + x.replace("/",os.sep)) ]
353+
for src in Utils.to_list(listoffiles):
354+
tgt = src + ".subst"
355+
inst = src # Utils.relpath(src,relative_to) <- disabled
356+
tgen = self(features='subst', source=src, target=tgt, **tgenkwargs)
357+
if dict is not None: tgen.dict = dict
358+
else: tgen.dict = self.env.get_merged_dict()
359+
self.path.find_or_declare(tgt)
360+
if install_to is not None: self.install_as("%s/%s"%(install_to,inst), tgt, **kwargs)
361+
Build.BuildContext.substitute = _substitute
362+
345363
def _setownership(ctx,path,owner,group,mode=None):
346364
def f(bld,path,owner,group,mode):
347365
dochown = not Options.options.NOCHOWN \
@@ -382,6 +400,8 @@ Build.BuildContext.setownership = _setownership
382400
def set_options(opt):
383401
"""Register command line options"""
384402
opt.tool_options('gnu_dirs')
403+
opt.tool_options('tar',tooldir='tools/waf')
404+
opt.tool_options('mkisofs',tooldir='tools/waf')
385405
if platform.system() not in ['Windows',"Darwin"]: opt.tool_options('compiler_cc')
386406
opt.tool_options('python')
387407

@@ -487,6 +507,18 @@ def showconfig(conf):
487507
continue
488508
Utils.pprint("BLUE"," %s: %s"%(key,val))
489509

510+
def _getconfig(self):
511+
lines = []
512+
for key,val in sorted(self.env.get_merged_dict().items()):
513+
if "CLASSPATH" in key:
514+
lines.append(" %s:"%key)
515+
for v in val.split(pathsep):
516+
lines.append(" %s"%v)
517+
continue
518+
lines.append(" %s: %s"%(key,val))
519+
return "\n".join(lines)
520+
Build.BuildContext.getconfig = _getconfig
521+
490522
def list_targets(ctx):
491523
"""return the list of buildable and installable targets"""
492524

0 commit comments

Comments
 (0)