build

#!/usr/bin/env python

import platform, sys, os, time, threading, subprocess, copy, codecs, glob, atexit, tempfile


config_file = 'config.default'


bcolors = {
    "candidate" : '\033[94m',
    "no known conversion" : '\033[94m',
    "expected" : '\033[93m',
    "^" : '\033[91m',
    "static assertion" : '\033[91m',
    "Linking" : '\033[01;32m',
    "In function" : '\033[01;32m',
    "WARNING" : '\033[95m',
    "Warning" : '\033[95m',
    "warning" : '\033[95m',
    "required from" : '\033[94m',
    "In instantiation of" : '\033[01;32m',
    "In member" : '\033[01;32m',
    "ERROR" : '\033[01;95m',
    "error" : '\033[01;31m',
    "failed" : '\033[91m',
    "note" : '\033[94m'}

def colorize(s):
  out = []
  for l in s.splitlines():
    for st in bcolors.keys():
      if st in l:
        l = l.replace (st, bcolors[st] + st) + '\033[0m'
        break
    out.append(l + '\n')
  return out



system = None
dependencies = False
verbose = False
targets = []
use_multiple_cores = True

global todo, headers, object_deps, file_flags, lock, print_lock, stop
global include_paths
todo, headers, object_deps, file_flags = {}, {}, {}, {}
lock = threading.Lock()
print_lock = threading.Lock()
stop = False
error_stream = None

logfile = open ('build.log', 'wb')



def pipe_errors_to_less_handler():
  global error_stream
  if len (error_stream):
    try:
      [ fid, name ] = tempfile.mkstemp()
      try:
        fid = os.fdopen (fid, 'wb')
        [ fid.write (x.encode (errors='ignore')) for x in colorize(error_stream) ]
        fid.close()
        os.system ("less -R " + name)
      except Exception as e:
        sys.stderr.write (str (e))
      os.unlink (name)
    except Exception as e:
      sys.stderr.write (str (e))
    except:
      raise



if sys.stderr.isatty():
  error_stream = ''
  atexit.register (pipe_errors_to_less_handler)




def disp (msg):
  print_lock.acquire()
  logfile.write (msg.encode (errors='ignore'))
  sys.stdout.write (msg)
  print_lock.release()
  
def log (msg):
  print_lock.acquire()
  logfile.write (msg.encode (errors='ignore'))
  if verbose:
    sys.stderr.write (msg)
  print_lock.release()

def error (msg):
  global error_stream
  print_lock.acquire()
  logfile.write (msg.encode (errors='ignore'))
  if error_stream is not None:
    error_stream += msg
  else:
    sys.stderr.write (msg)
  print_lock.release()
  


def split_path (filename):
  return filename.replace ('\\', '/').split ('/')



def get_git_lib_version (folder):
  log ('''
getting short git version in folder "''' + folder + '"... ')

  try: 
    process = subprocess.Popen ([ 'git', 'describe', '--abbrev=0', '--always' ], cwd=folder, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
    ( git_version, stderr ) = process.communicate()
    if process.returncode == 0:
      git_version = git_version.decode('utf-8', 'ignore').strip()
      log (git_version + '\n')
      return git_version
  except:
    pass
  log ('not found\n')
  return None


# common definitions:

bin_dir = 'bin'
cmd_dir = 'cmd'
lib_dir = 'lib'
misc_dir = 'src'
other_dir = '3rd_party'
doc_dir = 'doc'
dev_dir = 'dev'

cpp_suffix = '.cpp'
h_suffix = '.h'
libname = 'mrtrix'
config_suffix = ''




include_paths = [ misc_dir, cmd_dir ]
if os.path.exists (other_dir):
  include_paths += [ other_dir ]

# check if we are compiling a separate project:
mrtrix_dir = '.'
build_script = sys.argv[0]
separate_project = False

def get_real_name (path):
  if os.path.islink (path): return os.readlink (path)
  else: return (path)

while os.path.abspath (os.path.dirname (get_real_name (build_script))) != os.path.abspath (mrtrix_dir):
  if not separate_project:
    log ('compiling separate project against:' + os.linesep)
  separate_project = True
  build_script = os.path.normpath (os.path.join (mrtrix_dir, get_real_name (build_script)))
  mrtrix_dir = os.path.dirname (build_script)
  include_paths += [ os.path.join (mrtrix_dir, misc_dir) ]
  if os.path.exists (os.path.join (mrtrix_dir, other_dir)):
    include_paths += [ os.path.join (mrtrix_dir, other_dir) ]
  log ('    ' + mrtrix_dir + os.linesep)

if separate_project:
  config_file = os.path.join (mrtrix_dir, config_file)

  log (os.linesep);






# parse command-line:
for arg in sys.argv[1:]:
  if '-help'.startswith (arg):
    print ('''
usage: ./build [-verbose] [-dependencies] [target ...]

In most cases, a simple invocation is all that is required:

  $ ./build

If no targets are provided, the command will default to building 
all applications by scanning through the cmd/ folder. 

The special target 'clean' is used to remove all compiler-generated 
files, including objects, executables, and shared libraries. 

OPTIONS:

  -verbose           print each command as it is being invoked
  -dependencies      print the list of dependencies for every target 
  -disablethreading  disable multithreading and build using a single core only
''')
    exit (0)
  elif   '-verbose'.startswith(arg): 
    verbose = True
  elif '-dependencies'.startswith(arg): 
    dependencies = True
  elif '-disablethreading'.startswith(arg): 
    use_multiple_cores = False
  elif arg[0] == '-':
    sys.stderr.write ('unknown command-line option "' + arg + '"\n')
    sys.exit (1)
  elif arg == 'clean':
    targets = [ 'clean' ]
    break
  elif os.path.isfile (os.path.join (mrtrix_dir, "config." + arg)):
    config_file = os.path.join (mrtrix_dir, "config." + arg)
    config_suffix = '__' + arg
  else: targets.append(arg)





# load configuration file:
try:
  log ('reading configuration from "' + config_file + '"...' + os.linesep)
  exec (codecs.open (config_file, mode='r', encoding='utf-8').read())
except IOError:
  sys.stderr.write ('''no configuration file found!
please run "./configure" prior to invoking this script

''')
  sys.exit (1)


environ = os.environ.copy()
environ.update ({ 'PATH': PATH })


if ld_enabled and len(runpath):
    ld_flags += [ runpath+os.path.relpath (lib_dir,bin_dir) ]

if separate_project:
  lib_dir = os.path.join (mrtrix_dir, lib_dir)
  if ld_enabled and len(runpath):
    ld_flags += [ runpath+os.path.relpath (lib_dir,bin_dir) ]



# get version info:

lib_version = get_git_lib_version (mrtrix_dir)
if lib_version is not None:
  if lib_version.find('-') > 0:
    lib_version = lib_version[0:lib_version.find('-')]
  libname += '-' + lib_version

if ld_enabled:
  libname += config_suffix
  ld_flags.insert(0, '-l' + libname)
  libname = lib_prefix + libname + lib_suffix

obj_suffix = config_suffix + obj_suffix
exe_suffix = config_suffix + exe_suffix
moc_cpp_suffix = config_suffix + '_moc' + cpp_suffix
moc_obj_suffix = config_suffix + '_moc' + obj_suffix



# Qt4 settings:
#qt4_cflags += [ '-I' + entry for entry in qt4_include_path ]



# other settings:
include_paths += [ lib_dir, cmd_dir ]
cpp_flags += [ '-I' + entry for entry in include_paths ]
ld_flags += [ '-L' + lib_dir ]









###########################################################################
#                           TODO list Entry
###########################################################################

class Entry:
  def __init__ (self, name):
    global todo
    if name in todo.keys(): return
    todo[name] = self

    self.name = name
    self.cmd = []
    self.deps = set()
    self.action = 'NA'
    self.timestamp = mtime (self.name)
    self.dep_timestamp = self.timestamp
    self.currently_being_processed = False

    if is_executable (self.name): self.set_executable()
    elif is_icon (self.name): self.set_icon()
    elif is_object (self.name): self.set_object()
    elif is_library (self.name): self.set_library()
    elif is_moc (self.name): self.set_moc()
    elif not os.path.exists (self.name): 
      raise Exception ('unknown target "' + self.name + '"')

    [ Entry(item) for item in self.deps ]
    dep_timestamp = [ todo[item].timestamp for item in todo.keys() if item in self.deps and not is_library(item) ]
    dep_timestamp += [ todo[item].dep_timestamp for item in todo.keys() if item in self.deps and not is_library(item) ]
    if len(dep_timestamp): 
      self.dep_timestamp = max(dep_timestamp)

    

  def execute (self):
    if self.action == 'RCC':
      with codecs.open (self.cmd[1], mode='w', encoding='utf-8') as fd:
        fd.write ('<!DOCTYPE RCC><RCC version="1.0">\n<qresource>\n')
        for entry in self.deps:
          entry = os.path.basename (entry)
          if not entry.startswith ('config.'):
            fd.write ('<file>' + entry + '</file>\n')
        fd.write ('</qresource>\n</RCC>\n')
      
    if len(self.cmd) > 0: 
      return execute ('[' + self.action + '] ' + self.name, self.cmd)
    else: 
      return None


  def set_executable (self):
    self.action = 'LB'
    if len(exe_suffix) > 0: cc_file = self.name[:-len(exe_suffix)]
    else: cc_file = self.name
    cc_file = os.path.join (cmd_dir, os.sep.join (split_path(cc_file)[1:])) + cpp_suffix
    self.deps = list_cmd_deps(cc_file)

    skip = False
    flags = copy.copy (gsl_ldflags)
    if 'Q' in file_flags[cc_file]: flags += qt_ldflags

    if not skip: 
      if not os.path.isdir (bin_dir): 
        os.mkdir (bin_dir)

      if not ld_enabled: 
        self.deps = self.deps.union (list_lib_deps())

      self.cmd = fillin (ld, { 
        'LDFLAGS': [ s.replace ('LIBNAME', os.path.basename (self.name)) for s in ld_flags ] + flags,
        'OBJECTS': self.deps,
        'EXECUTABLE': [ self.name ] })

      try:
        if ld_use_shell: self.cmd = [ 'sh', '-c', ' '.join(self.cmd) ]
      except NameError: pass

      if ld_enabled:
        self.deps.add (os.path.join (lib_dir, libname))



  def set_object (self):
    self.action = 'CC'
    cc_file = self.name[:-len(obj_suffix)] + cpp_suffix
    self.deps = set([ cc_file ])
    flags = copy.copy (gsl_cflags)

    if is_moc (cc_file):
      src_header = cc_file[:-len(moc_cpp_suffix)] + h_suffix
      list_headers (src_header)
      file_flags[cc_file] = file_flags[src_header]
    else: self.deps = self.deps.union (list_headers (cc_file))
    self.deps.add (config_file)

    skip = False
    if 'Q' in file_flags[cc_file]: flags += qt_cflags

    if not skip: 
      self.cmd = fillin (cpp, { 
        'CFLAGS': cpp_flags + flags,
        'OBJECT': [ self.name ],
        'SRC': [ cc_file ] })


  def set_moc (self):
    self.action = 'MOC'
    src_file = self.name[:-len(moc_cpp_suffix)] + h_suffix
    self.deps = set([ src_file ])
    self.deps = self.deps.union (list_headers (src_file))
    self.deps.add (config_file)
    self.cmd = [ moc ]
    self.cmd += [ src_file, '-o', self.name ]

    
  def set_library (self):
    if not ld_enabled:
      error ('ERROR: shared library generation is disabled in this configuration')
      exit (1)

    self.action = 'LD'
    self.deps = list_lib_deps()

    self.cmd = fillin (ld_lib, {
      'LDLIB_FLAGS': [ s.replace ('LIBNAME', os.path.basename (self.name)) for s in ld_lib_flags ],
      'OBJECTS': [ item for item in self.deps ],
      'LIB': [ self.name ] })

    try:
      if ld_use_shell: self.cmd = [ 'sh', '-c', ' '.join(self.cmd) ]
    except NameError: pass


  def set_icon (self):
    self.action = 'RCC'
    with codecs.open (self.name[:-len(cpp_suffix)]+h_suffix, mode='r', encoding='utf-8') as fd:
      for line in fd:
        if line.startswith ('//RCC:'):
          for entry in line[6:].strip().split():
            self.deps = self.deps.union (glob.glob (os.path.join (mrtrix_dir, 'icons', entry)))
    self.deps.add (config_file)
    self.cmd = [ rcc, os.path.join ('icons', os.path.basename (os.path.dirname(self.name)) + '.qrc'), '-o', self.name ]


  def need_rebuild (self):
    return self.timestamp == float("inf") or self.timestamp < self.dep_timestamp

  def display (self):
    sys.stdout.write ('[' + self.action + '] ' + self.name)
    if self.need_rebuild():
      sys.stdout.write (' [REBUILD]')
    sys.stdout.write (''':
  deps: ''' + ' '.join(self.deps) + '''
  command: ''' + ' '.join(self.cmd) + '''
  timestamp: ''' + str(self.timestamp) + ', dep timestamp: ' + str(self.dep_timestamp) + '''
''')






###########################################################################
#                         FUNCTION DEFINITIONS
###########################################################################


def default_targets():
  if not os.path.isdir (cmd_dir): 
    sys.stderr.write ('ERROR: no "cmd" folder - unable to determine default targets' + os.linesep)
    sys.exit (1)
  for entry in os.listdir (cmd_dir):
    if entry.endswith(cpp_suffix):
      targets.append (os.path.join (bin_dir, entry[:-len(cpp_suffix)] + exe_suffix))
  return targets

def is_executable (target):
  return split_path (target)[0] == bin_dir and not is_moc (target)

def is_library (target):
  return target.endswith (lib_suffix) and split_path(target)[-1].startswith (lib_prefix)

def is_object (target):
  return target.endswith (obj_suffix)

def is_moc (target):
  return target.endswith (moc_cpp_suffix)

def is_icon (target):
  return os.path.basename (target) == 'icons'+cpp_suffix

def mtime (target):
  if not os.path.exists (target): return float('inf')
  return os.stat(target).st_mtime


def fillin (template, keyvalue):
  cmd = []
  for item in template:
    if item in keyvalue: 
      cmd += keyvalue[item]
    else: 
      cmd += [ item ]
  return cmd




def execute (message, cmd):
  disp (message + os.linesep)
  log (' '.join(cmd) + os.linesep)

  try: 
    process = subprocess.Popen (cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=environ)
    ( stdout, stderr ) = process.communicate()
    if process.returncode != 0:
      if error_stream is not None:
        disp ('ERROR: ' + message + os.linesep)
      error ('\nERROR: ' + message + '\n\n' + ' '.join(cmd) + '\n\nfailed with output\n\n' + stderr.decode (errors='ignore'))
      return 1

    errstr = None
    if len(stdout) or len(stderr):
      errstr = '\n' + ' '.join(cmd) + ':\n\n'

    if len(stdout):
      errstr += stdout.decode (errors='ignore') + '\n\n'
    if len(stderr):
      errstr += stderr.decode (errors='ignore') + '\n\n'

    if errstr:
      disp (errstr);

  except OSError:
    error (cmd[0] + ': command not found')
    return 1
  except:
    error ('unexpected exception: ' + str(sys.exc_info()))
    raise


def print_deps (current_file, indent=''):
  sys.stdout.write (indent + current_file)
  if current_file in file_flags: 
    if len(file_flags[current_file]): 
      sys.stdout.write (' [' + file_flags[current_file] + ']')
  sys.stdout.write (os.linesep)
  if len(todo[current_file].deps):
    for entry in todo[current_file].deps:
      print_deps (entry, indent + '    ')




def is_GUI_target (current_file):
  if 'gui' in split_path (current_file):
    return True
  if current_file in file_flags: 
    if 'Q' in file_flags[current_file]: 
      return True
  if len(todo[current_file].deps):
    for entry in todo[current_file].deps:
      if is_GUI_target (entry):
        return True
  return False



def list_headers (current_file):
  global headers, file_flags

  if current_file not in headers.keys(): 
    headers[current_file] = set()

    if current_file not in file_flags: 
      file_flags[current_file] = ''

    if 'gui' in split_path (current_file):
      if 'Q' not in file_flags[current_file]:
        file_flags[current_file] += 'Q'
    if not os.path.exists (current_file):
      if os.path.basename(current_file) == 'icons'+cpp_suffix: 
        return headers[current_file]
      sys.stderr.write ('ERROR: cannot find file "' + current_file + '"' + os.linesep)
      sys.exit(1)
    with codecs.open (current_file, mode='r', encoding='utf-8') as fd:
      for line in fd:
        line = line.strip()
        if line.startswith('#include'):
          line = line[8:].split ('//')[0].split ('/*')[0].strip()
          if line[0] == '"':
            line = line[1:].rstrip('"')
            for path in include_paths:
              if os.path.exists (os.path.join (path, line)):
                line = os.path.join (path, line)
                headers[current_file].add (line)
                [ headers[current_file].add(entry) for entry in list_headers(line) ]
                break
            else:
              sys.stderr.write ('ERROR: cannot find header file \"' + line + '\" (from file \"' + current_file + '\")' + os.linesep)
              sys.exit(1)
        elif line == 'Q_OBJECT':
          if 'M' not in file_flags[current_file]:
            file_flags[current_file] += 'M'

    for entry in headers[current_file]:
      for c in file_flags[entry]:
        if c != 'M' and c not in file_flags[current_file]:
          file_flags[current_file] += c

  return headers[current_file]






def list_cmd_deps (file_cc):
  global object_deps, file_flags

  if file_cc not in object_deps.keys():
    object_deps[file_cc] = set([ file_cc[:-len(cpp_suffix)] + obj_suffix ])
    for entry in list_headers (file_cc):
      if os.path.abspath(entry).startswith(os.path.abspath(lib_dir)): continue
      if 'M' in file_flags[entry]: 
        object_deps[file_cc] = object_deps[file_cc].union ([ entry[:-len(h_suffix)] + moc_obj_suffix ])
      entry_cc = entry[:-len(h_suffix)] + cpp_suffix
      if os.path.exists (entry_cc):
        object_deps[file_cc] = object_deps[file_cc].union (list_cmd_deps(entry_cc))
      if os.path.basename (entry) == 'icons'+h_suffix:
        object_deps[file_cc].add (entry[:-len(h_suffix)]+obj_suffix)
    if file_cc not in file_flags: file_flags[file_cc] = ''
    for entry in headers[file_cc]:
      for c in file_flags[entry]:
        if c != 'M' and c not in file_flags[file_cc]:
          file_flags[file_cc] += c

  return object_deps[file_cc]



def list_lib_deps ():
  deps = set()
  for root, dirs, files in os.walk (lib_dir):
    for current_file in files:
      if current_file[0] == '.': continue
      if current_file.endswith (cpp_suffix):
        deps.add (os.path.join (root, current_file[:-len(cpp_suffix)] + obj_suffix))

  return (deps)



def build_next (id):
  global todo, lock, stop

  try:
    while not stop:
      current = None
      lock.acquire()
      if len(todo):
        for item in todo.keys():
          if todo[item].currently_being_processed: continue
          unsatisfied_deps = set(todo[item].deps).intersection (todo.keys())
          if not len(unsatisfied_deps):
            todo[item].currently_being_processed = True
            current = item
            break
      else: stop = max (stop, 1)
      lock.release()
  
      if stop: return
      if current == None: 
        time.sleep (0.01)
        continue
  
      target = todo[current]
      if target.execute():
        stop = 2
        return

      lock.acquire()
      del todo[current]
      lock.release()

  except:
    stop = 2
    return
    
  stop = max(stop, 1)



def start_html (fid, title, left, up, home, right):
  fid.write ('<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.0 Transitional//EN">\n<html>\n<head>\n<meta http-equiv="Content-Type" content="text/html;charset=iso-8859-1">\n')
  fid.write ('<title>MRtrix documentation</title>\n<link rel="stylesheet" href="../stylesheet.css" type="text/css" media=screen>\n</head>\n<body>\n\n')
  fid.write ('<table class=nav>\n<tr>\n<td><a href="' + left + '.html"><img src="../left.png"></a></td>\n')
  fid.write ('<td><a href="' + up + '.html"><img src="../up.png"></a></td>\n')
  fid.write ('<td><a href="' + home + '.html"><img src="../home.png"></a></td>\n')
  fid.write ('<th>' + title + '</th>\n')
  fid.write ('<td><a href="' + right + '.html"><img src="../right.png"></a></td>\n</tr>\n</table>\n')




def gen_command_html_help ():
  binaries = os.listdir (bin_dir)
  binaries.sort()
  description = []

  # generate each program's individual page:
  for n in range (0, len(binaries)):
    sys.stderr.write ('[DOC] ' + binaries[n] + os.linesep)
     
    process = subprocess.Popen ([ binaries[n], '__print_full_usage__' ], stdout=subprocess.PIPE)
    if process.wait() != 0:
      sys.stderr.write ('ERROR: unable to execute', binaries[n], ' - aborting' + os.linesep)
      return 1
    H = process.stdout.read().decode (errors='ignore')
    H = H.splitlines()

    with codecs.open (os.path.join (doc_dir, 'commands', binaries[n] + '.html'), mode='w', encoding='utf-8') as fid:

      if n == 0: prev = 'index'
      else: prev = binaries[n-1]

      if n == len(binaries)-1: next = 'index'
      else: next = binaries[n+1]

      start_html (fid, binaries[n], prev, 'index', '../index', next)
      fid.write ('<h2>Description</h2>\n')

      line = 0
      while not H[line].startswith ('ARGUMENT') and not H[line].startswith('OPTION'):
        if len(description) <= n: description.append (H[line])
        fid.write ('<p>\n' + H[line] + '\n</p>\n')
        line += 1
  
      arg = []
      opt = []
      while line < len(H)-2:
        if not H[line].startswith ('ARGUMENT') and not H[line].startswith ('OPTION'):
          sys.stderr.write ('ERROR: malformed usage for executable "' + binaries[n] + '" - aborting' + os.linesep)
          return 1
  
        if H[line].startswith ('ARGUMENT'):
          S = H[line].split (None, 5)
          A = [ S[1], int(S[2]), int(S[3]), S[4], H[line+1], H[line+2] ]
          if len(opt) > 0: opt[-1].append (A)
          else: arg.append(A)
  
        elif H[line].startswith ('OPTION'):
          S = H[line].split (None, 4)
          A = [ S[1], int(S[2]), int(S[3]), H[line+1], H[line+2] ]
          opt.append (A)
  
        line += 3
  
      fid.write ('<p class=indented><strong>syntax:</strong> &nbsp; &nbsp; ' + binaries[n] + ' [ options ] ')
      for A in arg:
        if A[1] == 0: fid.write ('[ ')
        fid.write (A[0] + ' ')
        if A[2] == 1: fid.write ('[ ' + A[0] + ' ... ')
        if A[1] == 0 or A[2] == 1: fid.write ('] ')
      fid.write ('</p>\n<h2>Arguments</h2>\n<table class=args>\n')
  

      for A in arg:
        fid.write ('<tr><td><b>' + A[0] + '</b>')
        if A[1] == 0 or A[2] == 1:
          fid.write (' [ ')
          if A[1] == 0: 
            fid.write ('optional')
            if A[2] == 1: fid.write (', ')
          if A[2] == 1: fid.write ('multiples allowed')
          fid.write (' ]')
        fid.write ('</td>\n<td>' + A[5] + '</td></tr>\n')
      fid.write ('</table>\n')

      fid.write ('<h2>Options</h2>\n<table class=args>\n')
      for O in opt:
        fid.write ('<tr><td><b>-' + O[0] + '</b>')
        for A in O[5:]: fid.write ('&nbsp;<i>' + A[0] + '</i>')
        fid.write ('</td>\n<td>' + O[4])
        if len(O) > 5:
          fid.write ('\n<table class=opts>\n')
          for A in O[5:]:
            fid.write ('<tr><td><i>' + A[0] + '</i></td>\n<td>' + A[5] + '</td></tr>\n')
          fid.write ('</table>')
        fid.write ('</td></tr>\n')
      fid.write ('</table>\n')
  
      fid.write ('</body>\n</html>')
  

  with codecs.open (os.path.join (doc_dir, 'commands', 'index.html'), mode='w', encoding='utf-8') as fid:
    start_html (fid, 'list of MRtrix commands', '../faq', '../index', '../index', '../appendix/index')
    fid.write ('<table class=cmdindex width=100%>\n')

    for n in range (0,len(binaries)):
      fid.write ('<tr><td><a href="' + binaries[n] + '.html">' + binaries[n] + '</a></td><td>' + description[n] + '</td></tr>\n')
    fid.write ('</table>\n</body>\n</html>')




def clean_cmd ():
  files_to_remove = []
  for root, dirs, files in os.walk ('.'):
    for current_file in files:
      if current_file[0] == '.': continue
      if current_file.endswith (obj_suffix) or ( current_file.startswith (lib_prefix) and current_file.endswith (lib_suffix) ) or current_file.endswith (moc_cpp_suffix) or current_file.endswith ('.qrc'):
        files_to_remove.append (os.path.join (root, current_file))
        
  dirs_to_remove = []
  if os.path.isdir (bin_dir):
    for root, dirs, files in os.walk (bin_dir, topdown=False):
      for current_file in files: 
        files_to_remove.append (os.path.join (root, current_file))
      for entry in dirs: 
        dirs_to_remove.append (os.path.join (root, entry))
  
  if os.path.isdir (dev_dir):
    sys.stderr.write ('[RM] development doc' + os.linesep)
    for root, dirs, files in os.walk (dev_dir, topdown=False):
      for entry in files:
        os.remove (os.path.join (root, entry))
      for entry in dirs:
        os.rmdir (os.path.join (root, entry))
    os.rmdir (dev_dir)

  if len(files_to_remove):
    sys.stderr.write ('[RM] ' + ' '.join(files_to_remove) + os.linesep)
    for entry in files_to_remove: 
      os.remove (entry)

  if len(dirs_to_remove):
    sys.stderr.write ('[RM] ' + ' '.join (dirs_to_remove) + os.linesep)
    for entry in dirs_to_remove: 
      os.rmdir (entry)



def get_git_version (folder):
  log ('''
getting git version in folder "''' + folder + '"... ')

  try: 
    process = subprocess.Popen ([ 'git', 'describe', '--abbrev=8', '--dirty', '--always' ], cwd=folder, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
    ( git_version, stderr ) = process.communicate()
    if process.returncode == 0:
      git_version = git_version.decode(errors='ignore').strip()
      log (git_version + '\n')
      return git_version
  except:
    pass


  log ('not found\n')
  return 'unknown'


def update_git_version (folder, git_version_file, contents):
  git_version = get_git_version (folder)
  version_file_contents = contents.replace ('%%%', git_version)
  current_version_file_contents = ''
  try:
    with open (git_version_file) as fd:
      current_version_file_contents = fd.read()
  except:
    pass
    
  if not current_version_file_contents or (version_file_contents != current_version_file_contents and git_version != 'unknown'):
    log ('version file "' + git_version_file + '" is out of date - updating\n')
    with open (git_version_file, 'w') as fd:
      fd.write (version_file_contents)



###########################################################################
#                            START OF PROGRAM
###########################################################################



if 'clean' in targets:
  clean_cmd()
  sys.exit (0)

if doc_dir in targets: 
  sys.exit (gen_command_html_help());
    
if dev_dir in targets: 
  sys.exit (execute ('[DOC] development', [ 'doxygen' ]))
    
if len(targets) == 0: 
  targets = default_targets()



# get git version info:
update_git_version (mrtrix_dir, os.path.join (mrtrix_dir, 'lib/version.cpp'), '''
namespace MR { 
  namespace App { 
    const char* mrtrix_version = "%%%";
  } 
}
''')

if separate_project:
  if not os.path.exists (misc_dir):
    os.mkdir (misc_dir)
  git_version_file = os.path.join (misc_dir, 'project_version.h')
  update_git_version ('.', git_version_file, '#define MRTRIX_PROJECT_VERSION "%%%"\n');
  if not os.path.exists (git_version_file):
    with open (git_version_file, 'w'):
      pass


    





log ('''
compiling TODO list...
''')

try: [ Entry(item) for item in targets ]
except Exception as e: 
  sys.stderr.write ('ERROR: ' + str(e) + os.linesep)
  sys.exit (1)
except:
  raise



# for nogui config, remove GUI elements from targets and todo list:
if nogui:
  nogui_targets = []
  for entry in targets:
    if not is_GUI_target (entry):
      nogui_targets.append (entry)
  targets = nogui_targets
  
  nogui_todo = {}
  for item in todo.keys():
    if not is_GUI_target (todo[item].name):
       nogui_todo[item] = todo[item]
  todo = nogui_todo





log ('building targets: ' + ' '.join (targets) + os.linesep)


if dependencies:
  sys.stderr.write ('''
Printing dependencies:

''')
  for entry in targets:
    print_deps (entry)
  sys.exit (0)

todo_tmp = todo
todo = {}
for item in todo_tmp.keys():
  if todo_tmp[item].action != 'NA' and todo_tmp[item].need_rebuild():
    todo[item] = todo_tmp[item]

log ('TODO list contains ' + str(len(todo)) + ''' items

''')


#for entry in todo.values(): entry.display()

if not len(todo): sys.exit(0)


if use_multiple_cores:
  try: num_processors = os.sysconf('SC_NPROCESSORS_ONLN')
  except:
    try: num_processors = int(os.environ['NUMBER_OF_PROCESSORS'])
    except: num_processors = 1
else:
  num_processors = 1
  
log ('''

launching ''' + str(num_processors) + ''' threads

''')

threads = []
for i in range (1, num_processors):
  t = threading.Thread (target=build_next, args=(i,));
  t.start()
  threads.append (t)

build_next(0)

for t in threads: t.join()

sys.exit (stop > 1)