jpkg

#!/usr/bin/env python
# -*- coding: utf-8 -*-

from __future__ import print_function

import sys
import os
import re
import argparse
from subprocess import check_output
from string import Template
from glob import glob

# for python2 compatibility
try:
    input = raw_input
except:
    pass

"""
This installation script installs anaconda with a set of packages.

We also add our custom jupyter hub code to the jupyter configs.  All our configuration changes
and extensions are added to the anaconda subdirectory.
"""


def check_perm(p):
    """In each installation directory, check the permissions of files so
    we don't end up with packages users can't read."""
    cwd = os.getcwd()

    os.chdir(p)

    print("Checking file permissions in", p)
    bindir = os.path.join(p, 'bin')

    print("Finding and fixing files that are not world-readable")
    os.system('find . ! -perm -004 -exec chmod a+r {} \;')

    print("Finding non-searchable directories.  Should be none.")
    os.system('find . -type d -a ! -perm -011')

    print('Finding and fixing world-writable files.')
    os.system('find -L . -perm -o+w -exec chmod o-w {} \;')

    os.chdir(cwd)


def install_octave(p, env_dir, desktop):
    "Install octave kernel, if octave is found"
    octave_kstr = find_octave(p, desktop, env_dir, prompt=False)
    if octave_kstr is not None:
        dname = os.path.join(p, 'share', 'jupyter', 'kernels', 'octave')
        if not os.path.exists(dname):
            os.mkdir(dname)
        fname = os.path.join(dname, 'kernel.json')
        with open(fname, 'w') as f:
            f.write(octave_kstr)


def write_env(rver, env_dir, instpath):
    # write environ files for 'use' on the hubs
    bindir = os.path.join(instpath, 'bin')
    os.system('mkdir -p %s' % env_dir)

    fname = os.path.join(env_dir, os.path.basename(instpath))

    with open(fname, 'w') as f:
        f.write('\nconflict ANACONDA_CHOICE\n\n')
        f.write('desc "Anaconda is a package manager, an environment manager, a Python distribution, and a collection of over 1,000+ open source packages."\n\n')
        f.write('help "https://docs.anaconda.com/new-anaconda-start-here"\n\n')
        f.write('version=%s\n\n' % rver)
        f.write('prepend PATH %s\n\n' % bindir)
        f.write('tags MATHSCI\n')

    # needed for submit, which only works on python 2 for now
    #   hubzero = '/usr/lib/python2.7/dist-packages/hubzero'
    # libdir = os.path.join(p, 'lib', 'python2.7', 'site-packages')
    # os.system('ln -s %s %s' % (hubzero, libdir))


def install_extensions(p, desktop):
    global dirname
    cwd = os.getcwd()
    os.chdir(dirname)
    os.chdir('nbextensions')

    with open('jupyter_notebook_config.py', 'r') as f:
        intro = f.read()

    spath = os.path.join(p, 'share', 'jupyter')
    os.system('mkdir -p ' + os.path.join(p, 'etc', 'jupyter'))
    fname = os.path.join(p, 'etc', 'jupyter', 'jupyter_notebook_config.py')
    with open(fname, 'w') as f:
        f.write(intro)
        # f.write("c.Application.log_level = 'DEBUG'\n")
        f.write("c.IPKernelApp.pylab = 'inline'\n")  # always put plots inline
        f.write("import sys\n")
        f.write("sys.path.append('%s')\n" % os.path.join(spath, 'nbextensions'))
        f.write("\n")
        f.write("c.NotebookApp.extra_template_paths = ['%s']\n" % os.path.join(spath, 'templates'))
        f.write("c.NotebookApp.extra_static_paths = ['%s']\n" % os.path.join(spath, 'static'))
        f.write("c.NotebookApp.trust_xheaders = True\n")
        f.write("c.NotebookApp.mathjax_url = 'https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.3/MathJax.js'\n")
        if not desktop:
            f.write("c.NotebookApp.login_handler_class = 'hublogin.HubLoginHandler'\n")
        f.write("c.NotebookApp.disable_check_xsrf = True")


    if not desktop:
        os.system('jupyter nbextension install --sys-prefix prefs')
        os.system('jupyter nbextension enable --sys-prefix prefs/prefs')

    # install calysto document tools
    os.system('jupyter nbextension install --sys-prefix https://github.com/Calysto/notebook-extensions/archive/master.zip')


    # install hublogin and terminate
    dst = os.path.join(p, 'share', 'jupyter')
    ext = os.path.join(dst, 'nbextensions')
    # os.system('cp terminate.py %s' % ext)
    if not desktop:
        os.system('cp hublogin.py %s' % ext)

    # patch dashboards to work with our tool mode and autorun extensions
    # dashdir = os.path.join(ext, 'jupyter_dashboards', 'notebook', 'dashboard-view')
    # os.system('patch -f -d %s < dashboard-actions.patch' % dashdir)

    # install logos
    os.system('mkdir -p ' + dst + '/static')
    os.system('cp -r ../logo/* ' + dst + '/static')
    os.chdir(cwd)


def cleanup(p, desktop):
    # final cleanup

    # remove any local extensions because they may conflict with global
    os.system('rm -rf ~/.local/share/jupyter/nbextensions')


def get_scriptname(ver):
    global dirname
    cname = os.path.join(dirname, 'configs', 'base_conda_'+ver)
    with open(cname) as fp:
        for line in fp:
            if line[0] == '#':
                line = line[1:].split('=')
                if len(line) != 2:
                    continue
                line = [x.strip() for x in line]
                if line[0] == 'ver':
                    return line[1]
    return None


def make_new(args):
    global dirname  # where the install scripts are

#   print(args)
    print()
    print("Installing version %s" % (args.ver))
    print("Nanohub: %s" % (args.with_nanohub))
    print("Desktop: %s" % (args.desktop))
    print("Python2: %s" % (args.with_py2))
    print("ML: %s" % (args.with_ml))
    print("VNC: %s" % (args.with_vnc))
    print("Dash: %s" % (args.with_dash))
    if not args.desktop:
        print("Env Dir: %s" % (args.envdir))
    print("No notebooks: %s" % (args.no_notebooks))
    if args.themes:
        print("Themes: %s" % (args.themes))
    print()

    env_dir = args.envdir
    script = get_scriptname(args.ver)
    instpath = os.path.realpath('anaconda-%s' % (args.ver))
    print("script:%s - %s" % (script, instpath))
    if not os.path.isfile(script):
        print("Downloading", script)
        os.system("curl -O https://repo.continuum.io/archive/%s" % script)
        if not os.path.isfile(script):
            print("Error: %s not found and could not download." % script)
            sys.exit(1)

    print("Installing anaconda-%s in %s" % (args.ver, instpath))
    os.system('bash ' + script + ' -b -p ' + instpath)

    bindir = os.path.join(instpath, 'bin')

    # set PATH so we can use 'conda'  and 'pip'
    path = os.environ['PATH']
    os.environ['PATH'] = bindir + ':' + path

    cmd = '. ' + os.path.join(dirname, 'configs', 'base_conda_' + args.ver)
    print("********\n%s\n********" % cmd)
    os.system(cmd)
    if args.with_nanohub:
        cmd = '. ' + os.path.join(dirname, 'configs', 'nanohub_conda_' + args.ver)
        print("********\n%s\n********" % cmd)
        os.system(cmd)

    cmd = '. ' + os.path.join(dirname, 'configs', 'base_pip_' + args.ver)
    print("********\n%s\n********" % cmd)
    os.system(cmd)

    if args.with_nanohub:
        cmd = '. ' + os.path.join(dirname, 'configs', 'nanohub_pip_' + args.ver)
        print("********\n%s\n********" % cmd)
        os.system(cmd)

    if args.with_ml:
        cmd = '. ' + os.path.join(dirname, 'configs', 'ml_' + args.ver)
        print("********\n%s\n********" % cmd)
        os.system(cmd)

    # if args.desktop:
    #     cmd = '. ' + os.path.join(dirname, 'configs', 'desktop_' + args.ver)
    #     print("********\n%s\n********" % cmd)
    #     os.system(cmd)

    if args.with_py2:
        cmd = '. ' + os.path.join(dirname, 'configs', 'python2_' + args.ver)
        print("********\n%s\n********" % cmd)
        os.system(cmd)

    if args.with_r:
        cmd = '. ' + os.path.join(dirname, 'configs', 'r_' + args.ver)
        print("********\n%s\n********" % cmd)
        os.system(cmd)

    if args.with_vnc:
        cmd = '. ' + os.path.join(dirname, 'configs', 'nbnovnc')
        print("********\n%s\n********" % cmd)
        os.system(cmd)

    start_jupyter_main = """    sys.exit(main())


"""

    start_jupyter_main_themes = """    doThemes = False
    checkForThemes = False
    if len(sys.argv) > 1:
        for arg in sys.argv[1:]:
            if arg == '--themes':
                checkForThemes = True
                break
    if checkForThemes:
        sys.argv.remove('--themes')
        try:
            from {THEMEPACKAGE} import main as themesMain
            doThemes = True
        except ImportError:
            doThemes = False
        else:
            sys.argv[0] = re.sub(r'(-script\.pyw|\.exe)?$', '', sys.argv[0])
            sys.exit(themesMain())

    if not doThemes:
        sys.exit(main())


"""

    if not args.no_notebooks:
        pythonCmd = os.path.join(bindir,'python')
        # start_jupyter script
#       os.system('cp %s %s' % (os.path.join(dirname, 'start_jupyter'), bindir))
        with open(os.path.join(dirname,'start_jupyter'),'r') as fp:
            startJupyterTemplate = fp.read()
        if args.themes:
            main = start_jupyter_main_themes.format(THEMEPACKAGE=args.themes)
        else:
            main = start_jupyter_main
        with open(os.path.join(bindir,'start_jupyter'),'w') as fp:
            fp.write("#!%s\n" % (pythonCmd))
            fp.write(startJupyterTemplate)
            fp.write(main)
        os.system('chmod a+x %s' % (os.path.join(bindir,'start_jupyter')))

        # start_jupyterlab script
#       os.system('cp %s %s' % (os.path.join(dirname, 'start_jupyterlab'), bindir))
        with open(os.path.join(dirname,'start_jupyterlab'),'r') as fp:
            startJupyterLabTemplate = fp.read()
        with open(os.path.join(bindir,'start_jupyterlab'),'w') as fp:
            fp.write("#!%s\n" % (pythonCmd))
            fp.write(startJupyterLabTemplate)
        os.system('chmod a+x %s' % (os.path.join(bindir,'start_jupyterlab')))

        install_octave(instpath, env_dir, args.desktop)
        install_extensions(instpath, args.desktop)

    if not args.desktop:
        write_env(args.ver, env_dir, instpath)

    check_perm(instpath)
    cleanup(instpath, args.desktop)


def make_update(args):
    global dirname  # where the install scripts are

    print(args)
    print("Updating version %s" % args.ver)
    print("Nanohub: %s" % (args.with_nanohub))
    print("Desktop: %s" % (args.desktop))
    print("ML: %s" % (args.with_ml))
    print("VNC: %s" % (args.with_vnc))
    print("Dash: %s" % (args.with_dash))
    if not args.desktop:
        print("Env Dir: %s" % args.envdir)
    print()

    env_dir = args.envdir
    instpath = os.path.realpath('anaconda-%s' % (args.ver))
    if not os.path.isdir(instpath):
        print("Error: %s not found." % (instpath))
        sys.exit(1)

    bindir = os.path.join(instpath, 'bin')

    # set PATH so we can use 'conda'  and 'pip'
    path = os.environ['PATH']
    os.environ['PATH'] = bindir + ':' + path

    if args.with_nanohub:
        cmd = '. ' + os.path.join(dirname, 'configs', 'nanohub_conda_' + args.ver)
        print("********\n%s\n********" % cmd)
        os.system(cmd)

        cmd = '. ' + os.path.join(dirname, 'configs', 'nanohub_pip_' + args.ver)
        print("********\n%s\n********" % cmd)
        os.system(cmd)

    if args.with_ml:
        cmd = '. ' + os.path.join(dirname, 'configs', 'ml_' + args.ver)
        print("********\n%s\n********" % cmd)
        os.system(cmd)

    if args.with_r:
        cmd = '. ' + os.path.join(dirname, 'configs', 'r_' + args.ver)
        print("********\n%s\n********" % cmd)
        os.system(cmd)

    if args.with_vnc:
        cmd = '. ' + os.path.join(dirname, 'configs', 'nbnovnc')
        print("********\n%s\n********" % cmd)
        os.system(cmd)

    check_perm(instpath)
    cleanup(instpath, args.desktop)


def do_downloads(ver):
    ret = os.system("wget http://packages.hubzero.org/blobs/anaconda-%s.tar.xz -O - | tar -xJ" % ver)
    if ret:
        print("Download failed.")
        sys.exit(1)


def make_netinst(args):
    do_downloads(args.ver)

    instpath = os.path.realpath('anaconda-%s' % (args.ver))

    # write env modules
    write_env(args.ver, args.envdir, pathlist)

    if args.nopy2:
        # remove python2 kernel from menu
        kpath = os.path.join(instpath,"share","jupyter","kernels","python2")
        os.system("mv %s/kernel.json %s/kernel.json.disabled" % (kpath, kpath))

    try:
        check_output("octave --version", shell=True)
        octave = True
    except:
        octave = False
    if octave is False:
        print("Could not find octave.  Disabling octave kernel in menu.")
        kpath = os.path.join(instpath,"share","jupyter","kernels","octave")
        os.system("mv %s/kernel.json %s/kernel.json.disabled" % (kpath, kpath))

    hostimage = os.uname()[1].lower() + '.png'
    imagedir = os.path.join(instpath,"share","jupyter","static")
    hifull = os.path.join(imagedir, hostimage)
    hzfull = os.path.join(imagedir, 'hubzero.png')
    if not os.path.isfile(hifull):
        print("WARNING:")
        print("This installation will put an image representing your host in the Jupyter menubar.")
        print("No image for your host, \"%s\" could be found." % os.uname()[1].lower())
        print("This installation process will use a default hubzero image instead.")
        print("To update the image, copy a PNG image of your choice to")
        print(hifull)
        print()
        os.system('cp %s %s' % (hzfull, hifull))

def do_install(desktop, cname, env_dir, instpath, nopy2):
    install_updates(cname, instpath, nopy2)
    install_kernels(instpath, nopy2)
    install_octave(instpath, env_dir, desktop)
    install_R_pkgs(instpath)
    install_extensions(instpath, desktop)
    check_perm(instpath)
    cleanup(instpath, desktop)


def make_cleanup(args):
    instpath = os.path.realpath('anaconda-%s' % (args.ver))
    check_perm(instpath)


def find_octave(pypath, desktop, env_dir, prompt=True):
    global dirname

    pypath += "/bin/python"

    if desktop:
        try:
            oc = check_output("octave --version", shell=True).split('\n')[0]
            ver = oc.split()[-1]
            opath = check_output("which octave", shell=True).strip()
        except:
            if prompt:
                print("\nWARNING: Octave not found.  You may want to install it.")
                inp = ''
                while inp == '':
                    inp = input('Continue without installing Octave kernel? [Y/N]')
                if inp.upper() != 'Y':
                    sys.exit(0)
            return None

        kernel = open(os.path.join(dirname, 'octave_kernel_desktop')).read()
        s = Template(kernel)
        d = dict(python=pypath,
                 ver="Octave %s" % ver,
                 exe=opath)
        return s.substitute(d)

    try:
        olist = check_output("ls %s/octave*" % env_dir, shell=True).split()
    except:
        if prompt:
            print("\nWARNING: No octave module was found.")
            inp = ''
            while inp == '':
                inp = input('Continue without installing Octave kernel? [Y/N]')
            if inp.upper() != 'Y':
                sys.exit(0)
        return None

    with open(olist[-1]) as f:
        for line in f:
            if line.startswith('prepend PATH'):
                opath = line.split()[-1]
                ver = opath.split('-')[-1].split('/')[0]
                break

    kernel = open(os.path.join(dirname, 'octave_kernel')).read()
    s = Template(kernel)
    d = dict(python=pypath,
             ver="Octave %s" % ver,
             exe=opath + '/octave',
             home=os.path.dirname(opath))
    return s.substitute(d)


if __name__ == '__main__':
    global dirname
    dirname = os.path.dirname(os.path.realpath(sys.argv[0]))

    installBase    = os.path.dirname(os.path.dirname(os.path.realpath(os.path.abspath(__file__))))
    installPattern = os.path.join(os.sep,'apps','share64','([a-zA-Z0-9]+)','anaconda')
    patternMatch = re.match(installPattern,installBase)
    if patternMatch:
        installOS = patternMatch.group(1)
    else:
        installOS = 'debian10'
    installEnvDir = os.path.join(os.sep,'apps','share64',installOS,'environ.d')

    # at least one install fails if this is not set
    os.environ['LANG'] = 'en_US.UTF-8'

    parser = argparse.ArgumentParser()

    parser.add_argument('--with-py2', action='store_true', help='Install Python2')
    parser.add_argument('--with-nanohub', action='store_true', help='Install tools for materials simulations')
    parser.add_argument('--with-dash', action='store_true', help='Install Plotly Dash')
    parser.add_argument('--with-vnc', action='store_true', help='Install VNC')
    parser.add_argument('--with-ml', action='store_true', help='Install Machine Learning tools')
    parser.add_argument('--desktop', action='store_true', help='Install Desktop version')
    parser.add_argument('--envdir', type=str, default=installEnvDir,
                        help='Environment directory for hubs (default: %s)' % (installEnvDir))
    parser.add_argument('--with-r', action='store_true', 
                        help='Install the R kernel and packages')
    parser.add_argument('--no-notebooks', action='store_true', help='Do not install notebooks')
    parser.add_argument('--themes', type=str, help='Install with themes from package')


    subparsers = parser.add_subparsers()

    # create the parser for the "install" command
    parser_install = subparsers.add_parser('install')
    parser_install.add_argument('ver', type=str)
    parser_install.set_defaults(func=make_new)

    # create the parser for the "update" command
    parser_install = subparsers.add_parser('update')
    parser_install.add_argument('ver', type=str)
    parser_install.set_defaults(func=make_update)

    # create the parser for the "netinst" command
    parser_up = subparsers.add_parser('netinst')
    parser_up.add_argument('ver', type=str)
    parser_up.set_defaults(func=make_netinst)

    # create the parser for the "fix" command
    parser_fix = subparsers.add_parser('cleanup')
    parser_fix.add_argument('ver', default='',  nargs='?', type=str)
    parser_fix.set_defaults(func=make_cleanup)


    args = parser.parse_args()
    args.func(args)


# ./jupyter_notebook_setup/jpkg --desktop --with-py2 --with-r install 6  
# ./jupyter_notebook_setup/jpkg --with-nanohub --envdir='/apps/share64/debian9/environ.d' --with-r --with-vnc install 6

# typical nanohub
# ./jupyter_notebook_setup/jpkg --with-nanohub --with-py2 --with-r --with-vnc --with-ml install 6