new xp setup using venv

Author: AlexandreMary

Diff for: pyproject.toml

@@ -3,22 +3,15 @@ name = "davai"
 dynamic = ["version"]
 description = "Davai environment: commandlines for running DAVAI-tests experiments."
 readme = "README.md"
-requires-python = ">=3.7"
+requires-python = ">=3.10"
 keywords = []
 license = {text = "CECILL-C"}
 authors = [
     { name = "Alexandre Mary"  , email = "[email protected]"}
     ]
 dependencies=[
     "pyyaml",
-    # indirect dependencies: required by DAVAI-tests -> should be moved into DAVAI-tests, which should be cloned +
-    # checkout + pip install -e ?
-    "ial_build==1.3.0",
-    "ial_expertise==1.2.0",
-    "epygram==2.0.2",
-    # "ecbundle" is not on PyPI, to be installed manually (though not mandatory for incr packs)
-    "requests[socks]",
-    "six",
+    "ial_build==1.3.3",
     ]
 
 [project.urls]

Diff for: src/davai_env/conf/atos_bologna.ini

@@ -3,11 +3,6 @@ logs             = $SCRATCH/davai/logs
 default_mtooldir = $SCRATCH/mtool/$LOGNAME
 
 [packages]
-# vortex & epygram are special, the path has to be the root of the project
-#vortex        = ~sos/vortex/vortex-olive-dev
-vortex        = ~acrd/public/vortex/am_davai_ec_tmp
-epygram       = ~acrd/public/EPyGrAM/1.4.x-49t2_bf
-# for others, the path has to lead to the python package
-ial_expertise = ~acrd/public/IAL-expertise/1.1.6/src/ial_expertise
-ial_build     = ~acrd/public/IAL-build/1.2.6/src/ial_build
-ecbundle      = ~acrd/public/ecbundle/2.0.0/ecbundle
+# vortex is special, the path has to be the root of the project
+vortex        = ~sos/vortex/vortex-olive-dev
+#vortex        = ~acrd/public/vortex/am_davai_ec_tmp

Diff for: src/davai_env/conf/belenos.ini

@@ -3,10 +3,5 @@ logs        = $WORKDIR/davai/logs
 default_mtooldir = /scratch/mtool/$LOGNAME
 
 [packages]
-# vortex & epygram are special, the path has to be the root of the project
+# vortex is special, the path has to be the root of the project
 vortex        = ~verolive/vortex/vortex-olive-dev
-epygram       = ~mary/public/EPyGrAM/1.4.20-ecc2.38.0
-# for others, the path has to lead to the python package
-#ial_expertise = ~mary/public/IAL-expertise/1.2.0/src/ial_expertise
-#ial_build     = ~mary/public/IAL-build/1.3.0/src/ial_build
-#ecbundle      = ~mary/public/ecbundle/2.0.0/ecbundle

Diff for: src/davai_env/experiment.py

@@ -13,6 +13,7 @@
 import configparser
 import yaml
 import time
+import venv
 
 from ial_build.bundle import IALBundle, TmpIALbundleRepo
 
@@ -84,12 +85,13 @@ def new_xp(cls, sources_to_test,
         # now XP path is created, we move in for the continuation of the experiment setup
         os.chdir(xp_path)
         xp = ThisXP(new=True)
-        xp.setup(sources_to_test,
-                 davai_tests_version=davai_tests_version,
-                 davai_tests_origin=davai_tests_origin,
-                 usecase=usecase,
-                 host=host,
-                 bundle_src_dir=bundle_src_dir)
+        #xp.setup(sources_to_test,
+        xp.venv_setup(sources_to_test,
+                      davai_tests_version=davai_tests_version,
+                      davai_tests_origin=davai_tests_origin,
+                      usecase=usecase,
+                      host=host,
+                      bundle_src_dir=bundle_src_dir)
         if genesis_commandline:
             xp.write_genesis(genesis_commandline)
         return xp
@@ -108,6 +110,8 @@ class ThisXP(object):
 
     def __init__(self, new=False):
         self.xp_path = os.getcwd()
+        self.venv_path = os.path.join(self.xp_path, '.venv')
+        self.davai_tests_absdir = os.path.join(self.xp_path, self.davai_tests_dir)
         self.xpid = os.path.basename(os.path.dirname(os.path.dirname(self.xp_path)))
         self.vapp = os.path.basename(os.path.dirname(self.xp_path))
         self.vconf = os.path.basename(self.xp_path)
@@ -151,6 +155,39 @@ def setup(self, sources_to_test,
         self._setup_conf_general(host)
         self._setup_final_prompt()
 
+    def venv_setup(self, sources_to_test,
+                   davai_tests_version=None,
+                   davai_tests_origin=config['defaults']['davai_tests_origin'],
+                   usecase=config['defaults']['usecase'],
+                   host=guess_host(),
+                   bundle_src_dir=None):
+        """
+        Setup the experiment as a venv (at creation time).
+
+        :param sources_to_test: information about the sources to be tested, provided as a dict
+        :param davai_tests_version: version of the DAVAI-tests to be used. If not provided, try to guess from IAL repo
+        :param davai_tests_origin: remote repository of the DAVAI-tests to be cloned
+        :param usecase: type of set of tests to be prepared
+        :param host: host machine
+        :param bundle_src_dir: in case tests_version is not specified:
+            cache directory where to download/update bundle repositories,
+            in search for the tests_version, potentially stored in IAL.
+        """
+        os.makedirs('conf')
+        self._setup_conf_sources(sources_to_test)
+        if davai_tests_version is None:
+            # this will fail if the version is not known in IAL
+            davai_tests_version = self.guess_davai_tests_version(bundle_src_dir=bundle_src_dir)
+        # set DAVAI-tests repo
+        self._setup_DAVAI_tests(davai_tests_origin, davai_tests_version)
+        self._setup_venv()
+        self._setup_packages()  # remaining, not on PyPI: vortex
+        self._setup_logs()
+        # configuration files
+        self._setup_conf_usecase(usecase)
+        self._setup_conf_general(host)
+        self._setup_final_prompt()
+
     def guess_davai_tests_version(self, bundle_src_dir=None):
         """Guess davai_tests_version from IAL repo (potentially through bundle)."""
         if 'IAL_git_ref' in self.sources_to_test and 'IAL_repository' in self.sources_to_test:
@@ -225,8 +262,8 @@ def _checkout_davai_tests(gitref):
 
     def _setup_DAVAI_tests(self, remote, version):
         """Clone and checkout required version of the DAVAI-tests."""
-        subprocess.check_call(['git', 'clone', remote, self.davai_tests_dir])
-        os.chdir(self.davai_tests_dir)
+        subprocess.check_call(['git', 'clone', remote, self.davai_tests_absdir])
+        os.chdir(self.davai_tests_absdir)
         subprocess.check_call(['git', 'fetch', 'origin', version, '-q'])
         self._checkout_davai_tests(version)
         os.chdir(self.xp_path)
@@ -237,6 +274,21 @@ def check_sources_to_test(self, sources_to_test):
                            self.sources_to_test_minimal_keys)
         assert assertion_test, assertion_errmsg
 
+    def _setup_venv(self):
+        """Create a new venv it."""
+        # create venv within the xp
+        print("Create virtualenv ({})...".format(self.venv_path))
+        venv.create(self.venv_path,
+                    with_pip=True,
+                    symlinks=True,
+                    prompt='{}.venv'.format(self.xpid))
+        print("... virtualenv created.")
+        # install DAVAI-tests and dependencies in the venv
+        venv_python = os.path.join(self.venv_path, 'bin', 'python')
+        print("Setup virtualenv...")
+        subprocess.check_call([venv_python, '-m', 'pip', 'install', '-e', self.davai_tests_absdir])
+        print("... virtualenv set up.")
+
     def _setup_conf_sources(self, sources_to_test):
         """Sources config: information on sources to be tested."""
         self.check_sources_to_test(sources_to_test)
@@ -262,10 +314,6 @@ def _setup_tasks(self):
 
     def _setup_packages(self):
         """Link necessary packages in XP."""
-        # davai_taskutil from DAVAI-tests locally checkedout
-        os.symlink(os.path.join(self.davai_tests_dir, 'src', 'davai_taskutil'),
-                   'davai_taskutil')
-        # other packages
         packages = {p:expandpath(config['packages'][p]) for p in config['packages']}
         for package, path in packages.items():
             os.symlink(expandpath(path), package)
@@ -281,10 +329,10 @@ def _setup_final_prompt(self):
         """Final prompt for the setup of the experiment."""
         print("------------------------------------")
         print("DAVAI xp '{}' has been successfully setup !".format(self.xpid))
-        print("Now go to the XP path below and:")
-        print("- if necessary, tune experiment in '{}'".format(self.general_config_file))
-        print("- run experiment using: 'davai-run_xp'")
-        print("  =>", self.xp_path)
+        print("  => XP path:", self.xp_path)
+        print("  => XP config file: '{}'".format(self.general_config_file))
+        print("  => Activation venv: 'source {}/bin/activate'".format(self.venv_path))
+        print("  => Run experiment : 'davai-run_xp'")
         print("------------------------------------")
 
     def write_genesis(self, command):