Restructure repo as discussed in #7

.gitignore

@@ -1,9 +1,6 @@
-results/
-experiments/state/
-
 .ipynb_checkpoints
 
-inventory/aws_ec2.yml
+src/inventory/aws_ec2.yml
 
 ### Ansible ###
 *.retry

ansible.cfg

@@ -1,10 +1,10 @@
 [defaults]
-INVENTORY = inventory
+INVENTORY = src/inventory
+DEFAULT_ROLES_PATH = src/roles:demo_project/does_config/roles
 ANSIBLE_CALLBACKS_ENABLED = community.general.selective
 inventory_ignore_extensions = ~, .orig, .bak, .ini, .cfg, .retry, .pyc, .pyo, .j2
 
 ansible_ssh_common_args = '-o StrictHostKeyChecking=no -o ForwardAgent=yes'
 
-
 # TODO: activate / deactivate to only show the pretty log output
 stdout_callback = community.general.selective

demo_project/does_config/designs/demo.yml

@@ -0,0 +1,66 @@
+# AUTOGENERATED FROM experiments/table/{demo_exp1, demo_exp2}.yml WITH build_config_product(...)
+
+# TODO [nku] needs to be updated
+
+demo_exp1:
+  n_repetitions: 2
+  common_roles:
+  - setup-common
+  host_types:
+    client:
+      n: 1
+      init_role: setup-client
+    server:
+      n: 1
+      check_status: false
+      init_role: setup-server
+  base_experiment:
+    w: 13
+    x: $FACTOR$
+    host_vars:
+      client:
+        y: abc
+      server:
+        z: $FACTOR$
+  factor_levels:
+  - x: 4
+    host_vars:
+      server:
+        z: 1
+
+
+demo_exp2:
+  n_repetitions: 2
+  common_roles:
+  - setup-common
+  host_types:
+    client:
+      n: 1
+      init_role: setup-client
+    server:
+      n: 1
+      init_role: setup-server
+  base_experiment:
+    $CMD$: echo hello [% my_run.a %]
+    a: 10
+    b: info
+    c: $FACTOR$
+    d: $FACTOR$
+    host_vars:
+      client:
+        e: $FACTOR$
+        test:
+          f: cinfo
+      server:
+        g: ginfo
+  factor_levels:
+  - c: 1
+    d: 4
+    host_vars:
+      client:
+        e: V1
+  - c: 1
+    d: 4
+    host_vars:
+      client:
+        e: V2

demo_project/does_config/designs/simple.yml

@@ -0,0 +1,25 @@
+# AUTOGENERATED FROM experiments/table/{simple}.yml WITH build_config_product(...)
+
+# TODO [nku] needs to be updated
+
+
+simple:
+  n_repetitions: 2
+  common_roles:
+  - setup-common
+  host_types:
+    single:
+      n: 1
+  base_experiment:
+    seed: 1234
+    payload_size_mb: $FACTOR$
+    opt: $FACTOR$
+  factor_levels:
+  - payload_size_mb: 1
+    opt: true
+  - payload_size_mb: 1
+    opt: false
+  - payload_size_mb: 128
+    opt: true
+  - payload_size_mb: 128
+    opt: false

demo_project/does_config/group_vars/.gitignore

@@ -0,0 +1,7 @@
+# The following files are ignored because they are generated by the repotemplate.py script.
+# For your own experiments, remove this .gitignore file to commit hand-written experiment configurations.
+all
+ansible_controller
+client
+server
+simple

experiment-suite.yml → src/experiment-suite.yml

@@ -1,5 +1,15 @@
 ---
 
+
+##########################################################################
+#   Load Experiment State and Setup AWS                                  #
+##########################################################################
+- name: Load group_vars from non-standard location
+  hosts: localhost
+
+  tasks:
+    - include_vars: "{{ external_group_vars_dir }}/all/main.yml"
+
 ##########################################################################
 #   Load Experiment State and Setup AWS                                  #
 ##########################################################################

src/group_vars/all/main.yml

@@ -0,0 +1,3 @@
+does_project_dir: "{{ lookup('env', 'DOES_PROJECT_DIR') }}"
+does_config_dir: "{{ does_project_dir }}/does_config"
+external_group_vars_dir: "{{ does_config_dir }}/group_vars"

resources/repotemplate/group_vars/all/main.yml.j2 → src/resources/repotemplate/group_vars/all/main.yml.j2

@@ -10,7 +10,6 @@ git_remote_repository: <<git_remote_repository>> # TODO: set remote repository (
 # The experiments are mostly run concurrently (apart from the setup and cleanup parts). Thus, the experiment with the most jobs defines the
 # maximal duration. But as experiments usually use fewer than 'job_n_tries' tries, an experiment with few long-running jobs can be the bottleneck too.
 job_n_tries: <<job_n_tries>>   # should be max 1000 (otherwise playbook freezes -> unsure why)
-                               # TODO [mh]: must be tested if this is still an issue, I switched to an `until` loop
 job_check_wait_time: <<job_check_wait_time>>
 
 remote:
@@ -19,7 +18,8 @@ remote:
 exp_code_dir: "{{ remote.dir }}/code"
 
 local:
-  results_dir: "./results"
+  results_dir: "{{ does_project_dir }}/does_results"
+  designs_dir: "{{ does_config_dir }}/designs"
 
 exp_base:
   key_name: <<key_name>> # TODO: add key pair name
@@ -31,10 +31,3 @@ exp_base:
   vpc_subnet_cidr: 10.100.0.0/24
   sg_name: "{{ prj_id }}_sg"
   sg_desc: "{{ prj_id }} security group"
-
-separator: '_SEP_'
-
-# This prefix (incl. the capitalization) is chosen by the ec2 plugin
-ec2_tag_name_prefix: 'tag_Name_'
-ec2_tag_prj_prefix: 'tag_Prj_'
-ec2_tag_exp_prefix: 'tag_Exp_'

roles/abc/tasks/main.yml → src/roles/abc/tasks/main.yml

@@ -1,5 +1,4 @@
 
-
 - name: Create folder for ABC
   ansible.builtin.file:
     path: "{{ ABC_INSTALL_PATH }}"

src/roles/ansible-controller-clear/tasks/main.yml

@@ -0,0 +1,40 @@
+---
+
+- name: Collect info about running ec2 instances
+  community.aws.ec2_instance_info:
+      region: "{{ exp_base.aws_region }}"
+      filters:
+        "tag:name": "{{ ansible_controller }}"
+  register: ec2_instance_info
+
+# TODO: remove
+- debug:
+    msg: "ec2_instance_info: {{ ec2_instance_info }}"
+
+- name: Extract instance ids of ec2 instances to remove
+  ansible.builtin.set_fact:
+    ec2_instance_ids: "{{ ec2_instance_info | json_query('*.instances[*].instance_id') | list | flatten }}"
+
+# TODO: remove
+- debug:
+    msg: "ec2_instance_ids: {{ ec2_instance_ids }}"
+
+- ansible.builtin.pause:
+    seconds: 10
+    prompt: |
+      "Removing instances with the following ids:
+       {{ ec2_instance_ids }}
+
+       If the above instance IDs are wrong, abort now! (CTRL+C followed by 'A')"
+  tags: [print_action]
+
+- name: Cleanup AWS
+  community.aws.ec2_instance:
+    instance_ids: "{{ ec2_instance_ids }}"
+    region: "{{ exp_base.aws_region }}"
+    state: absent
+  when: (ec2_instance_ids | length) > 0
+
+- name: Remove AWS VPC
+  ansible.builtin.include_role:
+    name: suite-aws-vpc-delete

roles/experiment-job/tasks/main.yml → src/roles/experiment-job/tasks/main.yml

@@ -117,8 +117,8 @@
         cmd: "{{ job.exp_run_cmd }}"
         cmd_label: "{{ job.job_info | to_json | string }}"
         cmd_working_dir: "{{ job.exp_working_dir }}"
-        cmd_stdout_file: results/stdout.log # relative path compared to working dir
-        cmd_stderr_file: results/stderr.log
+        cmd_stdout_file: "{{ local.results_dir }}/stdout.log" # relative path compared to working dir
+        cmd_stderr_file: "{{ local.results_dir }}/stderr.log"
     delegate_to: "{{ job.host_info.public_dns_name }}"
     loop: "{{ exp_jobs_to_enqueue }}"
     loop_control:
@@ -266,4 +266,4 @@
     msg: "{{ suite_progress_info[item] }}"
   loop: "{{ suite_progress_info.keys() | sort }}"
   tags: [print_action]
-# TEMPORARY END
+# TEMPORARY END

roles/suite-aws-ec2-create/tasks/main.yml → src/roles/suite-aws-ec2-create/tasks/main.yml

@@ -17,7 +17,7 @@
   block:
   - name: Loop over different host types and load group vars from file.
     include_vars:
-      file: "group_vars/{{ host_type }}/main.yml"
+      file: "{{ external_group_vars_dir }}/{{ host_type }}/main.yml"
       name: host_type_specific_vars
     loop: "{{ host_types.keys() }}"
     loop_control:

roles/suite-load-pre-aws/tasks/main.yml → src/roles/suite-load-pre-aws/tasks/main.yml

@@ -81,7 +81,7 @@
 
 - name: validate the suite design and set default values
   suite_design_validate:
-    src: experiments/designs/example.yml
+    src: "{{ local.designs_dir }}/{{ suite }}.yml"
     dest: "{{ suite_design_dir }}/suite_design.yml"
   when: id == 'new' # only validate and resolve defaults for new (otherwise load the old already checked version)
 

scripts/expdesign.py → src/scripts/expdesign.py

@@ -10,7 +10,7 @@ def main():
 
     parser.add_argument('--exps', help='experiment name(s) (experiment file name without .yml ending)', nargs="+", type=str, required=True)
 
-    parser.add_argument('--inputdir', help='path to folder of experiment in "table" form', type=str, default="experiments/table")
+    parser.add_argument('--inputdir', help='path to folder of experiment in "table" form', type=str, default="does_config/table")
 
     parser.add_argument('--outpath', help='output path for generated of experiment in "design" form', type=str)
 
@@ -25,7 +25,7 @@ def main():
         suite = exps[0]
 
     if not outpath:
-        outpath = f"experiments/designs/{suite}.yml"
+        outpath = f"does_config/designs/{suite}.yml"
 
     with open(outpath, 'w') as out_fp:
         print(f"Writing experiment design to: {outpath}\n")

scripts/repotemplate.py → src/scripts/repotemplate.py

@@ -9,8 +9,9 @@
 
 DEFAULT_HOST_TYPE = "host_type"
 
-vars_base_path = "group_vars"
-templates_base_path = "resources/repotemplate/group_vars"
+vars_base_path = os.environ["DOES_PROJECT_FOLDER"] + "/does_config/group_vars"
+src_path = os.path.dirname(os.path.dirname(os.path.realpath(__file__))) 
+templates_base_path = src_path + "/resources/repotemplate/group_vars"
 groups = ["all", "server", "client", "ansible_controller"]
 template_name = "main.yml.j2"
 # The template path is assumed to be: f"{templates_base_path}/<<NAME>>", where