From bbf139afda1abc3c6fe82817c45238c075133080 Mon Sep 17 00:00:00 2001 From: Miro Haller Date: Fri, 8 Oct 2021 09:07:07 +0200 Subject: [PATCH] Ansible Controller and Restructuring (#14) * [WIP] ansible controller: untested implementation of setup and config; need to solve repo forking first before testing * Fixing setup and SSH keygen. Still WIP * replace deprecated ec2 role with ec2_instance * WIP bugfixing aws-controller * WIP bugfixing aws-controller, updating git/aws key generation * Fix regression bugs from rebase (mainly the tagging of instances) * Fix awscli, boto3, and ansible installation * Add untested roles to install seal and abc * Restructure repo as discussed in #7 * Removed deprecated examples * Fix demo example * Update repotemplate for project dir environment variable * Fix result storage * Remove pps specific roles * Fix minor bugs from restructuring * Fix bug from rebase Co-authored-by: Miro Haller --- .gitignore | 5 +- Pipfile | 2 +- README.md | 7 +- ansible.cfg | 4 +- {demo => demo_project}/client.py | 0 .../does_config}/designs/example-complex.yml | 0 .../does_config}/designs/example-minimal.yml | 0 .../does_config}/designs/example-multi.yml | 0 .../does_config}/designs/example-single.yml | 0 .../does_config}/designs/example.yml | 0 .../does_config/group_vars/.gitignore | 7 ++ .../roles}/setup-client/tasks/main.yml | 0 .../roles}/setup-common/tasks/main.yml | 0 .../roles}/setup-server/tasks/main.yml | 0 .../does_config}/table/demo_exp1.yml | 0 .../does_config}/table/demo_exp2.yml | 0 .../does_config}/table/simple.yml | 0 .../scripts-demo.ipynb | 0 {demo => demo_project}/server.py | 0 docs/troubleshooting.md | 2 +- group_vars/.gitignore | 4 - group_vars/single/main.yml | 7 -- src/ansible-controller.yml | 61 +++++++++++ clear.yml => src/clear.yml | 4 +- .../experiment-suite.yml | 14 +++ .../filter_plugins}/helpers.py | 0 src/group_vars/all/main.yml | 3 + src/inventory/.gitignore | 2 + .../inventory/.keep | 0 src/resources/awscli/aws_cli_team_gpg_key.pub | 29 +++++ {resources => src/resources}/flask.png | Bin .../resources}/inventory/aws_ec2.yml.j2 | 15 ++- .../repotemplate/group_vars/all/main.yml.j2 | 12 +-- .../group_vars/ansible_controller/main.yml.j2 | 32 ++++++ .../group_vars/host_type/main.yml.j2 | 2 +- .../ansible-controller-clear/tasks/main.yml | 40 +++++++ .../ansible-controller-ec2/tasks/main.yml | 38 +++++++ .../tasks/configure_aws.yml | 11 ++ .../tasks/fetch_does_repo.yml | 28 +++++ .../tasks/install_awscli.yml | 99 ++++++++++++++++++ .../tasks/install_boto3.yml | 6 ++ .../ansible-controller-setup/tasks/main.yml | 8 ++ .../tasks/prepare_ansible.yml | 12 +++ .../tasks/prerequisites.yml | 38 +++++++ .../tasks/ssh_setup.yml | 57 ++++++++++ .../experiment-job/filter_plugins/helpers.py | 14 +-- .../roles}/experiment-job/library/tsp.py | 0 .../roles}/experiment-job/library/tsp_info.py | 0 .../roles}/experiment-job/tasks/main.yml | 10 +- .../experiment-job/templates/config.json.j2 | 0 .../roles}/experiment-state/tasks/main.yml | 0 .../experiment-state/templates/state.yml.j2 | 0 src/roles/load-group-vars/tasks/main.yml | 8 ++ src/roles/setup-ssh/tasks/main.yml | 57 ++++++++++ .../roles}/setup-suite/tasks/main.yml | 0 .../filter_plugins/helpers.py | 0 .../suite-aws-ec2-create/tasks/main.yml | 30 +----- .../suite-aws-ec2-delete/tasks/main.yml | 0 src/roles/suite-aws-ec2-launch/tasks/main.yml | 30 ++++++ .../suite-aws-vpc-create/tasks/main.yml | 0 .../suite-aws-vpc-delete/tasks/main.yml | 0 .../action_plugins/suite_design_extend.py | 0 .../library/suite_design_extend.py | 0 .../roles}/suite-load-post-aws/tasks/main.yml | 0 .../action_plugins/suite_design_validate.py | 0 .../filter_plugins/helpers.py | 0 .../library/suite_design_validate.py | 0 .../roles}/suite-load-pre-aws/tasks/main.yml | 2 +- {scripts => src/scripts}/expdesign.py | 4 +- {scripts => src/scripts}/repotemplate.py | 50 ++++++--- {scripts => src/scripts}/results.py | 0 71 files changed, 658 insertions(+), 96 deletions(-) rename {demo => demo_project}/client.py (100%) rename {experiments => demo_project/does_config}/designs/example-complex.yml (100%) rename {experiments => demo_project/does_config}/designs/example-minimal.yml (100%) rename {experiments => demo_project/does_config}/designs/example-multi.yml (100%) rename {experiments => demo_project/does_config}/designs/example-single.yml (100%) rename {experiments => demo_project/does_config}/designs/example.yml (100%) create mode 100644 demo_project/does_config/group_vars/.gitignore rename {roles => demo_project/does_config/roles}/setup-client/tasks/main.yml (100%) rename {roles => demo_project/does_config/roles}/setup-common/tasks/main.yml (100%) rename {roles => demo_project/does_config/roles}/setup-server/tasks/main.yml (100%) rename {experiments => demo_project/does_config}/table/demo_exp1.yml (100%) rename {experiments => demo_project/does_config}/table/demo_exp2.yml (100%) rename {experiments => demo_project/does_config}/table/simple.yml (100%) rename scripts-demo.ipynb => demo_project/scripts-demo.ipynb (100%) rename {demo => demo_project}/server.py (100%) delete mode 100644 group_vars/.gitignore delete mode 100644 group_vars/single/main.yml create mode 100644 src/ansible-controller.yml rename clear.yml => src/clear.yml (92%) rename experiment-suite.yml => src/experiment-suite.yml (91%) rename {filter_plugins => src/filter_plugins}/helpers.py (100%) create mode 100644 src/group_vars/all/main.yml create mode 100644 src/inventory/.gitignore rename roles/suite-load-post-aws/library/suite_design_extend.py => src/inventory/.keep (100%) create mode 100644 src/resources/awscli/aws_cli_team_gpg_key.pub rename {resources => src/resources}/flask.png (100%) rename {resources => src/resources}/inventory/aws_ec2.yml.j2 (75%) rename {resources => src/resources}/repotemplate/group_vars/all/main.yml.j2 (83%) create mode 100644 src/resources/repotemplate/group_vars/ansible_controller/main.yml.j2 rename {resources => src/resources}/repotemplate/group_vars/host_type/main.yml.j2 (84%) create mode 100644 src/roles/ansible-controller-clear/tasks/main.yml create mode 100644 src/roles/ansible-controller-ec2/tasks/main.yml create mode 100644 src/roles/ansible-controller-setup/tasks/configure_aws.yml create mode 100644 src/roles/ansible-controller-setup/tasks/fetch_does_repo.yml create mode 100644 src/roles/ansible-controller-setup/tasks/install_awscli.yml create mode 100644 src/roles/ansible-controller-setup/tasks/install_boto3.yml create mode 100644 src/roles/ansible-controller-setup/tasks/main.yml create mode 100644 src/roles/ansible-controller-setup/tasks/prepare_ansible.yml create mode 100644 src/roles/ansible-controller-setup/tasks/prerequisites.yml create mode 100644 src/roles/ansible-controller-setup/tasks/ssh_setup.yml rename {roles => src/roles}/experiment-job/filter_plugins/helpers.py (93%) rename {roles => src/roles}/experiment-job/library/tsp.py (100%) rename {roles => src/roles}/experiment-job/library/tsp_info.py (100%) rename {roles => src/roles}/experiment-job/tasks/main.yml (98%) rename {roles => src/roles}/experiment-job/templates/config.json.j2 (100%) rename {roles => src/roles}/experiment-state/tasks/main.yml (100%) rename {roles => src/roles}/experiment-state/templates/state.yml.j2 (100%) create mode 100644 src/roles/load-group-vars/tasks/main.yml create mode 100644 src/roles/setup-ssh/tasks/main.yml rename {roles => src/roles}/setup-suite/tasks/main.yml (100%) rename {roles => src/roles}/suite-aws-ec2-create/filter_plugins/helpers.py (100%) rename {roles => src/roles}/suite-aws-ec2-create/tasks/main.yml (89%) rename {roles => src/roles}/suite-aws-ec2-delete/tasks/main.yml (100%) create mode 100644 src/roles/suite-aws-ec2-launch/tasks/main.yml rename {roles => src/roles}/suite-aws-vpc-create/tasks/main.yml (100%) rename {roles => src/roles}/suite-aws-vpc-delete/tasks/main.yml (100%) rename {roles => src/roles}/suite-load-post-aws/action_plugins/suite_design_extend.py (100%) rename roles/suite-load-pre-aws/library/suite_design_validate.py => src/roles/suite-load-post-aws/library/suite_design_extend.py (100%) rename {roles => src/roles}/suite-load-post-aws/tasks/main.yml (100%) rename {roles => src/roles}/suite-load-pre-aws/action_plugins/suite_design_validate.py (100%) rename {roles => src/roles}/suite-load-pre-aws/filter_plugins/helpers.py (100%) create mode 100644 src/roles/suite-load-pre-aws/library/suite_design_validate.py rename {roles => src/roles}/suite-load-pre-aws/tasks/main.yml (98%) rename {scripts => src/scripts}/expdesign.py (97%) rename {scripts => src/scripts}/repotemplate.py (84%) rename {scripts => src/scripts}/results.py (100%) diff --git a/.gitignore b/.gitignore index bfd0f28e..f819c139 100644 --- a/.gitignore +++ b/.gitignore @@ -1,9 +1,6 @@ -results/ -experiments/state/ - .ipynb_checkpoints -inventory/aws_ec2.yml +src/inventory/aws_ec2.yml ### Ansible ### *.retry diff --git a/Pipfile b/Pipfile index f1bd54b6..cfbb036b 100644 --- a/Pipfile +++ b/Pipfile @@ -18,4 +18,4 @@ pyinputplus = "*" argparse = "*" [requires] -python_version = "3.8" +python_version = "3.9" diff --git a/README.md b/README.md index 8d8f2107..a1a91473 100644 --- a/README.md +++ b/README.md @@ -185,7 +185,7 @@ After completing the getting started section, it should be possible to run the [ 7. Run the repository initialization helper script and configure the experiment suite and the example host types `client` and `server`. - This prompts user input to perform variable substitution in the `group_vars/*/main.yml.j2` variable templates for the groups [all](group_vars/all/main.yml.j2), [client](group_vars/all/client.yml.j2), and [server](group_vars/all/server.yml.j2). + This prompts user input to perform variable substitution using the `resources/repotemplate/group_vars/*/main.yml.j2` templates. By default, it creates four groups: `all`, `server`, `client`, and `ansible_controller`. When unsure, set the unique `project id` and the AWS `key name` from the prerequisites and otherwise use the default options. @@ -392,8 +392,7 @@ pipenv run ansible-playbook experiment.yml -e "suite=example id=last" ### Cleaning up AWS - -By default, after an experiment is complete, all resources created on AWS are terminated. +By default, after an experiment is complete, all _experiment_ resources created on AWS are terminated. To deactivate this default behavior, provide the flag: `awsclean=false`. Creating resources on AWS and setting up the environment takes a considerable amount of time. So, for debugging and short experiments, it can make sense not to terminate the instances. If you use this flag, be sure to check that instances are terminated when you are done. @@ -408,6 +407,8 @@ Furthermore, we also provide a playbook to terminate all AWS resources: pipenv run ansible-playbook clear.yml ``` +:warning: The ansible controller instance, if used, is not removed. It is intended to be left running and trigger individual experiment runs. To remove it, use the flag `awscleanall=true`. + ### Experimental Results The experiment suite creates a matching folder structure on the localhost and the remote EC2 instances. diff --git a/ansible.cfg b/ansible.cfg index fc0b82f7..2f3b33b9 100644 --- a/ansible.cfg +++ b/ansible.cfg @@ -1,10 +1,10 @@ [defaults] -INVENTORY = inventory +INVENTORY = src/inventory +roles_path = ${PWD}/src/roles:${DOES_PROJECT_DIR}/does_config/roles ANSIBLE_CALLBACKS_ENABLED = community.general.selective inventory_ignore_extensions = ~, .orig, .bak, .ini, .cfg, .retry, .pyc, .pyo, .j2 ansible_ssh_common_args = '-o StrictHostKeyChecking=no -o ForwardAgent=yes' - # TODO: activate / deactivate to only show the pretty log output stdout_callback = community.general.selective diff --git a/demo/client.py b/demo_project/client.py similarity index 100% rename from demo/client.py rename to demo_project/client.py diff --git a/experiments/designs/example-complex.yml b/demo_project/does_config/designs/example-complex.yml similarity index 100% rename from experiments/designs/example-complex.yml rename to demo_project/does_config/designs/example-complex.yml diff --git a/experiments/designs/example-minimal.yml b/demo_project/does_config/designs/example-minimal.yml similarity index 100% rename from experiments/designs/example-minimal.yml rename to demo_project/does_config/designs/example-minimal.yml diff --git a/experiments/designs/example-multi.yml b/demo_project/does_config/designs/example-multi.yml similarity index 100% rename from experiments/designs/example-multi.yml rename to demo_project/does_config/designs/example-multi.yml diff --git a/experiments/designs/example-single.yml b/demo_project/does_config/designs/example-single.yml similarity index 100% rename from experiments/designs/example-single.yml rename to demo_project/does_config/designs/example-single.yml diff --git a/experiments/designs/example.yml b/demo_project/does_config/designs/example.yml similarity index 100% rename from experiments/designs/example.yml rename to demo_project/does_config/designs/example.yml diff --git a/demo_project/does_config/group_vars/.gitignore b/demo_project/does_config/group_vars/.gitignore new file mode 100644 index 00000000..4930ae02 --- /dev/null +++ b/demo_project/does_config/group_vars/.gitignore @@ -0,0 +1,7 @@ +# The following files are ignored because they are generated by the repotemplate.py script. +# For your own experiments, remove this .gitignore file to commit hand-written experiment configurations. +all +ansible_controller +client +server +small diff --git a/roles/setup-client/tasks/main.yml b/demo_project/does_config/roles/setup-client/tasks/main.yml similarity index 100% rename from roles/setup-client/tasks/main.yml rename to demo_project/does_config/roles/setup-client/tasks/main.yml diff --git a/roles/setup-common/tasks/main.yml b/demo_project/does_config/roles/setup-common/tasks/main.yml similarity index 100% rename from roles/setup-common/tasks/main.yml rename to demo_project/does_config/roles/setup-common/tasks/main.yml diff --git a/roles/setup-server/tasks/main.yml b/demo_project/does_config/roles/setup-server/tasks/main.yml similarity index 100% rename from roles/setup-server/tasks/main.yml rename to demo_project/does_config/roles/setup-server/tasks/main.yml diff --git a/experiments/table/demo_exp1.yml b/demo_project/does_config/table/demo_exp1.yml similarity index 100% rename from experiments/table/demo_exp1.yml rename to demo_project/does_config/table/demo_exp1.yml diff --git a/experiments/table/demo_exp2.yml b/demo_project/does_config/table/demo_exp2.yml similarity index 100% rename from experiments/table/demo_exp2.yml rename to demo_project/does_config/table/demo_exp2.yml diff --git a/experiments/table/simple.yml b/demo_project/does_config/table/simple.yml similarity index 100% rename from experiments/table/simple.yml rename to demo_project/does_config/table/simple.yml diff --git a/scripts-demo.ipynb b/demo_project/scripts-demo.ipynb similarity index 100% rename from scripts-demo.ipynb rename to demo_project/scripts-demo.ipynb diff --git a/demo/server.py b/demo_project/server.py similarity index 100% rename from demo/server.py rename to demo_project/server.py diff --git a/docs/troubleshooting.md b/docs/troubleshooting.md index b99d2c3e..ae9dcecf 100644 --- a/docs/troubleshooting.md +++ b/docs/troubleshooting.md @@ -7,7 +7,7 @@ This document contains some errors that may occur and possible reasons for them. > ### Possible reason -Note that every host type defines `ec2_image` in the [group_vars](../group_vars). This instance ID is different in different AWS regions. It will also change with image updates (e.g., when Amazon updates the Ubuntu version of their standard image). +Note that every host type defines `ec2_image_id` in the [group_vars](../group_vars). This instance ID is different in different AWS regions. It will also change with image updates (e.g., when Amazon updates the Ubuntu version of their standard image). Amazon descibes [here](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/finding-an-ami.html#finding-an-ami-console) how to find the AMI ID for your desired region, OS, architecture, etc. diff --git a/group_vars/.gitignore b/group_vars/.gitignore deleted file mode 100644 index b135a564..00000000 --- a/group_vars/.gitignore +++ /dev/null @@ -1,4 +0,0 @@ -# Ignore auto-generated (see repotemplate) client and server group vars YML files -all/ -client/ -server/ diff --git a/group_vars/single/main.yml b/group_vars/single/main.yml deleted file mode 100644 index 5295d8ca..00000000 --- a/group_vars/single/main.yml +++ /dev/null @@ -1,7 +0,0 @@ ---- - -instance_type: t2.small -ec2_volume_size: 8 - -ec2_image: ami-02e5f497990930ec1 -ec2_volume_snapshot: snap-0ae96a390fca425bd diff --git a/src/ansible-controller.yml b/src/ansible-controller.yml new file mode 100644 index 00000000..50ccc1a7 --- /dev/null +++ b/src/ansible-controller.yml @@ -0,0 +1,61 @@ +--- + +################################################################################## +# This playbook sets up an ansible controller on AWS. This instance is able to # +# run the experiment playbook and gather the results. It is intended to be used # +# as a long-running instance where GitHub actions can send commands to, to # +# trigger experiment runs. # +################################################################################## + +########################################################################## +# Setup Ansible Controller # +########################################################################## +- name: Create ansible controller + hosts: localhost + tasks: + + - name: Template dynamic inventory config file + # we do this because we want to have the project id as an inventory filter (such that only ec2 instances with a corresponding tag are visible) + template: + src: resources/inventory/aws_ec2.yml.j2 + dest: inventory/aws_ec2.yml + mode: 0755 + vars: + prj_clear: True + is_ansible_controller: True + + # Load variables + - name: Load group variables + include_role: + name: load-group-vars + vars: + groups_to_load: + - all + + - name: Manually load variables for ansible_controller + include_vars: + file: "{{ external_group_vars_dir }}/ansible_controller/main.yml" + + - name: Setup AWS VPC + include_role: + name: suite-aws-vpc-create + + - name: Create AWS EC2 instance for ansible controller + include_role: + name: ansible-controller-ec2 + +- name: Setup ansible controller + hosts: ansible_controller + + tasks: + - name: Load group variables + include_role: + name: load-group-vars + vars: + groups_to_load: + - all + - ansible_controller + + - name: Configure the EC2 instance for running the AWS Ansible Experiment Suite + include_role: + name: ansible-controller-setup diff --git a/clear.yml b/src/clear.yml similarity index 92% rename from clear.yml rename to src/clear.yml index fcc7ff0c..5bffbc1b 100644 --- a/clear.yml +++ b/src/clear.yml @@ -11,12 +11,12 @@ mode: 0755 vars: prj_clear: True + is_ansible_controller: False - name: Clear EC2 instances include_role: name: suite-aws-ec2-delete - + - name: Clear VPC include_role: name: suite-aws-vpc-delete - diff --git a/experiment-suite.yml b/src/experiment-suite.yml similarity index 91% rename from experiment-suite.yml rename to src/experiment-suite.yml index 05f3d5b4..2f52f5ac 100644 --- a/experiment-suite.yml +++ b/src/experiment-suite.yml @@ -1,5 +1,15 @@ --- + +########################################################################## +# Load Experiment State and Setup AWS # +########################################################################## +- name: Load group_vars from non-standard location + hosts: localhost + + tasks: + - include_vars: "{{ external_group_vars_dir }}/all/main.yml" + ########################################################################## # Load Experiment State and Setup AWS # ########################################################################## @@ -15,6 +25,7 @@ mode: 0755 vars: prj_clear: False # controls whether to also filter for the suite or not (for a clear we want to include all instances of the project independent of the suite) + is_ansible_controller: False # TODO [nku] a design file should be able to contain an experiment design in table for in list form (could use info on whether list of factor levels is defined) - name: resolve suite_id, load and validate suite design, fill default values, and prepare variables @@ -60,6 +71,9 @@ strategy: free tasks: + - name: Load group_vars from non-standard location + include_vars: "{{ external_group_vars_dir }}/all/main.yml" + - name: Execute init roles (incl. common roles for all hosts) include_role: name: "{{ role_name }}" diff --git a/filter_plugins/helpers.py b/src/filter_plugins/helpers.py similarity index 100% rename from filter_plugins/helpers.py rename to src/filter_plugins/helpers.py diff --git a/src/group_vars/all/main.yml b/src/group_vars/all/main.yml new file mode 100644 index 00000000..0b174c58 --- /dev/null +++ b/src/group_vars/all/main.yml @@ -0,0 +1,3 @@ +does_project_dir: "{{ lookup('env', 'DOES_PROJECT_DIR') }}" +does_config_dir: "{{ does_project_dir }}/does_config" +external_group_vars_dir: "{{ does_config_dir }}/group_vars" diff --git a/src/inventory/.gitignore b/src/inventory/.gitignore new file mode 100644 index 00000000..a02f6f08 --- /dev/null +++ b/src/inventory/.gitignore @@ -0,0 +1,2 @@ +# This is generated dynamically by ansible +aws_ec2.yml diff --git a/roles/suite-load-post-aws/library/suite_design_extend.py b/src/inventory/.keep similarity index 100% rename from roles/suite-load-post-aws/library/suite_design_extend.py rename to src/inventory/.keep diff --git a/src/resources/awscli/aws_cli_team_gpg_key.pub b/src/resources/awscli/aws_cli_team_gpg_key.pub new file mode 100644 index 00000000..59518484 --- /dev/null +++ b/src/resources/awscli/aws_cli_team_gpg_key.pub @@ -0,0 +1,29 @@ +-----BEGIN PGP PUBLIC KEY BLOCK----- + +mQINBF2Cr7UBEADJZHcgusOJl7ENSyumXh85z0TRV0xJorM2B/JL0kHOyigQluUG +ZMLhENaG0bYatdrKP+3H91lvK050pXwnO/R7fB/FSTouki4ciIx5OuLlnJZIxSzx +PqGl0mkxImLNbGWoi6Lto0LYxqHN2iQtzlwTVmq9733zd3XfcXrZ3+LblHAgEt5G +TfNxEKJ8soPLyWmwDH6HWCnjZ/aIQRBTIQ05uVeEoYxSh6wOai7ss/KveoSNBbYz +gbdzoqI2Y8cgH2nbfgp3DSasaLZEdCSsIsK1u05CinE7k2qZ7KgKAUIcT/cR/grk +C6VwsnDU0OUCideXcQ8WeHutqvgZH1JgKDbznoIzeQHJD238GEu+eKhRHcz8/jeG +94zkcgJOz3KbZGYMiTh277Fvj9zzvZsbMBCedV1BTg3TqgvdX4bdkhf5cH+7NtWO +lrFj6UwAsGukBTAOxC0l/dnSmZhJ7Z1KmEWilro/gOrjtOxqRQutlIqG22TaqoPG +fYVN+en3Zwbt97kcgZDwqbuykNt64oZWc4XKCa3mprEGC3IbJTBFqglXmZ7l9ywG +EEUJYOlb2XrSuPWml39beWdKM8kzr1OjnlOm6+lpTRCBfo0wa9F8YZRhHPAkwKkX +XDeOGpWRj4ohOx0d2GWkyV5xyN14p2tQOCdOODmz80yUTgRpPVQUtOEhXQARAQAB +tCFBV1MgQ0xJIFRlYW0gPGF3cy1jbGlAYW1hem9uLmNvbT6JAlQEEwEIAD4WIQT7 +Xbd/1cEYuAURraimMQrMRnJHXAUCXYKvtQIbAwUJB4TOAAULCQgHAgYVCgkICwIE +FgIDAQIeAQIXgAAKCRCmMQrMRnJHXJIXEAChLUIkg80uPUkGjE3jejvQSA1aWuAM +yzy6fdpdlRUz6M6nmsUhOExjVIvibEJpzK5mhuSZ4lb0vJ2ZUPgCv4zs2nBd7BGJ +MxKiWgBReGvTdqZ0SzyYH4PYCJSE732x/Fw9hfnh1dMTXNcrQXzwOmmFNNegG0Ox +au+VnpcR5Kz3smiTrIwZbRudo1ijhCYPQ7t5CMp9kjC6bObvy1hSIg2xNbMAN/Do +ikebAl36uA6Y/Uczjj3GxZW4ZWeFirMidKbtqvUz2y0UFszobjiBSqZZHCreC34B +hw9bFNpuWC/0SrXgohdsc6vK50pDGdV5kM2qo9tMQ/izsAwTh/d/GzZv8H4lV9eO +tEis+EpR497PaxKKh9tJf0N6Q1YLRHof5xePZtOIlS3gfvsH5hXA3HJ9yIxb8T0H +QYmVr3aIUes20i6meI3fuV36VFupwfrTKaL7VXnsrK2fq5cRvyJLNzXucg0WAjPF +RrAGLzY7nP1xeg1a0aeP+pdsqjqlPJom8OCWc1+6DWbg0jsC74WoesAqgBItODMB +rsal1y/q+bPzpsnWjzHV8+1/EtZmSc8ZUGSJOPkfC7hObnfkl18h+1QtKTjZme4d +H17gsBJr+opwJw/Zio2LMjQBOqlm3K1A4zFTh7wBC7He6KPQea1p2XAMgtvATtNe +YLZATHZKTJyiqA== +=vYOk +-----END PGP PUBLIC KEY BLOCK----- diff --git a/resources/flask.png b/src/resources/flask.png similarity index 100% rename from resources/flask.png rename to src/resources/flask.png diff --git a/resources/inventory/aws_ec2.yml.j2 b/src/resources/inventory/aws_ec2.yml.j2 similarity index 75% rename from resources/inventory/aws_ec2.yml.j2 rename to src/resources/inventory/aws_ec2.yml.j2 index be2ebd62..5b0a7c86 100644 --- a/resources/inventory/aws_ec2.yml.j2 +++ b/src/resources/inventory/aws_ec2.yml.j2 @@ -3,12 +3,15 @@ plugin: aws_ec2 regions: - eu-central-1 filters: +{% if is_ansible_controller %} # for the ansible controller, we only filter for controllers but not projects + tag:Name: ansible_controller +{% else %} tag:prj_id: {{ prj_id }} {% if not prj_clear %} instance-state-name: ["running"] tag:suite: {{ suite }} {% endif %} - +{% endif %} # keyed_groups may be used to create custom groups #leading_separator: False @@ -17,7 +20,7 @@ keyed_groups: - prefix: "" separator: "" key: tags.prj_id - + - prefix: "" separator: "" key: tags.suite @@ -25,11 +28,11 @@ keyed_groups: - prefix: "" separator: "" key: tags.exp_name - + - prefix: "" separator: "" key: tags.host_type - + - prefix: "is_controller" separator: "_" key: tags.is_controller @@ -38,4 +41,6 @@ keyed_groups: separator: "_" key: tags.check_status - + - prefix: "" + separator: "" + key: tags.Name diff --git a/resources/repotemplate/group_vars/all/main.yml.j2 b/src/resources/repotemplate/group_vars/all/main.yml.j2 similarity index 83% rename from resources/repotemplate/group_vars/all/main.yml.j2 rename to src/resources/repotemplate/group_vars/all/main.yml.j2 index ee60e8d9..b1dff39c 100644 --- a/resources/repotemplate/group_vars/all/main.yml.j2 +++ b/src/resources/repotemplate/group_vars/all/main.yml.j2 @@ -10,16 +10,17 @@ git_remote_repository: <> # TODO: set remote repository ( # The experiments are mostly run concurrently (apart from the setup and cleanup parts). Thus, the experiment with the most jobs defines the # maximal duration. But as experiments usually use fewer than 'job_n_tries' tries, an experiment with few long-running jobs can be the bottleneck too. job_n_tries: <> # should be max 1000 (otherwise playbook freezes -> unsure why) - # TODO [mh]: must be tested if this is still an issue, I switched to an `until` loop job_check_wait_time: <> remote: dir: "/home/ubuntu" + results_dir: "/home/ubuntu/results" exp_code_dir: "{{ remote.dir }}/code" local: - results_dir: "./results" + results_dir: "{{ does_project_dir }}/does_results" + designs_dir: "{{ does_config_dir }}/designs" exp_base: key_name: <> # TODO: add key pair name @@ -31,10 +32,3 @@ exp_base: vpc_subnet_cidr: 10.100.0.0/24 sg_name: "{{ prj_id }}_sg" sg_desc: "{{ prj_id }} security group" - -separator: '_SEP_' - -# This prefix (incl. the capitalization) is chosen by the ec2 plugin -ec2_tag_name_prefix: 'tag_Name_' -ec2_tag_prj_prefix: 'tag_Prj_' -ec2_tag_exp_prefix: 'tag_Exp_' diff --git a/src/resources/repotemplate/group_vars/ansible_controller/main.yml.j2 b/src/resources/repotemplate/group_vars/ansible_controller/main.yml.j2 new file mode 100644 index 00000000..1ee0822c --- /dev/null +++ b/src/resources/repotemplate/group_vars/ansible_controller/main.yml.j2 @@ -0,0 +1,32 @@ +--- + +# The controller doesn't need to have beefy specs, it only distributes tasks. +# However, we need enough space for the results. + +instance_type: <> # change if you feel the controller is a bottleneck +ec2_volume_size: <> # choose large enough to store results (at least until they are downloaded) + +ec2_image_id: <> +ec2_volume_snapshot: <> + +ansible_controller_user: controller +ansible_controller_group: controller +ansible_controller_home: "/home/{{ ansible_controller_user }}" + +# AWS Ansible Experiment Suite repo options +ansible_exp_suite_git_repo: <> # TODO: change to the URL of your clone of the ansible experiment suite +ansible_exp_suite_private_repo: yes # Set to true if the repo is public. In that case we don't need to setup SSH keys for the repo. +ansible_controller_does_dir: "{{ ansible_controller_home }}/aws_ansible_experiment_suite" # Path to where AWS Ansible Experiment Suite is cloned to + +# SSH options +ansible_controller_ssh_key_dir: "{{ ansible_controller_home }}/.ssh" +ansible_controller_ssh_key_types: + git: ecdsa + aws: rsa +ansible_controller_ssh_key_sizes: + git: 521 + aws: 4096 + +ansible_controller_ssh_key_paths: + git: "{{ ansible_controller_ssh_key_dir }}/id_ssh_git" + aws: "{{ ansible_controller_ssh_key_dir }}/{{ exp_base.key_name }}" diff --git a/resources/repotemplate/group_vars/host_type/main.yml.j2 b/src/resources/repotemplate/group_vars/host_type/main.yml.j2 similarity index 84% rename from resources/repotemplate/group_vars/host_type/main.yml.j2 rename to src/resources/repotemplate/group_vars/host_type/main.yml.j2 index 76bf8bdf..3dcbb4c7 100644 --- a/resources/repotemplate/group_vars/host_type/main.yml.j2 +++ b/src/resources/repotemplate/group_vars/host_type/main.yml.j2 @@ -3,5 +3,5 @@ instance_type: <> # TODO: choose instance type ec2_volume_size: <> # TODO: choose volume size -ec2_image: <> +ec2_image_id: <> ec2_volume_snapshot: <> diff --git a/src/roles/ansible-controller-clear/tasks/main.yml b/src/roles/ansible-controller-clear/tasks/main.yml new file mode 100644 index 00000000..e4dbcd81 --- /dev/null +++ b/src/roles/ansible-controller-clear/tasks/main.yml @@ -0,0 +1,40 @@ +--- + +- name: Collect info about running ec2 instances + community.aws.ec2_instance_info: + region: "{{ exp_base.aws_region }}" + filters: + "tag:name": "{{ ansible_controller }}" + register: ec2_instance_info + +# TODO: remove +- debug: + msg: "ec2_instance_info: {{ ec2_instance_info }}" + +- name: Extract instance ids of ec2 instances to remove + ansible.builtin.set_fact: + ec2_instance_ids: "{{ ec2_instance_info | json_query('*.instances[*].instance_id') | list | flatten }}" + +# TODO: remove +- debug: + msg: "ec2_instance_ids: {{ ec2_instance_ids }}" + +- ansible.builtin.pause: + seconds: 10 + prompt: | + "Removing instances with the following ids: + {{ ec2_instance_ids }} + + If the above instance IDs are wrong, abort now! (CTRL+C followed by 'A')" + tags: [print_action] + +- name: Cleanup AWS + community.aws.ec2_instance: + instance_ids: "{{ ec2_instance_ids }}" + region: "{{ exp_base.aws_region }}" + state: absent + when: (ec2_instance_ids | length) > 0 + +- name: Remove AWS VPC + ansible.builtin.include_role: + name: suite-aws-vpc-delete diff --git a/src/roles/ansible-controller-ec2/tasks/main.yml b/src/roles/ansible-controller-ec2/tasks/main.yml new file mode 100644 index 00000000..97515995 --- /dev/null +++ b/src/roles/ansible-controller-ec2/tasks/main.yml @@ -0,0 +1,38 @@ +--- + +- name: Launching ansible controller EC2 instance + community.aws.ec2_instance: + state: running + instance_type: '{{ instance_type }}' + key_name: '{{ exp_base.key_name }}' + image_id: '{{ ec2_image_id }}' + region: '{{ exp_base.aws_region }}' + security_group: '{{ exp_base.sg_name }}' + vpc_subnet_id: '{{ exp_base.vpc_subnet_id }}' + wait: yes + network: + assign_public_ip: yes + tags: + Name: ansible_controller + volumes: + - device_name: /dev/sda1 + ebs: + snapshot_id: '{{ ec2_volume_snapshot }}' + delete_on_termination: true + volume_size: '{{ ec2_volume_size }}' + volume_type: gp2 + register: ec2_launch + +- name: Wait for SSH to come up + delegate_to: "{{ public_dns_name }}" + ansible.builtin.wait_for_connection: + connect_timeout: 3 + timeout: 320 + loop: "{{ ec2_launch.instances + | map(attribute='public_dns_name') + | list }}" + loop_control: + loop_var: public_dns_name + +- name: Refresh dynamic ec2 inventory + ansible.builtin.meta: refresh_inventory diff --git a/src/roles/ansible-controller-setup/tasks/configure_aws.yml b/src/roles/ansible-controller-setup/tasks/configure_aws.yml new file mode 100644 index 00000000..a5f0511d --- /dev/null +++ b/src/roles/ansible-controller-setup/tasks/configure_aws.yml @@ -0,0 +1,11 @@ + +- name: "Set AWS default region {{ exp_base.aws_region }}" + ansible.builtin.copy: + dest: "{{ ansible_controller_home/.aws/config }}" + content: | + [default] + region=eu-central-1 + +# We don't use .aws/credentials to store AWS credentials. Instead, the GitHub action +# uses the environment variables AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY, which +# it populates from GitHub secrets. diff --git a/src/roles/ansible-controller-setup/tasks/fetch_does_repo.yml b/src/roles/ansible-controller-setup/tasks/fetch_does_repo.yml new file mode 100644 index 00000000..769d7e0b --- /dev/null +++ b/src/roles/ansible-controller-setup/tasks/fetch_does_repo.yml @@ -0,0 +1,28 @@ + +- name: "Clone the AWS Ansible Experiment Suite from {{ ansible_exp_suite_git_repo }}" + ansible.builtin.git: + repo: "{{ ansible_exp_suite_git_repo }}" + dest: "{{ ansible_controller_does_dir }}" + key_file: "{{ ansible_controller_ssh_key_paths['git'] }}" + accept_hostkey: yes + become: yes + become_method: sudo + +# We cannot clone the repo directly as ansible_controller_user because then +# ansible fails when it tries to create temporary files and change their permissions, +# because ansible_controller_user does not have root privileges. +- name: Set permissions on cloned repo + ansible.builtin.file: + path: "{{ ansible_controller_does_dir }}" + owner: "{{ ansible_controller_user }}" + group: "{{ ansible_controller_group }}" + recurse: yes + become: yes + become_method: sudo + +- name: Install pipenv package dependencies + ansible.builtin.command: + cmd: pipenv install + chdir: "{{ ansible_controller_does_dir }}" + become: yes + become_method: sudo diff --git a/src/roles/ansible-controller-setup/tasks/install_awscli.yml b/src/roles/ansible-controller-setup/tasks/install_awscli.yml new file mode 100644 index 00000000..eb6dd491 --- /dev/null +++ b/src/roles/ansible-controller-setup/tasks/install_awscli.yml @@ -0,0 +1,99 @@ + +- name: Set variables for the CLI installation + set_fact: + ansible_controller_awscli_dir: "{{ ansible_controller_home }}/awscli" + awscli_url: https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip + awscli_zip_dest: "{{ ansible_controller_awscli_dir }}/awscli-exe-linux-x86_64.zip" + awscli_pub_key_path: "{{ ansible_controller_awscli_dir }}/aws_cli_team_gpg_key.pub" + +- block: + - name: Create a folder for the AWS CLI installer + ansible.builtin.file: + path: "{{ ansible_controller_awscli_dir }}" + state: directory + owner: "{{ ansible_controller_user }}" + group: "{{ ansible_controller_group }}" + mode: '0750' + + - name: Copy AWS CLI Team GPG public key to VM + ansible.builtin.copy: + src: resources/awscli/aws_cli_team_gpg_key.pub + dest: "{{ awscli_pub_key_path }}" + owner: "{{ ansible_controller_user }}" + group: "{{ ansible_controller_group }}" + mode: '0440' + + - name: Import AWS CLI Team GPG public key in GPG + ansible.builtin.command: "gpg --import {{ awscli_pub_key_path }}" + + - name: Download AWS CLI ZIP-file + ansible.builtin.get_url: + url: "{{ awscli_url }}" + dest: "{{ awscli_zip_dest }}" + owner: "{{ ansible_controller_user }}" + group: "{{ ansible_controller_group }}" + mode: '0440' + register: get_url + + - name: Download signature for AWS CLI ZIP-file + ansible.builtin.get_url: + url: "{{ awscli_url }}.sig" + dest: "{{ awscli_zip_dest }}.sig" + owner: "{{ ansible_controller_user }}" + group: "{{ ansible_controller_group }}" + mode: '0440' + + - name: Verify signature for AWS CLI ZIP-file + ansible.builtin.command: + cmd: "gpg --verify {{ awscli_zip_dest }}.sig {{ awscli_zip_dest }}" + register: gpg_cmd + + - name: Abort playbook for failed verification + ansible.builtin.fail: + msg: | + Failed to verify signature for AWS CLI ZIP-file! Check the official verification + guide [1] to understand why the verification failed. + + [1] https://docs.aws.amazon.com/cli/latest/userguide/install-cliv2-linux.html#v2-install-linux-validate + when: gpg_cmd.rc != 0 + + - name: Unzip AWS CLI tools + ansible.builtin.unarchive: + src: "{{ awscli_zip_dest }}" + dest: "{{ ansible_controller_awscli_dir }}" + owner: "{{ ansible_controller_user }}" + group: "{{ ansible_controller_group }}" + remote_src: yes + + - name: Check if AWS CLI is already installed + ansible.builtin.command: which aws + register: which_cmd + + - name: Install AWS CLI (if it is not yet installed) + ansible.builtin.command: + cmd: ./aws/install + chdir: "{{ ansible_controller_awscli_dir }}" + when: which_cmd.rc != 0 + + - name: Update AWS CLI (if awscli is already installed) + ansible.builtin.command: + cmd: ./aws/install --update + chdir: "{{ ansible_controller_awscli_dir }}" + when: which_cmd.rc == 0 + + - name: Verify installation + block: + - ansible.builtin.command: "sudo -u '{{ ansible_controller_user }}' aws --version" + register: awscli_cmd + + - ansible.builtin.debug: + msg: "Installed AWS CLI version {{ awscli_cmd.stdout }}" + tags: [print_action] + when: awscli_cmd.rc == 0 + + - ansible.builtin.fail: + msg: AWS CLI installation failed! + when: awscli_cmd.rc != 0 + + become: yes + become_method: sudo diff --git a/src/roles/ansible-controller-setup/tasks/install_boto3.yml b/src/roles/ansible-controller-setup/tasks/install_boto3.yml new file mode 100644 index 00000000..1d25ad8d --- /dev/null +++ b/src/roles/ansible-controller-setup/tasks/install_boto3.yml @@ -0,0 +1,6 @@ + +- name: Install boto3 Python package + ansible.builtin.pip: + name: boto3 + become: yes + become_method: sudo diff --git a/src/roles/ansible-controller-setup/tasks/main.yml b/src/roles/ansible-controller-setup/tasks/main.yml new file mode 100644 index 00000000..bb624e24 --- /dev/null +++ b/src/roles/ansible-controller-setup/tasks/main.yml @@ -0,0 +1,8 @@ +--- + +- include_tasks: prerequisites.yml +- include_tasks: ssh_setup.yml +- include_tasks: fetch_does_repo.yml +- include_tasks: install_boto3.yml +- include_tasks: install_awscli.yml +- include_tasks: prepare_ansible.yml diff --git a/src/roles/ansible-controller-setup/tasks/prepare_ansible.yml b/src/roles/ansible-controller-setup/tasks/prepare_ansible.yml new file mode 100644 index 00000000..830e2225 --- /dev/null +++ b/src/roles/ansible-controller-setup/tasks/prepare_ansible.yml @@ -0,0 +1,12 @@ + +- name: Install ansible + ansible.builtin.pip: + name: ansible + executable: pip3 + become: yes + become_method: sudo + +- name: Install the required Ansible collections + ansible.builtin.command: + cmd: pipenv run ansible-galaxy install -r requirements-collections.yml + chdir: "{{ ansible_controller_does_dir }}" diff --git a/src/roles/ansible-controller-setup/tasks/prerequisites.yml b/src/roles/ansible-controller-setup/tasks/prerequisites.yml new file mode 100644 index 00000000..d1923e98 --- /dev/null +++ b/src/roles/ansible-controller-setup/tasks/prerequisites.yml @@ -0,0 +1,38 @@ + +- name: Install required packages + ansible.builtin.apt: + name: + - git + - build-essential + - python3-dev + - python3 + - python3-pip + - gnupg # for verifying the signature of awscli + - unzip # to unpack awscli + update_cache: yes + state: present + + become: yes + become_method: sudo + +- name: "Add user and group {{ ansible_controller_user }}:{{ ansible_controller_group }}" + block: + - ansible.builtin.group: + name: "{{ ansible_controller_group }}" + state: present + + - ansible.builtin.user: + name: "{{ ansible_controller_user }}" + group: "{{ ansible_controller_group }}" + create_home: yes + state: present + + become: yes + become_method: sudo + +- name: Install pipenv + ansible.builtin.pip: + name: pipenv + executable: pip3 + become: yes + become_method: sudo diff --git a/src/roles/ansible-controller-setup/tasks/ssh_setup.yml b/src/roles/ansible-controller-setup/tasks/ssh_setup.yml new file mode 100644 index 00000000..30cb35fb --- /dev/null +++ b/src/roles/ansible-controller-setup/tasks/ssh_setup.yml @@ -0,0 +1,57 @@ + +- name: Set shared SSH variables + ansible.builtin.set_fact: + ssh_key_dir: "{{ ansible_controller_ssh_key_dir }}" + ssh_key_owner: "{{ ansible_controller_user }}" + ssh_key_group: "{{ ansible_controller_group }}" + +- name: Generate SSH key for Git + ansible.builtin.include_role: + name: setup-ssh + vars: + ssh_key_path: "{{ ansible_controller_ssh_key_paths['git'] }}" + ssh_key_type: "{{ ansible_controller_ssh_key_types['git'] }}" + ssh_key_size: "{{ ansible_controller_ssh_key_sizes['git'] }}" + when: ansible_exp_suite_private_repo + +- name: Add entry for git to ~/.ssh/config + ansible.builtin.blockinfile: + path: "{{ ansible_controller_ssh_key_dir }}/config" + block: | + Host github.com + Hostname github.com + PreferredAuthentications publickey + User {{ ansible_controller_git_username }} + IdentityFile {{ ansible_controller_ssh_key_paths['git'] }} + owner: "{{ ansible_controller_user }}" + group: "{{ ansible_controller_group }}" + mode: "0644" + create: yes + state: present + marker: "# {mark} ANSIBLE MANAGED BLOCK FOR GIT" + become: yes + become_method: sudo + +- name: Generate SSH key for AWS + ansible.builtin.include_role: + name: setup-ssh + vars: + ssh_key_path: "{{ ansible_controller_ssh_key_paths['aws'] }}" + ssh_key_type: "{{ ansible_controller_ssh_key_types['aws'] }}" + ssh_key_size: "{{ ansible_controller_ssh_key_sizes['aws'] }}" + +- name: Add entry for EC2 hosts to ~/.ssh/config + ansible.builtin.blockinfile: + path: "{{ ansible_controller_ssh_key_dir }}/config" + block: | + Host ec2* + IdentityFile {{ ansible_controller_ssh_key_paths['aws'] }} + User ubuntu + ForwardAgent yes + owner: "{{ ansible_controller_user }}" + group: "{{ ansible_controller_group }}" + mode: "0644" + state: present + marker: "# {mark} ANSIBLE MANAGED BLOCK FOR AWS EC2" + become: yes + become_method: sudo diff --git a/roles/experiment-job/filter_plugins/helpers.py b/src/roles/experiment-job/filter_plugins/helpers.py similarity index 93% rename from roles/experiment-job/filter_plugins/helpers.py rename to src/roles/experiment-job/filter_plugins/helpers.py index a687dcad..8a4fabc5 100644 --- a/roles/experiment-job/filter_plugins/helpers.py +++ b/src/roles/experiment-job/filter_plugins/helpers.py @@ -23,7 +23,7 @@ def tsp_job_finished(tsp_tasks, job_id): else: raise ValueError(f"tsp task with unknown task state = {task['state']} (task={task})") - + raise ValueError(f"no matching job found in tsp: {job_id} tsp_tasks={tsp_tasks}") def get_tsp_task_id(tsp_tasks, job_id): @@ -49,7 +49,7 @@ def to_job_schedule_lst(job_ids, exp_host_lst, exp_runs_ext, working_base_dir): for job_id in job_ids: for host_info in exp_host_lst: - + run_idx = int(job_id["exp_run"]) host_type = host_info["host_type"] host_type_idx = host_info["exp_host_type_idx"] @@ -79,12 +79,12 @@ def jobid2workingdir(job_id, base): return: path to the working directory for a job """ - exp_working_dir = os.path.join(base, "results", - f"{job_id['suite']}_{job_id['suite_id']}", - job_id['exp_name'], + exp_working_dir = os.path.join(base, + f"{job_id['suite']}_{job_id['suite_id']}", + job_id['exp_name'], f"run_{job_id['exp_run']}", f"rep_{job_id['exp_run_rep']}") - + return exp_working_dir @@ -99,4 +99,4 @@ def filters(self): "get_tsp_task_id": get_tsp_task_id, "to_job_schedule_lst": to_job_schedule_lst, "jobid2workingdir": jobid2workingdir, - } \ No newline at end of file + } diff --git a/roles/experiment-job/library/tsp.py b/src/roles/experiment-job/library/tsp.py similarity index 100% rename from roles/experiment-job/library/tsp.py rename to src/roles/experiment-job/library/tsp.py diff --git a/roles/experiment-job/library/tsp_info.py b/src/roles/experiment-job/library/tsp_info.py similarity index 100% rename from roles/experiment-job/library/tsp_info.py rename to src/roles/experiment-job/library/tsp_info.py diff --git a/roles/experiment-job/tasks/main.yml b/src/roles/experiment-job/tasks/main.yml similarity index 98% rename from roles/experiment-job/tasks/main.yml rename to src/roles/experiment-job/tasks/main.yml index 72583340..4d1f16d5 100644 --- a/roles/experiment-job/tasks/main.yml +++ b/src/roles/experiment-job/tasks/main.yml @@ -73,7 +73,7 @@ - name: build job schedule list (for each host of experiment, information about job and what is required to run) set_fact: - exp_jobs_to_enqueue: "{{ exp_job_ids_to_enqueue | to_job_schedule_lst(exp_host_lst, exp_runs_ext, remote.dir) }}" + exp_jobs_to_enqueue: "{{ exp_job_ids_to_enqueue | to_job_schedule_lst(exp_host_lst, exp_runs_ext, remote.results_dir) }}" ################################################################### @@ -229,9 +229,9 @@ vars: - - remote_results_dir: "{{ job_id_to_wait_for | jobid2workingdir(remote.dir) + '/results' }}" - - local_results_dir_base: "{{ job_id_to_wait_for | jobid2workingdir('') }}" - - remote_config_file: "{{ job_id_to_wait_for | jobid2workingdir(remote.dir) + '/config.json' }}" + - remote_results_dir: "{{ job_id_to_wait_for | jobid2workingdir(remote.results_dir) + '/results' }}" + - local_results_dir_base: "{{ job_id_to_wait_for | jobid2workingdir(local.results_dir) }}" + - remote_config_file: "{{ job_id_to_wait_for | jobid2workingdir(remote.results_dir) + '/config.json' }}" ################################################################### # Cleanup task spooler queue # @@ -286,4 +286,4 @@ msg: "{{ suite_progress_info[item] }}" loop: "{{ suite_progress_info.keys() | sort }}" tags: [print_action] -# TEMPORARY END \ No newline at end of file +# TEMPORARY END diff --git a/roles/experiment-job/templates/config.json.j2 b/src/roles/experiment-job/templates/config.json.j2 similarity index 100% rename from roles/experiment-job/templates/config.json.j2 rename to src/roles/experiment-job/templates/config.json.j2 diff --git a/roles/experiment-state/tasks/main.yml b/src/roles/experiment-state/tasks/main.yml similarity index 100% rename from roles/experiment-state/tasks/main.yml rename to src/roles/experiment-state/tasks/main.yml diff --git a/roles/experiment-state/templates/state.yml.j2 b/src/roles/experiment-state/templates/state.yml.j2 similarity index 100% rename from roles/experiment-state/templates/state.yml.j2 rename to src/roles/experiment-state/templates/state.yml.j2 diff --git a/src/roles/load-group-vars/tasks/main.yml b/src/roles/load-group-vars/tasks/main.yml new file mode 100644 index 00000000..0a04c596 --- /dev/null +++ b/src/roles/load-group-vars/tasks/main.yml @@ -0,0 +1,8 @@ +--- + +- name: Load variables for all groups of the host + include_vars: + file: "{{ external_group_vars_dir }}/{{ group_name }}/main.yml" + loop: "{{ groups_to_load }}" + loop_control: + loop_var: group_name diff --git a/src/roles/setup-ssh/tasks/main.yml b/src/roles/setup-ssh/tasks/main.yml new file mode 100644 index 00000000..204e28ee --- /dev/null +++ b/src/roles/setup-ssh/tasks/main.yml @@ -0,0 +1,57 @@ +--- + +- assert: + that: + - ssh_key_dir is defined + - ssh_key_owner is defined + - ssh_key_group is defined + - ssh_key_path is defined + - ssh_key_size is defined + +- block: + - name: Create folder for SSH keys + file: + path: "{{ ssh_key_dir }}" + owner: "{{ ssh_key_owner }}" + group: "{{ ssh_key_group }}" + mode: '0750' + state: directory + + - name: Generate SSH key + openssh_keypair: + path: "{{ ssh_key_path }}" + type: "{{ ssh_key_type }}" + size: "{{ ssh_key_size }}" + owner: "{{ ssh_key_owner }}" + group: "{{ ssh_key_group }}" + mode: '0600' + register: keygen + + become: yes + become_method: sudo + +- name: Read public key + slurp: + src: "{{ ssh_key_path }}.pub" + register: slurped_pub_key + + become: yes + become_method: sudo + +- name: Store public key in variable + set_fact: + new_pub_key: "{{ slurped_pub_key['content'] | b64decode }}" + +- pause: + prompt: "ACTION REQUIRED: A new key pair was generated, store the following public + key at the relevant service (Git, AWS, ...): + {{ new_pub_key }} + + Acknowledge once you added this key to continue the playbook." + + when: keygen.changed + +- debug: + msg: "Reuse existing public key: {{ new_pub_key }}" + tags: [print_action] + when: not keygen.changed diff --git a/roles/setup-suite/tasks/main.yml b/src/roles/setup-suite/tasks/main.yml similarity index 100% rename from roles/setup-suite/tasks/main.yml rename to src/roles/setup-suite/tasks/main.yml diff --git a/roles/suite-aws-ec2-create/filter_plugins/helpers.py b/src/roles/suite-aws-ec2-create/filter_plugins/helpers.py similarity index 100% rename from roles/suite-aws-ec2-create/filter_plugins/helpers.py rename to src/roles/suite-aws-ec2-create/filter_plugins/helpers.py diff --git a/roles/suite-aws-ec2-create/tasks/main.yml b/src/roles/suite-aws-ec2-create/tasks/main.yml similarity index 89% rename from roles/suite-aws-ec2-create/tasks/main.yml rename to src/roles/suite-aws-ec2-create/tasks/main.yml index f82ee767..eeeefdc9 100644 --- a/roles/suite-aws-ec2-create/tasks/main.yml +++ b/src/roles/suite-aws-ec2-create/tasks/main.yml @@ -17,7 +17,7 @@ block: - name: Loop over different host types and load group vars from file. include_vars: - file: "group_vars/{{ host_type }}/main.yml" + file: "{{ external_group_vars_dir }}/{{ host_type }}/main.yml" name: host_type_specific_vars loop: "{{ host_types.keys() }}" loop_control: @@ -47,30 +47,8 @@ ###################################################################### - name: Create EC2 Instances (only assign subset of tags yet) - ec2: - instance_type: '{{ ec2config.instance_type }}' - key_name: '{{ ec2config.key_name }}' - image: '{{ ec2config.ec2_image }}' - region: '{{ ec2config.aws_region }}' - group: '{{ ec2config.sg_name }}' - exact_count: '{{ n_sum }}' - count_tag: - prj_id: "{{ prj_id }}" - suite: "{{ suite }}" - host_type: "{{ host_type }}" - vpc_subnet_id: '{{ ec2config.vpc_subnet_id }}' - wait: no - assign_public_ip: yes - volumes: - - device_name: /dev/sda1 - volume_type: gp2 - snapshot: '{{ ec2config.ec2_volume_snapshot }}' - volume_size: '{{ ec2config.ec2_volume_size }}' - delete_on_termination: True - instance_tags: - prj_id: "{{ prj_id }}" - suite: "{{ suite }}" - host_type: "{{ host_type }}" + include_role: + name: suite-aws-ec2-launch vars: ec2config: "{{ host_type_specific_vars[host_type] | combine( exp_base ) }}" n_sum: "{{ host_types[host_type] | json_query('*.n') | sum }}" @@ -196,4 +174,4 @@ delegate_to: "{{ host_info.public_dns_name }}" loop: "{{ suite_hosts_lst }}" loop_control: - loop_var: host_info \ No newline at end of file + loop_var: host_info diff --git a/roles/suite-aws-ec2-delete/tasks/main.yml b/src/roles/suite-aws-ec2-delete/tasks/main.yml similarity index 100% rename from roles/suite-aws-ec2-delete/tasks/main.yml rename to src/roles/suite-aws-ec2-delete/tasks/main.yml diff --git a/src/roles/suite-aws-ec2-launch/tasks/main.yml b/src/roles/suite-aws-ec2-launch/tasks/main.yml new file mode 100644 index 00000000..50bc74bf --- /dev/null +++ b/src/roles/suite-aws-ec2-launch/tasks/main.yml @@ -0,0 +1,30 @@ + +- name: Create EC2 Instances (only assign subset of tags yet) + community.aws.ec2_instance: + instance_type: '{{ ec2config.instance_type }}' + key_name: '{{ ec2config.key_name }}' + image_id: '{{ ec2config.ec2_image_id }}' + region: '{{ ec2config.aws_region }}' + security_group: '{{ ec2config.sg_name }}' + vpc_subnet_id: '{{ ec2config.vpc_subnet_id }}' + wait: yes + network: + assign_public_ip: yes + volumes: + - device_name: /dev/sda1 + ebs: + volume_type: gp2 + snapshot_id: '{{ ec2config.ec2_volume_snapshot }}' + volume_size: '{{ ec2config.ec2_volume_size }}' + delete_on_termination: True + tags: + prj_id: "{{ prj_id }}" + suite: "{{ suite }}" + host_type: "{{ host_type }}" + + # This is a workaround for the missing "exact_count" feature that used to be + # in the deprecated community.aws.ec2 + # - https://github.com/ansible/ansible/issues/49944 + loop: "{{ range(0, n_sum | int) | list }}" + loop_control: + loop_var: instance_cnt diff --git a/roles/suite-aws-vpc-create/tasks/main.yml b/src/roles/suite-aws-vpc-create/tasks/main.yml similarity index 100% rename from roles/suite-aws-vpc-create/tasks/main.yml rename to src/roles/suite-aws-vpc-create/tasks/main.yml diff --git a/roles/suite-aws-vpc-delete/tasks/main.yml b/src/roles/suite-aws-vpc-delete/tasks/main.yml similarity index 100% rename from roles/suite-aws-vpc-delete/tasks/main.yml rename to src/roles/suite-aws-vpc-delete/tasks/main.yml diff --git a/roles/suite-load-post-aws/action_plugins/suite_design_extend.py b/src/roles/suite-load-post-aws/action_plugins/suite_design_extend.py similarity index 100% rename from roles/suite-load-post-aws/action_plugins/suite_design_extend.py rename to src/roles/suite-load-post-aws/action_plugins/suite_design_extend.py diff --git a/roles/suite-load-pre-aws/library/suite_design_validate.py b/src/roles/suite-load-post-aws/library/suite_design_extend.py similarity index 100% rename from roles/suite-load-pre-aws/library/suite_design_validate.py rename to src/roles/suite-load-post-aws/library/suite_design_extend.py diff --git a/roles/suite-load-post-aws/tasks/main.yml b/src/roles/suite-load-post-aws/tasks/main.yml similarity index 100% rename from roles/suite-load-post-aws/tasks/main.yml rename to src/roles/suite-load-post-aws/tasks/main.yml diff --git a/roles/suite-load-pre-aws/action_plugins/suite_design_validate.py b/src/roles/suite-load-pre-aws/action_plugins/suite_design_validate.py similarity index 100% rename from roles/suite-load-pre-aws/action_plugins/suite_design_validate.py rename to src/roles/suite-load-pre-aws/action_plugins/suite_design_validate.py diff --git a/roles/suite-load-pre-aws/filter_plugins/helpers.py b/src/roles/suite-load-pre-aws/filter_plugins/helpers.py similarity index 100% rename from roles/suite-load-pre-aws/filter_plugins/helpers.py rename to src/roles/suite-load-pre-aws/filter_plugins/helpers.py diff --git a/src/roles/suite-load-pre-aws/library/suite_design_validate.py b/src/roles/suite-load-pre-aws/library/suite_design_validate.py new file mode 100644 index 00000000..e69de29b diff --git a/roles/suite-load-pre-aws/tasks/main.yml b/src/roles/suite-load-pre-aws/tasks/main.yml similarity index 98% rename from roles/suite-load-pre-aws/tasks/main.yml rename to src/roles/suite-load-pre-aws/tasks/main.yml index 6b0b4f93..44fdd6be 100644 --- a/roles/suite-load-pre-aws/tasks/main.yml +++ b/src/roles/suite-load-pre-aws/tasks/main.yml @@ -81,7 +81,7 @@ - name: validate the suite design and set default values suite_design_validate: - src: experiments/designs/{{ suite }}.yml + src: "{{ local.designs_dir }}/{{ suite }}.yml" dest: "{{ suite_design_dir }}/suite_design.yml" when: id == 'new' # only validate and resolve defaults for new (otherwise load the old already checked version) diff --git a/scripts/expdesign.py b/src/scripts/expdesign.py similarity index 97% rename from scripts/expdesign.py rename to src/scripts/expdesign.py index 3e2e68c0..30d18e58 100644 --- a/scripts/expdesign.py +++ b/src/scripts/expdesign.py @@ -10,7 +10,7 @@ def main(): parser.add_argument('--exps', help='experiment name(s) (experiment file name without .yml ending)', nargs="+", type=str, required=True) - parser.add_argument('--inputdir', help='path to folder of experiment in "table" form', type=str, default="experiments/table") + parser.add_argument('--inputdir', help='path to folder of experiment in "table" form', type=str, default="does_config/table") parser.add_argument('--outpath', help='output path for generated of experiment in "design" form', type=str) @@ -25,7 +25,7 @@ def main(): suite = exps[0] if not outpath: - outpath = f"experiments/designs/{suite}.yml" + outpath = f"does_config/designs/{suite}.yml" with open(outpath, 'w') as out_fp: print(f"Writing experiment design to: {outpath}\n") diff --git a/scripts/repotemplate.py b/src/scripts/repotemplate.py similarity index 84% rename from scripts/repotemplate.py rename to src/scripts/repotemplate.py index 31eea3bc..564db4ae 100644 --- a/scripts/repotemplate.py +++ b/src/scripts/repotemplate.py @@ -9,9 +9,16 @@ DEFAULT_HOST_TYPE = "host_type" -vars_base_path = "group_vars" -templates_base_path = "resources/repotemplate/group_vars" -groups = ["all", "server", "client"] +does_proj_env_varname = "DOES_PROJECT_DIR" + +if does_proj_env_varname not in os.environ: + print(f"ERROR: {does_proj_env_varname} is not set to the path to the project directory") + exit(1) + +vars_base_path = os.environ[does_proj_env_varname] + "/does_config/group_vars" +src_path = os.path.dirname(os.path.dirname(os.path.realpath(__file__))) +templates_base_path = src_path + "/resources/repotemplate/group_vars" +groups = ["all", "server", "client", "ansible_controller"] template_name = "main.yml.j2" # The template path is assumed to be: f"{templates_base_path}/<>", where # <> is 'all' for the group 'all' and the string 'DEFAULT_HOST_TYPE' for all other host types. @@ -61,7 +68,7 @@ defaults = { "all": { #'prj_id': None, - 'git_remote_repository': 'git@github.com:pps-lab/aws-simple-ansible.git', + 'git_remote_repository': 'git@github.com:nicolas-kuechler/doe-suite.git', 'job_n_tries': 100, 'job_check_wait_time': 5, #'key_name': None, @@ -69,20 +76,27 @@ "server": { 'instance_type': 't2.medium', 'volume_size': 16, - 'ec2_image': ubuntu_ami, + 'ec2_image_id': ubuntu_ami, 'snapshot_id': None }, "client": { 'instance_type': 't2.medium', 'volume_size': 16, - 'ec2_image': ubuntu_ami, + 'ec2_image_id': ubuntu_ami, 'snapshot_id': None }, + "ansible_controller": { + 'instance_type': 't2.small', + 'volume_size': 64, + 'ec2_image_id': ubuntu_ami, + 'snapshot_id': None, + 'ansible_exp_suite_git_repo': 'git@github.com:nicolas-kuechler/doe-suite.git' + }, # General default values DEFAULT_HOST_TYPE: { 'instance_type': 't2.medium', 'volume_size': 16, - 'ec2_image': ubuntu_ami, + 'ec2_image_id': ubuntu_ami, 'snapshot_id': None } } @@ -130,7 +144,7 @@ def prompt_user(d, variables, host): input_num(d, "volume_size", "> EC2 volume size in GB", min=8, max=512) - input_str(d, "ec2_image", "> EC2 image AMI") + input_str(d, "ec2_image_id", "> EC2 image AMI") # Find the SnapshotId for this instance if "snapshot_id" not in d or d["snapshot_id"] == None: @@ -140,18 +154,20 @@ def prompt_user(d, variables, host): "ec2", "describe-images", "--image-ids", - d["ec2_image"], + d["ec2_image_id"], "--query", "Images[0].BlockDeviceMappings[*].Ebs.SnapshotId" ]).decode())[0] d["snapshot_id"] = snapshot_id except: - print(f"WARNING: could not fetch snapshot id for {d['ec2_image']}") + print(f"WARNING: could not fetch snapshot id for {d['ec2_image_id']}") pass input_str(d, "snapshot_id", "> Snapshot ID for this instance") + if host == "ansible_controller": + input_str(d, "ansible_exp_suite_git_repo", "> URL to AWS Ansible Experiment Suite git repository") remaining_variables = list(filter(lambda x: not (x in d), variables)) if len(remaining_variables) > 0: @@ -170,7 +186,6 @@ def get_env_and_template(template_path, template_name): variable_end_string=r'>>' ) - # print(env.list_templates()) template = env.get_template(template_name) return env, template @@ -220,6 +235,9 @@ def validate_group_list(group_list): # Catch CTRL+C, exit gracefully signal.signal(signal.SIGINT, signal_handler) +if not os.path.isdir(templates_base_path): + raise Exception("Invalid template path given. Change templates_base_path.") + while True: if not ask_confirmation(f"Create default host types ({', '.join(groups[1:])})?"): groups = pyip.inputCustom( @@ -236,9 +254,8 @@ def validate_group_list(group_list): configured_groups = groups[:] for group in groups: - if group == "all": - template_path = f"{templates_base_path}/all" - else: + template_path = f"{templates_base_path}/{group}" + if not os.path.isdir(template_path): template_path = f"{templates_base_path}/{DEFAULT_HOST_TYPE}" output_path = f"{vars_base_path}/{group}/{template_name.rstrip('.j2')}" @@ -251,7 +268,8 @@ def validate_group_list(group_list): env, template = get_env_and_template(template_path, template_name) # find all variables in the template file - with open(os.path.join(template_path, template_name), "r") as file: + template_full_path = os.path.join(template_path, template_name) + with open(template_full_path, "r") as file: ast = env.parse(file.read()) variables = meta.find_undeclared_variables(ast) @@ -260,7 +278,7 @@ def validate_group_list(group_list): group_defaults = defaults.get(group, defaults[DEFAULT_HOST_TYPE]) for k in group_defaults.keys(): if k not in variables: - raise ValueError(f"Variable {k} with default is missing in {template_name}") + raise ValueError(f"Variable {k} with default is missing in {template_full_path}") # prompt the user to select the configuration d[group] = prompt_user(group_defaults, variables, group) diff --git a/scripts/results.py b/src/scripts/results.py similarity index 100% rename from scripts/results.py rename to src/scripts/results.py