diff --git a/README.md b/README.md index 18fde85..b073197 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,7 @@ # Ansible playbook for Eucalyptus Cloud deployment +## Deploying Eucalyptus Cloud + Create an inventory for your environment: ``` @@ -40,3 +42,61 @@ ansible-playbook --skip-tags image -i inventory.yml playbook.yml which would run the playbook in two parts, the first installing packages and non-deployment specific configuration, and the second completing the deployment. + +## Maintaining Eucalyptus Cloud + +Maintenance playbooks support operations of a previously deployed cloud. + +* Add a new node controller +* Remove a node controller +* Passivate a node controller +* Activate a node controller +* Evict node controller instances + +### Adding or removing nodes + +Adding a node will install all the required software, and register the +node, while removing a node will deregister the node, and remove the +installed software. + +Maintenance playbooks use the same inventory as the initial deployment. +For the add/remove node playbooks some inventory changes should be made +as follows. + +To add a node controller you would update the inventory with information +for the new host and then run the corresponding playbook: + +``` +ansible-playbook -i inventory.yml node_add.yml +``` + +To remove a node controller you would update the inventory so the host +is not present in the `children`/`node` section but **is not** removed +from the main `hosts` section. Once the node removal is completed the +`hosts` section should then be updated. + +### Node activation and eviction + +A passive a node controller will not be available for launching new +instances. Activate a passive node controller to make it available for +new instances. + +Evicting instances from a node controller will migrate running instances +to other node controllers in the same availability zone. + +When using maintenance playbooks to activate, passivate or evict a node +an additional `pattern` is used: + +``` +ansible-playbook -i inventory.yml node_active.yml -e pattern='*.70' +``` + +the pattern controls the nodes that are acted on, and can be a host +group, such as the default `all` group: + +``` +ansible-playbook -i inventory.yml node_active.yml -e pattern=all +``` + +you can define host groups in the inventory if you need to act on +certain groups of nodes. diff --git a/host_groups.yml b/host_groups.yml index 9288bdd..c8a5f25 100644 --- a/host_groups.yml +++ b/host_groups.yml @@ -7,6 +7,7 @@ groups: node hostname: "{{ item }}" inventory_dir: "{{ hostvars[item].inventory_dir }}" + changed_when: False when: "'node' not in groups" with_items: "{{ groups.cloud }}" - name: zone group default @@ -14,14 +15,17 @@ groups: zone hostname: "{{ item }}" inventory_dir: "{{ hostvars[item].inventory_dir }}" + changed_when: False when: "'zone' not in groups" with_items: "{{ groups.cloud }}" - name: zone assignments default set_fact: "host_zone_key={{ item.0 + 1 }}" + changed_when: False when: "'host_zone_key' not in hostvars[inventory_hostname] and inventory_hostname == item.1" with_indexed_items: "{{ groups.zone|sort }}" - name: node zone assignments default set_fact: "host_zone_key={{ (item.0 % (groups.zone|length)) + 1 }}" + changed_when: False when: "'host_zone_key' not in hostvars[inventory_hostname] and inventory_hostname == item.1" with_indexed_items: "{{ groups.node|sort }}" @@ -31,6 +35,7 @@ groups: console hostname: "{{ item }}" inventory_dir: "{{ hostvars[item].inventory_dir }}" + changed_when: False when: "'console' not in groups and not eucalyptus_console_cloud_deploy|default(False)" with_items: "{{ groups.cloud }}" @@ -40,6 +45,7 @@ groups: ceph hostname: "{{ item }}" inventory_dir: "{{ hostvars[item].inventory_dir }}" + changed_when: False when: ceph_converged|default(False) with_items: "{{ groups.node }}" - name: cloud to ceph_object_gateway group for converged deployments @@ -47,6 +53,7 @@ groups: ceph_object_gateway hostname: "{{ item }}" inventory_dir: "{{ hostvars[item].inventory_dir }}" + changed_when: False when: "ceph_converged|default(False) and ('ceph_object_gateway' not in groups)" with_items: "{{ groups.cloud }}" - name: ceph_deploy group default @@ -54,6 +61,7 @@ groups: ceph_deploy hostname: "{{ item }}" inventory_dir: "{{ hostvars[item].inventory_dir }}" + changed_when: False when: "'ceph' in groups and groups.ceph and 'ceph_deploy' not in groups" with_items: "{{ groups.ceph[0] }}" - name: ceph_object_gateway group default @@ -61,6 +69,7 @@ groups: ceph_object_gateway hostname: "{{ item }}" inventory_dir: "{{ hostvars[item].inventory_dir }}" + changed_when: False when: "'ceph' in groups and groups.ceph and 'ceph_object_gateway' not in groups" with_items: "{{ groups.ceph[0] }}" @@ -70,6 +79,7 @@ groups: midonet_nsdb hostname: "{{ item }}" inventory_dir: "{{ hostvars[item].inventory_dir }}" + changed_when: False when: ceph_converged|default(False) and net_mode|default('EDGE') == 'VPCMIDO' with_items: "{{ groups.node }}" - name: cloud to midonet_nsdb group for default vpcmido deployments @@ -77,5 +87,6 @@ groups: midonet_nsdb hostname: "{{ item }}" inventory_dir: "{{ hostvars[item].inventory_dir }}" + changed_when: False when: "'midonet_nsdb' not in groups and net_mode|default('EDGE') == 'VPCMIDO'" with_items: "{{ groups.cloud }}" diff --git a/node_active.yml b/node_active.yml new file mode 100644 index 0000000..26bb9d4 --- /dev/null +++ b/node_active.yml @@ -0,0 +1,32 @@ +# Eucalyptus maintenance playbook to allow a node to run new instances +# +# Use this playbook to start some or all nodes of a previously deployed +# eucalyptus cloud. +--- + +# Start node services +- hosts: "node:&{{ pattern | default('none') }}" + name: allow new instance launch + tasks: + - name: set eucalyptus node facts + include_role: + name: common + vars: + ceph_facts: no + facts_only: yes + net_mode: "{{ eucalyptus_net_mode | default('EDGE') }}" + - name: start node + shell: | + set -eu + eval $(clcadmin-assume-system-credentials) + euserv-modify-service -s ENABLED "{{ eucalyptus_host_cluster_ipv4 | quote }}" + sleep 11 + euserv-describe-services -a "{{ eucalyptus_host_cluster_ipv4 | quote }}" + register: shell_result + failed_when: + - shell_result.rc != 0 + - '"enabled" not in shell_result.stdout' + until: shell_result is succeeded + retries: 5 + delegate_to: "{{ groups.cloud[0] }}" + diff --git a/node_add.yml b/node_add.yml new file mode 100644 index 0000000..073336b --- /dev/null +++ b/node_add.yml @@ -0,0 +1,119 @@ +# Eucalyptus maintenance playbook for adding nodes to a deployment +# +# Use this playbook to add nodes (not a cluster/zone) to a previously +# deployed eucalyptus cloud. +--- + +# Initialize hosts and facts +- hosts: all + name: initialize host groups + gather_facts: no + tasks: + - name: read eucalyptus configuration + slurp: + path: /etc/eucalyptus/eucalyptus.conf + register: slurp_result + delegate_to: "{{ groups.cloud[0] }}" + - name: extract nodes from eucalyptus configuration + set_fact: + eucalyptus_net_mode: "{{ slurp_result.content | b64decode | regex_findall('^\\s*VNET_MODE\\s*=\\s*\"(.*)\"\\s*$', multiline=True) | join('') }}" + - import_tasks: host_groups.yml + vars: + net_mode: "{{ eucalyptus_net_mode | default('EDGE') }}" + +- hosts: ceph_object_gateway:midonet_nsdb + name: initialize ceph / midonet facts + roles: + - common + vars: + facts_only: yes + +# Detect new nodes +- hosts: zone + name: collect list of currently configured nodes + tasks: + - name: set eucalyptus zone facts + include_role: + name: common + vars: + facts_only: yes + net_mode: "{{ eucalyptus_net_mode | default('EDGE') }}" + - name: read eucalyptus configuration + slurp: + path: /etc/eucalyptus/eucalyptus.conf + register: slurp_result + - name: extract nodes from eucalyptus configuration + set_fact: + eucalyptus_zone_node_ips: "{{ slurp_result.content | b64decode | regex_findall('^\\s*NODES\\s*=.*$', multiline=True) | join(' ') | regex_findall('\\b(?:[0-9]{1,3}\\.){3}[0-9]{1,3}\\b') }}" + +- hosts: node + name: collect list of new node hosts + tasks: + - name: set eucalyptus node facts + include_role: + name: common + vars: + facts_only: yes + key_facts: yes + net_mode: "{{ eucalyptus_net_mode | default('EDGE') }}" + - name: currently configured node list + set_fact: + eucalyptus_node_ips: "{{ (eucalyptus_node_ips | default([])) + hostvars[item]['eucalyptus_zone_node_ips'] }}" + with_items: "{{ groups.zone }}" + - name: add new node to host group + add_host: + groups: new_node + hostname: "{{ item }}" + inventory_dir: "{{ hostvars[item].inventory_dir }}" + changed_when: False + when: "hostvars[item].eucalyptus_host_cluster_ipv4 not in eucalyptus_node_ips" + with_items: "{{ groups.node }}" + +# Verify new nodes are clean +- hosts: new_node + name: verify new node preconditions + tasks: + - name: check for existing eucalyptus configuration + stat: + path: /etc/eucalyptus/eucalyptus.conf + register: stat_result + - name: fail on new node configured + fail: + msg: "ERROR: eucalyptus configuration already present on node {{ inventory_hostname }}, cannot add" + when: stat_result.stat.exists + +# Install / configure node +- hosts: new_node + module_defaults: + yum: + disablerepo: "{{ eucalyptus_yum_disablerepo | default(omit) }}" + roles: + - node + vars: + net_mode: "{{ eucalyptus_net_mode | default('EDGE') }}" + +# Add new nodes to eucalyptus configuration on clusters +- hosts: zone + name: register new nodes for zone + tasks: + - name: eucalyptus configuration + include_role: + name: common + tasks_from: base_config + vars: + facts_only: yes + net_mode: "{{ eucalyptus_net_mode | default('EDGE') }}" + when: "eucalyptus_zone_name in (groups.new_node | default([]) | map('extract', hostvars, ['eucalyptus_zone_name']) | list)" + +# Add new nodes to vpc tunnel zone +- hosts: cloud + name: vpc tunnel zone update + tasks: + - name: eucalyptus tunnel zone update + command: + cmd: /usr/local/bin/eucalyptus-vpcmidotz-up.sh + environment: + PYTHONPATH: /usr/lib/python2.7/site-packages/WebOb-1.4.1-py2.7.egg/ + EUCA_TZ_HOST_EXTRAS: "{{ groups.new_node | default([]) | map('extract', hostvars, ['eucalyptus_host_cluster_ipv4']) | list | sort | join(' ') }}" + when: (eucalyptus_net_mode | default('EDGE')) == "VPCMIDO" + diff --git a/node_evict.yml b/node_evict.yml new file mode 100644 index 0000000..5a81f21 --- /dev/null +++ b/node_evict.yml @@ -0,0 +1,24 @@ +# Eucalyptus maintenance playbook to move instances from a node +# +# Use this playbook to move running instances in a previously deployed +# eucalyptus cloud. +--- + +# Evict nodes +- hosts: node:&{{ pattern | default('none') }} + name: evict instances + tasks: + - name: set eucalyptus node facts + include_role: + name: common + vars: + ceph_facts: no + facts_only: yes + net_mode: "{{ eucalyptus_net_mode | default('EDGE') }}" + - name: evict node instances + shell: | + set -eu + eval $(clcadmin-assume-system-credentials) + euserv-migrate-instances -s "{{ eucalyptus_host_cluster_ipv4 | quote }}" + delegate_to: "{{ groups.cloud[0] }}" + diff --git a/node_passive.yml b/node_passive.yml new file mode 100644 index 0000000..d009c28 --- /dev/null +++ b/node_passive.yml @@ -0,0 +1,32 @@ +# Eucalyptus maintenance playbook to stop a node running new instances +# +# Use this playbook to stop some or all nodes of a previously deployed +# eucalyptus cloud. +--- + +# Stop node services +- hosts: node:&{{ pattern | default('none') }} + name: prevent new instance launch + tasks: + - name: set eucalyptus node facts + include_role: + name: common + vars: + ceph_facts: no + facts_only: yes + net_mode: "{{ eucalyptus_net_mode | default('EDGE') }}" + - name: stop node + shell: | + set -eu + eval $(clcadmin-assume-system-credentials) + euserv-modify-service -s STOPPED "{{ eucalyptus_host_cluster_ipv4 | quote }}" + sleep 11 + euserv-describe-services -a "{{ eucalyptus_host_cluster_ipv4 | quote }}" + register: shell_result + failed_when: + - shell_result.rc != 0 + - '"stopped" not in shell_result.stdout' + until: shell_result is succeeded + retries: 5 + delegate_to: "{{ groups.cloud[0] }}" + diff --git a/node_remove.yml b/node_remove.yml new file mode 100644 index 0000000..b82292b --- /dev/null +++ b/node_remove.yml @@ -0,0 +1,111 @@ +# Eucalyptus maintenance playbook for removing nodes from a deployment +# +# Use this playbook to remove nodes (not a cluster/zone) from a +# previously deployed eucalyptus cloud. +--- + +# Initialize hosts and facts +- hosts: all + name: initialize host groups + gather_facts: no + tasks: + - name: read eucalyptus configuration + slurp: + path: /etc/eucalyptus/eucalyptus.conf + register: slurp_result + delegate_to: "{{ groups.cloud[0] }}" + - name: extract nodes from eucalyptus configuration + set_fact: + eucalyptus_net_mode: "{{ slurp_result.content | b64decode | regex_findall('^\\s*VNET_MODE\\s*=\\s*\"(.*)\"\\s*$', multiline=True) | join('') }}" + - import_tasks: host_groups.yml + vars: + net_mode: "{{ eucalyptus_net_mode | default('EDGE') }}" + +- hosts: ceph_object_gateway + name: initialize ceph facts + roles: + - common + vars: + facts_only: yes + +# Detect removed nodes +- hosts: zone + name: collect list of currently configured nodes + tasks: + - name: set eucalyptus zone facts + include_role: + name: common + vars: + facts_only: yes + net_mode: "{{ eucalyptus_net_mode | default('EDGE') }}" + - name: read eucalyptus configuration + slurp: + path: /etc/eucalyptus/eucalyptus.conf + register: slurp_result + - name: extract nodes from eucalyptus configuration + set_fact: + eucalyptus_zone_node_ips: "{{ slurp_result.content | b64decode | regex_findall('^\\s*NODES\\s*=.*$', multiline=True) | join(' ') | regex_findall('\\b(?:[0-9]{1,3}\\.){3}[0-9]{1,3}\\b') }}" + +- hosts: all + name: collect list of removed node hosts + tasks: + - name: check for eucalyptus configuration + stat: + path: /etc/eucalyptus/eucalyptus.conf + register: stat_result + - name: eucalyptus configured fact + set_fact: + eucalyptus_configuration: yes + when: stat_result.stat.exists + - name: set eucalyptus node facts + include_role: + name: common + vars: + facts_only: yes + net_mode: "{{ eucalyptus_net_mode | default('EDGE') }}" + when: stat_result.stat.exists + - name: currently configured node list + set_fact: + eucalyptus_node_ips: "{{ (eucalyptus_node_ips | default([])) + hostvars[item]['eucalyptus_zone_node_ips'] }}" + with_items: "{{ groups.zone }}" + - name: add removed node to host group + add_host: + groups: old_node + hostname: "{{ item }}" + inventory_dir: "{{ hostvars[item].inventory_dir }}" + changed_when: False + when: "hostvars[item].inventory_hostname not in groups.node and hostvars[item].eucalyptus_host_cluster_ipv4 in eucalyptus_node_ips" + with_items: "{{ groups.all }}" + +# Verify nodes being removed are not in use +- hosts: old_node + name: verify remove node preconditions + tasks: + - name: check for running instances + find: + path: /var/run/libvirt/qemu/ + register: find_result + - name: fail on node in use + fail: + msg: "ERROR: node is in use {{ inventory_hostname }}, cannot remove" + when: "find_result.matched > 0" + +# Remove old nodes from eucalyptus configuration on clusters +- hosts: zone + name: deregister removed nodes for zone + tasks: + - name: eucalyptus configuration + include_role: + name: common + tasks_from: base_config + vars: + facts_only: yes + net_mode: "{{ eucalyptus_net_mode | default('EDGE') }}" + when: "groups.old_node | default([]) | map('extract', hostvars, ['eucalyptus_host_cluster_ipv4']) | list | intersect(eucalyptus_zone_node_ips)" + +# Clean old nodes +- hosts: old_node + name: clean old nodes + roles: + - none + diff --git a/roles/ceph-common/tasks/main.yml b/roles/ceph-common/tasks/main.yml index 38ed287..7da7a0f 100644 --- a/roles/ceph-common/tasks/main.yml +++ b/roles/ceph-common/tasks/main.yml @@ -2,13 +2,16 @@ - import_tasks: host_facts.yml - import_tasks: yum_repos_ceph.yml + when: not (facts_only | default(False)) - import_tasks: base_packages.yml + when: not (facts_only | default(False)) - import_tasks: base_config.yml + when: not (facts_only | default(False)) - import_tasks: base_firewall.yml - when: cloud_firewalld_configure + when: cloud_firewalld_configure and not (facts_only | default(False)) - import_tasks: ceph_facts.yml delegate_to: "{{ ceph_facts_delegate|default(groups['ceph_deploy'][0], true) }}" diff --git a/roles/common/tasks/main.yml b/roles/common/tasks/main.yml index e5f4cf0..6a6f39f 100644 --- a/roles/common/tasks/main.yml +++ b/roles/common/tasks/main.yml @@ -1,5 +1,6 @@ --- - import_tasks: yum_repos.yml + when: not (facts_only | default(False)) tags: - image - packages @@ -8,13 +9,15 @@ tags: - image - packages - when: net_mode == "VPCMIDO" + when: not (facts_only | default(False)) and net_mode == "VPCMIDO" - import_tasks: host_facts.yml - import_tasks: base_packages.yml + when: not (facts_only | default(False)) - import_tasks: base_config.yml + when: not (facts_only | default(False)) - name: zone key facts include_tasks: key_facts.yml diff --git a/roles/common/templates/eucalyptus.conf.j2 b/roles/common/templates/eucalyptus.conf.j2 index 067d5e0..cd013ac 100644 --- a/roles/common/templates/eucalyptus.conf.j2 +++ b/roles/common/templates/eucalyptus.conf.j2 @@ -1,7 +1,7 @@ # Eucalyptus cloud configuration LOGLEVEL="{{ cloud_log_level }}" CLOUD_OPTS="{{ cloud_opts }}" -NODES="{% for node in groups.node if (inventory_hostname in groups.zone and eucalyptus_zone_name == hostvars[node]['eucalyptus_zone_name']) %}{{ hostvars[node].eucalyptus_host_cluster_ipv4 }} {% endfor %}" +NODES="{% for node in groups.node|sort if (inventory_hostname in groups.zone and eucalyptus_zone_name == hostvars[node]['eucalyptus_zone_name']) %}{{ hostvars[node].eucalyptus_host_cluster_ipv4 }} {% endfor %}" HYPERVISOR="kvm" INSTANCE_PATH="{{ cloud_instances_state_dir }}" LIBVIRT_USE_POLICY_KIT="-1"