[Playbook] ansible system check playbook

현재 OpenStack Compute system 상태 체크용으로 사용하는 플레이북입니다.

 

1. openstack-system-check.yml

[root@hk-controller etech]#cat openstack-system-check.yml
---
- name: Main Openstack System Check
  hosts: dell
  become: false

  tasks:
  - name: Add Info
    shell:
      cmd: |
        cat > /home/heat-admin/{{ ansible_hostname }}-compute-system-info-$(date "+%Y%m%d").txt << EOF
         #################################################
         #                                               #
         #    Gathering OpenStack Host System info       #
         #    Generated by HK Jeon (E-tech)              #
         #    Last Update                                #
         #                                               #
         #################################################


         ################ System Uptime Info ######################

  - name: Check System Uptime
    shell: uptime | awk '{print $1,$2,$3,$4,$5, "\n "}'
    register: uptime
  - debug:
      var: uptime.stdout_lines

  - name: save System Uptime Info in txt file
    shell:
      cmd: |
        echo -e "\n" >> /home/heat-admin/{{ ansible_hostname}}-compute-system-info-$(date "+%Y%m%d").txt
        uptime | awk '{print $1,$2,$3,$4,$5, "\n "}' >> /home/heat-admin/{{ ansible_hostname}}-compute-system-info-$(date "+%Y%m%d").txt
        echo "############   System Top info   ################" >> /home/heat-admin/{{ ansible_hostname}}-compute-system-info-$(date "+%Y%m%d").txt

  - name: Check System top
    shell: top -b -n1 | grep -Po '[0-9.]+ id' | awk '{print 100-$1, "\n "}'
    register: top
  - debug:
      var: top.stdout_lines

  - name: save System Top info in txt file
    shell:
      cmd: |
        top -b -n1 | grep -Po '[0-9.]+ id' | awk '{print 100-$1, "\n "}'  >> /home/heat-admin/{{ ansible_hostname}}-compute-system-info-$(date "+%Y%m%d").txt
        echo "############   System Mem Usage Info   ################" >> /home/heat-admin/{{ ansible_hostname}}-compute-system-info-$(date "+%Y%m%d").txt

  - name: Check System memory usage Info
    shell: free -h | grep ^Mem | awk '{print 100*($2-$7)/$2, "\n "}'
    register: free
  - debug:
      var: free.stdout_lines

  - name: save System memory info in txt file
    shell:
      cmd: |
        free -h | grep ^Mem | awk '{print 100*($2-$7)/$2, "\n "}'  >> /home/heat-admin/{{ ansible_hostname}}-compute-system-info-$(date "+%Y%m%d").txt
        echo "############   System Disk Usage Info   ################" >> /home/heat-admin/{{ ansible_hostname}}-compute-system-info-$(date "+%Y%m%d").txt

  - name: Check System Disk usage Info
    shell: df -h | grep sda; df -h | grep sdb
    register: disk
  - debug:
      var: disk.stdout_lines

  - name: save System Disk info in txt file
    shell:
      cmd: |
        df -h | grep sda  >> /home/heat-admin/{{ ansible_hostname}}-compute-system-info-$(date "+%Y%m%d").txt
        df -h | grep sdb  >> /home/heat-admin/{{ ansible_hostname}}-compute-system-info-$(date "+%Y%m%d").txt
        echo -e "\n " >> /home/heat-admin/{{ ansible_hostname}}-compute-system-info-$(date "+%Y%m%d").txt
        echo "############   System fail msg count in the messages Info   ################" >> /home/heat-admin/{{ ansible_hostname}}-compute-system-info-$(date "+%Y%m%d").txt

  - name: Check System fail msg count in the messages Info
    shell: sudo cat /var/log/messages | grep -i fail | wc -l
    register: fail
  - debug:
      var: fail.stdout_lines

  - name: save System fail msg count in the messages info in txt file
    shell:
      cmd: |
        sudo cat /var/log/messages | grep -i fail | wc -l  >> /home/heat-admin/{{ ansible_hostname}}-compute-system-info-$(date "+%Y%m%d").txt
        echo -e "\n " >> /home/heat-admin/{{ ansible_hostname}}-compute-system-info-$(date "+%Y%m%d").txt
        echo "############   System Bonding Info   ################" >> /home/heat-admin/{{ ansible_hostname}}-compute-system-info-$(date "+%Y%m%d").txt

  - name: Check System Bonding Info
    shell: cat /proc/net/bonding/bond{0,1,2,3} | grep -B 1 Status
    register: bond
  - debug:
      var: bond.stdout_lines

  - name: save System Bonding info in txt file
    shell:
      cmd: |
        echo "############   System Bond0 Info   ################" >> /home/heat-admin/{{ ansible_hostname}}-compute-system-info-$(date "+%Y%m%d").txt
        cat /proc/net/bonding/bond0 | grep -B 1 Status  >> /home/heat-admin/{{ ansible_hostname}}-compute-system-info-$(date "+%Y%m%d").txt
        echo "############   System Bond1 Info   ################" >> /home/heat-admin/{{ ansible_hostname}}-compute-system-info-$(date "+%Y%m%d").txt
        cat /proc/net/bonding/bond1 | grep -B 1 Status  >> /home/heat-admin/{{ ansible_hostname}}-compute-system-info-$(date "+%Y%m%d").txt
        echo "############   System Bond2 Info   ################" >> /home/heat-admin/{{ ansible_hostname}}-compute-system-info-$(date "+%Y%m%d").txt
        cat /proc/net/bonding/bond2 | grep -B 1 Status  >> /home/heat-admin/{{ ansible_hostname}}-compute-system-info-$(date "+%Y%m%d").txt
        echo "############   System Bond3 Info   ################" >> /home/heat-admin/{{ ansible_hostname}}-compute-system-info-$(date "+%Y%m%d").txt
        cat /proc/net/bonding/bond3 | grep -B 1 Status  >> /home/heat-admin/{{ ansible_hostname}}-compute-system-info-$(date "+%Y%m%d").txt
        echo -e "\n " >> /home/heat-admin/{{ ansible_hostname}}-compute-system-info-$(date "+%Y%m%d").txt

  - name: Check System IP Info
    shell: sudo ip a | grep -w inet
    register: ip
  - debug:
      var: ip.stdout_lines

  - name: save System IP info in txt file
    shell:
      cmd: |
        echo "############   System IP Info   ################" >> /home/heat-admin/{{ ansible_hostname}}-compute-system-info-$(date "+%Y%m%d").txt
        sudo ip a | grep -w inet >> /home/heat-admin/{{ ansible_hostname}}-compute-system-info-$(date "+%Y%m%d").txt
        echo -e "\n " >> /home/heat-admin/{{ ansible_hostname}}-compute-system-info-$(date "+%Y%m%d").txt

  - name: Check System IP Link Info
    shell: sudo ip link show | grep -i state
    register: link
  - debug:
      var: link.stdout_lines

  - name: save System IP Link info in txt file
    shell:
      cmd: |
        echo "############   System IP Link Info   ################" >> /home/heat-admin/{{ ansible_hostname}}-compute-system-info-$(date "+%Y%m%d").txt
        sudo ip link show | grep -i state >> /home/heat-admin/{{ ansible_hostname}}-compute-system-info-$(date "+%Y%m%d").txt
        echo -e "\n " >> /home/heat-admin/{{ ansible_hostname}}-compute-system-info-$(date "+%Y%m%d").txt

  - name: Check System Routing Info
    shell: sudo route -n
    register: route
  - debug:
      var: route.stdout_lines

  - name: save System Routing info in txt file
    shell:
      cmd: |
        echo "############   System Routing Info   ################" >> /home/heat-admin/{{ ansible_hostname}}-compute-system-info-$(date "+%Y%m%d").txt
        sudo route -n >> /home/heat-admin/{{ ansible_hostname}}-compute-system-info-$(date "+%Y%m%d").txt
        echo -e "\n " >> /home/heat-admin/{{ ansible_hostname}}-compute-system-info-$(date "+%Y%m%d").txt

  - name: Check System NTP Info
    shell: sudo ntpq -p
    register: ntpq
  - debug:
      var: ntpq.stdout_lines

  - name: save System NTP info in txt file
    shell:
      cmd: |
        echo "############   System NTP Info   ################" >> /home/heat-admin/{{ ansible_hostname}}-compute-system-info-$(date "+%Y%m%d").txt
        sudo ntpq -p >> /home/heat-admin/{{ ansible_hostname}}-compute-system-info-$(date "+%Y%m%d").txt
        echo -e "\n " >> /home/heat-admin/{{ ansible_hostname}}-compute-system-info-$(date "+%Y%m%d").txt

  - name: Check System Container Info
    shell: sudo docker ps
    register: docker
  - debug:
      var: docker.stdout_lines

  - name: save System Container info in txt file
    shell:
      cmd: |
        echo "############   System Container Info   ################" >> /home/heat-admin/{{ ansible_hostname}}-compute-system-info-$(date "+%Y%m%d").txt
        sudo docker ps >> /home/heat-admin/{{ ansible_hostname}}-compute-system-info-$(date "+%Y%m%d").txt
        echo -e "\n " >> /home/heat-admin/{{ ansible_hostname}}-compute-system-info-$(date "+%Y%m%d").txt

  - name: Check Virtual Machine Info
    shell: sudo virsh list --all
    register: virsh
  - debug:
      var: virsh.stdout_lines

  - name: save Virtual Machine info in txt file
    shell:
      cmd: |
        echo "############   Virtual Machine Info   ################" >> /home/heat-admin/{{ ansible_hostname}}-compute-system-info-$(date "+%Y%m%d").txt
        sudo virsh list --all >> /home/heat-admin/{{ ansible_hostname}}-compute-system-info-$(date "+%Y%m%d").txt
        echo -e "\n " >> /home/heat-admin/{{ ansible_hostname}}-compute-system-info-$(date "+%Y%m%d").txt

  - name: Check openvswitch Service Info
    shell: sudo systemctl status openvswitch | grep -E "openvswitch.service|Loaded:|Active:"
    register: openvswitch
  - debug:
      var: openvswitch.stdout_lines

  - name: save openvswitch Service info in txt file
    shell:
      cmd: |
        echo "############   openvswitch Service Info   ################" >> /home/heat-admin/{{ ansible_hostname}}-compute-system-info-$(date "+%Y%m%d").txt
        sudo systemctl status openvswitch | grep -E "openvswitch.service | Loaded: | Active:" >> /home/heat-admin/{{ ansible_hostname}}-compute-system-info-$(date "+%Y%m%d").txt
        echo -e "\n " >> /home/heat-admin/{{ ansible_hostname}}-compute-system-info-$(date "+%Y%m%d").txt

  - name: Check DPDK bond Status
    shell: sudo ovs-appctl bond/show
    register: dpdk
  - debug:
      var: dpdk.stdout_lines

  - name: save DPDK Bond Status info in txt file
    shell:
      cmd: |
        echo "############   DPDK Bond Status Info   ################" >> /home/heat-admin/{{ ansible_hostname}}-compute-system-info-$(date "+%Y%m%d").txt
        sudo ovs-appctl bond/show >> /home/heat-admin/{{ ansible_hostname}}-compute-system-info-$(date "+%Y%m%d").txt
        echo -e "\n " >> /home/heat-admin/{{ ansible_hostname}}-compute-system-info-$(date "+%Y%m%d").txt

  - name: Check fail and error msg count in the nova-compute.log file
    shell: sudo cat /var/log/containers/nova/nova-compute.log | grep -iE "fail|error" | wc -l
    register: nova
  - debug:
      var: nova.stdout_lines

  - name: save fail and error msg count in the nova-compute.log in txt file
    shell:
      cmd: |
        echo "############   nova-compute.log fail and error Info   ################" >> /home/heat-admin/{{ ansible_hostname}}-compute-system-info-$(date "+%Y%m%d").txt
        sudo cat /var/log/containers/nova/nova-compute.log | grep -iE "fail|error" | wc -l >> /home/heat-admin/{{ ansible_hostname}}-compute-system-info-$(date "+%Y%m%d").txt
        echo -e "\n " >> /home/heat-admin/{{ ansible_hostname}}-compute-system-info-$(date "+%Y%m%d").txt

  - name: Check fail and error msg count in the dhcp-agent.log file
    shell: sudo cat /var/log/containers/neutron/dhcp-agent.log | grep -iE "fail|error" | wc -l
    register: dhcp
  - debug:
      var: dhcp.stdout_lines

  - name: save fail and error msg count in the dhcp-agent.log in txt file
    shell:
      cmd: |
        echo "############   dhcp-agent.log fail and error Info   ################" >> /home/heat-admin/{{ ansible_hostname}}-compute-system-info-$(date "+%Y%m%d").txt
        sudo cat /var/log/containers/neutron/dhcp-agent.log | grep -iE "fail|error" | wc -l >> /home/heat-admin/{{ ansible_hostname}}-compute-system-info-$(date "+%Y%m%d").txt
        echo -e "\n " >> /home/heat-admin/{{ ansible_hostname}}-compute-system-info-$(date "+%Y%m%d").txt

  - name: Check fail and error msg count in the metadata-agent.log file
    shell: sudo cat /var/log/containers/neutron/metadata-agent.log | grep -iE "fail|error" | wc -l
    register: meta
  - debug:
      var: meta.stdout_lines

  - name: save fail and error msg count in the metadata-agent.log in txt file
    shell:
      cmd: |
        echo "############   metadata-agent.log fail and error Info   ################" >> /home/heat-admin/{{ ansible_hostname}}-compute-system-info-$(date "+%Y%m%d").txt
        sudo cat /var/log/containers/neutron/metadata-agent.log | grep -iE "fail|error" | wc -l >> /home/heat-admin/{{ ansible_hostname}}-compute-system-info-$(date "+%Y%m%d").txt
        echo -e "\n " >> /home/heat-admin/{{ ansible_hostname}}-compute-system-info-$(date "+%Y%m%d").txt

  - name: Check fail and error msg count in the openvswitch-agent.log file
    shell: sudo cat /var/log/containers/neutron/openvswitch-agent.log | grep -iE "fail|error" | wc -l
    register: ovs
  - debug:
      var: ovs.stdout_lines

  - name: save fail and error msg count in the openvswitch-agent.log in txt file
    shell:
      cmd: |
        echo "############   openvswitch-agent.log fail and error Info   ################" >> /home/heat-admin/{{ ansible_hostname}}-compute-system-info-$(date "+%Y%m%d").txt
        sudo cat /var/log/containers/neutron/openvswitch-agent.log | grep -iE "fail|error" | wc -l >> /home/heat-admin/{{ ansible_hostname}}-compute-system-info-$(date "+%Y%m%d").txt
        echo -e "\n " >> /home/heat-admin/{{ ansible_hostname}}-compute-system-info-$(date "+%Y%m%d").txt


  - name: Create Directory for OpenStack Compute Host System Check result file.
    become: true
    file:
      path: /home/stack/os-compute-check-result/{{ ansible_hostname }}/
      state: directory
      owner: stack
      group: stack

  - name: find Result File
    find:
      paths: "/home/heat-admin"
      recurse: yes
      patterns: "*.txt"
    register: files_to_copy

  - name: Copy Result files to director server.
    fetch:
      src: "{{ item.path }}"
      dest: /home/stack/os-compute-check-result/{{ ansible_hostname }}/
      flat: yes
    with_items: "{{ files_to_copy.files }}"

  - name: Note Info
    shell: echo "OpenStack Compute Host System Info Gathering is done. But, If ping test between Compute host and gateway, you have to run on a remote server.!!!!"
    register: note
  - debug:
      var: note.stdout_lines