diff --git a/ansible.cfg b/ansible.cfg index d91952a..43a3803 100644 --- a/ansible.cfg +++ b/ansible.cfg @@ -1,4 +1,4 @@ [defaults] inventory = ./inventory.yml -host_key_checking = False +host_key_checking = True #vault_password_file = ./.password_file \ No newline at end of file diff --git a/playbooks/deploy.yml b/playbooks/deploy.yml index 2a88057..1f10020 100644 --- a/playbooks/deploy.yml +++ b/playbooks/deploy.yml @@ -12,13 +12,13 @@ path: ~/docker/caddy state: directory mode: '0755' - + - name: Create ddns folder ansible.builtin.file: path: ~/docker/ddns state: directory mode: '0755' - + - name: Copy encrypted docker-compose ansible.builtin.copy: src: ./vault/compose/docker-compose.yml @@ -38,15 +38,21 @@ ansible.builtin.command: cmd: docker compose up -d chdir: ~/docker - ignore_errors: yes - name: Pause for 30 seconds to allow containers to stabilize ansible.builtin.pause: seconds: 30 - - name: Check container status - ansible.builtin.shell: docker compose ps -q | xargs -n1 docker container inspect --format '{{ "{{" }} .State.Running {{ "}}" }}' + - name: Verify all containers are running + ansible.builtin.shell: | + expected=$(docker compose config --services | wc -l | tr -d ' ') + running=$(docker compose ps --status running -q | wc -l | tr -d ' ') + if [ "$expected" != "$running" ]; then + echo "FAIL: $running/$expected containers running" + docker compose ps + exit 1 + fi + echo "OK: all $running containers running" args: chdir: ~/docker - register: container_status - ignore_errors: yes \ No newline at end of file + changed_when: false diff --git a/playbooks/disaster.yml b/playbooks/disaster.yml index 768a90f..2783d48 100644 --- a/playbooks/disaster.yml +++ b/playbooks/disaster.yml @@ -2,35 +2,44 @@ hosts: DR tasks: - include_vars: ./vault/restic/restic.yml + - name: Create folder ansible.builtin.file: path: ~/docker state: directory mode: '0755' - + - name: Pull backups - shell: | + ansible.builtin.shell: | unset HISTFILE export RESTIC_REPOSITORY={{ RESTIC_REPOSITORY }} export AWS_ACCESS_KEY_ID={{ AWS_ACCESS_KEY_ID }} export AWS_SECRET_ACCESS_KEY={{ AWS_SECRET_ACCESS_KEY }} export RESTIC_PASSWORD={{ RESTIC_PASSWORD }} cd ~/docker - restic restore latest:/source/gcloud --target ./ + # RESTIC_RESTORE_PATH: verify with 'restic snapshots' before running + # Updated from /source/gcloud — set the correct snapshot path in vault/restic/restic.yml + restic restore latest:{{ RESTIC_RESTORE_PATH }} --target ./ --overwrite always - name: Start container using Docker Compose ansible.builtin.command: cmd: docker compose up -d chdir: ~/docker - ignore_errors: true - name: Pause for 30 seconds to allow containers to stabilize ansible.builtin.pause: seconds: 30 - - name: Check container status - ansible.builtin.shell: docker compose ps -q | xargs -n1 docker container inspect --format '{{ "{{" }} .State.Running {{ "}}" }}' + - name: Verify all containers are running + ansible.builtin.shell: | + expected=$(docker compose config --services | wc -l | tr -d ' ') + running=$(docker compose ps --status running -q | wc -l | tr -d ' ') + if [ "$expected" != "$running" ]; then + echo "FAIL: $running/$expected containers running" + docker compose ps + exit 1 + fi + echo "OK: all $running containers running" args: chdir: ~/docker - register: container_status - ignore_errors: true \ No newline at end of file + changed_when: false diff --git a/playbooks/docker_status.yml b/playbooks/docker_status.yml index d99697a..806eee8 100644 --- a/playbooks/docker_status.yml +++ b/playbooks/docker_status.yml @@ -2,6 +2,7 @@ - name: Check and Report Status of Docker Containers hosts: Docker gather_facts: true + become: true vars: exited_containers: [] diff --git a/playbooks/docker_update_containers.yml b/playbooks/docker_update_containers.yml new file mode 100644 index 0000000..702525e --- /dev/null +++ b/playbooks/docker_update_containers.yml @@ -0,0 +1,37 @@ +--- +- name: Pull latest images and recreate updated containers + hosts: Docker + tasks: + - name: Pull latest images + ansible.builtin.command: + cmd: docker compose pull + chdir: ~/docker + register: pull_result + changed_when: "'Downloaded newer image' in pull_result.stdout or 'Pulled' in pull_result.stdout" + + - name: Recreate containers with updated images + ansible.builtin.command: + cmd: docker compose up -d --remove-orphans + chdir: ~/docker + + - name: Pause for 30 seconds to allow containers to stabilize + ansible.builtin.pause: + seconds: 30 + + - name: Verify all containers are running + ansible.builtin.shell: | + expected=$(docker compose config --services | wc -l | tr -d ' ') + running=$(docker compose ps --status running -q | wc -l | tr -d ' ') + if [ "$expected" != "$running" ]; then + echo "FAIL: $running/$expected containers running" + docker compose ps + exit 1 + fi + echo "OK: all $running containers running" + args: + chdir: ~/docker + changed_when: false + + - name: Remove dangling images + ansible.builtin.command: + cmd: docker image prune -f diff --git a/playbooks/fresh_install.yml b/playbooks/fresh_install.yml index 07a8c18..613b4a1 100644 --- a/playbooks/fresh_install.yml +++ b/playbooks/fresh_install.yml @@ -5,6 +5,22 @@ gather_facts: true tasks: + - name: Check if Docker containers are already running (safety guard) + ansible.builtin.shell: docker compose ps -q 2>/dev/null || true + args: + chdir: ~/docker + register: running_containers + changed_when: false + failed_when: false + + - name: Abort if containers are already running on this host + ansible.builtin.fail: + msg: > + Docker containers are already running on {{ inventory_hostname }}. + Use redeploy.yml to restore from backup or deploy.yml to redeploy config. + Only run fresh_install.yml on hosts with no active containers. + when: running_containers.stdout != "" + - name: Update apt cache (Debian/Ubuntu) apt: update_cache: yes @@ -61,7 +77,7 @@ name: '*' state: latest when: ansible_os_family == 'RedHat' - + - name: Update Restic Binaries ansible.builtin.command: cmd: restic self-update diff --git a/playbooks/redeploy.yml b/playbooks/redeploy.yml index 9bee8c1..2f0db78 100644 --- a/playbooks/redeploy.yml +++ b/playbooks/redeploy.yml @@ -2,35 +2,44 @@ hosts: Prod tasks: - include_vars: ./vault/restic/restic.yml + - name: Create folder ansible.builtin.file: path: ~/docker state: directory mode: '0755' - + - name: Pull backups - shell: | + ansible.builtin.shell: | unset HISTFILE export RESTIC_REPOSITORY={{ RESTIC_REPOSITORY }} export AWS_ACCESS_KEY_ID={{ AWS_ACCESS_KEY_ID }} export AWS_SECRET_ACCESS_KEY={{ AWS_SECRET_ACCESS_KEY }} export RESTIC_PASSWORD={{ RESTIC_PASSWORD }} cd ~/docker - restic restore latest:/source/gcloud --target ./ + # RESTIC_RESTORE_PATH: verify with 'restic snapshots' before running + # Updated from /source/gcloud — set the correct snapshot path in vault/restic/restic.yml + restic restore latest:{{ RESTIC_RESTORE_PATH }} --target ./ --overwrite always - name: Start container using Docker Compose ansible.builtin.command: cmd: docker compose up -d chdir: ~/docker - ignore_errors: true - name: Pause for 30 seconds to allow containers to stabilize ansible.builtin.pause: seconds: 30 - - name: Check container status - ansible.builtin.shell: docker compose ps -q | xargs -n1 docker container inspect --format '{{ "{{" }} .State.Running {{ "}}" }}' + - name: Verify all containers are running + ansible.builtin.shell: | + expected=$(docker compose config --services | wc -l | tr -d ' ') + running=$(docker compose ps --status running -q | wc -l | tr -d ' ') + if [ "$expected" != "$running" ]; then + echo "FAIL: $running/$expected containers running" + docker compose ps + exit 1 + fi + echo "OK: all $running containers running" args: chdir: ~/docker - register: container_status - ignore_errors: true \ No newline at end of file + changed_when: false diff --git a/playbooks/update_upgrade.yml b/playbooks/update_upgrade.yml index 0524f6c..c8aa365 100644 --- a/playbooks/update_upgrade.yml +++ b/playbooks/update_upgrade.yml @@ -1,5 +1,5 @@ --- -- hosts: all +- hosts: "all:!AnsibleHost" gather_facts: true tasks: @@ -19,7 +19,7 @@ - name: Reboot the server (if required). ansible.builtin.reboot: - when: reboot_required_file.stat.exists == true + when: reboot_required_file is defined and reboot_required_file.stat.exists - name: "Updating and Upgrading Yum Packages" yum: @@ -42,7 +42,7 @@ category_names: '*' reboot: true when: ansible_os_family == "Windows" - + - name: Upgrade installed packages win_chocolatey: name: all