Introduce sysupgrade playbook

Playbook sysupgrade.yml is a attempt to do automatic full system
upgrade. Currently logic is completely automated for happy path.

1. Shutdown machine.
2. Take snapshot from the machine.
3. Power on machine back.
4. Update archlinux-keyring.
5. Upgrade everything.
6. Reboot.

If something is working, then we are done. Otherwise, restore from
snapshot manually and try to figure out what went wrong.

10 files changed, 90 insertions, 0 deletions

diff --git a/TODO.txt b/TODO.txt
index 0cde275..953dae9 100644
--- a/TODO.txt
+++ b/TODO.txt
@@ -1,10 +1,14 @@
 TODO
 
 * Automate snapshots and reboots.
+* Rename vaults -> secrets.
+* Fix quotes: use either single quotes or double quotes.
+* Replace true -> yes.
 * Enable IPv6 for wireguard.
 * Setup L2TP/IPsec VPN server.
 * Set correct hostname for each host.
 * Setup irssi client.
+* Migrate from DigitalOcean snapshots to avoid vendor lock.
 
 
 DOTFILES
diff --git a/host_vars/gate.ilvokhin.com/digitalocean.yml b/host_vars/gate.ilvokhin.com/digitalocean.yml
new file mode 100644
index 0000000..3fc5cdf
--- /dev/null
+++ b/host_vars/gate.ilvokhin.com/digitalocean.yml
@@ -0,0 +1 @@
+droplet_id: 2970512
diff --git a/hosts.yml b/hosts.yml
index 92c10c4..32ef319 100644
--- a/hosts.yml
+++ b/hosts.yml
@@ -1,3 +1,7 @@
+digitalocean:
+  hosts:
+    gate.ilvokhin.com:
+
 web:
   hosts:
     sun.ilvokhin.com:
diff --git a/misc/vaults/digitalocean.yml b/misc/vaults/digitalocean.yml
new file mode 100644
index 0000000..1e5b743
--- /dev/null
+++ b/misc/vaults/digitalocean.yml
@@ -0,0 +1,10 @@
+$ANSIBLE_VAULT;1.1;AES256
+61663065346133353434353361363232643234313338633837383664353339336264306666633934
+3165336163373766613534396339616431613565303262660a353164326533383331646430333733
+31346434326261303863616533386234353531396362626432366538613135613361353332653564
+3934336239353166300a643031666435663431383839616661626463626365326332303766313734
+64646634663461636338333733626166373138633638333431346465343263366538356466353436
+66313735613862366334663163646230343165303765353732386135643239653261623231643164
+65383161653638333336613132346264363430313835373537396261383231363032393731323733
+30386638363930653535666531386465656537623937333962333638343736393239373836653233
+65383139646662636135636531646162363939643166303137396336313832646439
diff --git a/roles/poweron/tasks/main.yml b/roles/poweron/tasks/main.yml
new file mode 100644
index 0000000..7aac651
--- /dev/null
+++ b/roles/poweron/tasks/main.yml
@@ -0,0 +1,15 @@
+- name: Power on droplet
+  local_action:
+    module: ansible.builtin.uri
+    url: "{{ digitalocean_api_url }}/droplets/{{ droplet_id }}/actions"
+    method: POST
+    headers:
+      Content-Type: application/json
+      Authorization: Bearer {{ digitalocean_api_token }}
+    body_format: json
+    status_code: 201
+    body:
+      type: power_on
+
+- name: Wait for host to back up
+  ansible.builtin.wait_for_connection:
diff --git a/roles/reboot/tasks/main.yml b/roles/reboot/tasks/main.yml
new file mode 100644
index 0000000..50d20e7
--- /dev/null
+++ b/roles/reboot/tasks/main.yml
@@ -0,0 +1,2 @@
+- name: Reboot host
+  ansible.builtin.reboot:
diff --git a/roles/shutdown/tasks/main.yml b/roles/shutdown/tasks/main.yml
new file mode 100644
index 0000000..39cdda5
--- /dev/null
+++ b/roles/shutdown/tasks/main.yml
@@ -0,0 +1,2 @@
+- name: Shutdown host
+  community.general.shutdown:
diff --git a/roles/snapshot/tasks/main.yml b/roles/snapshot/tasks/main.yml
new file mode 100644
index 0000000..a46e9d6
--- /dev/null
+++ b/roles/snapshot/tasks/main.yml
@@ -0,0 +1,31 @@
+- name: Make droplet snapshot
+  local_action:
+    module: ansible.builtin.uri
+    url: "{{ digitalocean_api_url }}/droplets/{{ droplet_id }}/actions"
+    method: POST
+    headers:
+      Content-Type: application/json
+      Authorization: Bearer {{ digitalocean_api_token }}
+    body_format: json
+    status_code: 201
+    body:
+      type: snapshot
+      name: "{{ inventory_hostname_short }}-\
+        {{ (now(utc=true).timestamp() * 1000) | int }}"
+  register: snapshot
+
+- name: Wait for droplet snapshot to finish
+  local_action:
+    module: ansible.builtin.uri
+    url: "{{ digitalocean_api_url }}/droplets/{{ droplet_id }}\
+      /actions/{{ snapshot.json.action.id }}"
+    method: GET
+    headers:
+      Content-Type: application/json
+      Authorization: Bearer {{ digitalocean_api_token }}
+    body_format: json
+    status_code: 200
+  register: snapshot_info
+  until: snapshot_info.json.action.completed_at != None
+  retries: 16
+  delay: 60
diff --git a/roles/sysupgrade/tasks/main.yml b/roles/sysupgrade/tasks/main.yml
new file mode 100644
index 0000000..77f2b1c
--- /dev/null
+++ b/roles/sysupgrade/tasks/main.yml
@@ -0,0 +1,9 @@
+- name: Update archlinux-keyring
+  ansible.builtin.package:
+    name: archlinux-keyring
+    state: latest
+
+- name: Upgrade all packages in the system
+  community.general.pacman:
+    update_cache: yes
+    upgrade: yes
diff --git a/sysupgrade.yml b/sysupgrade.yml
new file mode 100644
index 0000000..95bae48
--- /dev/null
+++ b/sysupgrade.yml
@@ -0,0 +1,12 @@
+- name: Update DigitalOcean machines
+  hosts: digitalocean
+  vars:
+    digitalocean_api_url: https://api.digitalocean.com/v2
+  vars_files:
+    - misc/vaults/digitalocean.yml
+  roles:
+    - { role: shutdown }
+    - { role: snapshot }
+    - { role: poweron }
+    - { role: sysupgrade }
+    - { role: reboot }