From dedb9def51a69deee888ab7d79a632a505a5ce3a Mon Sep 17 00:00:00 2001 From: Artur Czepiel Date: Sat, 11 Jan 2025 23:49:51 +0100 Subject: [PATCH 1/9] add ansible/lint --- Makefile | 6 ++++++ deploy/playbooks/01_setup.yml | 33 +++++++++++++++++---------------- deploy/playbooks/02_nginx.yml | 16 +++++++++++----- deploy/playbooks/03_app.yml | 25 +++++++++++++++++-------- 4 files changed, 51 insertions(+), 29 deletions(-) diff --git a/Makefile b/Makefile index 7b3ab94..227ee58 100644 --- a/Makefile +++ b/Makefile @@ -15,6 +15,7 @@ CI_RUN=cd intbot && DJANGO_SETTINGS_MODULE="intbot.settings" DJANGO_ENV="ci" # Deployment DEPLOY_CMD=cd deploy && uvx --from "ansible-core" ansible-playbook -i hosts.yml +DEPLOY_LINT_CMD=cd deploy && uvx --from "ansible-lint" ansible-lint # mostly useful for docker and deployment current_git_hash=$(shell git rev-parse HEAD) @@ -158,3 +159,8 @@ deploy/app: @echo "Deploying version $(V) to a remote server" $(DEPLOY_CMD) playbooks/03_app.yml --extra-vars "app_version=$(V)" $(DEPLOY_CMD) playbooks/04_cron.yml + +deploy/lint: + $(DEPLOY_LINT_CMD) playbooks/01_setup.yml + $(DEPLOY_LINT_CMD) playbooks/02_nginx.yml + $(DEPLOY_LINT_CMD) playbooks/03_app.yml diff --git a/deploy/playbooks/01_setup.yml b/deploy/playbooks/01_setup.yml index 0e2be55..bcb0743 100644 --- a/deploy/playbooks/01_setup.yml +++ b/deploy/playbooks/01_setup.yml @@ -1,14 +1,14 @@ - name: Deploy nginx and Let's Encrypt SSL certificate hosts: intbot_setup - become: yes - gather_facts: yes + become: true + gather_facts: true tasks: - name: Install Docker dependencies - apt: + ansible.builtin.apt: name: "{{ package }}" state: present - update_cache: yes + update_cache: true vars: package: - apt-transport-https @@ -21,22 +21,22 @@ - name: Install Docker block: - name: Add Docker GPG key - apt_key: + ansible.builtin.apt_key: url: https://download.docker.com/linux/ubuntu/gpg state: present - name: Add Docker repository - apt_repository: + ansible.builtin.apt_repository: repo: deb [arch=amd64] https://download.docker.com/linux/ubuntu {{ ansible_lsb.codename }} stable state: present - name: Install Docker - apt: + ansible.builtin.apt: name: docker-ce state: present - name: Combine non-root users to a single list - set_fact: + ansible.builtin.set_fact: non_root_user_names: ["{{ nginx_user }}", "{{ app_user }}"] - name: Create non-root users @@ -45,10 +45,10 @@ ansible.builtin.user: name: "{{ username }}" shell: "/bin/bash" - generate_ssh_key: yes + generate_ssh_key: true ssh_key_type: ed25519 ssh_key_comment: "{{ username }}@{{ inventory_hostname }}" - create_home: yes + create_home: true loop: "{{ non_root_user_names }}" loop_control: loop_var: username @@ -59,36 +59,37 @@ state: directory owner: "{{ username }}" group: "{{ username }}" + mode: "0755" loop: "{{ non_root_user_names }}" loop_control: loop_var: username - name: Then copy the authorized_keys from root so you can ssh later to the user - copy: + ansible.builtin.copy: src: "/root/.ssh/authorized_keys" dest: "/home/{{ username }}/.ssh/authorized_keys" owner: "{{ username }}" group: "{{ username }}" mode: "0600" - remote_src: "yes" + remote_src: true loop: "{{ non_root_user_names }}" loop_control: loop_var: username - name: Add the non root users (both nginx and app) to docker group - user: + ansible.builtin.user: name: "{{ username }}" groups: docker - append: yes + append: true loop: "{{ non_root_user_names }}" loop_control: loop_var: username - name: Read the deploy public key - slurp: + ansible.builtin.slurp: src: "/home/{{ app_user }}/.ssh/id_ed25519.pub" register: deploy_key - name: Display the public key - debug: + ansible.builtin.debug: msg: "For private repositories, make sure to put this key as deploy key on github: {{ deploy_key.content | b64decode }}" diff --git a/deploy/playbooks/02_nginx.yml b/deploy/playbooks/02_nginx.yml index d2b4a60..2422783 100644 --- a/deploy/playbooks/02_nginx.yml +++ b/deploy/playbooks/02_nginx.yml @@ -4,22 +4,28 @@ tasks: - name: Copy nginx configuration file ansible.builtin.template: - src: ../templates/nginx/nginx.conf.j2 + src: nginx/nginx.conf.j2 dest: ./nginx.conf + mode: "0644" - name: Create a server Makefile (for nginx) to manage on-server tasks ansible.builtin.template: - src: ../templates/nginx/Makefile.nginx.j2 + src: nginx/Makefile.nginx.j2 dest: ./Makefile + mode: "0644" - name: Set up docker-compose.yml on the remote server ansible.builtin.template: - src: ../templates/nginx/docker-compose.nginx.yml.j2 + src: nginx/docker-compose.nginx.yml.j2 dest: ./docker-compose.yml + mode: "0644" - name: Make sure the directory structure for certs exist - shell: mkdir -p ./data/certbot/conf + ansible.builtin.file: + path: "/home/{{ ansible_user }}/data/cerbot/conf" + state: directory + mode: "0755" - name: Display info at the end - debug: + ansible.builtin.debug: msg: "Go to /home/{{ ansible_user }} and run make certbot/init-staging; then make certbot/upgrade-to-prod" diff --git a/deploy/playbooks/03_app.yml b/deploy/playbooks/03_app.yml index 2a90d57..b43ff14 100644 --- a/deploy/playbooks/03_app.yml +++ b/deploy/playbooks/03_app.yml @@ -3,26 +3,28 @@ tasks: - name: Clone the repository to a specific version (to a temp location) - git: + ansible.builtin.git: repo: "{{ repository_url }}" dest: /tmp/src - accept_hostkey: yes + accept_hostkey: true version: "{{ app_version }}" - name: Build with a given commit hash # This will be stored in local registry, and available as version to docker-compose # where we can just reference correct version - shell: "cd /tmp/src && make docker/build V={{ app_version }}" + ansible.builtin.shell: "cd /tmp/src && make docker/build V={{ app_version }}" - name: Create a server Makefile to manage app tasks ansible.builtin.template: - src: ../templates/app/Makefile.app.j2 + src: app/Makefile.app.j2 dest: ./Makefile + mode: "0644" - name: Set up docker-compose.yml for the app ansible.builtin.template: - src: ../templates/app/docker-compose.app.yml.j2 + src: app/docker-compose.app.yml.j2 dest: ./docker-compose.yml + mode: "0644" - name: Copy env file example ansible.builtin.copy: @@ -34,16 +36,23 @@ path: intbot.env register: env_file + - name: If env file doesn't exist - copy the example + ansible.builtin.copy: + src: app/intbot.env.example + dest: intbot.env.example + mode: "0644" + when: not env_file.stat.exists + - name: If the env file doesn't exist - fail with error message ansible.builtin.fail: msg: "The env file doesn't exist. Please ssh, copy the example and adjust" when: not env_file.stat.exists - name: Start docker compose to see if everything is running - shell: "docker compose up -d" + ansible.builtin.shell: "docker compose up -d" - name: Migrate on prod - shell: "make prod/migrate" + ansible.builtin.shell: "make prod/migrate" - name: Restart everything and finish - shell: "docker compose up -d" + ansible.builtin.shell: "docker compose up -d" From 7584b1962713d49d00b23caef351d0f0a9bf3a3b Mon Sep 17 00:00:00 2001 From: Artur Czepiel Date: Sat, 18 Jan 2025 00:40:20 +0100 Subject: [PATCH 2/9] Update deploy/playbooks/02_nginx.yml Co-authored-by: Cyril Bitterich --- deploy/playbooks/02_nginx.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deploy/playbooks/02_nginx.yml b/deploy/playbooks/02_nginx.yml index 2422783..223d76a 100644 --- a/deploy/playbooks/02_nginx.yml +++ b/deploy/playbooks/02_nginx.yml @@ -22,7 +22,7 @@ - name: Make sure the directory structure for certs exist ansible.builtin.file: - path: "/home/{{ ansible_user }}/data/cerbot/conf" + path: "/home/{{ ansible_user }}/data/certbot/conf" state: directory mode: "0755" From 750cdacfec13cddfabde04935602141111faa9b7 Mon Sep 17 00:00:00 2001 From: Artur Czepiel Date: Sat, 18 Jan 2025 00:41:11 +0100 Subject: [PATCH 3/9] Update deploy/playbooks/03_app.yml Co-authored-by: Cyril Bitterich --- deploy/playbooks/03_app.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deploy/playbooks/03_app.yml b/deploy/playbooks/03_app.yml index b43ff14..06931d3 100644 --- a/deploy/playbooks/03_app.yml +++ b/deploy/playbooks/03_app.yml @@ -18,7 +18,7 @@ ansible.builtin.template: src: app/Makefile.app.j2 dest: ./Makefile - mode: "0644" + mode: "0640" - name: Set up docker-compose.yml for the app ansible.builtin.template: From 69ab6ca83ca2b2766bba22f4b5fbd87d4767b511 Mon Sep 17 00:00:00 2001 From: Artur Czepiel Date: Sun, 19 Jan 2025 17:03:34 +0100 Subject: [PATCH 4/9] Apply suggestions from code review Co-authored-by: Cyril Bitterich --- deploy/playbooks/02_nginx.yml | 4 ++-- deploy/playbooks/03_app.yml | 18 +++++++++++++----- 2 files changed, 15 insertions(+), 7 deletions(-) diff --git a/deploy/playbooks/02_nginx.yml b/deploy/playbooks/02_nginx.yml index 223d76a..884dbad 100644 --- a/deploy/playbooks/02_nginx.yml +++ b/deploy/playbooks/02_nginx.yml @@ -12,13 +12,13 @@ ansible.builtin.template: src: nginx/Makefile.nginx.j2 dest: ./Makefile - mode: "0644" + mode: "0640" - name: Set up docker-compose.yml on the remote server ansible.builtin.template: src: nginx/docker-compose.nginx.yml.j2 dest: ./docker-compose.yml - mode: "0644" + mode: "0640" - name: Make sure the directory structure for certs exist ansible.builtin.file: diff --git a/deploy/playbooks/03_app.yml b/deploy/playbooks/03_app.yml index 06931d3..f698a53 100644 --- a/deploy/playbooks/03_app.yml +++ b/deploy/playbooks/03_app.yml @@ -12,7 +12,9 @@ - name: Build with a given commit hash # This will be stored in local registry, and available as version to docker-compose # where we can just reference correct version - ansible.builtin.shell: "cd /tmp/src && make docker/build V={{ app_version }}" + ansible.builtin.shell: + chdir: /tmp/src + cmd: "/usr/bin/make docker/build V={{ app_version }}" - name: Create a server Makefile to manage app tasks ansible.builtin.template: @@ -24,7 +26,7 @@ ansible.builtin.template: src: app/docker-compose.app.yml.j2 dest: ./docker-compose.yml - mode: "0644" + mode: "0640" - name: Copy env file example ansible.builtin.copy: @@ -49,10 +51,16 @@ when: not env_file.stat.exists - name: Start docker compose to see if everything is running - ansible.builtin.shell: "docker compose up -d" + ansible.builtin.command: + chdir: {{ ansible_user_dir }} + cmd: "docker compose up -d" - name: Migrate on prod - ansible.builtin.shell: "make prod/migrate" + ansible.builtin.shell: + chdir: /tmp/src + cmd: "/usr/bin/make prod/migrate" - name: Restart everything and finish - ansible.builtin.shell: "docker compose up -d" + ansible.builtin.command: + chdir: {{ ansible_user_dir }} + cmd: "docker compose up -d" From 94fec58f679d315810a8f2e1b621dcc4fb8d7290 Mon Sep 17 00:00:00 2001 From: Artur Czepiel Date: Sun, 19 Jan 2025 17:12:56 +0100 Subject: [PATCH 5/9] tweak ansible lint --- Makefile | 2 +- deploy/playbooks/03_app.yml | 10 +++++----- deploy/{ => playbooks}/templates/app/.env.example | 0 deploy/{ => playbooks}/templates/app/Makefile.app.j2 | 0 .../templates/app/docker-compose.app.yml.j2 | 0 .../{ => playbooks}/templates/app/intbot.env.example | 0 .../{ => playbooks}/templates/nginx/Makefile.nginx.j2 | 0 .../templates/nginx/docker-compose.nginx.yml.j2 | 0 deploy/{ => playbooks}/templates/nginx/nginx.conf.j2 | 0 9 files changed, 6 insertions(+), 6 deletions(-) rename deploy/{ => playbooks}/templates/app/.env.example (100%) rename deploy/{ => playbooks}/templates/app/Makefile.app.j2 (100%) rename deploy/{ => playbooks}/templates/app/docker-compose.app.yml.j2 (100%) rename deploy/{ => playbooks}/templates/app/intbot.env.example (100%) rename deploy/{ => playbooks}/templates/nginx/Makefile.nginx.j2 (100%) rename deploy/{ => playbooks}/templates/nginx/docker-compose.nginx.yml.j2 (100%) rename deploy/{ => playbooks}/templates/nginx/nginx.conf.j2 (100%) diff --git a/Makefile b/Makefile index 227ee58..d3618a3 100644 --- a/Makefile +++ b/Makefile @@ -160,7 +160,7 @@ deploy/app: $(DEPLOY_CMD) playbooks/03_app.yml --extra-vars "app_version=$(V)" $(DEPLOY_CMD) playbooks/04_cron.yml -deploy/lint: +lint/deploy: $(DEPLOY_LINT_CMD) playbooks/01_setup.yml $(DEPLOY_LINT_CMD) playbooks/02_nginx.yml $(DEPLOY_LINT_CMD) playbooks/03_app.yml diff --git a/deploy/playbooks/03_app.yml b/deploy/playbooks/03_app.yml index f698a53..9ed283f 100644 --- a/deploy/playbooks/03_app.yml +++ b/deploy/playbooks/03_app.yml @@ -12,7 +12,7 @@ - name: Build with a given commit hash # This will be stored in local registry, and available as version to docker-compose # where we can just reference correct version - ansible.builtin.shell: + ansible.builtin.command: chdir: /tmp/src cmd: "/usr/bin/make docker/build V={{ app_version }}" @@ -52,15 +52,15 @@ - name: Start docker compose to see if everything is running ansible.builtin.command: - chdir: {{ ansible_user_dir }} + chdir: "{{ ansible_user_dir }}" cmd: "docker compose up -d" - name: Migrate on prod - ansible.builtin.shell: - chdir: /tmp/src + ansible.builtin.command: + chdir: "{{ ansible_user_dir }}" cmd: "/usr/bin/make prod/migrate" - name: Restart everything and finish ansible.builtin.command: - chdir: {{ ansible_user_dir }} + chdir: "{{ ansible_user_dir }}" cmd: "docker compose up -d" diff --git a/deploy/templates/app/.env.example b/deploy/playbooks/templates/app/.env.example similarity index 100% rename from deploy/templates/app/.env.example rename to deploy/playbooks/templates/app/.env.example diff --git a/deploy/templates/app/Makefile.app.j2 b/deploy/playbooks/templates/app/Makefile.app.j2 similarity index 100% rename from deploy/templates/app/Makefile.app.j2 rename to deploy/playbooks/templates/app/Makefile.app.j2 diff --git a/deploy/templates/app/docker-compose.app.yml.j2 b/deploy/playbooks/templates/app/docker-compose.app.yml.j2 similarity index 100% rename from deploy/templates/app/docker-compose.app.yml.j2 rename to deploy/playbooks/templates/app/docker-compose.app.yml.j2 diff --git a/deploy/templates/app/intbot.env.example b/deploy/playbooks/templates/app/intbot.env.example similarity index 100% rename from deploy/templates/app/intbot.env.example rename to deploy/playbooks/templates/app/intbot.env.example diff --git a/deploy/templates/nginx/Makefile.nginx.j2 b/deploy/playbooks/templates/nginx/Makefile.nginx.j2 similarity index 100% rename from deploy/templates/nginx/Makefile.nginx.j2 rename to deploy/playbooks/templates/nginx/Makefile.nginx.j2 diff --git a/deploy/templates/nginx/docker-compose.nginx.yml.j2 b/deploy/playbooks/templates/nginx/docker-compose.nginx.yml.j2 similarity index 100% rename from deploy/templates/nginx/docker-compose.nginx.yml.j2 rename to deploy/playbooks/templates/nginx/docker-compose.nginx.yml.j2 diff --git a/deploy/templates/nginx/nginx.conf.j2 b/deploy/playbooks/templates/nginx/nginx.conf.j2 similarity index 100% rename from deploy/templates/nginx/nginx.conf.j2 rename to deploy/playbooks/templates/nginx/nginx.conf.j2 From 2d4f785555222c2d61cf7190a3456409076d2dd5 Mon Sep 17 00:00:00 2001 From: Artur Czepiel Date: Sun, 19 Jan 2025 20:01:23 +0100 Subject: [PATCH 6/9] lint deployment playbooks on CI --- .github/workflows/build_and_deploy.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/build_and_deploy.yml b/.github/workflows/build_and_deploy.yml index cb089bf..bb1a900 100644 --- a/.github/workflows/build_and_deploy.yml +++ b/.github/workflows/build_and_deploy.yml @@ -41,6 +41,7 @@ jobs: run: | docker run --rm intbot make ci/lint docker run --rm intbot make ci/type-check + docker run --rm intbot make lint/deploy services: postgres: From dcc001438daf33419aec574fb26fd29cceb82830 Mon Sep 17 00:00:00 2001 From: Artur Czepiel Date: Sun, 19 Jan 2025 20:04:52 +0100 Subject: [PATCH 7/9] deploy is not pat of the image, so lets run it with uvx --- .github/workflows/build_and_deploy.yml | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build_and_deploy.yml b/.github/workflows/build_and_deploy.yml index bb1a900..cda0f8c 100644 --- a/.github/workflows/build_and_deploy.yml +++ b/.github/workflows/build_and_deploy.yml @@ -14,6 +14,10 @@ jobs: - name: Checkout code uses: actions/checkout@v3 + # Uv is needed for the deployment lint + - name: Install uv + uses: astral-sh/setup-uv@v5 + - name: Set up Docker Buildx uses: docker/setup-buildx-action@v2 @@ -41,7 +45,10 @@ jobs: run: | docker run --rm intbot make ci/lint docker run --rm intbot make ci/type-check - docker run --rm intbot make lint/deploy + + + - name: Run deployment playbooks lint + run: make lint/deploy services: postgres: From 4ac68ba6c09a3aa6b6e329b4edb9958bc675b281 Mon Sep 17 00:00:00 2001 From: Artur Czepiel Date: Sun, 19 Jan 2025 20:11:04 +0100 Subject: [PATCH 8/9] fix remaining ansible lint errors --- deploy/playbooks/03_app.yml | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/deploy/playbooks/03_app.yml b/deploy/playbooks/03_app.yml index 9ed283f..1e4461e 100644 --- a/deploy/playbooks/03_app.yml +++ b/deploy/playbooks/03_app.yml @@ -15,6 +15,9 @@ ansible.builtin.command: chdir: /tmp/src cmd: "/usr/bin/make docker/build V={{ app_version }}" + register: output + changed_when: output.rc != 0 + failed_when: output.rc != 0 - name: Create a server Makefile to manage app tasks ansible.builtin.template: @@ -54,13 +57,22 @@ ansible.builtin.command: chdir: "{{ ansible_user_dir }}" cmd: "docker compose up -d" + register: output + changed_when: output.rc != 0 + failed_when: output.rc != 0 - name: Migrate on prod ansible.builtin.command: chdir: "{{ ansible_user_dir }}" cmd: "/usr/bin/make prod/migrate" + register: output + changed_when: output.rc != 0 + failed_when: output.rc != 0 - name: Restart everything and finish ansible.builtin.command: chdir: "{{ ansible_user_dir }}" cmd: "docker compose up -d" + register: output + changed_when: output.rc != 0 + failed_when: output.rc != 0 From 067a09c31b11c519891c001151856b89f7b2e633 Mon Sep 17 00:00:00 2001 From: Artur Czepiel Date: Fri, 12 Sep 2025 21:00:18 +0200 Subject: [PATCH 9/9] fix lint --- .github/workflows/build_and_deploy.yml | 1 - deploy/playbooks/03_app.yml | 1 + 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build_and_deploy.yml b/.github/workflows/build_and_deploy.yml index cda0f8c..9e340e9 100644 --- a/.github/workflows/build_and_deploy.yml +++ b/.github/workflows/build_and_deploy.yml @@ -46,7 +46,6 @@ jobs: docker run --rm intbot make ci/lint docker run --rm intbot make ci/type-check - - name: Run deployment playbooks lint run: make lint/deploy diff --git a/deploy/playbooks/03_app.yml b/deploy/playbooks/03_app.yml index 1e4461e..21ea6fb 100644 --- a/deploy/playbooks/03_app.yml +++ b/deploy/playbooks/03_app.yml @@ -35,6 +35,7 @@ ansible.builtin.copy: src: ../templates/app/intbot.env.example dest: intbot.env.example + mode: "0640" - name: Check if the env file exists ansible.builtin.stat: