feat!: Expose consensus debug port, restart EN on config file change (#42)

## What ❔

Subj

## Why ❔

QoL

## Checklist

<!-- Check your PR fulfills the following items. -->
<!-- For draft PRs check the boxes as you complete them. -->

- [x] PR title corresponds to the body of PR (we generate changelog
entries from PRs).
- [ ] Documentation comments have been added / updated.

README.md

@@ -99,10 +99,10 @@ Basic auth secret can be generated by `htpasswd` and `sed` for interpolation:
 2. Prepare the latest database backup on your host. you can download it from our public GCS buckets:
 Skip this step if you are recovering from a snapshot!
 
-* [Era Mainnet latest dump](https://storage.googleapis.com/zksync-era-mainnet-external-node-backups/external_node_latest.pgdump)
+* [Era Mainnet latest dump](https://en-backups.matterlabs.dev/)
 * [Era Sepolia Testnet latest dump](https://storage.googleapis.com/zksync-era-testnet-sepolia-external-node-backups/external_node_latest.pgdump)
 
-Downloaded dump file should be placed into `{{ storage_directory }}/pg_backups` directory (`/usr/src/en/pg_backups` by default)
+Downloaded dump, if needed, should be unarchived and named `external_node_latest.pgdump`. File should be placed into `{{ storage_directory }}/pg_backups` directory (`/usr/src/en/pg_backups` by default).
 
 3. **OPTIONAL**: If you already have running node, you can copy its tree and state directory to a new host's `{{ storage_directory }}/db`. (`/usr/src/en/db` by default)
 Skip this step if you are recovering from a snapshot!

defaults/main.yml

@@ -11,7 +11,7 @@ docker_compose_version: "v2.23.0"
 # Versions of External Node and 3rd party components
 traefik_version: 2.11
 postgres_version: 14
-external_node_version: 24.26.0
+external_node_version: 27.2.0
 external_node_raw_docker_tag: ""
 vmagent_version: 1.100.1
 cadvisor_version: 0.47.2
@@ -89,6 +89,10 @@ enable_consensus: false
 consensus_secrets_file: ""
 consensus_port: 3054
 consensus_outbound: []
+consensus_debug_port: 5000
+enable_consensus_debug_port: false
+expose_consensus_debug_port: false
+consensus_debug_port_path_prefix: "/consensus_debug"
 
 # External Node and database options
 database_name: ""

example_playbooks/mainnet_with_snapshots_recovery/requirements.yml

@@ -8,6 +8,8 @@ roles:
     version: "v3.3.0"
 
 collections:
+  - name: community.docker
+      version: 4.5.2
   - name: community.general
     version: 8.4.0
   # Collection for the replication only.

handlers/main.yml

@@ -0,0 +1,8 @@
+---
+- name: Restart external-node service
+  community.docker.docker_compose_v2:
+    project_src: "{{ configuration_directory }}"
+    files: "{{ docker_compose_files }}"
+    state: restarted
+    services:
+      - external_node

tasks/provision.yml

@@ -54,6 +54,13 @@
     msg: "Variable '{{ item }}' is empty"
   when: vars[item] == ""
   with_items: "{{ en_required_variables }}"
+- name: "Verify consensus debug port configuration"
+  ansible.builtin.fail:
+    msg: "Cannot expose consensus debug port (expose_consensus_debug_port=true) if it is not enabled (enable_consensus_debug_port=false)."
+  when:
+    - enable_consensus
+    - expose_consensus_debug_port
+    - not enable_consensus_debug_port
 
 - name: Create main configs
   ansible.builtin.template:
@@ -67,6 +74,8 @@
       dest: "{{ configuration_directory }}/external_node.env"
     - src: "templates/postgres.env.j2"
       dest: "{{ configuration_directory }}/postgres.env"
+  loop_control:
+    label: "{{ item.dest }}"
 
 - name: Create restore script
   register: restore_dump_script
@@ -99,6 +108,7 @@
     src: "templates/consensus_config.yaml.j2"
     dest: "{{ configuration_directory }}/consensus_config.yaml"
     mode: '0644'
+  notify: Restart external-node service
 
 - name: Decrypt consensus_secrets
   when: enable_consensus
@@ -107,24 +117,17 @@
     dest: "{{ configuration_directory }}/consensus_secrets.yaml"
     decrypt: true
     mode: '0600'
+  notify: Restart external-node service
 
-- name: Run docker-compose without monitoring
+- name: Set docker compose files list
-  when: not enable_monitoring
+  ansible.builtin.set_fact:
-  ansible.builtin.shell:
+    docker_compose_files: "{{ ['docker-compose.yaml'] + (['monitoring.yaml'] if enable_monitoring else []) }}"
-    cmd: nohup docker compose -f docker-compose.yaml up -d </dev/null >/dev/null 2>&1 &
-    chdir: "{{ configuration_directory }}"
-  changed_when: false
 
-- name: Run docker-compose with monitoring
+- name: Run docker compose services (non-blocking)
-  when: enable_monitoring and (not restore_dump_script.changed)
+  community.docker.docker_compose_v2:
-  ansible.builtin.shell:
+    project_src: "{{ configuration_directory }}"
-    cmd: nohup docker compose -f monitoring.yaml -f docker-compose.yaml up -d </dev/null >/dev/null 2>&1 &
+    files: "{{ docker_compose_files }}"
-    chdir: "{{ configuration_directory }}"
+    state: present
-  changed_when: false
+    pull: "{{ docker_pull_policy | default('missing') }}"
-
+    recreate: "{{ 'always' if restore_dump_script.changed else 'auto' }}"
-- name: Run docker-compose with monitoring with recreation
+    wait: false
-  when: enable_monitoring and restore_dump_script.changed
-  ansible.builtin.shell:
-    cmd: nohup docker compose -f monitoring.yaml -f docker-compose.yaml up -d --force-recreate </dev/null >/dev/null 2>&1 &
-    chdir: "{{ configuration_directory }}"
-  changed_when: false

templates/consensus_config.yaml.j2

@@ -2,6 +2,9 @@ server_addr: '0.0.0.0:3054'
 public_addr: '{{ ansible_default_ipv4.address }}:{{ consensus_port }}'
 max_payload_size: 5000000
 gossip_dynamic_inbound_limit: 200
+{% if enable_consensus_debug_port %}
+debug_page_addr: "0.0.0.0:{{ consensus_debug_port }}"
+{% endif %}
 rpc_config:
   get_block_rate:
     burst: 5

templates/docker-compose.yaml.j2

@@ -80,7 +80,6 @@ services:
       - "traefik.http.routers.external_node_main.entrypoints=web"
 {% endif %}
       - "traefik.http.routers.external_node_main.service=external_node_main"
-
       - "traefik.http.services.external_node_health.loadbalancer.server.port={{ healthcheck_port }}"
       - "traefik.http.routers.external_node_health.rule=PathPrefix(`/`)"
       - "traefik.http.routers.external_node_health.entrypoints=external_node_health"
@@ -94,6 +93,20 @@ services:
       - "traefik.tcp.routers.external_node_consensus.rule=HostSNI(`*`)"
       - "traefik.tcp.routers.external_node_consensus.entrypoints=external_node_consensus"
       - "traefik.tcp.routers.external_node_consensus.service=external_node_consensus"
+{% endif %}
+{% if enable_consensus and expose_consensus_debug_port %}
+      - "traefik.http.services.external_node_consensus_debug.loadbalancer.server.port={{ consensus_debug_port }}"
+      - "traefik.http.routers.external_node_consensus_debug.rule=PathPrefix(`{{ consensus_debug_port_path_prefix }}`)"
+{% if enable_tls %}
+      - "traefik.http.routers.external_node_consensus_debug.entrypoints=websecure"
+      - "traefik.http.routers.external_node_consensus_debug.tls.certresolver=myresolver"
+{% else %}
+      - "traefik.http.routers.external_node_consensus_debug.entrypoints=web"
+{% endif %}
+      - "traefik.http.routers.external_node_consensus_debug.service=external_node_consensus_debug"
+{% if enable_basic_auth %}
+      - "traefik.http.routers.external_node_consensus_debug.middlewares=external_node_auth"
+{% endif %}
 {% endif %}
     expose:
       - {{ rpc_http_port }}
@@ -102,6 +115,9 @@ services:
       - {{ metrics_port }}
 {% if enable_consensus %}
       - {{ consensus_port }}
+{% if expose_consensus_debug_port %}
+      - {{ consensus_debug_port }}
+{% endif %}
 {% endif %}
     environment:
       ZKSYNC_HOME: "/"