feat!: Expose consensus debug port, restart EN on config file change (#42)

## What ❔

Subj

## Why ❔

QoL

## Checklist

<!-- Check your PR fulfills the following items. -->
<!-- For draft PRs check the boxes as you complete them. -->

- [x] PR title corresponds to the body of PR (we generate changelog
entries from PRs).
- [ ] Documentation comments have been added / updated.

README.md

@@ -99,11 +99,10 @@ Basic auth secret can be generated by `htpasswd` and `sed` for interpolation:
 2. Prepare the latest database backup on your host. you can download it from our public GCS buckets:
 Skip this step if you are recovering from a snapshot!
 
-* [Era Mainnet latest dump](https://storage.googleapis.com/zksync-era-mainnet-external-node-backups/external_node_latest.pgdump)
-* [Era Sepolia Testnet latest dump](https://storage.googleapis.com/zksync-era-boojnet-external-node-snapshots/external_node_latest.pgdump)
-* [Era Goerli Testnet latest dump](https://storage.googleapis.com/zksync-era-testnet-external-node-backups/external_node_latest.pgdump)
+* [Era Mainnet latest dump](https://en-backups.matterlabs.dev/)
+* [Era Sepolia Testnet latest dump](https://storage.googleapis.com/zksync-era-testnet-sepolia-external-node-backups/external_node_latest.pgdump)
 
-Downloaded dump file should be placed into `{{ storage_directory }}/pg_backups` directory (`/usr/src/en/pg_backups` by default)
+Downloaded dump, if needed, should be unarchived and named `external_node_latest.pgdump`. File should be placed into `{{ storage_directory }}/pg_backups` directory (`/usr/src/en/pg_backups` by default).
 
 3. **OPTIONAL**: If you already have running node, you can copy its tree and state directory to a new host's `{{ storage_directory }}/db`. (`/usr/src/en/db` by default)
 Skip this step if you are recovering from a snapshot!
@@ -124,13 +123,18 @@ vm_auth_password
 
 ## Snapshots Recovery
 
-example config enabling recovery from a snapshot
+Example config enabling recovery from a snapshot:
 
 ```yaml
 - enable_snapshots_recovery: true
-- snapshots_bucket_base_url: "zksync-era-mainnet-external-node-snapshots"
+- snapshots_bucket_base_url: "snapshots-bucket-name"
 ```
 
+Snapshot buckets:
+
+* Era Mainnet: `zksync-era-mainnet-external-node-snapshots`
+* Era Sepolia Testnet: `zksync-era-boojnet-external-node-snapshots`
+
 ## Example Playbook
 
 ```yaml

defaults/main.yml

@@ -11,8 +11,9 @@ docker_compose_version: "v2.23.0"
 # Versions of External Node and 3rd party components
 traefik_version: 2.11
 postgres_version: 14
-external_node_version: 21.0.2
-vmagent_version: 1.95.1
+external_node_version: 27.2.0
+external_node_raw_docker_tag: ""
+vmagent_version: 1.100.1
 cadvisor_version: 0.47.2
 postgres_exporter_version: 0.15.0
 
@@ -55,6 +56,17 @@ postgres_arguments:
   - max_parallel_maintenance_workers=4
   - -c
   - checkpoint_timeout=1800
+enable_postgres_replication: false
+# IP address of the interface replication
+postgres_replications_arguments: []
+postgres_replica_user_name: ""
+postgres_replica_user_password: ""
+postgres_replica_auth_method: "scram-sha-256"
+postgres_replication_bind_address: ""
+postgres_replica_address: ""
+backup_db_user: ""
+backup_db_password: ""
+backup_db_name: ""
 
 # Enable TLS for traefik
 enable_tls: false
@@ -72,6 +84,16 @@ force_pg_restore: false
 enable_snapshots_recovery: false
 snapshots_bucket_base_url: ""
 
+# https://github.com/matter-labs/zksync-era/blob/main/docs/guides/external-node/09_decentralization.md
+enable_consensus: false
+consensus_secrets_file: ""
+consensus_port: 3054
+consensus_outbound: []
+consensus_debug_port: 5000
+enable_consensus_debug_port: false
+expose_consensus_debug_port: false
+consensus_debug_port_path_prefix: "/consensus_debug"
+
 # External Node and database options
 database_name: ""
 database_username: ""

example_playbooks/mainnet_with_snapshots_recovery/README.md

@@ -8,16 +8,16 @@ To run this playbook, first install dependencies
 
 ```shell
  ansible-galaxy install -r requirements.yml
- ``` 
+ ```
 
 and then you can run the playbook using
 
 ```shell
  ansible-playbook playbook.yml -i hosts.ini -K
- ``` 
+ ```
 
 To see logs you can use
 
 ```shell
- docker logs en-external_node-1 
- ``` 
+ docker logs en-external_node-1
+ ```

example_playbooks/mainnet_with_snapshots_recovery/requirements.yml

@@ -5,8 +5,13 @@ roles:
     version: "7.1.0"
   - name: external_node
     src: https://github.com/matter-labs/ansible-en-role
-    version: "v3.0.0"
+    version: "v3.3.0"
 
 collections:
+  - name: community.docker
+      version: 4.5.2
   - name: community.general
     version: 8.4.0
+  # Collection for the replication only.
+  - name: community.postgresql
+    version: 3.7.0

handlers/main.yml

@@ -0,0 +1,8 @@
+---
+- name: Restart external-node service
+  community.docker.docker_compose_v2:
+    project_src: "{{ configuration_directory }}"
+    files: "{{ docker_compose_files }}"
+    state: restarted
+    services:
+      - external_node

tasks/firewall.yml

@@ -49,6 +49,23 @@
     source: "{{ loadbalancer_ip | mandatory }}"
     jump: ACCEPT
 
+- name: Allow consensus port traffic from any IP
+  when: enable_consensus
+  ansible.builtin.iptables:
+    chain: INPUT
+    protocol: tcp
+    destination_port: "{{ consensus_port }}"
+    jump: ACCEPT
+
+- name: Allow postgres replication traffic from replica only
+  when: enable_postgres_replication
+  ansible.builtin.iptables:
+    chain: INPUT
+    protocol: tcp
+    destination_port: 5432
+    source: "{{ postgres_replica_address }}"
+    jump: ACCEPT
+
 - name: Set default policy to DROP
   ansible.builtin.iptables:
     chain: INPUT

tasks/main.yml

@@ -9,3 +9,7 @@
 
 - name: Prepare configs
   ansible.builtin.include_tasks: provision.yml
+
+- name: Configure replication on main instance
+  ansible.builtin.include_tasks: replication.yml
+  when: enable_postgres_replication

tasks/provision.yml

@@ -32,13 +32,37 @@
     - l2_chain_id
     - l1_chain_id
 
+- name: "Verify that required variables for replication is set"
+  when: enable_postgres_replication
+  ansible.builtin.assert:
+    that:
+      - postgress_replication_required_var != ""
+    fail_msg: "{{ postgress_replication_required_var }} needs to be set for the role for postgres replication to work"
+    success_msg: "Required variable for postgres replication {{ postgress_replication_required_var }} isn't empty"
+  loop_control:
+    loop_var: postgress_replication_required_var
+  with_items:
+    - enable_postgres_replication
+    - postgres_replication_bind_address
+    - postgres_replica_address
+    - postgres_replications_arguments
+    - postgres_replica_user_name
+    - postgres_replica_user_password
+
 - name: Check required en vars empty
   ansible.builtin.fail:
     msg: "Variable '{{ item }}' is empty"
   when: vars[item] == ""
   with_items: "{{ en_required_variables }}"
+- name: "Verify consensus debug port configuration"
+  ansible.builtin.fail:
+    msg: "Cannot expose consensus debug port (expose_consensus_debug_port=true) if it is not enabled (enable_consensus_debug_port=false)."
+  when:
+    - enable_consensus
+    - expose_consensus_debug_port
+    - not enable_consensus_debug_port
 
-- name: Copy main configs
+- name: Create main configs
   ansible.builtin.template:
     src: '{{ item.src }}'
     dest: '{{ item.dest }}'
@@ -50,8 +74,10 @@
       dest: "{{ configuration_directory }}/external_node.env"
     - src: "templates/postgres.env.j2"
       dest: "{{ configuration_directory }}/postgres.env"
+  loop_control:
+    label: "{{ item.dest }}"
 
-- name: Copy restore script
+- name: Create restore script
   register: restore_dump_script
   ansible.builtin.template:
     src: 'templates/restore_dump.sh.j2'
@@ -64,7 +90,7 @@
   when: enable_monitoring and ( vars[item] == "" )
   with_items: "{{ monitoring_required_variables }}"
 
-- name: Copy monitoring configs
+- name: Create monitoring configs
   when: enable_monitoring
   ansible.builtin.template:
     src: '{{ item.src }}'
@@ -76,23 +102,32 @@
     - src: "templates/vmagent-config.yml.j2"
       dest: "{{ configuration_directory }}/vmagent-config.yml"
 
-- name: Run docker-compose without monitoring
-  when: not enable_monitoring
-  ansible.builtin.shell:
-    cmd: nohup docker compose -f docker-compose.yaml up -d &
-    chdir: "{{ configuration_directory }}"
-  changed_when: false
+- name: Create consensus config
+  when: enable_consensus
+  ansible.builtin.template:
+    src: "templates/consensus_config.yaml.j2"
+    dest: "{{ configuration_directory }}/consensus_config.yaml"
+    mode: '0644'
+  notify: Restart external-node service
 
-- name: Run docker-compose with monitoring
-  when: enable_monitoring and (not restore_dump_script.changed)
-  ansible.builtin.shell:
-    cmd: nohup docker compose -f monitoring.yaml -f docker-compose.yaml up -d &
-    chdir: "{{ configuration_directory }}"
-  changed_when: false
+- name: Decrypt consensus_secrets
+  when: enable_consensus
+  ansible.builtin.copy:
+    src: "{{ consensus_secrets_file }}"
+    dest: "{{ configuration_directory }}/consensus_secrets.yaml"
+    decrypt: true
+    mode: '0600'
+  notify: Restart external-node service
 
-- name: Run docker-compose with monitoring with recreation
-  when: enable_monitoring and restore_dump_script.changed
-  ansible.builtin.shell:
-    cmd: nohup docker compose -f monitoring.yaml -f docker-compose.yaml up -d --force-recreate &
-    chdir: "{{ configuration_directory }}"
-  changed_when: false
+- name: Set docker compose files list
+  ansible.builtin.set_fact:
+    docker_compose_files: "{{ ['docker-compose.yaml'] + (['monitoring.yaml'] if enable_monitoring else []) }}"
+
+- name: Run docker compose services (non-blocking)
+  community.docker.docker_compose_v2:
+    project_src: "{{ configuration_directory }}"
+    files: "{{ docker_compose_files }}"
+    state: present
+    pull: "{{ docker_pull_policy | default('missing') }}"
+    recreate: "{{ 'always' if restore_dump_script.changed else 'auto' }}"
+    wait: false

tasks/replication.yml

@@ -0,0 +1,60 @@
+---
+
+- name: Install libpq-dev packages
+  ansible.builtin.apt:
+    update_cache: true
+    name: libpq-dev
+
+- name: Install psycopg2 python package
+  ansible.builtin.pip:
+    name: psycopg2
+
+- name: Grant user replication access for replication.
+  community.postgresql.postgresql_pg_hba:
+    dest: "{{ storage_directory }}/postgres/pg_hba.conf"
+    contype: host
+    users: "{{ postgres_replica_user_name }}"
+    source: "{{ postgres_replica_address }}/32"
+    databases: replication
+    method: "{{ postgres_replica_auth_method }}"
+
+- name: Create postgres replication user
+  community.postgresql.postgresql_user:
+    login_host: "{{ postgres_replication_bind_address }}"
+    login_user: "{{ database_username }}"
+    login_password: "{{ database_password }}"
+    name: "{{ postgres_replica_user_name }}"
+    password: "{{ postgres_replica_user_password }}"
+    role_attr_flags: "REPLICATION"
+
+- name: Create replication slot if doesn't exist
+  community.postgresql.postgresql_slot:
+    login_host: "{{ postgres_replication_bind_address }}"
+    login_user: "{{ database_username }}"
+    login_password: "{{ database_password }}"
+    slot_name: replica
+
+- name: Reload postgres configuration
+  community.postgresql.postgresql_query:
+    login_host: "{{ postgres_replication_bind_address }}"
+    login_user: "{{ database_username }}"
+    login_password: "{{ database_password }}"
+    query: "SELECT pg_reload_conf()"
+
+- name: Create postgres backup user
+  community.postgresql.postgresql_user:
+    login_host: "{{ postgres_replication_bind_address }}"
+    login_user: "{{ database_username }}"
+    login_password: "{{ database_password }}"
+    name: "{{ backup_db_user }}"
+    password: "{{ backup_db_password }}"
+
+- name: Grant role pg_read_all_data to backup user
+  community.postgresql.postgresql_membership:
+    login_host: "{{ postgres_replication_bind_address }}"
+    login_user: "{{ database_username }}"
+    login_password: "{{ database_password }}"
+    group: pg_read_all_data
+    target_roles:
+      - "{{ backup_db_user }}"
+    state: present

templates/consensus_config.yaml.j2

@@ -0,0 +1,13 @@
+server_addr: '0.0.0.0:3054'
+public_addr: '{{ ansible_default_ipv4.address }}:{{ consensus_port }}'
+max_payload_size: 5000000
+gossip_dynamic_inbound_limit: 200
+{% if enable_consensus_debug_port %}
+debug_page_addr: "0.0.0.0:{{ consensus_debug_port }}"
+{% endif %}
+rpc_config:
+  get_block_rate:
+    burst: 5
+    refresh: # 0.2s
+      seconds: 0
+      nanos: 200000000

templates/docker-compose.yaml.j2

@@ -8,10 +8,13 @@ services:
       - "--log.level=INFO"
       - "--providers.docker=true"
       - "--providers.docker.exposedbydefault=false"
-      - "--entrypoints.web.address=:80"
-      - "--entrypoints.external_node_health.address=:3080"
+      - "--entryPoints.web.address=:80"
+      - "--entryPoints.external_node_health.address=:3080"
+{% if enable_consensus %}
+      - "--entryPoints.external_node_consensus.address=:{{ consensus_port }}"
+{% endif %}
 {% if enable_tls %}
-      - "--entrypoints.websecure.address=:443"
+      - "--entryPoints.websecure.address=:443"
       - "--certificatesresolvers.en_resolver.acme.tlschallenge=true"
       - "--certificatesresolvers.en_resolver.acme.storage=/letsencrypt/acme.json"
       - "--certificatesresolvers.myresolver.acme.email={{ acme_email }}"
@@ -40,12 +43,21 @@ services:
       - ./restore_dump.sh:/docker-entrypoint-initdb.d/restore_dump.sh
     env_file:
       - postgres.env
+{% if enable_postgres_replication %}
+    ports:
+      - "{{ postgres_replication_bind_address }}:5432:5432"
+{% endif %}
     command:
       - postgres
       - -c
 {% for argument in postgres_arguments %}
       - {{ argument }}
 {% endfor %}
+{% if enable_postgres_replication %}
+{% for repl_argument in postgres_replications_arguments %}
+      - {{ repl_argument }}
+{% endfor %}
+{% endif %}
   external_node:
 {% if not external_node_raw_docker_tag %}
     image: "matterlabs/external-node:v{{ external_node_version }}"
@@ -68,7 +80,6 @@ services:
       - "traefik.http.routers.external_node_main.entrypoints=web"
 {% endif %}
       - "traefik.http.routers.external_node_main.service=external_node_main"
-
       - "traefik.http.services.external_node_health.loadbalancer.server.port={{ healthcheck_port }}"
       - "traefik.http.routers.external_node_health.rule=PathPrefix(`/`)"
       - "traefik.http.routers.external_node_health.entrypoints=external_node_health"
@@ -76,12 +87,38 @@ services:
 {% if enable_basic_auth %}
       - "traefik.http.routers.external_node_main.middlewares=external_node_auth"
       - "traefik.http.middlewares.external_node_auth.basicauth.users={{ basic_auth_secret }}"
+{% endif %}
+{% if enable_consensus %}
+      - "traefik.tcp.services.external_node_consensus.loadbalancer.server.port={{ consensus_port }}"
+      - "traefik.tcp.routers.external_node_consensus.rule=HostSNI(`*`)"
+      - "traefik.tcp.routers.external_node_consensus.entrypoints=external_node_consensus"
+      - "traefik.tcp.routers.external_node_consensus.service=external_node_consensus"
+{% endif %}
+{% if enable_consensus and expose_consensus_debug_port %}
+      - "traefik.http.services.external_node_consensus_debug.loadbalancer.server.port={{ consensus_debug_port }}"
+      - "traefik.http.routers.external_node_consensus_debug.rule=PathPrefix(`{{ consensus_debug_port_path_prefix }}`)"
+{% if enable_tls %}
+      - "traefik.http.routers.external_node_consensus_debug.entrypoints=websecure"
+      - "traefik.http.routers.external_node_consensus_debug.tls.certresolver=myresolver"
+{% else %}
+      - "traefik.http.routers.external_node_consensus_debug.entrypoints=web"
+{% endif %}
+      - "traefik.http.routers.external_node_consensus_debug.service=external_node_consensus_debug"
+{% if enable_basic_auth %}
+      - "traefik.http.routers.external_node_consensus_debug.middlewares=external_node_auth"
+{% endif %}
 {% endif %}
     expose:
       - {{ rpc_http_port }}
       - {{ rpc_ws_port }}
       - {{ healthcheck_port }}
       - {{ metrics_port }}
+{% if enable_consensus %}
+      - {{ consensus_port }}
+{% if expose_consensus_debug_port %}
+      - {{ consensus_debug_port }}
+{% endif %}
+{% endif %}
     environment:
       ZKSYNC_HOME: "/"
       EN_STATE_CACHE_PATH: /db/state_keeper
@@ -96,6 +133,10 @@ services:
       EN_MAX_BLOCKS_PER_TREE_BATCH: 200
       MISC_LOG_FORMAT: json
       RUST_LOG: {{ rust_log }}
+{% if enable_consensus %}
+      EN_CONSENSUS_CONFIG_PATH: /etc/consensus_config.yaml
+      EN_CONSENSUS_SECRETS_PATH: /run/secrets/consensus_secrets
+{% endif %}
     healthcheck:
       test: [ "CMD", "curl", "-f", "http://localhost:{{ healthcheck_port }}/health" ]
       interval: 1m
@@ -104,10 +145,21 @@ services:
       start_period: 1m
     volumes:
       - "{{ storage_directory }}/db:/db"
+{% if enable_consensus %}
+      - "{{ configuration_directory }}/consensus_config.yaml:/etc/consensus_config.yaml"
+{% endif %}
     env_file:
       - "external_node.env"
       - "postgres.env"
-{% if enable_snapshots_recovery %}
     command:
-      - --enable-snapshots-recovery
+{% if enable_consensus %}
+      - --enable-consensus
+    secrets:
+      - consensus_secrets
+{% endif %}
+
+{% if enable_consensus %}
+secrets:
+  consensus_secrets:
+    file: consensus_secrets.yaml
 {% endif %}

templates/vmagent-config.yml.j2

@@ -33,7 +33,8 @@ scrape_configs:
   - job_name: traefik
     static_configs:
       - targets:
-          - "traefik:8080"
+          # traefik uses network host, so docker DNS wouldn't work.
+          - "127.0.0.1:8080"
     relabel_configs:
       - source_labels: [instance]
         target_label: instance