Merge pull request #27 from matter-labs/ya-zkd-1817-upgrade-external-nodes-on-hetzner4

feat: Add traefik configuration for consensus TCP port, open it in firewall

.ansible-lint

@@ -2,3 +2,6 @@ skip_list:
   - 'yaml'
   - 'risky-shell-pipe'
   - 'role-name'
+
+exclude_paths:
+  - example_playbooks

.github/ISSUE_TEMPLATE/bug_report.md

@@ -26,10 +26,11 @@ Describe what actually happened.
 
 #### 🖥️ Environment
 
-Any relevant environment details like: 
+Any relevant environment details like:
+
 * Ansible version
 * Operating system
-* External node version
+* External Node version
 
 #### 📋 Additional Context
 

CONTRIBUTING.md

@@ -5,7 +5,7 @@ sovereignty! We welcome contributions from anyone on the internet, and are grate
 
 ## Ways to contribute
 
-There are many ways to contribute to the external node role:
+There are many ways to contribute to the External Node role:
 
 1. Open issues: if you find a bug, have something you believe needs to be fixed, or have an idea for a feature, please
    open an issue.

README.md

@@ -14,7 +14,9 @@ This role has been tested on:
 
 ## Usage
 
-Minimal required variables that has to be set:
+For a very simple minimal working example, see example_playbooks directory
+
+Minimal required variables that have to be set:
 
 ```yaml
 database_name: ""
@@ -26,6 +28,15 @@ l1_chain_id: ""
 l2_chain_id: ""
 ```
 
+Additional arbitrary environment variables can be passed to External Node container:
+
+```yaml
+additional_env_vars:
+  - { name: "EN_ADDITIONAL_VAR1", value: "Value1" }
+  - { name: "EN_ADDITIONAL_VAR2", value: "Value2" }
+  - { name: "EN_ADDITIONAL_VAR3", value: "Value3" }
+```
+
 Please refer to [External Node docs](https://github.com/matter-labs/zksync-era/tree/main/docs/guides/external-node/prepared_configs) to find values for different zkSync Era chains.
 
 If you want to use monitoring (which we highly recommend), you have to change these variables:
@@ -73,7 +84,6 @@ We recommend using pgtune [online](https://pgtune.leopard.in.ua/) or [self-hoste
 If you want to use basic auth for inbound requests, you have to change next variables:
 
 ```yaml
-# Enable basic auth for external node
 enable_basic_auth: true
 basic_auth_secret: "htpasswd-generated-secret"
 ```
@@ -87,20 +97,21 @@ Basic auth secret can be generated by `htpasswd` and `sed` for interpolation:
 `ansible-galaxy collection install community.general`
 
 2. Prepare the latest database backup on your host. you can download it from our public GCS buckets:
+Skip this step if you are recovering from a snapshot!
 
 * [Era Mainnet latest dump](https://storage.googleapis.com/zksync-era-mainnet-external-node-backups/external_node_latest.pgdump)
-* [Era Sepolia Testnet latest dump](https://storage.googleapis.com/zksync-era-boojnet-external-node-snapshots/external_node_latest.pgdump)
-* [Era Goerli Testnet latest dump](https://storage.googleapis.com/zksync-era-testnet-external-node-backups/external_node_latest.pgdump)
+* [Era Sepolia Testnet latest dump](https://storage.googleapis.com/zksync-era-testnet-sepolia-external-node-backups/external_node_latest.pgdump)
 
 Downloaded dump file should be placed into `{{ storage_directory }}/pg_backups` directory (`/usr/src/en/pg_backups` by default)
 
 3. **OPTIONAL**: If you already have running node, you can copy its tree and state directory to a new host's `{{ storage_directory }}/db`. (`/usr/src/en/db` by default)
+Skip this step if you are recovering from a snapshot!
 
 **Keep in mind that tree and state should be older than PostgreSQL database backup.**
 
 4. Run ansible-playbook using this role. We recommend encrypting next variables with ansible-vault or some another way:
 
-```
+```yaml
 database_username
 database_password
 eth_l1_url
@@ -110,6 +121,20 @@ vm_auth_password
 
 5. Connect to your host, and see status of `postgres` container. It can take a lot of time before PostgreSQL database backup will be restored (hours to days, depending on your disk throughput and IOPS), after which PostgreSQL server will be ready for use. Once `postgres` becomes "healthy", `external_node` runs automatically.
 
+## Snapshots Recovery
+
+Example config enabling recovery from a snapshot:
+
+```yaml
+- enable_snapshots_recovery: true
+- snapshots_bucket_base_url: "snapshots-bucket-name"
+```
+
+Snapshot buckets:
+
+* Era Mainnet: `zksync-era-mainnet-external-node-snapshots`
+* Era Sepolia Testnet: `zksync-era-boojnet-external-node-snapshots`
+
 ## Example Playbook
 
 ```yaml
@@ -125,7 +150,6 @@ vm_auth_password
     l2_chain_id: "324"
     l1_chain_id: "1"
     enable_tls: false
-    partner_id: matterlabs
   vars_files:
     - secrets/mainnet_secrets.yml
   roles:

defaults/main.yml

@@ -8,11 +8,12 @@ docker_install_compose: true
 docker_version: "25.0.3"
 docker_compose_version: "v2.23.0"
 
-# Versions of external node and 3rd party components
+# Versions of External Node and 3rd party components
 traefik_version: 2.11
 postgres_version: 14
-external_node_version: 21.0.2
-vmagent_version: 1.95.1
+external_node_version: 24.16.0
+external_node_raw_docker_tag: ""
+vmagent_version: 1.100.1
 cadvisor_version: 0.47.2
 postgres_exporter_version: 0.15.0
 
@@ -61,14 +62,24 @@ enable_tls: false
 acme_email: ""
 domain_name: ""
 
-# Enable basic auth for external node
+# Enable basic auth for External Node
 enable_basic_auth: false
 basic_auth_secret: ""
 
 # Force restore pg database
 force_pg_restore: false
 
-# External node and database options
+# Use a snapshot to recover
+enable_snapshots_recovery: false
+snapshots_bucket_base_url: ""
+
+# https://github.com/matter-labs/zksync-era/blob/main/docs/guides/external-node/09_decentralization.md
+enable_consensus: false
+consensus_secrets_file: ""
+consensus_port: 3054
+consensus_outbound: []
+
+# External Node and database options
 database_name: ""
 database_username: ""
 database_password: ""
@@ -80,6 +91,10 @@ rpc_http_port: 3060
 rpc_ws_port: 3061
 healthcheck_port: 3081
 metrics_port: 3082
+rust_log: zksync_external_node=info,zksync_core=info,zksync_core::sync_layer=info,zksync_server=info,zksync_prover=info,zksync_contract_verifier=info,zksync_dal=info,zksync_eth_client=info,zksync_storage=info,zksync_db_manager=info,zksync_merkle_tree=info,zksync_state=info,zksync_utils=info,zksync_types=info,loadnext=info,dev_ticker=info,vm=info,block_sizes_test=info,zksync_verification_key_generator_and_server=info,zksync_object_store=info,setup_key_generator_and_server=info,zksync_circuit_synthesizer=info,zksync_queued_job_processor=info,zksync_health_check=info
+
+# Additional env vars passed to External Node
+additional_env_vars: []
 
 # Monitoring options section
 enable_monitoring: false

example_playbooks/mainnet_with_snapshots_recovery/README.md

@@ -0,0 +1,23 @@
+# Mainnet Snapshots Recovery playbook
+
+This directory is simple example how to set up EN using this role. It comes with snapshots recovery enabled by default.\
+**Note that for simplicity it's using postgres database
+with a very unsecure password and the EN is just started on the same machine**
+
+To run this playbook, first install dependencies
+
+```shell
+ ansible-galaxy install -r requirements.yml
+ ``` 
+
+and then you can run the playbook using
+
+```shell
+ ansible-playbook playbook.yml -i hosts.ini -K
+ ``` 
+
+To see logs you can use
+
+```shell
+ docker logs en-external_node-1 
+ ``` 

example_playbooks/mainnet_with_snapshots_recovery/hosts.ini

@@ -0,0 +1,2 @@
+[local]
+localhost ansible_connection=local

example_playbooks/mainnet_with_snapshots_recovery/playbook.yml

@@ -0,0 +1,16 @@
+---
+- hosts: all
+  become: true
+  vars:
+    database_name: "zksync_ext_node_mainnet"
+    database_username: "postgres"
+    database_password: "notsecurepassword"
+    eth_l1_url: "https://ethereum-rpc.publicnode.com"
+    main_node_url: "https://zksync2-mainnet.zksync.io"
+    l1_chain_id: "1"
+    l2_chain_id: "324"
+    enable_snapshots_recovery: true
+    snapshots_bucket_base_url: "zksync-era-mainnet-external-node-snapshots"
+
+  roles:
+    - external_node

example_playbooks/mainnet_with_snapshots_recovery/requirements.yml

@@ -0,0 +1,12 @@
+---
+roles:
+  - name: geerlingguy.docker
+    src: https://github.com/geerlingguy/ansible-role-docker
+    version: "7.1.0"
+  - name: external_node
+    src: https://github.com/matter-labs/ansible-en-role
+    version: "v3.3.0"
+
+collections:
+  - name: community.general
+    version: 8.4.0

meta/main.yml

@@ -2,12 +2,12 @@
 dependencies:
   - src: geerlingguy.docker
     version: "7.1.0"
-    when: docker_install
+    when: docker_install_compose
 
 galaxy_info:
   role_name: external_node
   author: matter-labs
-  description: External node setup
+  description: External Node setup
   license: "license (MIT, APACHE)"
   min_ansible_version: "2.13.9"
   platforms:

tasks/firewall.yml

@@ -49,6 +49,14 @@
     source: "{{ loadbalancer_ip | mandatory }}"
     jump: ACCEPT
 
+- name: Allow consensus port traffic from any IP
+  when: enable_consensus
+  ansible.builtin.iptables:
+    chain: INPUT
+    protocol: tcp
+    destination_port: "{{ consensus_port }}"
+    jump: ACCEPT
+
 - name: Set default policy to DROP
   ansible.builtin.iptables:
     chain: INPUT

tasks/provision.yml

@@ -38,7 +38,7 @@
   when: vars[item] == ""
   with_items: "{{ en_required_variables }}"
 
-- name: Copy main configs
+- name: Create main configs
   ansible.builtin.template:
     src: '{{ item.src }}'
     dest: '{{ item.dest }}'
@@ -51,7 +51,7 @@
     - src: "templates/postgres.env.j2"
       dest: "{{ configuration_directory }}/postgres.env"
 
-- name: Copy restore script
+- name: Create restore script
   register: restore_dump_script
   ansible.builtin.template:
     src: 'templates/restore_dump.sh.j2'
@@ -64,7 +64,7 @@
   when: enable_monitoring and ( vars[item] == "" )
   with_items: "{{ monitoring_required_variables }}"
 
-- name: Copy monitoring configs
+- name: Create monitoring configs
   when: enable_monitoring
   ansible.builtin.template:
     src: '{{ item.src }}'
@@ -76,23 +76,38 @@
     - src: "templates/vmagent-config.yml.j2"
       dest: "{{ configuration_directory }}/vmagent-config.yml"
 
+- name: Create consensus config
+  when: enable_consensus
+  ansible.builtin.template:
+    src: "templates/consensus_config.yaml.j2"
+    dest: "{{ configuration_directory }}/consensus_config.yaml"
+    mode: '0644'
+
+- name: Decrypt consensus_secrets
+  when: enable_consensus
+  ansible.builtin.copy:
+    src: "{{ consensus_secrets_file }}"
+    dest: "{{ configuration_directory }}/consensus_secrets.yaml"
+    decrypt: true
+    mode: '0600'
+
 - name: Run docker-compose without monitoring
   when: not enable_monitoring
   ansible.builtin.shell:
-    cmd: nohup docker compose -f docker-compose.yaml up -d &
+    cmd: nohup docker compose -f docker-compose.yaml up -d </dev/null >/dev/null 2>&1 &
     chdir: "{{ configuration_directory }}"
   changed_when: false
 
 - name: Run docker-compose with monitoring
   when: enable_monitoring and (not restore_dump_script.changed)
   ansible.builtin.shell:
-    cmd: nohup docker compose -f monitoring.yaml -f docker-compose.yaml up -d &
+    cmd: nohup docker compose -f monitoring.yaml -f docker-compose.yaml up -d </dev/null >/dev/null 2>&1 &
     chdir: "{{ configuration_directory }}"
   changed_when: false
 
 - name: Run docker-compose with monitoring with recreation
   when: enable_monitoring and restore_dump_script.changed
   ansible.builtin.shell:
-    cmd: nohup docker compose -f monitoring.yaml -f docker-compose.yaml up -d --force-recreate &
+    cmd: nohup docker compose -f monitoring.yaml -f docker-compose.yaml up -d --force-recreate </dev/null >/dev/null 2>&1 &
     chdir: "{{ configuration_directory }}"
   changed_when: false

templates/consensus_config.yaml.j2

@@ -0,0 +1,9 @@
+server_addr: '0.0.0.0:3054'
+public_addr: '{{ ansible_default_ipv4.address }}:{{ consensus_port }}'
+max_payload_size: 5000000
+gossip_dynamic_inbound_limit: 100
+gossip_static_outbound:
+{% for item in consensus_outbound %}
+  - key: {{ item.key }}
+    addr: {{ item.addr }}
+{% endfor %}

templates/docker-compose.yaml.j2

@@ -8,10 +8,13 @@ services:
       - "--log.level=INFO"
       - "--providers.docker=true"
       - "--providers.docker.exposedbydefault=false"
-      - "--entrypoints.web.address=:80"
-      - "--entrypoints.external_node_health.address=:3080"
+      - "--entryPoints.web.address=:80"
+      - "--entryPoints.external_node_health.address=:3080"
+{% if enable_consensus %}
+      - "--entryPoints.external_node_consensus.address=:{{ consensus_port }}"
+{% endif %}
 {% if enable_tls %}
-      - "--entrypoints.websecure.address=:443"
+      - "--entryPoints.websecure.address=:443"
       - "--certificatesresolvers.en_resolver.acme.tlschallenge=true"
       - "--certificatesresolvers.en_resolver.acme.storage=/letsencrypt/acme.json"
       - "--certificatesresolvers.myresolver.acme.email={{ acme_email }}"
@@ -47,7 +50,11 @@ services:
       - {{ argument }}
 {% endfor %}
   external_node:
+{% if not external_node_raw_docker_tag %}
     image: "matterlabs/external-node:v{{ external_node_version }}"
+{% else %}
+    image: "matterlabs/external-node:{{ external_node_raw_docker_tag }}"
+{% endif %}
     restart: unless-stopped
     depends_on:
       postgres:
@@ -72,12 +79,20 @@ services:
 {% if enable_basic_auth %}
       - "traefik.http.routers.external_node_main.middlewares=external_node_auth"
       - "traefik.http.middlewares.external_node_auth.basicauth.users={{ basic_auth_secret }}"
+{% endif %}
+{% if enable_consensus %}
+      - "traefik.tcp.services.external_node_consensus.loadbalancer.server.port={{ consensus_port }}"
+      - "traefik.tcp.routers.external_node_consensus.entrypoints=external_node_consensus"
+      - "traefik.tcp.routers.external_node_consensus.service=external_node_consensus"
 {% endif %}
     expose:
       - {{ rpc_http_port }}
       - {{ rpc_ws_port }}
       - {{ healthcheck_port }}
       - {{ metrics_port }}
+{% if enable_consensus %}
+      - {{ consensus_port }}
+{% endif %}
     environment:
       ZKSYNC_HOME: "/"
       EN_STATE_CACHE_PATH: /db/state_keeper
@@ -90,7 +105,12 @@ services:
       CHAIN_STATE_KEEPER_VALIDATION_COMPUTATIONAL_GAS_LIMIT: 2000000
       DATABASE_POOL_SIZE: 200
       EN_MAX_BLOCKS_PER_TREE_BATCH: 200
-      RUST_LOG: zksync_core=debug,zksync_dal=info,zksync_eth_client=info,zksync_merkle_tree=info,zksync_storage=info,zksync_state=debug,zksync_types=info,vm=info,zksync_external_node=info,zksync_utils=debug
+      MISC_LOG_FORMAT: json
+      RUST_LOG: {{ rust_log }}
+{% if enable_consensus %}
+      EN_CONSENSUS_CONFIG_PATH: /etc/consensus_config.yaml
+      EN_CONSENSUS_SECRETS_PATH: /run/secrets/consensus_secrets
+{% endif %}
     healthcheck:
       test: [ "CMD", "curl", "-f", "http://localhost:{{ healthcheck_port }}/health" ]
       interval: 1m
@@ -99,6 +119,24 @@ services:
       start_period: 1m
     volumes:
       - "{{ storage_directory }}/db:/db"
+{% if enable_consensus %}
+      - "{{ configuration_directory }}/consensus_config.yaml:/etc/consensus_config.yaml"
+{% endif %}
     env_file:
       - "external_node.env"
       - "postgres.env"
+    command:
+{% if enable_snapshots_recovery %}
+      - --enable-snapshots-recovery
+{% endif %}
+{% if enable_consensus %}
+      - --enable-consensus
+    secrets:
+      - consensus_secrets
+{% endif %}
+
+{% if enable_consensus %}
+secrets:
+  consensus_secrets:
+    file: consensus_secrets.yaml
+{% endif %}

templates/external_node.env.j2

@@ -2,4 +2,16 @@ EN_ETH_CLIENT_URL="{{ eth_l1_url | mandatory }}"
 EN_MAIN_NODE_URL="{{ main_node_url | mandatory }}"
 EN_L2_CHAIN_ID="{{ l2_chain_id | mandatory }}"
 EN_L1_CHAIN_ID="{{ l1_chain_id | mandatory }}"
+{% if enable_snapshots_recovery %}
+EN_SNAPSHOTS_RECOVERY_ENABLED="true"
+EN_SNAPSHOTS_OBJECT_STORE_MODE="GCSAnonymousReadOnly"
+EN_SNAPSHOTS_OBJECT_STORE_BUCKET_BASE_URL="{{ snapshots_bucket_base_url | mandatory }}"
+{% endif %}
+
 DATABASE_URL="postgres://{{ database_username | mandatory }}:{{ database_password | mandatory }}@postgres/{{ database_name | mandatory }}"
+
+{% if additional_env_vars is defined and additional_env_vars|length > 0 %}
+{% for env_var in additional_env_vars %}
+{{ env_var.name }}="{{ env_var.value }}"
+{% endfor %}
+{% endif %}

templates/restore_dump.sh.j2

@@ -3,7 +3,7 @@ set -e
 
 {% if force_pg_restore %}
 pg_restore --clean --exit-on-error -j $(nproc --all) -d postgres -U $POSTGRES_USER --no-owner --no-privileges --disable-triggers --create /pg_backups/external_node_latest.pgdump
-{% else %}
+{% elif not enable_snapshots_recovery %}
 if psql -U $POSTGRES_USER -d postgres -lqt | cut -d \| -f 1 | grep -qw "{{ database_name }}"; then
     echo "Database already exists"
 else

templates/vmagent-config.yml.j2

@@ -33,7 +33,8 @@ scrape_configs:
   - job_name: traefik
     static_configs:
       - targets:
-          - "traefik:8080"
+          # traefik uses network host, so docker DNS wouldn't work.
+          - "127.0.0.1:8080"
     relabel_configs:
       - source_labels: [instance]
         target_label: instance