Merge pull request #24 from matter-labs/ya-zkd-1817-upgrade-external-nodes-on-hetzner

feat: Add consensus support

README.md

@@ -100,8 +100,7 @@ Basic auth secret can be generated by `htpasswd` and `sed` for interpolation:
 Skip this step if you are recovering from a snapshot!
 
 * [Era Mainnet latest dump](https://storage.googleapis.com/zksync-era-mainnet-external-node-backups/external_node_latest.pgdump)
-* [Era Sepolia Testnet latest dump](https://storage.googleapis.com/zksync-era-boojnet-external-node-snapshots/external_node_latest.pgdump)
-* [Era Goerli Testnet latest dump](https://storage.googleapis.com/zksync-era-testnet-external-node-backups/external_node_latest.pgdump)
+* [Era Sepolia Testnet latest dump](https://storage.googleapis.com/zksync-era-testnet-sepolia-external-node-backups/external_node_latest.pgdump)
 
 Downloaded dump file should be placed into `{{ storage_directory }}/pg_backups` directory (`/usr/src/en/pg_backups` by default)
 
@@ -124,13 +123,18 @@ vm_auth_password
 
 ## Snapshots Recovery
 
-example config enabling recovery from a snapshot
+Example config enabling recovery from a snapshot:
 
 ```yaml
 - enable_snapshots_recovery: true
-- snapshots_bucket_base_url: "zksync-era-mainnet-external-node-snapshots"
+- snapshots_bucket_base_url: "snapshots-bucket-name"
 ```
 
+Snapshot buckets:
+
+* Era Mainnet: `zksync-era-mainnet-external-node-snapshots`
+* Era Sepolia Testnet: `zksync-era-boojnet-external-node-snapshots`
+
 ## Example Playbook
 
 ```yaml

defaults/main.yml

@@ -11,9 +11,9 @@ docker_compose_version: "v2.23.0"
 # Versions of External Node and 3rd party components
 traefik_version: 2.11
 postgres_version: 14
-external_node_version: 21.0.2
+external_node_version: 24.16.0
 external_node_raw_docker_tag: ""
-vmagent_version: 1.95.1
+vmagent_version: 1.100.1
 cadvisor_version: 0.47.2
 postgres_exporter_version: 0.15.0
 
@@ -73,6 +73,12 @@ force_pg_restore: false
 enable_snapshots_recovery: false
 snapshots_bucket_base_url: ""
 
+# https://github.com/matter-labs/zksync-era/blob/main/docs/guides/external-node/09_decentralization.md
+enable_consensus: false
+consensus_secrets_file: ""
+consensus_port: 3054
+consensus_outbound: []
+
 # External Node and database options
 database_name: ""
 database_username: ""

example_playbooks/mainnet_with_snapshots_recovery/requirements.yml

@@ -5,7 +5,7 @@ roles:
     version: "7.1.0"
   - name: external_node
     src: https://github.com/matter-labs/ansible-en-role
-    version: "v3.0.0"
+    version: "v3.3.0"
 
 collections:
   - name: community.general

tasks/provision.yml

@@ -76,23 +76,38 @@
     - src: "templates/vmagent-config.yml.j2"
       dest: "{{ configuration_directory }}/vmagent-config.yml"
 
+- name: Copy main configs
+  when: enable_consensus
+  ansible.builtin.template:
+    src: "templates/consensus_config.yaml.j2"
+    dest: "{{ configuration_directory }}/consensus_config.yaml"
+    mode: '0644'
+
+- name: Decrypt consensus_secrets
+  when: enable_consensus
+  ansible.builtin.copy:
+    src: "{{ consensus_secrets_file }}"
+    dest: "{{ configuration_directory }}/consensus_secrets.yaml"
+    decrypt: yes
+    mode: '0600'
+
 - name: Run docker-compose without monitoring
   when: not enable_monitoring
   ansible.builtin.shell:
-    cmd: nohup docker compose -f docker-compose.yaml up -d &
+    cmd: nohup docker compose -f docker-compose.yaml up -d </dev/null >/dev/null 2>&1 &
     chdir: "{{ configuration_directory }}"
   changed_when: false
 
 - name: Run docker-compose with monitoring
   when: enable_monitoring and (not restore_dump_script.changed)
   ansible.builtin.shell:
-    cmd: nohup docker compose -f monitoring.yaml -f docker-compose.yaml up -d &
+    cmd: nohup docker compose -f monitoring.yaml -f docker-compose.yaml up -d </dev/null >/dev/null 2>&1 &
     chdir: "{{ configuration_directory }}"
   changed_when: false
 
 - name: Run docker-compose with monitoring with recreation
   when: enable_monitoring and restore_dump_script.changed
   ansible.builtin.shell:
-    cmd: nohup docker compose -f monitoring.yaml -f docker-compose.yaml up -d --force-recreate &
+    cmd: nohup docker compose -f monitoring.yaml -f docker-compose.yaml up -d --force-recreate </dev/null >/dev/null 2>&1 &
     chdir: "{{ configuration_directory }}"
   changed_when: false

templates/consensus_config.yaml.j2

@@ -0,0 +1,9 @@
+server_addr: '0.0.0.0:3054'
+public_addr: '{{ ansible_default_ipv4.address }}:{{ consensus_port }}'
+max_payload_size: 5000000
+gossip_dynamic_inbound_limit: 100
+gossip_static_outbound:
+{% for item in consensus_outbound %}
+  - key: {{ item.key }}
+    addr: {{ item.addr }}
+{% endfor %}

templates/docker-compose.yaml.j2

@@ -82,6 +82,9 @@ services:
       - {{ rpc_ws_port }}
       - {{ healthcheck_port }}
       - {{ metrics_port }}
+{% if enable_consensus %}
+      - {{ consensus_port }}
+{% endif %}
     environment:
       ZKSYNC_HOME: "/"
       EN_STATE_CACHE_PATH: /db/state_keeper
@@ -96,6 +99,10 @@ services:
       EN_MAX_BLOCKS_PER_TREE_BATCH: 200
       MISC_LOG_FORMAT: json
       RUST_LOG: {{ rust_log }}
+{% if enable_consensus %}
+      EN_CONSENSUS_CONFIG_PATH: /etc/consensus_config.yaml
+      EN_CONSENSUS_SECRETS_PATH: /run/secrets/consensus_secrets.yaml
+{% endif %}
     healthcheck:
       test: [ "CMD", "curl", "-f", "http://localhost:{{ healthcheck_port }}/health" ]
       interval: 1m
@@ -104,10 +111,24 @@ services:
       start_period: 1m
     volumes:
       - "{{ storage_directory }}/db:/db"
+{% if enable_consensus %}
+      - "consensus_config.yaml:/etc/consensus_config.yaml"
+{% endif %}
     env_file:
       - "external_node.env"
       - "postgres.env"
-{% if enable_snapshots_recovery %}
     command:
+{% if enable_snapshots_recovery %}
       - --enable-snapshots-recovery
 {% endif %}
+{% if enable_consensus %}
+      - --enable-consensus
+    secrets:
+      - consensus_secrets
+{% endif %}
+
+{% if enable_consensus %}
+secrets:
+  consensus_secrets:
+    file: consensus_secrets.yaml
+{% endif %}

templates/vmagent-config.yml.j2

@@ -33,7 +33,8 @@ scrape_configs:
   - job_name: traefik
     static_configs:
       - targets:
-          - "traefik:8080"
+          # traefik uses network host, so docker DNS wouldn't work.
+          - "127.0.0.1:8080"
     relabel_configs:
       - source_labels: [instance]
         target_label: instance