Merge pull request #22 from matter-labs/betterdetach

fix: Better handle detached docker command

.ansible-lint

@@ -2,3 +2,6 @@ skip_list:
   - 'yaml'
   - 'risky-shell-pipe'
   - 'role-name'
+
+exclude_paths:
+  - example_playbooks

.github/ISSUE_TEMPLATE/bug_report.md

@@ -27,9 +27,10 @@ Describe what actually happened.
 #### 🖥️ Environment
 
 Any relevant environment details like:
+
 * Ansible version
 * Operating system
-* External node version
+* External Node version
 
 #### 📋 Additional Context
 

CONTRIBUTING.md

@@ -5,7 +5,7 @@ sovereignty! We welcome contributions from anyone on the internet, and are grate
 
 ## Ways to contribute
 
-There are many ways to contribute to the external node role:
+There are many ways to contribute to the External Node role:
 
 1. Open issues: if you find a bug, have something you believe needs to be fixed, or have an idea for a feature, please
    open an issue.

README.md

@@ -14,7 +14,9 @@ This role has been tested on:
 
 ## Usage
 
-Minimal required variables that has to be set:
+For a very simple minimal working example, see example_playbooks directory
+
+Minimal required variables that have to be set:
 
 ```yaml
 database_name: ""
@@ -26,6 +28,15 @@ l1_chain_id: ""
 l2_chain_id: ""
 ```
 
+Additional arbitrary environment variables can be passed to External Node container:
+
+```yaml
+additional_env_vars:
+  - { name: "EN_ADDITIONAL_VAR1", value: "Value1" }
+  - { name: "EN_ADDITIONAL_VAR2", value: "Value2" }
+  - { name: "EN_ADDITIONAL_VAR3", value: "Value3" }
+```
+
 Please refer to [External Node docs](https://github.com/matter-labs/zksync-era/tree/main/docs/guides/external-node/prepared_configs) to find values for different zkSync Era chains.
 
 If you want to use monitoring (which we highly recommend), you have to change these variables:
@@ -73,7 +84,6 @@ We recommend using pgtune [online](https://pgtune.leopard.in.ua/) or [self-hoste
 If you want to use basic auth for inbound requests, you have to change next variables:
 
 ```yaml
-# Enable basic auth for external node
 enable_basic_auth: true
 basic_auth_secret: "htpasswd-generated-secret"
 ```
@@ -87,20 +97,21 @@ Basic auth secret can be generated by `htpasswd` and `sed` for interpolation:
 `ansible-galaxy collection install community.general`
 
 2. Prepare the latest database backup on your host. you can download it from our public GCS buckets:
+Skip this step if you are recovering from a snapshot!
 
 * [Era Mainnet latest dump](https://storage.googleapis.com/zksync-era-mainnet-external-node-backups/external_node_latest.pgdump)
 * [Era Sepolia Testnet latest dump](https://storage.googleapis.com/zksync-era-boojnet-external-node-snapshots/external_node_latest.pgdump)
-* [Era Goerli Testnet latest dump](https://storage.googleapis.com/zksync-era-testnet-external-node-backups/external_node_latest.pgdump)
 
 Downloaded dump file should be placed into `{{ storage_directory }}/pg_backups` directory (`/usr/src/en/pg_backups` by default)
 
 3. **OPTIONAL**: If you already have running node, you can copy its tree and state directory to a new host's `{{ storage_directory }}/db`. (`/usr/src/en/db` by default)
+Skip this step if you are recovering from a snapshot!
 
 **Keep in mind that tree and state should be older than PostgreSQL database backup.**
 
 4. Run ansible-playbook using this role. We recommend encrypting next variables with ansible-vault or some another way:
 
-```
+```yaml
 database_username
 database_password
 eth_l1_url
@@ -110,6 +121,20 @@ vm_auth_password
 
 5. Connect to your host, and see status of `postgres` container. It can take a lot of time before PostgreSQL database backup will be restored (hours to days, depending on your disk throughput and IOPS), after which PostgreSQL server will be ready for use. Once `postgres` becomes "healthy", `external_node` runs automatically.
 
+## Snapshots Recovery
+
+Example config enabling recovery from a snapshot:
+
+```yaml
+- enable_snapshots_recovery: true
+- snapshots_bucket_base_url: "snapshots-bucket-name"
+```
+
+Snapshot buckets:
+
+* Era Mainnet: `zksync-era-mainnet-external-node-snapshots`
+* Era Sepolia Testnet: `zksync-era-boojnet-external-node-snapshots`
+
 ## Example Playbook
 
 ```yaml
@@ -125,7 +150,6 @@ vm_auth_password
     l2_chain_id: "324"
     l1_chain_id: "1"
     enable_tls: false
-    partner_id: matterlabs
   vars_files:
     - secrets/mainnet_secrets.yml
   roles:

defaults/main.yml

@@ -8,11 +8,12 @@ docker_install_compose: true
 docker_version: "25.0.3"
 docker_compose_version: "v2.23.0"
 
-# Versions of external node and 3rd party components
+# Versions of External Node and 3rd party components
 traefik_version: 2.11
 postgres_version: 14
-external_node_version: 21.0.2
-vmagent_version: 1.95.1
+external_node_version: 24.6.0
+external_node_raw_docker_tag: ""
+vmagent_version: 1.100.1
 cadvisor_version: 0.47.2
 postgres_exporter_version: 0.15.0
 
@@ -61,14 +62,18 @@ enable_tls: false
 acme_email: ""
 domain_name: ""
 
-# Enable basic auth for external node
+# Enable basic auth for External Node
 enable_basic_auth: false
 basic_auth_secret: ""
 
 # Force restore pg database
 force_pg_restore: false
 
-# External node and database options
+# Use a snapshot to recover
+enable_snapshots_recovery: false
+snapshots_bucket_base_url: ""
+
+# External Node and database options
 database_name: ""
 database_username: ""
 database_password: ""
@@ -80,7 +85,10 @@ rpc_http_port: 3060
 rpc_ws_port: 3061
 healthcheck_port: 3081
 metrics_port: 3082
-log_verbosity: info
+rust_log: zksync_external_node=info,zksync_core=info,zksync_core::sync_layer=info,zksync_server=info,zksync_prover=info,zksync_contract_verifier=info,zksync_dal=info,zksync_eth_client=info,zksync_storage=info,zksync_db_manager=info,zksync_merkle_tree=info,zksync_state=info,zksync_utils=info,zksync_types=info,loadnext=info,dev_ticker=info,vm=info,block_sizes_test=info,zksync_verification_key_generator_and_server=info,zksync_object_store=info,setup_key_generator_and_server=info,zksync_circuit_synthesizer=info,zksync_queued_job_processor=info,zksync_health_check=info
+
+# Additional env vars passed to External Node
+additional_env_vars: []
 
 # Monitoring options section
 enable_monitoring: false

example_playbooks/mainnet_with_snapshots_recovery/README.md

@@ -0,0 +1,23 @@
+# Mainnet Snapshots Recovery playbook
+
+This directory is simple example how to set up EN using this role. It comes with snapshots recovery enabled by default.\
+**Note that for simplicity it's using postgres database
+with a very unsecure password and the EN is just started on the same machine**
+
+To run this playbook, first install dependencies
+
+```shell
+ ansible-galaxy install -r requirements.yml
+ ``` 
+
+and then you can run the playbook using
+
+```shell
+ ansible-playbook playbook.yml -i hosts.ini -K
+ ``` 
+
+To see logs you can use
+
+```shell
+ docker logs en-external_node-1 
+ ``` 

example_playbooks/mainnet_with_snapshots_recovery/hosts.ini

@@ -0,0 +1,2 @@
+[local]
+localhost ansible_connection=local

example_playbooks/mainnet_with_snapshots_recovery/playbook.yml

@@ -0,0 +1,16 @@
+---
+- hosts: all
+  become: true
+  vars:
+    database_name: "zksync_ext_node_mainnet"
+    database_username: "postgres"
+    database_password: "notsecurepassword"
+    eth_l1_url: "https://ethereum-rpc.publicnode.com"
+    main_node_url: "https://zksync2-mainnet.zksync.io"
+    l1_chain_id: "1"
+    l2_chain_id: "324"
+    enable_snapshots_recovery: true
+    snapshots_bucket_base_url: "zksync-era-mainnet-external-node-snapshots"
+
+  roles:
+    - external_node

example_playbooks/mainnet_with_snapshots_recovery/requirements.yml

@@ -0,0 +1,12 @@
+---
+roles:
+  - name: geerlingguy.docker
+    src: https://github.com/geerlingguy/ansible-role-docker
+    version: "7.1.0"
+  - name: external_node
+    src: https://github.com/matter-labs/ansible-en-role
+    version: "v3.3.0"
+
+collections:
+  - name: community.general
+    version: 8.4.0

meta/main.yml

@@ -2,12 +2,12 @@
 dependencies:
   - src: geerlingguy.docker
     version: "7.1.0"
-    when: docker_install
+    when: docker_install_compose
 
 galaxy_info:
   role_name: external_node
   author: matter-labs
-  description: External node setup
+  description: External Node setup
   license: "license (MIT, APACHE)"
   min_ansible_version: "2.13.9"
   platforms:

tasks/provision.yml

@@ -79,20 +79,20 @@
 - name: Run docker-compose without monitoring
   when: not enable_monitoring
   ansible.builtin.shell:
-    cmd: nohup docker compose -f docker-compose.yaml up -d &
+    cmd: nohup docker compose -f docker-compose.yaml up -d </dev/null >/dev/null 2>&1 &
     chdir: "{{ configuration_directory }}"
   changed_when: false
 
 - name: Run docker-compose with monitoring
   when: enable_monitoring and (not restore_dump_script.changed)
   ansible.builtin.shell:
-    cmd: nohup docker compose -f monitoring.yaml -f docker-compose.yaml up -d &
+    cmd: nohup docker compose -f monitoring.yaml -f docker-compose.yaml up -d </dev/null >/dev/null 2>&1 &
     chdir: "{{ configuration_directory }}"
   changed_when: false
 
 - name: Run docker-compose with monitoring with recreation
   when: enable_monitoring and restore_dump_script.changed
   ansible.builtin.shell:
-    cmd: nohup docker compose -f monitoring.yaml -f docker-compose.yaml up -d --force-recreate &
+    cmd: nohup docker compose -f monitoring.yaml -f docker-compose.yaml up -d --force-recreate </dev/null >/dev/null 2>&1 &
     chdir: "{{ configuration_directory }}"
   changed_when: false

templates/docker-compose.yaml.j2

@@ -47,7 +47,11 @@ services:
       - {{ argument }}
 {% endfor %}
   external_node:
+{% if not external_node_raw_docker_tag %}
     image: "matterlabs/external-node:v{{ external_node_version }}"
+{% else %}
+    image: "matterlabs/external-node:{{ external_node_raw_docker_tag }}"
+{% endif %}
     restart: unless-stopped
     depends_on:
       postgres:
@@ -91,7 +95,7 @@ services:
       DATABASE_POOL_SIZE: 200
       EN_MAX_BLOCKS_PER_TREE_BATCH: 200
       MISC_LOG_FORMAT: json
-      RUST_LOG: zksync_external_node={{ log_verbosity }}
+      RUST_LOG: {{ rust_log }}
     healthcheck:
       test: [ "CMD", "curl", "-f", "http://localhost:{{ healthcheck_port }}/health" ]
       interval: 1m
@@ -103,3 +107,7 @@ services:
     env_file:
       - "external_node.env"
       - "postgres.env"
+{% if enable_snapshots_recovery %}
+    command:
+      - --enable-snapshots-recovery
+{% endif %}

templates/external_node.env.j2

@@ -2,4 +2,16 @@ EN_ETH_CLIENT_URL="{{ eth_l1_url | mandatory }}"
 EN_MAIN_NODE_URL="{{ main_node_url | mandatory }}"
 EN_L2_CHAIN_ID="{{ l2_chain_id | mandatory }}"
 EN_L1_CHAIN_ID="{{ l1_chain_id | mandatory }}"
+{% if enable_snapshots_recovery %}
+EN_SNAPSHOTS_RECOVERY_ENABLED="true"
+EN_SNAPSHOTS_OBJECT_STORE_MODE="GCSAnonymousReadOnly"
+EN_SNAPSHOTS_OBJECT_STORE_BUCKET_BASE_URL="{{ snapshots_bucket_base_url | mandatory }}"
+{% endif %}
+
 DATABASE_URL="postgres://{{ database_username | mandatory }}:{{ database_password | mandatory }}@postgres/{{ database_name | mandatory }}"
+
+{% if additional_env_vars is defined and additional_env_vars|length > 0 %}
+{% for env_var in additional_env_vars %}
+{{ env_var.name }}="{{ env_var.value }}"
+{% endfor %}
+{% endif %}

templates/restore_dump.sh.j2

@@ -3,7 +3,7 @@ set -e
 
 {% if force_pg_restore %}
 pg_restore --clean --exit-on-error -j $(nproc --all) -d postgres -U $POSTGRES_USER --no-owner --no-privileges --disable-triggers --create /pg_backups/external_node_latest.pgdump
-{% else %}
+{% elif not enable_snapshots_recovery %}
 if psql -U $POSTGRES_USER -d postgres -lqt | cut -d \| -f 1 | grep -qw "{{ database_name }}"; then
     echo "Database already exists"
 else

templates/vmagent-config.yml.j2

@@ -33,7 +33,8 @@ scrape_configs:
   - job_name: traefik
     static_configs:
       - targets:
-          - "traefik:8080"
+          # traefik uses network host, so docker DNS wouldn't work.
+          - "127.0.0.1:8080"
     relabel_configs:
       - source_labels: [instance]
         target_label: instance