Merge pull request #12 from matter-labs/snapshots-recovery

feat: Support for snapshots recovery and a simple example how to run an EN using this repo

.ansible-lint

@@ -2,3 +2,6 @@ skip_list:
   - 'yaml'
   - 'risky-shell-pipe'
   - 'role-name'
+
+exclude_paths:
+  - example_playbooks

README.md

@@ -14,7 +14,9 @@ This role has been tested on:
 
 ## Usage
 
-Minimal required variables that has to be set:
+For a very simple minimal working example, see example_playbooks directory 
+
+Minimal required variables that have to be set:
 
 ```yaml
 database_name: ""
@@ -87,6 +89,7 @@ Basic auth secret can be generated by `htpasswd` and `sed` for interpolation:
 `ansible-galaxy collection install community.general`
 
 2. Prepare the latest database backup on your host. you can download it from our public GCS buckets:
+Skip this step if you are recovering from a snapshot!
 
 * [Era Mainnet latest dump](https://storage.googleapis.com/zksync-era-mainnet-external-node-backups/external_node_latest.pgdump)
 * [Era Sepolia Testnet latest dump](https://storage.googleapis.com/zksync-era-boojnet-external-node-snapshots/external_node_latest.pgdump)
@@ -95,6 +98,7 @@ Basic auth secret can be generated by `htpasswd` and `sed` for interpolation:
 Downloaded dump file should be placed into `{{ storage_directory }}/pg_backups` directory (`/usr/src/en/pg_backups` by default)
 
 3. **OPTIONAL**: If you already have running node, you can copy its tree and state directory to a new host's `{{ storage_directory }}/db`. (`/usr/src/en/db` by default)
+Skip this step if you are recovering from a snapshot!
 
 **Keep in mind that tree and state should be older than PostgreSQL database backup.**
 
@@ -110,6 +114,14 @@ vm_auth_password
 
 5. Connect to your host, and see status of `postgres` container. It can take a lot of time before PostgreSQL database backup will be restored (hours to days, depending on your disk throughput and IOPS), after which PostgreSQL server will be ready for use. Once `postgres` becomes "healthy", `external_node` runs automatically.
 
+## Snapshots Recovery
+
+example config enabling recovery from a snapshot
+```yaml
+- enable_snapshots_recovery: true
+- snapshots_bucket_base_url: "zksync-era-mainnet-external-node-snapshots"
+```
+
 ## Example Playbook
 
 ```yaml
@@ -125,7 +137,6 @@ vm_auth_password
     l2_chain_id: "324"
     l1_chain_id: "1"
     enable_tls: false
-    partner_id: matterlabs
   vars_files:
     - secrets/mainnet_secrets.yml
   roles:

defaults/main.yml

@@ -68,6 +68,10 @@ basic_auth_secret: ""
 # Force restore pg database
 force_pg_restore: false
 
+# Use a snapshot to recover
+enable_snapshots_recovery: false
+snapshots_bucket_base_url: ""
+
 # External node and database options
 database_name: ""
 database_username: ""

example_playbooks/mainnet_with_snapshots_recovery/README.md

@@ -0,0 +1,23 @@
+# Mainnet Snapshots Recovery playbook
+
+This directory is simple example how to set up EN using this role. It comes with snapshots recovery enabled by default.\
+**Note that for simplicity it's using postgres database
+with a very unsecure password and the EN is just started on the same machine**
+
+To run this playbook, first install dependencies
+
+```shell
+ ansible-galaxy install -r requirements.yml
+ ``` 
+
+and then you can run the playbook using
+
+```shell
+ ansible-playbook playbook.yml -i hosts.ini -K
+ ``` 
+
+To see logs you can use
+
+```shell
+ docker logs en-external_node-1 
+ ``` 

example_playbooks/mainnet_with_snapshots_recovery/hosts.ini

@@ -0,0 +1,2 @@
+[local]
+localhost ansible_connection=local

example_playbooks/mainnet_with_snapshots_recovery/playbook.yml

@@ -0,0 +1,16 @@
+---
+- hosts: all
+  become: true
+  vars:
+    database_name: "zksync_ext_node_mainnet"
+    database_username: "postgres"
+    database_password: "notsecurepassword"
+    eth_l1_url: "https://ethereum-rpc.publicnode.com"
+    main_node_url: "https://zksync2-mainnet.zksync.io"
+    l1_chain_id: "1"
+    l2_chain_id: "324"
+    enable_snapshots_recovery: true
+    snapshots_bucket_base_url: "zksync-era-mainnet-external-node-snapshots"
+
+  roles:
+    - external_node

example_playbooks/mainnet_with_snapshots_recovery/requirements.yml

@@ -0,0 +1,12 @@
+---
+roles:
+  - name: geerlingguy.docker
+    src: https://github.com/geerlingguy/ansible-role-docker
+    version: "7.1.0"
+  - name: external_node
+    src: https://github.com/matter-labs/ansible-en-role
+    version: "v3.0.0"
+
+collections:
+  - name: community.general
+    version: 8.4.0

meta/main.yml

@@ -2,7 +2,7 @@
 dependencies:
   - src: geerlingguy.docker
     version: "7.1.0"
-    when: docker_install
+    when: docker_install_compose
 
 galaxy_info:
   role_name: external_node

templates/docker-compose.yaml.j2

@@ -47,7 +47,11 @@ services:
       - {{ argument }}
 {% endfor %}
   external_node:
+{% if not external_node_raw_docker_tag %}
     image: "matterlabs/external-node:v{{ external_node_version }}"
+{% else %}
+    image: "matterlabs/external-node:{{ external_node_raw_docker_tag }}"
+{% endif %}
     restart: unless-stopped
     depends_on:
       postgres:
@@ -103,3 +107,7 @@ services:
     env_file:
       - "external_node.env"
       - "postgres.env"
+{% if enable_snapshots_recovery %}
+    command:
+      - --enable-snapshots-recovery
+{% endif %}

templates/external_node.env.j2

@@ -2,4 +2,10 @@ EN_ETH_CLIENT_URL="{{ eth_l1_url | mandatory }}"
 EN_MAIN_NODE_URL="{{ main_node_url | mandatory }}"
 EN_L2_CHAIN_ID="{{ l2_chain_id | mandatory }}"
 EN_L1_CHAIN_ID="{{ l1_chain_id | mandatory }}"
+{% if enable_snapshots_recovery %}
+EN_SNAPSHOTS_RECOVERY_ENABLED="true"
+EN_SNAPSHOTS_OBJECT_STORE_MODE="GCSAnonymousReadOnly"
+EN_SNAPSHOTS_OBJECT_STORE_BUCKET_BASE_URL="{{ snapshots_bucket_base_url | mandatory }}"
+{% endif %}
+
 DATABASE_URL="postgres://{{ database_username | mandatory }}:{{ database_password | mandatory }}@postgres/{{ database_name | mandatory }}"

templates/restore_dump.sh.j2

@@ -3,7 +3,7 @@ set -e
 
 {% if force_pg_restore %}
 pg_restore --clean --exit-on-error -j $(nproc --all) -d postgres -U $POSTGRES_USER --no-owner --no-privileges --disable-triggers --create /pg_backups/external_node_latest.pgdump
-{% else %}
+{% elif not enable_snapshots_recovery %}
 if psql -U $POSTGRES_USER -d postgres -lqt | cut -d \| -f 1 | grep -qw "{{ database_name }}"; then
     echo "Database already exists"
 else