From 281068a7529fd4f008d3acb50ad843bf37c23226 Mon Sep 17 00:00:00 2001 From: tomg10 Date: Thu, 21 Mar 2024 16:01:41 +0100 Subject: [PATCH 1/4] support for snapshots recovery --- README.md | 11 ++++++++++- defaults/main.yml | 4 ++++ templates/external_node.env.j2 | 6 ++++++ templates/restore_dump.sh.j2 | 2 +- 4 files changed, 21 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 1e09f9c..4dc73fe 100644 --- a/README.md +++ b/README.md @@ -87,6 +87,7 @@ Basic auth secret can be generated by `htpasswd` and `sed` for interpolation: `ansible-galaxy collection install community.general` 2. Prepare the latest database backup on your host. you can download it from our public GCS buckets: +Skip this step if you are recovering from a snapshot! * [Era Mainnet latest dump](https://storage.googleapis.com/zksync-era-mainnet-external-node-backups/external_node_latest.pgdump) * [Era Sepolia Testnet latest dump](https://storage.googleapis.com/zksync-era-boojnet-external-node-snapshots/external_node_latest.pgdump) @@ -95,6 +96,7 @@ Basic auth secret can be generated by `htpasswd` and `sed` for interpolation: Downloaded dump file should be placed into `{{ storage_directory }}/pg_backups` directory (`/usr/src/en/pg_backups` by default) 3. **OPTIONAL**: If you already have running node, you can copy its tree and state directory to a new host's `{{ storage_directory }}/db`. (`/usr/src/en/db` by default) +Skip this step if you are recovering from a snapshot! **Keep in mind that tree and state should be older than PostgreSQL database backup.** @@ -110,6 +112,14 @@ vm_auth_password 5. Connect to your host, and see status of `postgres` container. It can take a lot of time before PostgreSQL database backup will be restored (hours to days, depending on your disk throughput and IOPS), after which PostgreSQL server will be ready for use. Once `postgres` becomes "healthy", `external_node` runs automatically. +## Snapshots Recovery + +example config enabling recovery from a snapshot +```yaml +- enable_snapshots_recovery: true +- snapshots_bucket_base_url: "zksync-era-mainnet-external-node-snapshots" +``` + ## Example Playbook ```yaml @@ -125,7 +135,6 @@ vm_auth_password l2_chain_id: "324" l1_chain_id: "1" enable_tls: false - partner_id: matterlabs vars_files: - secrets/mainnet_secrets.yml roles: diff --git a/defaults/main.yml b/defaults/main.yml index 464f8a9..58aa4bc 100644 --- a/defaults/main.yml +++ b/defaults/main.yml @@ -68,6 +68,10 @@ basic_auth_secret: "" # Force restore pg database force_pg_restore: false +# Use a snapshot to recover +enable_snapshots_recovery: false +snapshots_bucket_base_url: "" + # External node and database options database_name: "" database_username: "" diff --git a/templates/external_node.env.j2 b/templates/external_node.env.j2 index 3931794..20c5a50 100644 --- a/templates/external_node.env.j2 +++ b/templates/external_node.env.j2 @@ -2,4 +2,10 @@ EN_ETH_CLIENT_URL="{{ eth_l1_url | mandatory }}" EN_MAIN_NODE_URL="{{ main_node_url | mandatory }}" EN_L2_CHAIN_ID="{{ l2_chain_id | mandatory }}" EN_L1_CHAIN_ID="{{ l1_chain_id | mandatory }}" +{% if enable_snapshots_recovery %} +EN_SNAPSHOTS_RECOVERY_ENABLED="true" +EN_SNAPSHOTS_OBJECT_STORE_MODE="GCSAnonymousReadOnly" +EN_SNAPSHOTS_OBJECT_STORE_BUCKET_BASE_URL="{{ snapshots_bucket_base_url | mandatory }}" +{% endif %} + DATABASE_URL="postgres://{{ database_username | mandatory }}:{{ database_password | mandatory }}@postgres/{{ database_name | mandatory }}" diff --git a/templates/restore_dump.sh.j2 b/templates/restore_dump.sh.j2 index 43db63e..6f1295d 100644 --- a/templates/restore_dump.sh.j2 +++ b/templates/restore_dump.sh.j2 @@ -3,7 +3,7 @@ set -e {% if force_pg_restore %} pg_restore --clean --exit-on-error -j $(nproc --all) -d postgres -U $POSTGRES_USER --no-owner --no-privileges --disable-triggers --create /pg_backups/external_node_latest.pgdump -{% else %} +{% elif not enable_snapshots_recovery %} if psql -U $POSTGRES_USER -d postgres -lqt | cut -d \| -f 1 | grep -qw "{{ database_name }}"; then echo "Database already exists" else From 69435131f78931de51b9a9e5709d5ef32f53be95 Mon Sep 17 00:00:00 2001 From: tomg10 Date: Fri, 22 Mar 2024 01:35:00 +0100 Subject: [PATCH 2/4] added an example --- README.md | 4 +++- defaults/main.yml | 4 ---- .../mainnet_with_snapshots_recovery/README.md | 23 +++++++++++++++++++ .../mainnet_with_snapshots_recovery/hosts.ini | 2 ++ .../playbook.yml | 15 ++++++++++++ .../requirements.yml | 13 +++++++++++ meta/main.yml | 2 +- templates/docker-compose.yaml.j2 | 8 +++++++ 8 files changed, 65 insertions(+), 6 deletions(-) create mode 100644 example_playbooks/mainnet_with_snapshots_recovery/README.md create mode 100644 example_playbooks/mainnet_with_snapshots_recovery/hosts.ini create mode 100644 example_playbooks/mainnet_with_snapshots_recovery/playbook.yml create mode 100644 example_playbooks/mainnet_with_snapshots_recovery/requirements.yml diff --git a/README.md b/README.md index 4dc73fe..7551fce 100644 --- a/README.md +++ b/README.md @@ -14,7 +14,9 @@ This role has been tested on: ## Usage -Minimal required variables that has to be set: +For a very simple minimal working example, see example_playbooks directory + +Minimal required variables that have to be set: ```yaml database_name: "" diff --git a/defaults/main.yml b/defaults/main.yml index 58aa4bc..2ae94ba 100644 --- a/defaults/main.yml +++ b/defaults/main.yml @@ -22,10 +22,6 @@ postgres_arguments: - -c - max_connections=256 - -c - - shared_buffers=47616MB - - -c - - effective_cache_size=142848MB - - -c - maintenance_work_mem=2GB - -c - checkpoint_completion_target=0.9 diff --git a/example_playbooks/mainnet_with_snapshots_recovery/README.md b/example_playbooks/mainnet_with_snapshots_recovery/README.md new file mode 100644 index 0000000..50acff3 --- /dev/null +++ b/example_playbooks/mainnet_with_snapshots_recovery/README.md @@ -0,0 +1,23 @@ +# Mainnet Snapshots Recovery playbook + +This directory is simple example how to set up EN using this role. It comes with snapshots recovery enabled by default.\ +**Note that for simplicity it's using postgres database +with a very unsecure password and the EN is just started on the same machine** + +To run this playbook, first install dependencies + +```shell + ansible-galaxy install -r requirements.yml + ``` + +and then you can run the playbook using + +```shell + ansible-playbook playbook.yml -i hosts.ini -K + ``` + +To see logs you can use + +```shell + docker logs en-external_node-1 + ``` diff --git a/example_playbooks/mainnet_with_snapshots_recovery/hosts.ini b/example_playbooks/mainnet_with_snapshots_recovery/hosts.ini new file mode 100644 index 0000000..13cfabe --- /dev/null +++ b/example_playbooks/mainnet_with_snapshots_recovery/hosts.ini @@ -0,0 +1,2 @@ +[local] +localhost ansible_connection=local diff --git a/example_playbooks/mainnet_with_snapshots_recovery/playbook.yml b/example_playbooks/mainnet_with_snapshots_recovery/playbook.yml new file mode 100644 index 0000000..ae36758 --- /dev/null +++ b/example_playbooks/mainnet_with_snapshots_recovery/playbook.yml @@ -0,0 +1,15 @@ +- hosts: all + become: true + vars: + database_name: "zksync_ext_node_mainnet" + database_username: "postgres" + database_password: "notsecurepassword" + eth_l1_url: "https://ethereum-rpc.publicnode.com" + main_node_url: "https://zksync2-mainnet.zksync.io" + l1_chain_id: "1" + l2_chain_id: "324" + enable_snapshots_recovery: true + snapshots_bucket_base_url: "zksync-era-mainnet-external-node-snapshots" + + roles: + - external_node diff --git a/example_playbooks/mainnet_with_snapshots_recovery/requirements.yml b/example_playbooks/mainnet_with_snapshots_recovery/requirements.yml new file mode 100644 index 0000000..71da07e --- /dev/null +++ b/example_playbooks/mainnet_with_snapshots_recovery/requirements.yml @@ -0,0 +1,13 @@ +--- +roles: + - name: geerlingguy.docker + src: https://github.com/geerlingguy/ansible-role-docker + version: "7.1.0" + + - name: external_node + src: https://github.com/matter-labs/ansible-en-role + version: "v3.0.0" + +collections: + - name: community.general + version: 8.4.0 diff --git a/meta/main.yml b/meta/main.yml index f63c924..f940170 100644 --- a/meta/main.yml +++ b/meta/main.yml @@ -2,7 +2,7 @@ dependencies: - src: geerlingguy.docker version: "7.1.0" - when: docker_install + when: docker_install_compose galaxy_info: role_name: external_node diff --git a/templates/docker-compose.yaml.j2 b/templates/docker-compose.yaml.j2 index 1aef1de..cdcb46b 100644 --- a/templates/docker-compose.yaml.j2 +++ b/templates/docker-compose.yaml.j2 @@ -47,7 +47,11 @@ services: - {{ argument }} {% endfor %} external_node: +{% if not external_node_raw_docker_tag %} image: "matterlabs/external-node:v{{ external_node_version }}" +{% else %} + image: "matterlabs/external-node:{{ external_node_raw_docker_tag }}" +{% endif %} restart: unless-stopped depends_on: postgres: @@ -103,3 +107,7 @@ services: env_file: - "external_node.env" - "postgres.env" +{% if enable_snapshots_recovery %} + command: + - --enable-snapshots-recovery +{% endif %} From 5df4c691bbae6e508e2089171fda15e2ee79b354 Mon Sep 17 00:00:00 2001 From: tomg10 Date: Fri, 22 Mar 2024 01:47:26 +0100 Subject: [PATCH 3/4] fix --- .ansible-lint | 3 +++ example_playbooks/mainnet_with_snapshots_recovery/playbook.yml | 1 + .../mainnet_with_snapshots_recovery/requirements.yml | 1 - 3 files changed, 4 insertions(+), 1 deletion(-) diff --git a/.ansible-lint b/.ansible-lint index 2cd8c9f..f4439ae 100644 --- a/.ansible-lint +++ b/.ansible-lint @@ -2,3 +2,6 @@ skip_list: - 'yaml' - 'risky-shell-pipe' - 'role-name' + +exclude_paths: + - example_playbooks diff --git a/example_playbooks/mainnet_with_snapshots_recovery/playbook.yml b/example_playbooks/mainnet_with_snapshots_recovery/playbook.yml index ae36758..1c07779 100644 --- a/example_playbooks/mainnet_with_snapshots_recovery/playbook.yml +++ b/example_playbooks/mainnet_with_snapshots_recovery/playbook.yml @@ -1,3 +1,4 @@ +--- - hosts: all become: true vars: diff --git a/example_playbooks/mainnet_with_snapshots_recovery/requirements.yml b/example_playbooks/mainnet_with_snapshots_recovery/requirements.yml index 71da07e..53819f4 100644 --- a/example_playbooks/mainnet_with_snapshots_recovery/requirements.yml +++ b/example_playbooks/mainnet_with_snapshots_recovery/requirements.yml @@ -3,7 +3,6 @@ roles: - name: geerlingguy.docker src: https://github.com/geerlingguy/ansible-role-docker version: "7.1.0" - - name: external_node src: https://github.com/matter-labs/ansible-en-role version: "v3.0.0" From f0e128860c57c6c0f078c9194d5a42dc20c9fba2 Mon Sep 17 00:00:00 2001 From: tomg10 Date: Fri, 22 Mar 2024 09:55:34 +0100 Subject: [PATCH 4/4] revert --- defaults/main.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/defaults/main.yml b/defaults/main.yml index 2ae94ba..58aa4bc 100644 --- a/defaults/main.yml +++ b/defaults/main.yml @@ -22,6 +22,10 @@ postgres_arguments: - -c - max_connections=256 - -c + - shared_buffers=47616MB + - -c + - effective_cache_size=142848MB + - -c - maintenance_work_mem=2GB - -c - checkpoint_completion_target=0.9