support for snapshots recovery

This commit is contained in:
tomg10
2024-03-21 16:01:41 +01:00
parent 97a71926d7
commit 281068a752
4 changed files with 21 additions and 2 deletions

View File

@ -87,6 +87,7 @@ Basic auth secret can be generated by `htpasswd` and `sed` for interpolation:
`ansible-galaxy collection install community.general` `ansible-galaxy collection install community.general`
2. Prepare the latest database backup on your host. you can download it from our public GCS buckets: 2. Prepare the latest database backup on your host. you can download it from our public GCS buckets:
Skip this step if you are recovering from a snapshot!
* [Era Mainnet latest dump](https://storage.googleapis.com/zksync-era-mainnet-external-node-backups/external_node_latest.pgdump) * [Era Mainnet latest dump](https://storage.googleapis.com/zksync-era-mainnet-external-node-backups/external_node_latest.pgdump)
* [Era Sepolia Testnet latest dump](https://storage.googleapis.com/zksync-era-boojnet-external-node-snapshots/external_node_latest.pgdump) * [Era Sepolia Testnet latest dump](https://storage.googleapis.com/zksync-era-boojnet-external-node-snapshots/external_node_latest.pgdump)
@ -95,6 +96,7 @@ Basic auth secret can be generated by `htpasswd` and `sed` for interpolation:
Downloaded dump file should be placed into `{{ storage_directory }}/pg_backups` directory (`/usr/src/en/pg_backups` by default) Downloaded dump file should be placed into `{{ storage_directory }}/pg_backups` directory (`/usr/src/en/pg_backups` by default)
3. **OPTIONAL**: If you already have running node, you can copy its tree and state directory to a new host's `{{ storage_directory }}/db`. (`/usr/src/en/db` by default) 3. **OPTIONAL**: If you already have running node, you can copy its tree and state directory to a new host's `{{ storage_directory }}/db`. (`/usr/src/en/db` by default)
Skip this step if you are recovering from a snapshot!
**Keep in mind that tree and state should be older than PostgreSQL database backup.** **Keep in mind that tree and state should be older than PostgreSQL database backup.**
@ -110,6 +112,14 @@ vm_auth_password
5. Connect to your host, and see status of `postgres` container. It can take a lot of time before PostgreSQL database backup will be restored (hours to days, depending on your disk throughput and IOPS), after which PostgreSQL server will be ready for use. Once `postgres` becomes "healthy", `external_node` runs automatically. 5. Connect to your host, and see status of `postgres` container. It can take a lot of time before PostgreSQL database backup will be restored (hours to days, depending on your disk throughput and IOPS), after which PostgreSQL server will be ready for use. Once `postgres` becomes "healthy", `external_node` runs automatically.
## Snapshots Recovery
example config enabling recovery from a snapshot
```yaml
- enable_snapshots_recovery: true
- snapshots_bucket_base_url: "zksync-era-mainnet-external-node-snapshots"
```
## Example Playbook ## Example Playbook
```yaml ```yaml
@ -125,7 +135,6 @@ vm_auth_password
l2_chain_id: "324" l2_chain_id: "324"
l1_chain_id: "1" l1_chain_id: "1"
enable_tls: false enable_tls: false
partner_id: matterlabs
vars_files: vars_files:
- secrets/mainnet_secrets.yml - secrets/mainnet_secrets.yml
roles: roles:

View File

@ -68,6 +68,10 @@ basic_auth_secret: ""
# Force restore pg database # Force restore pg database
force_pg_restore: false force_pg_restore: false
# Use a snapshot to recover
enable_snapshots_recovery: false
snapshots_bucket_base_url: ""
# External node and database options # External node and database options
database_name: "" database_name: ""
database_username: "" database_username: ""

View File

@ -2,4 +2,10 @@ EN_ETH_CLIENT_URL="{{ eth_l1_url | mandatory }}"
EN_MAIN_NODE_URL="{{ main_node_url | mandatory }}" EN_MAIN_NODE_URL="{{ main_node_url | mandatory }}"
EN_L2_CHAIN_ID="{{ l2_chain_id | mandatory }}" EN_L2_CHAIN_ID="{{ l2_chain_id | mandatory }}"
EN_L1_CHAIN_ID="{{ l1_chain_id | mandatory }}" EN_L1_CHAIN_ID="{{ l1_chain_id | mandatory }}"
{% if enable_snapshots_recovery %}
EN_SNAPSHOTS_RECOVERY_ENABLED="true"
EN_SNAPSHOTS_OBJECT_STORE_MODE="GCSAnonymousReadOnly"
EN_SNAPSHOTS_OBJECT_STORE_BUCKET_BASE_URL="{{ snapshots_bucket_base_url | mandatory }}"
{% endif %}
DATABASE_URL="postgres://{{ database_username | mandatory }}:{{ database_password | mandatory }}@postgres/{{ database_name | mandatory }}" DATABASE_URL="postgres://{{ database_username | mandatory }}:{{ database_password | mandatory }}@postgres/{{ database_name | mandatory }}"

View File

@ -3,7 +3,7 @@ set -e
{% if force_pg_restore %} {% if force_pg_restore %}
pg_restore --clean --exit-on-error -j $(nproc --all) -d postgres -U $POSTGRES_USER --no-owner --no-privileges --disable-triggers --create /pg_backups/external_node_latest.pgdump pg_restore --clean --exit-on-error -j $(nproc --all) -d postgres -U $POSTGRES_USER --no-owner --no-privileges --disable-triggers --create /pg_backups/external_node_latest.pgdump
{% else %} {% elif not enable_snapshots_recovery %}
if psql -U $POSTGRES_USER -d postgres -lqt | cut -d \| -f 1 | grep -qw "{{ database_name }}"; then if psql -U $POSTGRES_USER -d postgres -lqt | cut -d \| -f 1 | grep -qw "{{ database_name }}"; then
echo "Database already exists" echo "Database already exists"
else else