From 2af6a243ec5398af99e02681186945d44fdbd7b7 Mon Sep 17 00:00:00 2001 From: Maksym Date: Tue, 5 Mar 2024 17:05:57 +0200 Subject: [PATCH] feat!: create ansible EN role (#2) --- .ansible-lint | 4 + .github/ISSUE_TEMPLATE/bug_report.md | 42 ++++++ .github/ISSUE_TEMPLATE/feature_request.md | 21 +++ .github/pull_request_template.md | 18 +++ .github/semantic.yml | 6 + .github/workflows/ci.yml | 43 ++++++ .github/workflows/pr-title.yml | 33 ++++ .github/workflows/release.yml | 29 ++++ .github/workflows/secrets_scanner.yml | 19 +++ .releaserc.json | 23 +++ .yamllint | 10 ++ CONTRIBUTING.md | 44 ++++++ LICENSE-APACHE | 176 ++++++++++++++++++++++ LICENSE-MIT | 21 +++ README.md | 139 ++++++++++++++++- SECURITY.md | 74 +++++++++ defaults/main.yml | 109 ++++++++++++++ meta/main.yml | 18 +++ tasks/firewall.yml | 80 ++++++++++ tasks/main.yml | 7 + tasks/provision.yml | 98 ++++++++++++ templates/docker-compose.yaml.j2 | 97 ++++++++++++ templates/external_node.env.j2 | 5 + templates/monitoring.yaml.j2 | 50 ++++++ templates/postgres.env.j2 | 3 + templates/restore_dump.sh.j2 | 13 ++ templates/vmagent-config.yml.j2 | 40 +++++ 27 files changed, 1221 insertions(+), 1 deletion(-) create mode 100644 .ansible-lint create mode 100644 .github/ISSUE_TEMPLATE/bug_report.md create mode 100644 .github/ISSUE_TEMPLATE/feature_request.md create mode 100644 .github/pull_request_template.md create mode 100644 .github/semantic.yml create mode 100644 .github/workflows/ci.yml create mode 100644 .github/workflows/pr-title.yml create mode 100644 .github/workflows/release.yml create mode 100644 .github/workflows/secrets_scanner.yml create mode 100644 .releaserc.json create mode 100644 .yamllint create mode 100644 CONTRIBUTING.md create mode 100644 LICENSE-APACHE create mode 100644 LICENSE-MIT create mode 100644 SECURITY.md create mode 100644 defaults/main.yml create mode 100644 meta/main.yml create mode 100644 tasks/firewall.yml create mode 100644 tasks/main.yml create mode 100644 tasks/provision.yml create mode 100644 templates/docker-compose.yaml.j2 create mode 100644 templates/external_node.env.j2 create mode 100644 templates/monitoring.yaml.j2 create mode 100644 templates/postgres.env.j2 create mode 100644 templates/restore_dump.sh.j2 create mode 100644 templates/vmagent-config.yml.j2 diff --git a/.ansible-lint b/.ansible-lint new file mode 100644 index 0000000..2cd8c9f --- /dev/null +++ b/.ansible-lint @@ -0,0 +1,4 @@ +skip_list: + - 'yaml' + - 'risky-shell-pipe' + - 'role-name' diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md new file mode 100644 index 0000000..8e4115a --- /dev/null +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -0,0 +1,42 @@ +--- +name: Bug report +about: Use this template for reporting issues +title: '' +labels: bug +assignees: '' +--- + +### 🐛 Bug Report + +#### 📝 Description + +Provide a clear and concise description of the bug. + +#### 🔄 Reproduction Steps + +Steps to reproduce the behaviour + +#### 🤔 Expected Behavior + +Describe what you expected to happen. + +#### 😯 Current Behavior + +Describe what actually happened. + +#### 🖥️ Environment + +Any relevant environment details like: +* Ansible version +* Operating system +* External node version + +#### 📋 Additional Context + +Add any other context about the problem here. If applicable, add screenshots to help explain. + +#### 📎 Log Output + +``` +Paste any relevant log output here. +``` diff --git a/.github/ISSUE_TEMPLATE/feature_request.md b/.github/ISSUE_TEMPLATE/feature_request.md new file mode 100644 index 0000000..d921e06 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/feature_request.md @@ -0,0 +1,21 @@ +--- +name: Feature request +about: Use this template for requesting features +title: '' +labels: feat +assignees: '' +--- + +### 🌟 Feature Request + +#### 📝 Description + +Provide a clear and concise description of the feature you'd like to see. + +#### 🤔 Rationale + +Explain why this feature is important and how it benefits the project. + +#### 📋 Additional Context + +Add any other context or information about the feature request here. diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md new file mode 100644 index 0000000..69bc372 --- /dev/null +++ b/.github/pull_request_template.md @@ -0,0 +1,18 @@ +## What ❔ + + + + + +## Why ❔ + + + + +## Checklist + + + + +- [ ] PR title corresponds to the body of PR (we generate changelog entries from PRs). +- [ ] Documentation comments have been added / updated. diff --git a/.github/semantic.yml b/.github/semantic.yml new file mode 100644 index 0000000..a28c301 --- /dev/null +++ b/.github/semantic.yml @@ -0,0 +1,6 @@ +--- +# Always validate the PR title AND all the commits +titleAndCommits: true +# Allows use of Merge commits (e.g. on GitHub: "Merge branch 'master' into feature/ride-unicorns") +# this is only relevant when using commitsOnly: true (or titleAndCommits: true) +allowMergeCommits: true diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..bba6e62 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,43 @@ +--- +name: CI +'on': + pull_request: + + schedule: + - cron: "0 7 * * 0" + +jobs: + yaml-lint: + name: Lint + runs-on: ubuntu-latest + steps: + - name: Check out the codebase. + uses: actions/checkout@v4 + + - name: Set up Python 3. + uses: actions/setup-python@v5 + with: + python-version: '3.x' + + - name: Install test dependencies. + run: pip3 install yamllint + + - name: Lint code. + run: | + yamllint . + + + ansible-lint: + name: Ansible lint + runs-on: ubuntu-latest + steps: + - name: Check out the codebase. + uses: actions/checkout@v4 + + - name: Set up Python 3. + uses: actions/setup-python@v5 + with: + python-version: '3.x' + + - name: Ansible-lint + uses: ansible/ansible-lint@v24.2.0 diff --git a/.github/workflows/pr-title.yml b/.github/workflows/pr-title.yml new file mode 100644 index 0000000..c0ab14b --- /dev/null +++ b/.github/workflows/pr-title.yml @@ -0,0 +1,33 @@ +--- +name: CI Ansible Module Validate PR title + +on: + pull_request_target: + types: + - opened + - edited + - synchronize + +jobs: + lint: + name: Validate PR title + runs-on: ubuntu-latest + steps: + - uses: amannn/action-semantic-pull-request@c3cd5d1ea3580753008872425915e343e351ab54 # v5.2.0 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + with: + types: | + fix + feat + docs + ci + chore + requireScope: false + subjectPattern: ^[A-Za-z].+$ + subjectPatternError: | + The subject "{subject}" found in the pull request title "{title}" + didn't match the configured pattern. Please ensure that the subject + starts with an uppercase character. + wip: true + validateSingleCommit: false diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 0000000..847bd88 --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,29 @@ +--- +name: Release + +on: + push: + branches: + - main + +jobs: + release: + name: Release + runs-on: ubuntu-latest + if: github.repository_owner == 'matter-labs' + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + persist-credentials: false + fetch-depth: 0 + - name: Release + uses: cycjimmy/semantic-release-action@v4 + with: + semantic_version: 23.0.2 + extra_plugins: | + @semantic-release/changelog@6.0.3 + @semantic-release/git@10.0.1 + conventional-changelog-conventionalcommits@7.0.2 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/secrets_scanner.yml b/.github/workflows/secrets_scanner.yml new file mode 100644 index 0000000..47518b6 --- /dev/null +++ b/.github/workflows/secrets_scanner.yml @@ -0,0 +1,19 @@ +--- +name: Leaked Secrets Scan +on: + pull_request: +jobs: + TruffleHog: + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v3 + with: + fetch-depth: 0 + - name: TruffleHog OSS + uses: trufflesecurity/trufflehog@0c66d30c1f4075cee1aada2e1ab46dabb1b0071a + with: + path: ./ + base: ${{ github.event.repository.default_branch }} + head: HEAD + extra_args: --debug --only-verified diff --git a/.releaserc.json b/.releaserc.json new file mode 100644 index 0000000..4bce97c --- /dev/null +++ b/.releaserc.json @@ -0,0 +1,23 @@ +{ + "branches": [ + "main" + ], + "ci": false, + "plugins": [ + [ + "@semantic-release/commit-analyzer", + { + "preset": "conventionalcommits" + } + ], + [ + "@semantic-release/release-notes-generator", + { + "preset": "conventionalcommits" + } + ], + [ + "@semantic-release/github" + ] + ] +} diff --git a/.yamllint b/.yamllint new file mode 100644 index 0000000..15eaa3e --- /dev/null +++ b/.yamllint @@ -0,0 +1,10 @@ +--- +extends: default + +rules: + line-length: + max: 200 + level: warning + +ignore: | + .github/ diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 0000000..c909593 --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,44 @@ +# Contribution Guidelines + +Hello! Thanks for your interest in joining the mission to accelerate the mass adoption of crypto for personal +sovereignty! We welcome contributions from anyone on the internet, and are grateful for even the smallest of fixes! + +## Ways to contribute + +There are many ways to contribute to the external node role: + +1. Open issues: if you find a bug, have something you believe needs to be fixed, or have an idea for a feature, please + open an issue. +2. Add color to existing issues: provide screenshots, code snippets, and whatever you think would be helpful to resolve + issues. +3. Resolve issues: either by showing an issue isn't a problem and the current state is ok as is or by fixing the problem + and opening a PR. +4. Report security issues, see [our security policy](./SECURITY.md). +5. [Join the team!](https://matterlabs.notion.site/Shape-the-future-of-Ethereum-at-Matter-Labs-dfb3b5a037044bb3a8006af2eb0575e0) + +## Fixing issues + +To contribute code fixing issues, please fork the repo, fix an issue, commit, add documentation as per the PR template, +and the repo's maintainers will review the PR. +[here](https://docs.github.com/en/pull-requests/collaborating-with-pull-requests/proposing-changes-to-your-work-with-pull-requests/creating-a-pull-request-from-a-fork) +for guidance how to work with PRs created from a fork. + +## Licenses + +If you contribute to this project, your contributions will be made to the project under both Apache 2.0 and the MIT +license. + +## Code of Conduct + +Be polite and respectful. + +## FAQ + +**Q**: I have a small contribution that's not getting traction/being merged? + +**A**: Due to capacity, contributions that are simple renames of variables or stylistic/minor text improvements, one-off +typo fix will not be merged. If you do find any typos or grammar errors, the preferred avenue is to improve the existing +spellchecker. Given you have no technical prowess to do so, please create an issue. Please note that issues will be +resolved on a best effort basis. + +### Thank you diff --git a/LICENSE-APACHE b/LICENSE-APACHE new file mode 100644 index 0000000..d9a10c0 --- /dev/null +++ b/LICENSE-APACHE @@ -0,0 +1,176 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS diff --git a/LICENSE-MIT b/LICENSE-MIT new file mode 100644 index 0000000..2739ea6 --- /dev/null +++ b/LICENSE-MIT @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2019 Matter Labs + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/README.md b/README.md index 55163ba..b003593 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,139 @@ # ansible-en-role -Ansible role for setup external node +Ansible role for setup external node. + +## Requirements +This role has been tested on: +* Ubuntu 22.04, Jammy Jellyfish; Ansible 2.13.9 + +## Usage +This role contains variables which has to be set: +```yaml +database_name: "" +database_username: "" +database_password: "" +eth_l1_url: "" +main_node_url: "" +l1_chain_id: "" +l2_chain_id: "" +``` + +If you want to use monitoring, you can use next variables: +```yaml +# Monitoring options section +enable_monitoring: false +node_name: "" +prometheus_remote_write: false +prometheus_remote_write_url: "" +prometheus_remote_write_auth: false +prometheus_remote_write_auth_username: "" +prometheus_remote_write_auth_password: "" +prometheus_remote_write_label: "" +``` + +This role also has option to secure your server and allow traffic only from specified ip in case if you want +to use some load balancer in front of your node: + +```yaml +# Security options +use_predefined_iptables: false +disable_ssh_password_auth: false +iptables_packages: + - iptables + - iptables-persistent +# Variable can be used in case with accept external traffic only from one ip +loadbalancer_ip: "" +``` + +In some cases, you may need to change postgres parameters, so you can do it using `postgres_arguments` variable: +```yaml + +postgres_arguments: + - log_error_verbosity=terse + - -c + - max_connections=256 + - -c + - shared_buffers=47616MB + - -c + - effective_cache_size=142848MB + - -c + - maintenance_work_mem=2GB + - -c + - checkpoint_completion_target=0.9 + - -c + - wal_buffers=16MB + - -c + - default_statistics_target=500 + - -c + - random_page_cost=1.1 + - -c + - effective_io_concurrency=200 + - -c + - work_mem=2573kB + - -c + - huge_pages=try + - -c + - min_wal_size=4GB + - -c + - max_wal_size=16GB + - -c + - max_worker_processes=74 + - -c + - max_parallel_workers_per_gather=37 + - -c + - max_parallel_workers=74 + - -c + - max_parallel_maintenance_workers=4 + - -c + - checkpoint_timeout=1800 +``` +We recommend to use [pgtune](https://github.com/le0pard/pgtune) to choose optimal config for your hardware. + +## Step-by-step guide + +1. Install ansible collection on your machine from where you will run ansible: +`ansible-galaxy collection install community.general` +2. Prepare latest database backup on your host. you can download it from our [public GCS bucket](https://storage.googleapis.com/zksync-era-mainnet-external-node-backups/external_node_latest.pgdump). +you should place it to `{{ storage_directory }}/pg_backups` directory. By default, `{{ storage_directory }}` is `/usr/src/en` +3. **OPTIONAL**: If you already have external-node, you can copy tree directory to new host. Copy external-node database tree to `{{ storage_directory }}/db`. +**Keep in mind, tree should be older than postgres database backup.** +4. Run ansible-playbook using this role. We recommend to encrypt next variables with ansible-vault or some another way: +``` +database_username +database_password +eth_l1_url +vm_auth_username +vm_auth_password +``` +5. Connect to your host, and see status of postgres container. It can take a lot of time before postgres database backup will be restored +and postgres server will be ready for use. After postgres goes healty status, external-node runs automatically. + +## Example Playbook + +```yaml +--- +- hosts: all + become: true + vars: + loadbalancer_ip: "1.2.3.4" + use_predefined_iptables: true + enable_monitoring: false + database_name: "mainnet2" + main_node_url: "https://zksync2-mainnet.zksync.io" + l2_chain_id: "324" + l1_chain_id: "1" + enable_tls: false + partner_id: matterlabs + vars_files: + - secrets/mainnet_secrets.yml + roles: + - external_node +``` + +## License + +Ansible role for external node is distributed under the terms of either + +- Apache License, Version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) or ) +- MIT license ([LICENSE-MIT](LICENSE-MIT) or ) + +at your option. diff --git a/SECURITY.md b/SECURITY.md new file mode 100644 index 0000000..3889e68 --- /dev/null +++ b/SECURITY.md @@ -0,0 +1,74 @@ +# Security Policy + +We truly appreciate efforts to discover and disclose security issues responsibly! + +## Vulnerabilities + +If you'd like to report a security issue in the repositories of matter-labs organization, please proceed to our +[Bug Bounty Program on Immunefi](https://era.zksync.io/docs/reference/troubleshooting/audit-bug-bounty.html#bug-bounty-program). + +## Other Security Issues + +We take an impact-first approach instead of a rules-first approach. Therefore, if you believe you found the impactful +issue but can't report it via the Bug Bounty, please email us at +[security@matterlabs.dev](mailto:security@matterlabs.dev). + +### PGP Key + +The following PGP key may be used to communicate sensitive information to developers: + +Fingerprint: `5FED B2D0 EA2C 4906 DD66 71D7 A2C5 0B40 CE3C F297` + +```text +-----BEGIN PGP PUBLIC KEY BLOCK----- + +mQINBGEBmQkBEAD6tlkBEZFMvR8kOgxXX857nC2+oTik6TopJz4uCskuqDaeldMy +l+26BBzLkIeO1loS+bzVgnNFJRrGt9gv98MzNEHJVv6D7GsSLlUX/pz7Lxn0J4ry +o5XIk3MQTCUBdaXGs6GBLl5Xe8o+zNj4MKd4zjgDLinITNlE/YZCDsXyvYS3YFTQ +cwaUTNlawkKgw4BLaEqwB2JuyEhI9wx5X7ibjFL32sWMolYsNAlzFQzM09HCurTn +q0DYau9kPJARcEk9/DK2iq0z3gMCQ8iRTDaOWd8IbSP3HxcEoM5j5ZVAlULmjmUE +StDaMPLj0Kh01Tesh/j+vjchPXHT0n4zqi1+KOesAOk7SIwLadHfQMTpkU7G2fR1 +BrA5MtlzY+4Rm6o7qu3dpZ+Nc4iM3FUnaQRpvn4g5nTh8vjG94OCzX8DXWrCKyxx +amCs9PLDYOpx84fXYv4frkWpKh2digDSUGKhoHaOSnqyyvu3BNWXBCQZJ20rqEIu +sXOQMxWIoWCOOPRRvrHrKDA2hpoKjs3pGsProfpVRzb9702jhWpTfbDp9WjQlFtX +2ZIDxlwAxcugClgrp5JiUxvhg2A9lDNwCF7r1e68uNv5usBZQVKPJmnvS2nWgKy8 +x9oJsnwrEjxwiRHd34UvfMkwY9RENSJ+NoXqBdS7Lwz4m6vgbzq6K56WPQARAQAB +tCRaa1N5bmMgU2VjdXJpdHkgPHNlY3VyaXR5QHprc3luYy5pbz6JAk4EEwEKADgW +IQRf7bLQ6ixJBt1mcdeixQtAzjzylwUCYQGZCQIbAwULCQgHAgYVCgkICwIEFgID +AQIeAQIXgAAKCRCixQtAzjzyl5y8EAC/T3oq88Dak2b+5TlWdU2Gpm6924eAqlMt +y1KksDezzNQUlPiCUVllpin2PIjU/S+yzMWKXJA04LoVkEPfPOWjAaavLOjRumxu +MR6P2dVUg1InqzYVsJuRhKSpeexzNA5qO2BPM7/I2Iea1IoJPjogGbfXCo0r5kne +KU7a5GEa9eDHxpHTsbphQe2vpQ1239mUJrFpzAvILn6jV1tawMn5pNCXbsa8l6l2 +gtlyQPdOQECy77ZJxrgzaUBcs/RPzUGhwA/qNuvpF0whaCvZuUFMVuCTEu5LZka2 +I9Rixy+3jqBeONBgb+Fiz5phbiMX33M9JQwGONFaxdvpFTerLwPK2N1T8zcufa01 +ypzkWGheScFZemBxUwXwK4x579wjsnfrY11w0p1jtDgPTnLlXUA2mom4+7MyXPg0 +F75qh6vU1pdXaCVkruFgPVtIw+ccw2AxD50iZQ943ZERom9k165dR9+QxOVMXQ4P +VUxsFZWvK70/s8TLjsGljvSdSOa85iEUqSqh0AlCwIAxLMiDwh5s/ZgiHoIM6Xih +oCpuZyK9p0dn+DF/XkgAZ/S91PesMye3cGm6M5r0tS26aoc2Pk6X37Hha1pRALwo +MOHyaGjc/jjcXXxv6o55ALrOrzS0LQmLZ+EHuteCT15kmeY3kqYJ3og62KgiDvew +dKHENvg7d7kCDQRhAZleARAA6uD6WfdqGeKV5i170+kLsxR3QGav0qGNAbxpSJyn +iHQ8u7mQk3S+ziwN2AAopfBk1je+vCWtEGC3+DWRRfJSjLbtaBG8e6kLP3/cGA75 +qURz6glTG4nl5fcEAa6B1st0OxjVWiSLX3g/yjz8lznQb9awuRjdeHMnyx5DsJUN +d+Iu5KxGupQvKGOMKivSvC8VWk9taaQRpRF+++6stLCDk3ZtlxiopMs3X2jAp6xG +sOBbix1cv9BTsfaiL7XDL/gviqBPXYY5L42x6+jnPo5lROfnlLYkWrv6KZr7HD4k +tRXeaSwxLD2EkUyb16Jpp0be/ofvBtITGUDDLCGBiaXtx/v8d52MARjsyLJSYloj +1yiW01LfAiWHUC4z5jl2T7E7sicrlLH1M8Z6WbuqjdeaYwtfyPA2YCKr/3fn6pIo +D+pYaBSESmhA92P+XVaf5y2BZ6Qf8LveDpWwsVGdBGh9T0raA1ooe1GESLjmIjUa +z5AeQ/uXL5Md9I6bpMUUJYQiH19RPcFlJriI3phXyyf6Wlkk8oVEeCWyzcmw+x1V +deRTvE2x4WIwKGLXRNjin2j1AP7vU2HaNwlPrLijqdyi68+0irRQONoH7Qonr4ca +xWgL+pAaa3dWxf0xqK7uZFp4aTVWlr2uXtV/eaUtLmGMCU0jnjb109wg5L0F7WRT +PfEAEQEAAYkCNgQYAQoAIBYhBF/tstDqLEkG3WZx16LFC0DOPPKXBQJhAZleAhsM +AAoJEKLFC0DOPPKXAAEP/jK7ch9GkoaYlsuqY/aHtxEwVddUDOxjyn3FMDoln85L +/n8AmLQb2bcpKSqpaJwMbmfEyr5MDm8xnsBTfx3u6kgaLOWfKxjLQ6PM7kgIMdi4 +bfaRRuSEI1/R6c/hNpiGnzAeeexldH1we+eH1IVmh4crdat49S2xh7Qlv9ahvgsP +LfKl3rJ+aaX/Ok0AHzhvSfhFpPr1gAaGeaRt+rhlZsx2QyG4Ez8p2nDAcAzPiB3T +73ENoBIX6mTPfPm1UgrRyFKBqtUzAodz66j3r6ebBlWzIRg8iZenVMAxzjINAsxN +w1Bzfgsi5ZespfsSlmEaa7jJkqqDuEcLa2YuiFAue7Euqwz1aGeq1GfTicQioSCb +Ur/LGyz2Mj3ykbaP8p5mFVcUN51yQy6OcpvR/W1DfRT9SHFT/bCf9ixsjB2HlZGo +uxPJowwqmMgHd755ZzPDUM9YDgLI1yXdcYshObv3Wq537JAxnZJCGRK4Y8SwrMSh +8WRxlaM0AGWXiJFIDD4bQPIdnF3X8w0cGWE5Otkb8mMHOT+rFTVlDODwm1zF6oIG +PTwfVrpiZBwiUtfJol1exr/MzSPyGoJnYs3cRf2E3O+D1LbcR8w0LbjGuUy38Piz +ZO/vCeyJ3JZC5kE8nD+XBA4idwzh0BKEfH9t+WchQ3Up9rxyzLyQamoqt5Xby4pY +=xkM3 +-----END PGP PUBLIC KEY BLOCK----- +``` diff --git a/defaults/main.yml b/defaults/main.yml new file mode 100644 index 0000000..c18fe9d --- /dev/null +++ b/defaults/main.yml @@ -0,0 +1,109 @@ +--- +# Places for store configurations and databases +configuration_directory: "/opt/en" +storage_directory: "/usr/src/en" + +# Install docker-compose +docker_install_compose: true +docker_version: "25.0.3" +docker_compose_version: "v2.23.0" + +# Versions of external node and 3rd party components +traefik_version: 2.11 +postgres_version: 14 +external_node_version: 21.0.2 +vmagent_version: 1.95.1 +node_exporter_version: 1.7.0 +cadvisor_version: 0.47.2 +postgres_exporter_version: 0.15.0 + +# Postgres configuration +postgres_arguments: + - log_error_verbosity=terse + - -c + - max_connections=256 + - -c + - shared_buffers=47616MB + - -c + - effective_cache_size=142848MB + - -c + - maintenance_work_mem=2GB + - -c + - checkpoint_completion_target=0.9 + - -c + - wal_buffers=16MB + - -c + - default_statistics_target=500 + - -c + - random_page_cost=1.1 + - -c + - effective_io_concurrency=200 + - -c + - work_mem=2573kB + - -c + - huge_pages=try + - -c + - min_wal_size=4GB + - -c + - max_wal_size=16GB + - -c + - max_worker_processes=74 + - -c + - max_parallel_workers_per_gather=37 + - -c + - max_parallel_workers=74 + - -c + - max_parallel_maintenance_workers=4 + - -c + - checkpoint_timeout=1800 + + +# Enable TLS for traefik +enable_tls: false +acme_email: "" +domain_name: "" + +# Force restore pg database +force_pg_restore: false + +# External node and database options +database_name: "" +database_username: "" +database_password: "" +eth_l1_url: "" +main_node_url: "" +l1_chain_id: "" +l2_chain_id: "" +rpc_http_port: 3060 +rpc_ws_port: 3061 +healthcheck_port: 3081 +metrics_port: 3082 + +# Monitoring options section +enable_monitoring: false +node_name: "" +prometheus_remote_write: false +prometheus_remote_write_url: "" +prometheus_remote_write_auth: false +prometheus_remote_write_auth_username: "" +prometheus_remote_write_auth_password: "" +prometheus_remote_write_label: "" + +# Security options +use_predefined_iptables: false +disable_ssh_password_auth: false +iptables_packages: + - iptables + - iptables-persistent +# Variable can be used in case with accept external traffic only from one ip +loadbalancer_ip: "" + +# For internal usage +en_required_variables: + - eth_l1_url + - main_node_url + - l1_chain_id + - l2_chain_id + +monitoring_required_variables: + - node_name diff --git a/meta/main.yml b/meta/main.yml new file mode 100644 index 0000000..f63c924 --- /dev/null +++ b/meta/main.yml @@ -0,0 +1,18 @@ +--- +dependencies: + - src: geerlingguy.docker + version: "7.1.0" + when: docker_install + +galaxy_info: + role_name: external_node + author: matter-labs + description: External node setup + license: "license (MIT, APACHE)" + min_ansible_version: "2.13.9" + platforms: + - name: Ubuntu + versions: + - jammy + galaxy_tags: + - en diff --git a/tasks/firewall.yml b/tasks/firewall.yml new file mode 100644 index 0000000..ba1fae6 --- /dev/null +++ b/tasks/firewall.yml @@ -0,0 +1,80 @@ +--- +- name: Install iptables packages + ansible.builtin.apt: + update_cache: true + name: "{{ iptables_packages }}" + +- name: Allow loopback traffic + ansible.builtin.iptables: + chain: INPUT + in_interface: lo + jump: ACCEPT + +- name: Allow related and established connections + ansible.builtin.iptables: + chain: INPUT + match: state + ctstate: RELATED,ESTABLISHED + jump: ACCEPT + +- name: Allow SSH traffic + ansible.builtin.iptables: + chain: INPUT + protocol: tcp + destination_port: 22 + jump: ACCEPT + +- name: Allow HTTP traffic from specific IP to http port + ansible.builtin.iptables: + chain: INPUT + protocol: tcp + destination_port: 80 + source: "{{ loadbalancer_ip | mandatory }}" + jump: ACCEPT + +- name: Allow HTTP traffic from specific IP to https port + when: enable_tls + ansible.builtin.iptables: + chain: INPUT + protocol: tcp + destination_port: 443 + source: "{{ loadbalancer_ip | mandatory }}" + jump: ACCEPT + +- name: Allow healthcheck port traffic from specific IP + ansible.builtin.iptables: + chain: INPUT + protocol: tcp + destination_port: 3080 + source: "{{ loadbalancer_ip | mandatory }}" + jump: ACCEPT + +- name: Set default policy to DROP + ansible.builtin.iptables: + chain: INPUT + policy: DROP + +- name: Save ipv4 current state of the firewall in system file + community.general.iptables_state: + ip_version: ipv4 + state: saved + path: /etc/iptables/rules.v4 + +- name: Save ipv6 current state of the firewall in system file + community.general.iptables_state: + ip_version: ipv6 + state: saved + path: /etc/iptables/rules.v6 + +- name: Disable SSH password authentication + when: disable_ssh_password_auth + ansible.builtin.lineinfile: + path: /etc/ssh/sshd_config + regexp: '^#PasswordAuthentication yes' + line: 'PasswordAuthentication no' + +- name: Restart ssh + when: disable_ssh_password_auth + ansible.builtin.service: + name: ssh + state: restarted diff --git a/tasks/main.yml b/tasks/main.yml new file mode 100644 index 0000000..a705f30 --- /dev/null +++ b/tasks/main.yml @@ -0,0 +1,7 @@ +--- +- name: Setup firewall + ansible.builtin.include_tasks: firewall.yml + when: use_predefined_iptables + +- name: Prepare configs + ansible.builtin.include_tasks: provision.yml diff --git a/tasks/provision.yml b/tasks/provision.yml new file mode 100644 index 0000000..895b76c --- /dev/null +++ b/tasks/provision.yml @@ -0,0 +1,98 @@ +--- +- name: Create configuration directory + ansible.builtin.file: + path: "{{ configuration_directory }}" + state: directory + mode: '0755' + +- name: Create storage directories + ansible.builtin.file: + path: "{{ storage_directory }}/{{ item }}" + state: directory + mode: '0755' + loop: + - db + - db/lightweight-new + - db/state_keeper + +- name: "Verify that required variables are defined" + ansible.builtin.assert: + that: + - required_var != "" + fail_msg: "{{ required_var }} needs to be set for the role to work" + success_msg: "Required variable {{ required_var }} isn't empty" + loop_control: + loop_var: required_var + with_items: + - database_name + - database_username + - database_password + - eth_l1_url + - main_node_url + - l2_chain_id + - l1_chain_id + +- name: Check required en vars empty + ansible.builtin.fail: + msg: "Variable '{{ item }}' is empty" + when: vars[item] == "" + with_items: "{{ en_required_variables }}" + +- name: Copy main configs + ansible.builtin.template: + src: '{{ item.src }}' + dest: '{{ item.dest }}' + mode: '0644' + loop: + - src: "templates/docker-compose.yaml.j2" + dest: "{{ configuration_directory }}/docker-compose.yaml" + - src: "templates/external_node.env.j2" + dest: "{{ configuration_directory }}/external_node.env" + - src: "templates/postgres.env.j2" + dest: "{{ configuration_directory }}/postgres.env" + +- name: Copy restore script + register: restore_dump_script + ansible.builtin.template: + src: 'templates/restore_dump.sh.j2' + dest: '{{ configuration_directory }}/restore_dump.sh' + mode: "a+x" + +- name: Check required monitoring vars empty + ansible.builtin.fail: + msg: "Variable '{{ item }}' is empty" + when: enable_monitoring and ( vars[item] == "" ) + with_items: "{{ monitoring_required_variables }}" + +- name: Copy monitoring configs + when: enable_monitoring + ansible.builtin.template: + src: '{{ item.src }}' + dest: '{{ item.dest }}' + mode: '0644' + loop: + - src: "templates/monitoring.yaml.j2" + dest: "{{ configuration_directory }}/monitoring.yaml" + - src: "templates/vmagent-config.yml.j2" + dest: "{{ configuration_directory }}/vmagent-config.yml" + +- name: Run docker-compose without monitoring + when: not enable_monitoring + ansible.builtin.shell: + cmd: nohup docker compose -f docker-compose.yaml up -d & + chdir: "{{ configuration_directory }}" + changed_when: false + +- name: Run docker-compose with monitoring + when: enable_monitoring and (not restore_dump_script.changed) + ansible.builtin.shell: + cmd: nohup docker compose -f monitoring.yaml -f docker-compose.yaml up -d & + chdir: "{{ configuration_directory }}" + changed_when: false + +- name: Run docker-compose with monitoring with recreation + when: enable_monitoring and restore_dump_script.changed + ansible.builtin.shell: + cmd: nohup docker compose -f monitoring.yaml -f docker-compose.yaml up -d --force-recreate & + chdir: "{{ configuration_directory }}" + changed_when: false diff --git a/templates/docker-compose.yaml.j2 b/templates/docker-compose.yaml.j2 new file mode 100644 index 0000000..4fab4db --- /dev/null +++ b/templates/docker-compose.yaml.j2 @@ -0,0 +1,97 @@ +version: "3.9" +services: + traefik: + image: "traefik:{{ traefik_version }}" + network_mode: "host" + restart: unless-stopped + command: + - "--log.level=INFO" + - "--providers.docker=true" + - "--providers.docker.exposedbydefault=false" + - "--entrypoints.web.address=:80" + - "--entrypoints.external_node_health.address=:3080" +{% if enable_tls %} + - "--entrypoints.websecure.address=:443" + - "--certificatesresolvers.en_resolver.acme.tlschallenge=true" + - "--certificatesresolvers.en_resolver.acme.storage=/letsencrypt/acme.json" + - "--certificatesresolvers.myresolver.acme.email={{ acme_email }}" +{% endif %} + volumes: + - "/var/run/docker.sock:/var/run/docker.sock" +{% if enable_tls %} + - "./letsencrypt:/letsencrypt" +{% endif %} + + postgres: + image: "postgres:{{ postgres_version }}" + restart: unless-stopped + healthcheck: + interval: 1m + timeout: 30s + retries: 288000 + start_period: 4h + test: psql -U postgres -c "select exists (select * from pg_stat_activity where datname = '{{ database_name }}' and application_name = 'pg_restore')" | grep -e ".f$" + volumes: + - "{{ storage_directory }}/postgres:/var/lib/postgresql/data" + - "{{ storage_directory }}/pg_backups:/pg_backups" + - ./restore_dump.sh:/docker-entrypoint-initdb.d/restore_dump.sh + env_file: + - postgres.env + command: + - postgres + - -c +{% for argument in postgres_arguments %} + - {{ argument }} +{% endfor %} + external_node: + image: "matterlabs/external-node:v{{ external_node_version }}" + restart: unless-stopped + depends_on: + postgres: + condition: service_healthy + labels: + - "traefik.enable=true" + - "traefik.http.services.external_node_main.loadbalancer.server.port={{ rpc_http_port }}" + - "traefik.http.routers.external_node_main.rule=PathPrefix(`/`)" +{% if enable_tls %} + - "traefik.http.routers.whoami.entrypoints=websecure" + - "traefik.http.routers.whoami.tls.certresolver=myresolver" + - "traefik.http.routers.whoami.rule=Host(`{{ domain_name }}`)" +{% else %} + - "traefik.http.routers.external_node_main.entrypoints=web" +{% endif %} + - "traefik.http.routers.external_node_main.service=external_node_main" + + - "traefik.http.services.external_node_health.loadbalancer.server.port={{ healthcheck_port }}" + - "traefik.http.routers.external_node_health.rule=PathPrefix(`/`)" + - "traefik.http.routers.external_node_health.entrypoints=external_node_health" + - "traefik.http.routers.external_node_health.service=external_node_health" + expose: + - {{ rpc_http_port }} + - {{ rpc_ws_port }} + - {{ healthcheck_port }} + - {{ metrics_port }} + environment: + ZKSYNC_HOME: "/" + EN_STATE_CACHE_PATH: /db/state_keeper + EN_MERKLE_TREE_PATH: /db/lightweight-new + EN_HTTP_PORT: {{ rpc_http_port }} + EN_WS_PORT: {{ rpc_ws_port }} + EN_HEALTHCHECK_PORT: {{ healthcheck_port }} + EN_PROMETHEUS_PORT: {{ metrics_port }} + EN_THREADS_PER_SERVER: 64 + CHAIN_STATE_KEEPER_VALIDATION_COMPUTATIONAL_GAS_LIMIT: 2000000 + DATABASE_POOL_SIZE: 200 + EN_MAX_BLOCKS_PER_TREE_BATCH: 200 + RUST_LOG: zksync_core=debug,zksync_dal=info,zksync_eth_client=info,zksync_merkle_tree=info,zksync_storage=info,zksync_state=debug,zksync_types=info,vm=info,zksync_external_node=info,zksync_utils=debug + healthcheck: + test: [ "CMD", "curl", "-f", "http://localhost:{{ healthcheck_port }}/health" ] + interval: 1m + timeout: 30s + retries: 60 + start_period: 1m + volumes: + - "{{ storage_directory }}/db:/db" + env_file: + - "external_node.env" + - "postgres.env" diff --git a/templates/external_node.env.j2 b/templates/external_node.env.j2 new file mode 100644 index 0000000..3931794 --- /dev/null +++ b/templates/external_node.env.j2 @@ -0,0 +1,5 @@ +EN_ETH_CLIENT_URL="{{ eth_l1_url | mandatory }}" +EN_MAIN_NODE_URL="{{ main_node_url | mandatory }}" +EN_L2_CHAIN_ID="{{ l2_chain_id | mandatory }}" +EN_L1_CHAIN_ID="{{ l1_chain_id | mandatory }}" +DATABASE_URL="postgres://{{ database_username | mandatory }}:{{ database_password | mandatory }}@postgres/{{ database_name | mandatory }}" diff --git a/templates/monitoring.yaml.j2 b/templates/monitoring.yaml.j2 new file mode 100644 index 0000000..608b3ea --- /dev/null +++ b/templates/monitoring.yaml.j2 @@ -0,0 +1,50 @@ +version: "3.9" +services: + vmagent: + container_name: vmagent + image: "victoriametrics/vmagent:v{{ vmagent_version }}" + volumes: + - vmagentdata:/vmagentdata + - ./vmagent-config.yml:/etc/vmagent/config.yml + command: +{% if prometheus_remote_write %} + - "--remoteWrite.url={{ prometheus_remote_write_url }}" + - "--remoteWrite.label={{ prometheus_remote_write_label }}" +{% if prometheus_remote_write_auth %} + - "--remoteWrite.basicAuth.username={{ prometheus_remote_write_auth_username }}" + - "--remoteWrite.basicAuth.password={{ prometheus_remote_write_auth_password }}" + - "--remoteWrite.tmpDataPath=/tmp/vmagent" +{% endif %} +{% endif %} + - "--promscrape.config=/etc/vmagent/config.yml" + - "--promscrape.streamParse=true" + - "--remoteWrite.vmProtoCompressLevel=2" + restart: always + + node-exporter: + image: "prom/node-exporter:v{{ node_exporter_version }}" + volumes: + - /proc:/host/proc:ro + - /sys:/host/sys:ro + - /:/rootfs:ro + restart: unless-stopped + command: + - '--path.procfs=/host/proc' + - '--path.sysfs=/host/sys' + - '--collector.filesystem.ignored-mount-points=^/(sys|proc|dev|host|etc)($$|/)' + + cadvisor: + image: "gcr.io/cadvisor/cadvisor:v{{ cadvisor_version }}" + volumes: + - /:/rootfs:ro + - /var/run:/var/run:rw + - /sys:/sys:ro + - /var/lib/docker/:/var/lib/docker:ro + + postgres-exporter: + image: "quay.io/prometheuscommunity/postgres-exporter:v{{ postgres_exporter_version }}" + env_file: + - postgres.env + +volumes: + vmagentdata: diff --git a/templates/postgres.env.j2 b/templates/postgres.env.j2 new file mode 100644 index 0000000..e6e599b --- /dev/null +++ b/templates/postgres.env.j2 @@ -0,0 +1,3 @@ +POSTGRES_USER="{{ database_username | mandatory }}" +POSTGRES_PASSWORD="{{ database_password | mandatory }}" +DATA_SOURCE_NAME="postgresql://${POSTGRES_USER}:${POSTGRES_PASSWORD}@postgres:5432/postgres?sslmode=disable" diff --git a/templates/restore_dump.sh.j2 b/templates/restore_dump.sh.j2 new file mode 100644 index 0000000..43db63e --- /dev/null +++ b/templates/restore_dump.sh.j2 @@ -0,0 +1,13 @@ +#!/usr/bin/env bash +set -e + +{% if force_pg_restore %} +pg_restore --clean --exit-on-error -j $(nproc --all) -d postgres -U $POSTGRES_USER --no-owner --no-privileges --disable-triggers --create /pg_backups/external_node_latest.pgdump +{% else %} +if psql -U $POSTGRES_USER -d postgres -lqt | cut -d \| -f 1 | grep -qw "{{ database_name }}"; then + echo "Database already exists" +else + echo "Database does not exist" + pg_restore --exit-on-error -j $(nproc --all) -d postgres -U $POSTGRES_USER --no-owner --no-privileges --disable-triggers --create /pg_backups/external_node_latest.pgdump +fi +{% endif %} diff --git a/templates/vmagent-config.yml.j2 b/templates/vmagent-config.yml.j2 new file mode 100644 index 0000000..ad4966d --- /dev/null +++ b/templates/vmagent-config.yml.j2 @@ -0,0 +1,40 @@ +scrape_configs: + - job_name: external-node + static_configs: + - targets: ['external_node:{{ metrics_port }}'] + relabel_configs: + - source_labels: [instance] + target_label: instance + replacement: '{{ node_name | mandatory }}' + - job_name: vmagent + static_configs: + - targets: + - "127.0.0.1:8429" + relabel_configs: + - source_labels: [instance] + target_label: instance + replacement: '{{ node_name | mandatory }}' + - job_name: node-exporter + static_configs: + - targets: + - "node-exporter:9100" + relabel_configs: + - source_labels: [instance] + target_label: instance + replacement: '{{ node_name | mandatory }}' + - job_name: cadvisor + static_configs: + - targets: + - "cadvisor:8080" + relabel_configs: + - source_labels: [instance] + target_label: instance + replacement: '{{ node_name | mandatory }}' + - job_name: postgres-exporter + static_configs: + - targets: + - "postgres-exporter:9187" + relabel_configs: + - source_labels: [instance] + target_label: instance + replacement: '{{ node_name | mandatory }}'