fix(sol): bind validator and RPC to same address

ansible/group_vars/sol_rpc.yml

@@ -0,0 +1,41 @@
+solana_user: solana
+solana_group: solana
+solana_home: /var/lib/solana
+
+solana_install_script_url: https://release.anza.xyz/stable/install
+solana_active_release_bin_dir: "{{ solana_home }}/.local/share/solana/install/active_release/bin"
+solana_validator_bin: /opt/solana/bin/agave-validator
+solana_keygen_bin: /opt/solana/bin/solana-keygen
+solana_rpc_service_name: solana-rpc
+
+agave_repo_url: https://github.com/anza-xyz/agave.git
+agave_version_tag: v2.3.13
+agave_src_dir: "{{ solana_home }}/src/agave"
+agave_rust_toolchain: "1.86.0"
+
+solana_identity_path: /var/lib/solana/identity.json
+solana_ledger_dir: /var/lib/solana/ledger
+solana_accounts_dir: /var/lib/solana/accounts
+solana_log_dir: /var/log/solana
+
+# Note: agave-validator expects all sockets to be bound to the same IP.
+# For now we bind validator + RPC to 0.0.0.0 and rely on network hardening in a later etap.
+solana_bind_address: 0.0.0.0
+solana_rpc_port: 8899
+solana_rpc_pubsub_port: 8900
+solana_dynamic_port_range: "8000-8020"
+
+solana_entrypoints:
+  - entrypoint.mainnet-beta.solana.com:8001
+solana_known_validators: []
+
+solana_geyser_enabled: false
+solana_geyser_plugin_config_path: /etc/solana/yellowstone-geyser.json
+
+solana_rpc_extra_args:
+  - --full-rpc-api
+  - --limit-ledger-size
+
+solana_rpc_manage_service: true
+solana_rpc_enable_on_boot: true
+solana_rpc_start_now: true

ansible/inventory/hosts.ini

@@ -1,2 +1,2 @@
 [sol_rpc]
-mevnode ansible_host=n8c1e71.mevnode.com ansible_user=root
+mevnode ansible_host=n8c1e71.mevnode.com ansible_user=root ansible_ssh_private_key_file=/home/runner/.ssh/mevnode_baremetal

ansible/playbooks/doc-rpc-sol-min.yml

@@ -3,6 +3,8 @@
   hosts: sol_rpc
   gather_facts: true
   become: true
+  vars_files:
+    - ../group_vars/sol_rpc.yml
 
   tasks:
     - name: Install operator packages (Debian/Ubuntu)
@@ -16,6 +18,32 @@
         update_cache: true
       when: ansible_facts.os_family == "Debian"
 
+    - name: Install Solana host base packages (Debian/Ubuntu)
+      ansible.builtin.apt:
+        name:
+          - chrony
+          - curl
+          - jq
+          - smartmontools
+          - nvme-cli
+          - prometheus-node-exporter
+        state: present
+        update_cache: true
+      when: ansible_facts.os_family == "Debian"
+
+    - name: Ensure solana group exists
+      ansible.builtin.group:
+        name: "{{ solana_group }}"
+        system: true
+
+    - name: Ensure solana user exists
+      ansible.builtin.user:
+        name: "{{ solana_user }}"
+        group: "{{ solana_group }}"
+        home: "{{ solana_home }}"
+        system: true
+        create_home: true
+
     - name: Ensure root config directories exist
       ansible.builtin.file:
         path: "{{ item }}"
@@ -29,6 +57,38 @@
         - /root/.config/nvim
         - /root/.config/nvim/lua
 
+    - name: Ensure Solana directories exist
+      ansible.builtin.file:
+        path: "{{ item.path }}"
+        state: directory
+        owner: "{{ item.owner }}"
+        group: "{{ item.group }}"
+        mode: "{{ item.mode }}"
+      loop:
+        - { path: "/etc/solana", owner: "root", group: "root", mode: "0755" }
+        - { path: "{{ solana_home }}", owner: "{{ solana_user }}", group: "{{ solana_group }}", mode: "0750" }
+        - { path: "{{ solana_ledger_dir }}", owner: "{{ solana_user }}", group: "{{ solana_group }}", mode: "0750" }
+        - { path: "{{ solana_accounts_dir }}", owner: "{{ solana_user }}", group: "{{ solana_group }}", mode: "0750" }
+        - { path: "{{ solana_log_dir }}", owner: "{{ solana_user }}", group: "{{ solana_group }}", mode: "0750" }
+        - { path: "/opt/solana/bin", owner: "root", group: "root", mode: "0755" }
+
+    - name: Deploy Solana sysctl tuning (network buffers)
+      ansible.builtin.copy:
+        dest: /etc/sysctl.d/99-solana-rpc.conf
+        owner: root
+        group: root
+        mode: "0644"
+        content: |
+          # Required by agave-validator startup checks
+          net.core.rmem_max = 134217728
+          net.core.wmem_max = 134217728
+      register: solana_sysctl_conf
+
+    - name: Apply sysctl tuning when changed
+      ansible.builtin.command: sysctl --system
+      changed_when: true
+      when: solana_sysctl_conf.changed
+
     - name: Deploy tmux config (Ctrl+a prefix)
       ansible.builtin.copy:
         src: ../files/operator-dotfiles/tmux.conf
@@ -64,6 +124,197 @@
         - { src: "lua/utils.lua", dest: "lua/utils.lua" }
         - { src: "lua/hazard3_dap.lua", dest: "lua/hazard3_dap.lua" }
 
+    - name: Deploy solana-rpc systemd unit (runs as solana user)
+      ansible.builtin.template:
+        src: ../templates/solana-rpc.service.j2
+        dest: /etc/systemd/system/{{ solana_rpc_service_name }}.service
+        owner: root
+        group: root
+        mode: "0644"
+      register: solana_rpc_unit
+
+    - name: Reload systemd after unit change
+      ansible.builtin.systemd:
+        daemon_reload: true
+      when: solana_rpc_unit.changed
+
+    - name: Check validator binary exists
+      ansible.builtin.stat:
+        path: "{{ solana_validator_bin }}"
+      register: solana_validator_bin_stat
+
+    - name: Install Agave toolchain for solana user when validator missing
+      ansible.builtin.shell: |
+        set -euo pipefail
+        sh -c "$(curl -sSfL {{ solana_install_script_url }})"
+      become_user: "{{ solana_user }}"
+      environment:
+        HOME: "{{ solana_home }}"
+      args:
+        executable: /bin/bash
+      when: not solana_validator_bin_stat.stat.exists
+
+    - name: Link Agave binaries into /opt/solana/bin
+      ansible.builtin.shell: |
+        set -euo pipefail
+        if [ ! -d "{{ solana_active_release_bin_dir }}" ]; then
+          echo "Active release bin dir missing: {{ solana_active_release_bin_dir }}" >&2
+          exit 1
+        fi
+        for bin in "{{ solana_active_release_bin_dir }}"/*; do
+          name="$(basename "$bin")"
+          ln -sfn "$bin" "/opt/solana/bin/$name"
+        done
+      args:
+        executable: /bin/bash
+      when: not solana_validator_bin_stat.stat.exists
+
+    - name: Re-check validator binary after install
+      ansible.builtin.stat:
+        path: "{{ solana_validator_bin }}"
+      register: solana_validator_bin_stat
+
+    - name: Install Agave build dependencies (Debian/Ubuntu) when validator missing
+      ansible.builtin.apt:
+        name:
+          - build-essential
+          - pkg-config
+          - libssl-dev
+          - clang
+          - llvm-dev
+          - libclang-dev
+          - cmake
+          - protobuf-compiler
+          - libudev-dev
+          - zlib1g-dev
+        state: present
+        update_cache: true
+      when:
+        - ansible_facts.os_family == "Debian"
+        - not solana_validator_bin_stat.stat.exists
+
+    - name: Ensure Agave source parent directory exists
+      ansible.builtin.file:
+        path: "{{ agave_src_dir | dirname }}"
+        state: directory
+        owner: "{{ solana_user }}"
+        group: "{{ solana_group }}"
+        mode: "0755"
+      when: not solana_validator_bin_stat.stat.exists
+
+    - name: Install rustup (Agave build) for solana user when validator missing
+      ansible.builtin.shell: |
+        set -euo pipefail
+        curl -sSfL https://sh.rustup.rs | sh -s -- -y --default-toolchain "{{ agave_rust_toolchain }}"
+      become_user: "{{ solana_user }}"
+      environment:
+        HOME: "{{ solana_home }}"
+      args:
+        executable: /bin/bash
+        creates: "{{ solana_home }}/.cargo/bin/cargo"
+      when: not solana_validator_bin_stat.stat.exists
+
+    - name: Ensure Rust toolchain is installed for solana user when validator missing
+      ansible.builtin.shell: |
+        set -euo pipefail
+        export PATH="{{ solana_home }}/.cargo/bin:$PATH"
+        rustup toolchain install "{{ agave_rust_toolchain }}"
+      become_user: "{{ solana_user }}"
+      environment:
+        HOME: "{{ solana_home }}"
+      args:
+        executable: /bin/bash
+      when: not solana_validator_bin_stat.stat.exists
+
+    - name: Clone Agave sources when validator missing
+      ansible.builtin.git:
+        repo: "{{ agave_repo_url }}"
+        dest: "{{ agave_src_dir }}"
+        version: "{{ agave_version_tag }}"
+        depth: 1
+        update: true
+      become_user: "{{ solana_user }}"
+      environment:
+        HOME: "{{ solana_home }}"
+      when: not solana_validator_bin_stat.stat.exists
+
+    - name: Build agave-validator from sources when validator missing
+      ansible.builtin.shell: |
+        set -euo pipefail
+        export PATH="{{ solana_home }}/.cargo/bin:$PATH"
+        cd "{{ agave_src_dir }}"
+        cargo +{{ agave_rust_toolchain }} build --release -p agave-validator
+      become_user: "{{ solana_user }}"
+      environment:
+        HOME: "{{ solana_home }}"
+      args:
+        executable: /bin/bash
+        creates: "{{ agave_src_dir }}/target/release/agave-validator"
+      when: not solana_validator_bin_stat.stat.exists
+
+    - name: Install agave-validator to /opt/solana/bin when built
+      ansible.builtin.copy:
+        remote_src: true
+        src: "{{ agave_src_dir }}/target/release/agave-validator"
+        dest: "{{ solana_validator_bin }}"
+        owner: root
+        group: root
+        mode: "0755"
+      when: not solana_validator_bin_stat.stat.exists
+
+    - name: Re-check validator binary after source build
+      ansible.builtin.stat:
+        path: "{{ solana_validator_bin }}"
+      register: solana_validator_bin_stat
+
+    - name: Ensure identity key exists
+      ansible.builtin.shell: |
+        set -euo pipefail
+        "{{ solana_keygen_bin }}" new --no-passphrase -o "{{ solana_identity_path }}"
+      become_user: "{{ solana_user }}"
+      environment:
+        HOME: "{{ solana_home }}"
+      args:
+        executable: /bin/bash
+        creates: "{{ solana_identity_path }}"
+
+    - name: Check identity key exists
+      ansible.builtin.stat:
+        path: "{{ solana_identity_path }}"
+      register: solana_identity_stat
+
+    - name: Ensure solana-rpc service state when prerequisites exist
+      ansible.builtin.systemd:
+        name: "{{ solana_rpc_service_name }}"
+        enabled: "{{ solana_rpc_enable_on_boot | bool }}"
+        state: "{{ 'started' if (solana_rpc_start_now | bool) else 'stopped' }}"
+      when:
+        - solana_rpc_manage_service | bool
+        - solana_validator_bin_stat.stat.exists
+        - solana_identity_stat.stat.exists
+
+    - name: Verify solana-rpc is active
+      ansible.builtin.command: systemctl is-active {{ solana_rpc_service_name }}
+      register: solana_rpc_is_active
+      changed_when: false
+      retries: 10
+      delay: 3
+      until: solana_rpc_is_active.stdout.strip() == "active"
+      when:
+        - solana_rpc_manage_service | bool
+        - solana_validator_bin_stat.stat.exists
+        - solana_identity_stat.stat.exists
+
+    - name: Report skipped solana-rpc start due to missing prerequisites
+      ansible.builtin.debug:
+        msg:
+          - "solana-rpc start skipped: missing prerequisites"
+          - "validator_bin={{ solana_validator_bin }} exists={{ solana_validator_bin_stat.stat.exists }}"
+          - "identity={{ solana_identity_path }} exists={{ solana_identity_stat.stat.exists }}"
+      when:
+        - solana_rpc_manage_service | bool
+        - not (solana_validator_bin_stat.stat.exists and solana_identity_stat.stat.exists)
+
     - name: Validate Ansible transport
       ansible.builtin.ping:
 

ansible/templates/solana-rpc.service.j2

@@ -0,0 +1,35 @@
+[Unit]
+Description=Solana RPC node (Agave)
+After=network-online.target
+Wants=network-online.target
+
+[Service]
+Type=simple
+User={{ solana_user }}
+Group={{ solana_group }}
+WorkingDirectory={{ solana_home }}
+Environment=RUST_LOG=info
+LimitNOFILE=1048576
+Restart=always
+RestartSec=5
+TimeoutStopSec=120
+ExecStart={{ solana_validator_bin }} \
+  --identity {{ solana_identity_path }} \
+  --ledger {{ solana_ledger_dir }} \
+  --accounts {{ solana_accounts_dir }} \
+  --bind-address {{ solana_bind_address }} \
+  --rpc-bind-address {{ solana_bind_address }} \
+  --rpc-port {{ solana_rpc_port }} \
+  --dynamic-port-range {{ solana_dynamic_port_range }}{% for ep in solana_entrypoints %} \
+  --entrypoint {{ ep }}{% endfor %}{% for kv in solana_known_validators %} \
+  --known-validator {{ kv }}{% endfor %}{% if solana_geyser_enabled | bool %} \
+  --geyser-plugin-config {{ solana_geyser_plugin_config_path }}{% endif %}{% for arg in solana_rpc_extra_args %} \
+  {{ arg }}{% endfor %} \
+  --log {{ solana_log_dir }}/validator.log
+NoNewPrivileges=true
+PrivateTmp=true
+ProtectSystem=full
+ReadWritePaths={{ solana_ledger_dir }} {{ solana_accounts_dir }} {{ solana_log_dir }} /var/lib/solana
+
+[Install]
+WantedBy=multi-user.target

doc/etap-001-doc-rpc-sol-ansible.md

@@ -4,7 +4,7 @@ Cel etapu: uruchomić pierwszy, bezpieczny playbook Ansible dla hosta RPC (`mevn
 
 ## Zakres
 
-1. Sprawdzić i zainstalować Ansible na VPS (control node), jeśli brak.
+1. Uruchamiać Ansible na VPS jako runner w Dockerze (bez instalacji Ansible na hoście), jeśli to jest preferowany model operacyjny.
 2. Dodać minimalną strukturę Ansible w `trade-iac`:
    - inventory dla `mevnode`,
    - minimalny playbook testowy (bez zmian destrukcyjnych).
@@ -18,6 +18,23 @@ Cel etapu: uruchomić pierwszy, bezpieczny playbook Ansible dla hosta RPC (`mevn
 
 ## Kryteria akceptacji
 
-- `ansible-playbook --version` działa na VPS.
+- `ansible-playbook --version` działa w kontenerze na VPS.
 - Playbook kończy się statusem success dla grupy `sol_rpc`.
 - Wynik zawiera podstawowe fakty hosta i potwierdzenie łączności Ansible.
+
+## Jak uruchomić na VPS (Docker)
+
+Przykład (image: `quay.io/ansible/ansible-runner:latest`):
+
+```bash
+cd /opt/trade-iac
+docker run --rm -t \
+  -v "$PWD/ansible:/ansible" \
+  -v "$HOME/.ssh:/home/runner/.ssh:ro" \
+  -w /ansible \
+  quay.io/ansible/ansible-runner:latest \
+  ansible-playbook -i inventory/hosts.ini playbooks/doc-rpc-sol-min.yml
+```
+
+Ważne: inventory w `ansible/inventory/hosts.ini` używa jawnie pliku klucza
+`/home/runner/.ssh/mevnode_baremetal` (czyli wymaga montowania `~/.ssh` do `/home/runner/.ssh` w kontenerze).

doc/etap-005-solana-rpc-user-svc.md

@@ -0,0 +1,22 @@
+# Etap 005: Solana RPC jako usługa użytkownika `solana`
+
+Cel etapu: przygotować i wdrożyć baseline pod `solana-rpc`, uruchamiany jako dedykowany użytkownik systemowy `solana` (nie `root`).
+
+## Zakres
+
+1. Rozszerzyć playbook o:
+   - pakiety bazowe dla hosta RPC,
+   - utworzenie użytkownika i katalogów `solana`,
+   - deployment unitu `systemd` `solana-rpc.service` z `User=solana`.
+2. Dodać zmienne grupowe `sol_rpc` (ścieżki, porty, opcje startu).
+3. Dodać bezpieczną logikę startu:
+   - unit jest wdrażany zawsze,
+   - start usługi tylko gdy istnieją wymagane artefakty (`agave-validator`, identity keypair).
+4. Wdrożyć na `mevnode` i zweryfikować stan.
+
+## Kryteria akceptacji
+
+- `id solana` istnieje na `mevnode`.
+- `/etc/systemd/system/solana-rpc.service` istnieje i zawiera `User=solana`.
+- Playbook kończy się bez błędów.
+- Jeśli binarka/identity nie istnieją, playbook raportuje to jawnie i nie wymusza startu.

doc/etap-006-agave-install-identity-start.md

@@ -0,0 +1,31 @@
+# Etap 006: Instalacja Agave + identity + start `solana-rpc`
+
+Cel etapu: domknąć bootstrap uruchomienia `solana-rpc` jako `solana` przez:
+
+1. instalację binarki `agave-validator`,
+2. wygenerowanie `identity.json` (jeśli brak),
+3. start usługi `solana-rpc` i test endpointu RPC.
+
+## Zakres
+
+- Rozszerzyć playbook o zadania instalacyjne Agave (idempotentnie).
+- Dodać provisioning `identity` jako użytkownik `solana`.
+- Dodać minimalny tuning OS wymagany przez startup check (`sysctl`).
+- Utrzymać bezpieczny start: usługa uruchamiana tylko przy komplecie prereq.
+- Wykonać testy powdrożeniowe (`systemd`, port, JSON-RPC).
+
+## Założenia
+
+- W tej wersji `agave-validator` wszystkie sockety (gossip/TPU/RPC) muszą być zbindowane do tego samego IP.
+- Na czas bootstrapu bind jest na `0.0.0.0` (żeby validator przeszedł check reachability i wystartował).
+- Produkcyjny bind na WG IP i hardening sieciowy będzie osobnym etapem.
+- Release tar z `agave-install` nie zawiera `agave-validator`, więc `agave-validator` budujemy ze źródeł (tag `v2.x`) i instalujemy do `/opt/solana/bin`.
+- Build wymaga pakietów dev, w tym `libclang`/`llvm` (Ansible instaluje je w playbooku).
+
+## Kryteria akceptacji
+
+- `agave-validator` istnieje pod `/opt/solana/bin/agave-validator`.
+- `identity` istnieje pod `/var/lib/solana/identity.json` (owner `solana`).
+- `systemctl is-active solana-rpc` zwraca `active`.
+- Endpoint `http://127.0.0.1:8899` odpowiada na JSON-RPC.
+- Websocket (pubsub) jest na porcie `solana_rpc_port + 1` (czyli domyślnie `8900` przy `8899`).

doc/workflow.md

@@ -0,0 +1,67 @@
+# Workflow `trade-iac`
+
+Cel: utrzymywać konfigurację IaC w Git i wdrażać ją kontrolowanie na `mevnode`.
+
+## 1. Dodanie klucza SSH na hoście (operator)
+
+Host operatora musi mieć klucz do dostępu do repo `trade/trade-iac`.
+
+Przykład:
+
+```bash
+ssh-keygen -t ed25519 -C "trade-iac-host" -f ~/.ssh/trade_iac
+cat ~/.ssh/trade_iac.pub
+```
+
+Publiczny klucz dodajemy w Gitei (`Settings -> SSH Keys`), a potem test:
+
+```bash
+ssh -T git@gitea.mpabi.pl
+```
+
+## 2. Dodanie klucza SSH na VPS
+
+VPS (runner/deployer) musi mieć osobny klucz do klonowania/pull z `trade/trade-iac`.
+
+Przykład:
+
+```bash
+ssh-keygen -t ed25519 -C "trade-iac-vps" -f ~/.ssh/trade_iac
+cat ~/.ssh/trade_iac.pub
+ssh -T git@gitea.mpabi.pl
+```
+
+Ten publiczny klucz też dodajemy do Gitei.
+
+## 3. Wdrożenie na `mevnode` (runner na VPS)
+
+Po zmianach w repo:
+
+1. `doc/` (opis etapu),
+2. implementacja (playbook/vars),
+3. test/syntax-check,
+4. commit + push,
+5. wdrożenie z VPS na `mevnode`,
+6. testy powdrożeniowe.
+
+Minimalny przepływ:
+
+```bash
+# VPS
+git -C /opt/trade-iac pull --ff-only origin main
+cd /opt/trade-iac
+
+# uruchom Ansible w kontenerze (bez instalacji Ansible na hoście)
+docker run --rm -t \
+  -v "$PWD/ansible:/ansible" \
+  -v "$HOME/.ssh:/home/runner/.ssh:ro" \
+  -w /ansible \
+  quay.io/ansible/ansible-runner:latest \
+  ansible-playbook -i inventory/hosts.ini playbooks/doc-rpc-sol-min.yml
+```
+
+## Zasada pracy
+
+- Zmiany zawsze przez PR/commit (bez ręcznych zmian na `mevnode` poza awarią).
+- `mevnode` traktujemy jako target deploymentu, nie źródło prawdy.
+- Źródłem prawdy jest repo `trade/trade-iac`.