trade-iac/doc/step-002-plan-solana-rpc-mainnet-private.txt

Step 002: Plan wdrozenia Solana RPC (Agave mainnet, non-voting, private RPC + Yellowstone gRPC + monitoring + NOC tmux)

Zalozenia (default):
- WG iface: wg0, WG subnet: 10.66.66.0/24, WG IP baremetala: 10.66.66.1
- Public iface: eth0, public IP: 149.50.116.219
- RPC: 8899 (HTTP), WS: 8900
- Yellowstone gRPC: 10000 (tylko po WG)
- Prometheus: 9090, Grafana: 3000, node_exporter: 9100 (tylko po WG)
- Dynamic port range (cluster): 8000-8020/udp (public eth0)
- User: solana
- Dirs:
  - /var/lib/solana (ledger/accounts)
  - /etc/solana (configs)
  - /var/log/solana (logs)

0) Preconditions (check)
  # WG dziala
  ip -4 a show wg0
  wg show

  # SSH po WG dziala (z laptop/VPS):
  #   ssh user@10.66.66.1

1) Packages
  sudo apt-get update
  sudo apt-get install -y \
    tmux btop \
    sysstat iotop \
    ethtool nstat \
    chrony curl jq git ca-certificates \
    smartmontools nvme-cli \
    build-essential pkg-config libssl-dev \
    clang llvm-dev libclang-dev \
    cmake protobuf-compiler libudev-dev zlib1g-dev

2) Users/Dirs
  sudo getent group solana >/dev/null || sudo groupadd --system solana
  sudo id -u solana >/dev/null 2>&1 || sudo useradd --system --gid solana \
    --home-dir /var/lib/solana --create-home --shell /bin/bash solana

  sudo install -d -o root -g root -m 0755 /etc/solana /opt/solana/bin
  sudo install -d -o solana -g solana -m 0750 \
    /var/lib/solana /var/lib/solana/ledger /var/lib/solana/accounts /var/log/solana

  # identity permissions (po utworzeniu pliku):
  #   sudo chown solana:solana /var/lib/solana/identity.json
  #   sudo chmod 600 /var/lib/solana/identity.json

3) Firewall/WG binding (private-only)
  # UFW: deny everything inbound by default
  sudo ufw default deny incoming
  sudo ufw default allow outgoing

  # WireGuard (public)
  sudo ufw allow 51820/udp

  # SSH only via WG
  sudo ufw allow in on wg0 to any port 22 proto tcp
  sudo ufw deny  in on eth0 to any port 22 proto tcp

  # RPC/WS only via WG
  sudo ufw allow in on wg0 to any port 8899 proto tcp
  sudo ufw allow in on wg0 to any port 8900 proto tcp
  sudo ufw deny  in on eth0 to any port 8899 proto tcp
  sudo ufw deny  in on eth0 to any port 8900 proto tcp

  # Yellowstone gRPC only via WG
  sudo ufw allow in on wg0 to any port 10000 proto tcp
  sudo ufw deny  in on eth0 to any port 10000 proto tcp

  # Monitoring only via WG
  sudo ufw allow in on wg0 to any port 9100 proto tcp
  sudo ufw allow in on wg0 to any port 9090 proto tcp
  sudo ufw allow in on wg0 to any port 3000 proto tcp
  sudo ufw deny  in on eth0 to any port 9100 proto tcp
  sudo ufw deny  in on eth0 to any port 9090 proto tcp
  sudo ufw deny  in on eth0 to any port 3000 proto tcp

  # Cluster ports (public, inbound) for sync (UDP)
  sudo ufw allow in on eth0 to any port 8000:8020 proto udp
  # (opcjonalnie) jesli bedziesz mial problemy z reachability:
  # sudo ufw allow in on eth0 to any port 8000:8020 proto tcp

  sudo ufw enable
  sudo ufw status verbose

4) Validator (Agave) config + systemd (non-voting RPC-only)
  # sysctl baseline (Agave startup checks)
  sudo tee /etc/sysctl.d/90-agave.conf >/dev/null <<'EOF'
net.core.rmem_max = 134217728
net.core.wmem_max = 134217728
EOF
  sudo sysctl --system

  # ulimit baseline
  sudo tee /etc/security/limits.d/90-solana.conf >/dev/null <<'EOF'
* soft nofile 1000000
* hard nofile 1000000
EOF

  # Agave tools (solana-keygen etc.) for solana user
  sudo -u solana -H bash -lc 'sh -c "$(curl -sSfL https://release.anza.xyz/stable/install)"'

  # Copy binaries to /opt/solana/bin (avoid symlink permission gotchas)
  sudo bash -lc '
  set -e
  SRC="/var/lib/solana/.local/share/solana/install/active_release/bin"
  DST="/opt/solana/bin"
  for f in "$SRC"/*; do
    [ -f "$f" ] || continue
    install -m 0755 "$f" "$DST/$(basename "$f")"
  done
  '

  /opt/solana/bin/solana --version || true
  /opt/solana/bin/solana-keygen --version

  # Build agave-validator (pin tag + toolchain)
  AGAVE_VERSION_TAG="v2.3.13"
  AGAVE_RUST_TOOLCHAIN="1.86.0"
  AGAVE_SRC_DIR="/var/lib/solana/src/agave"

  sudo -u solana -H bash -lc "curl -sSfL https://sh.rustup.rs | sh -s -- -y --default-toolchain '${AGAVE_RUST_TOOLCHAIN}'"
  sudo -u solana -H bash -lc "
    set -euo pipefail
    export PATH=/var/lib/solana/.cargo/bin:\$PATH
    rustup toolchain install '${AGAVE_RUST_TOOLCHAIN}'
    mkdir -p '$(dirname \"$AGAVE_SRC_DIR\")'
    rm -rf '${AGAVE_SRC_DIR}'
    git clone --depth 1 --branch '${AGAVE_VERSION_TAG}' https://github.com/anza-xyz/agave.git '${AGAVE_SRC_DIR}'
    cd '${AGAVE_SRC_DIR}'
    cargo +${AGAVE_RUST_TOOLCHAIN} build --release -p agave-validator
  "

  sudo install -o root -g root -m 0755 \
    "${AGAVE_SRC_DIR}/target/release/agave-validator" \
    /opt/solana/bin/agave-validator

  # Identity (create once)
  if [ ! -f /var/lib/solana/identity.json ]; then
    sudo -u solana -H /opt/solana/bin/solana-keygen new --no-passphrase -o /var/lib/solana/identity.json
    sudo chown solana:solana /var/lib/solana/identity.json
    sudo chmod 600 /var/lib/solana/identity.json
  fi

  # Genesis hash (mainnet)
  GENESIS_HASH="$(curl -sS -H 'content-type: application/json' \
    --data '{"jsonrpc":"2.0","id":1,"method":"getGenesisHash"}' \
    https://api.mainnet-beta.solana.com | jq -r .result)"
  echo "$GENESIS_HASH"

  # systemd unit (bind RPC to WG/localhost only)
  # NOTE: agave-validator expects sockets on the same IP in many setups.
  # We keep bind 0.0.0.0 but rely on UFW to ensure private-only access to RPC/WS.
  sudo tee /etc/systemd/system/agave-validator.service >/dev/null <<EOF
[Unit]
Description=Agave validator (mainnet, RPC-only, non-voting)
After=network-online.target
Wants=network-online.target

[Service]
Type=simple
User=solana
Group=solana
WorkingDirectory=/var/lib/solana
Environment=RUST_LOG=info
LimitNOFILE=1048576
Restart=always
RestartSec=5
TimeoutStopSec=120
ExecStart=/opt/solana/bin/agave-validator \\
  --identity /var/lib/solana/identity.json \\
  --no-voting \\
  --ledger /var/lib/solana/ledger \\
  --accounts /var/lib/solana/accounts \\
  --log /var/log/solana/validator.log \\
  --bind-address 0.0.0.0 \\
  --rpc-bind-address 0.0.0.0 \\
  --rpc-port 8899 \\
  --rpc-pubsub-bind-address 0.0.0.0 \\
  --rpc-pubsub-port 8900 \\
  --dynamic-port-range 8000-8020 \\
  --entrypoint entrypoint.mainnet-beta.solana.com:8001 \\
  --expected-genesis-hash ${GENESIS_HASH} \\
  --full-rpc-api \\
  --limit-ledger-size
NoNewPrivileges=true
PrivateTmp=true
ProtectSystem=full
ReadWritePaths=/var/lib/solana/ledger /var/lib/solana/accounts /var/log/solana /var/lib/solana

[Install]
WantedBy=multi-user.target
EOF

  sudo systemctl daemon-reload
  sudo systemctl enable --now agave-validator

  systemctl is-active agave-validator
  journalctl -u agave-validator -n 80 --no-pager

5) Yellowstone gRPC (plugin) (placeholder)
  # TODO (next step): install/build yellowstone geyser plugin matching Agave 3.0.15,
  # put plugin .so under /opt/solana/plugins/, write /etc/solana/yellowstone-geyser.json,
  # and add to ExecStart:
  #   --geyser-plugin-config /etc/solana/yellowstone-geyser.json
  #
  # Verification (after enable):
  #   journalctl -u agave-validator | rg -i 'geyser|yellowstone|plugin'
  #   ss -lntp | rg ':10000\\b'

6) Drift DLOB (2 markets) (on VPS, not baremetal) (placeholder)
  # VPS env:
  #   PERP_MARKETS_TO_LOAD=0,75
  #   USE_GRPC=true
  #   GRPC_ENDPOINT=http://10.66.66.1:10000

7) Monitoring (bind to WG only) (placeholder)
  # node_exporter (prefer bind to WG):
  #   sudo apt-get install -y prometheus-node-exporter
  #   sudo systemctl enable --now prometheus-node-exporter
  #
  # Prometheus/Grafana (bind to 10.66.66.1):
  #   sudo apt-get install -y prometheus grafana
  #   sudo systemctl enable --now prometheus grafana-server

8) NOC tmux (btop) (placeholder)
  # Create tmux session:
  #   btop
  #   iostat -x 1
  #   vmstat 1
  #   nstat -az
  #   ethtool -S eth0 | head -n 40 (wrap in watch)
  #   journalctl -fu agave-validator
  #   while true; do curl getHealth/getSlot; sleep 1; done

9) Troubleshooting checklist
  # Services:
  systemctl status agave-validator --no-pager
  journalctl -u agave-validator -n 200 --no-pager

  # Sockets and binds:
  ss -lntp | egrep ':22\\b|:8899\\b|:8900\\b|:10000\\b|:9100\\b|:9090\\b|:3000\\b' || true

  # Firewall:
  sudo ufw status verbose

  # RPC local:
  curl -sS -H 'content-type: application/json' \
    --data '{"jsonrpc":"2.0","id":1,"method":"getHealth"}' \
    http://127.0.0.1:8899 | jq .

5-min quick verification checklist
  wg show
  sudo ufw status verbose
  systemctl is-active agave-validator
  ss -lntp | egrep ':8899\\b|:8900\\b|:10000\\b|:9090\\b|:3000\\b|:9100\\b' || true
  curl -sS -H 'content-type: application/json' \
    --data '{"jsonrpc":"2.0","id":1,"method":"getSlot"}' \
    http://10.66.66.1:8899 | jq .