add infra
Some checks are pending
CI / tongo-core (push) Waiting to run
CI / web-terminal (push) Waiting to run
Deploy / build-and-push (push) Waiting to run
Deploy / deploy (push) Blocked by required conditions
Infrastructure / plan (push) Waiting to run
Infrastructure / apply (push) Blocked by required conditions

This commit is contained in:
Pedro Faria da Costa 2026-07-05 23:44:51 +01:00
parent caa0e4225f
commit b6bc68de23
No known key found for this signature in database
14 changed files with 1390 additions and 1 deletions

60
.forgejo/workflows/ci.yml Normal file
View file

@ -0,0 +1,60 @@
name: CI
on:
push:
branches: ['*']
pull_request:
branches: [main]
jobs:
# ── Go Backend ──────────────────────────────────────────
tongo-core:
runs-on: self-hosted
steps:
- uses: actions/checkout@v4
- name: Setup Go
uses: actions/setup-go@v5
with:
go-version: '1.23'
- name: Lint
working-directory: tongo-core
run: |
go install github.com/golangci/golangci-lint/cmd/golangci-lint@latest
golangci-lint run ./...
- name: Test
working-directory: tongo-core
run: go test -race -coverprofile=coverage.out ./...
- name: Build
working-directory: tongo-core
run: go build -o bin/tongo-core ./cmd/tongo-core
# ── Frontend ────────────────────────────────────────────
web-terminal:
runs-on: self-hosted
steps:
- uses: actions/checkout@v4
- name: Setup Node
uses: actions/setup-node@v4
with:
node-version: '22'
- name: Install
working-directory: web-terminal
run: npm ci
- name: Lint & Format Check
working-directory: web-terminal
run: npm run format
- name: Type Check
working-directory: web-terminal
run: npx tsc --noEmit
- name: Build
working-directory: web-terminal
run: npm run build

View file

@ -0,0 +1,88 @@
name: Deploy
on:
push:
branches: [main]
tags: ['v*']
env:
REGISTRY: git.holo-grid.bar
IMAGE_TONGO: git.holo-grid.bar/costap/quarkshologridledger/tongo-core
IMAGE_WEB: git.holo-grid.bar/costap/quarkshologridledger/web-terminal
jobs:
build-and-push:
runs-on: self-hosted
steps:
- uses: actions/checkout@v4
- name: Set image tag
run: |
if [ "${{ github.ref_type }}" = "tag" ]; then
echo "TAG=${{ github.ref_name }}" >> $GITHUB_ENV
else
echo "TAG=sha-$(git rev-parse --short HEAD)" >> $GITHUB_ENV
fi
- name: Login to Forgejo Registry
run: echo "${{ secrets.REGISTRY_TOKEN }}" | docker login $REGISTRY -u ${{ secrets.REGISTRY_USER }} --password-stdin
- name: Build & Push Tongo Core
run: |
docker build -t $IMAGE_TONGO:$TAG -t $IMAGE_TONGO:latest -f tongo-core/Dockerfile .
docker push $IMAGE_TONGO:$TAG
docker push $IMAGE_TONGO:latest
- name: Build & Push Web Terminal
run: |
docker build -t $IMAGE_WEB:$TAG -t $IMAGE_WEB:latest -f web-terminal/Dockerfile .
docker push $IMAGE_WEB:$TAG
docker push $IMAGE_WEB:latest
deploy:
runs-on: self-hosted
needs: build-and-push
steps:
- uses: actions/checkout@v4
- name: Set environment target variables
id: target
run: |
if [[ "${{ github.ref }}" == refs/tags/v* ]]; then
echo "env_name=production" >> $GITHUB_OUTPUT
echo "deploy_host=${{ secrets.DEPLOY_HOST_PROD }}" >> $GITHUB_OUTPUT
echo "deploy_key<<EOF" >> $GITHUB_OUTPUT
echo "${{ secrets.DEPLOY_SSH_KEY_PROD }}" >> $GITHUB_OUTPUT
echo "EOF" >> $GITHUB_OUTPUT
else
echo "env_name=staging" >> $GITHUB_OUTPUT
echo "deploy_host=${{ secrets.DEPLOY_HOST_STAGING }}" >> $GITHUB_OUTPUT
echo "deploy_key<<EOF" >> $GITHUB_OUTPUT
echo "${{ secrets.DEPLOY_SSH_KEY_STAGING }}" >> $GITHUB_OUTPUT
echo "EOF" >> $GITHUB_OUTPUT
fi
- name: Deploy to Hetzner target environment
env:
ENV_NAME: ${{ steps.target.outputs.env_name }}
DEPLOY_HOST: ${{ steps.target.outputs.deploy_host }}
DEPLOY_KEY: ${{ steps.target.outputs.deploy_key }}
run: |
mkdir -p ~/.ssh
echo "$DEPLOY_KEY" > ~/.ssh/deploy_key
chmod 600 ~/.ssh/deploy_key
# Copy production compose file and configuration
scp -i ~/.ssh/deploy_key -o StrictHostKeyChecking=no \
deploy/docker-compose.prod.yml \
deploy/Caddyfile \
deploy@$DEPLOY_HOST:/opt/hologrid/
# Pull new images and restart the stack
ssh -i ~/.ssh/deploy_key -o StrictHostKeyChecking=no deploy@$DEPLOY_HOST << 'EOF'
cd /opt/hologrid
echo "${{ secrets.REGISTRY_TOKEN }}" | docker login ${{ env.REGISTRY }} -u ${{ secrets.REGISTRY_USER }} --password-stdin
docker compose -f docker-compose.prod.yml pull
docker compose -f docker-compose.prod.yml up -d --remove-orphans
docker image prune -f
EOF

View file

@ -0,0 +1,73 @@
name: Infrastructure
on:
push:
branches: [main]
paths: ['infra/**']
pull_request:
branches: [main]
paths: ['infra/**']
jobs:
plan:
runs-on: self-hosted
steps:
- uses: actions/checkout@v4
- name: Setup OpenTofu
uses: opentofu/setup-opentofu@v1
- name: Set environment target
id: target
run: |
if [[ "${{ github.ref }}" == refs/tags/v* ]]; then
echo "env_name=production" >> $GITHUB_OUTPUT
else
echo "env_name=staging" >> $GITHUB_OUTPUT
fi
- name: Init and Plan
working-directory: infra
env:
HCLOUD_TOKEN: ${{ secrets.HCLOUD_TOKEN }}
ENV_NAME: ${{ steps.target.outputs.env_name }}
run: |
tofu init
tofu workspace select $ENV_NAME || tofu workspace new $ENV_NAME
tofu plan -out=tfplan
- name: Upload plan artifact
if: github.event_name == 'pull_request'
uses: actions/upload-artifact@v4
with:
name: tfplan
path: infra/tfplan
apply:
runs-on: self-hosted
needs: plan
if: github.ref == 'refs/heads/main' && github.event_name == 'push'
steps:
- uses: actions/checkout@v4
- name: Setup OpenTofu
uses: opentofu/setup-opentofu@v1
- name: Set environment target
id: target
run: |
if [[ "${{ github.ref }}" == refs/tags/v* ]]; then
echo "env_name=production" >> $GITHUB_OUTPUT
else
echo "env_name=staging" >> $GITHUB_OUTPUT
fi
- name: Init and Apply
working-directory: infra
env:
HCLOUD_TOKEN: ${{ secrets.HCLOUD_TOKEN }}
ENV_NAME: ${{ steps.target.outputs.env_name }}
run: |
tofu init
tofu workspace select $ENV_NAME || tofu workspace new $ENV_NAME
tofu apply -auto-approve

View file

@ -1,9 +1,23 @@
.PHONY: build run clean help infra-up infra-down infra-logs run-engine run-web .PHONY: build run clean help infra-up infra-down infra-logs run-engine run-web infra-init infra-plan infra-apply
# Command Variables # Command Variables
DOCKER_COMPOSE=docker compose DOCKER_COMPOSE=docker compose
GO=go GO=go
NPM=npm NPM=npm
TOFU=tofu
ENV?=staging
## infra-init: Initialize OpenTofu provider configurations
infra-init:
cd infra && $(TOFU) init
## infra-plan: Generate and show an OpenTofu execution plan (use ENV=staging|production)
infra-plan:
cd infra && ($(TOFU) workspace select $(ENV) || $(TOFU) workspace new $(ENV)) && $(TOFU) plan
## infra-apply: Build or change infrastructure (use ENV=staging|production)
infra-apply:
cd infra && ($(TOFU) workspace select $(ENV) || $(TOFU) workspace new $(ENV)) && $(TOFU) apply -auto-approve
## infra-up: Start Valkey and Redis Insight containers in the background ## infra-up: Start Valkey and Redis Insight containers in the background
infra-up: infra-up:
@ -33,3 +47,6 @@ clean: infra-down
## help: Show this help ## help: Show this help
help: help:
@grep -E '^## ' Makefile | sed 's/## //' | column -t -s ':' @grep -E '^## ' Makefile | sed 's/## //' | column -t -s ':'
ssh-git:
@ssh -i ~/.ssh/hologrid_deploy deploy@git.holo-grid.bar

13
infra/.gitignore vendored Normal file
View file

@ -0,0 +1,13 @@
# OpenTofu / Terraform state files
*.tfstate
*.tfstate.*
# Provider plugins and cache
.terraform/
.terraform.lock.hcl
# Sensitive variables — never commit real tokens
terraform.tfvars
# Plan files
*.tfplan

58
infra/app.tf Normal file
View file

@ -0,0 +1,58 @@
#
# Quark's Holo-Grid Ledger App server (Caddy + Tongo + Valkey)
#
# Persistent volume for Valkey data
resource "hcloud_volume" "data" {
count = (terraform.workspace == "staging" || terraform.workspace == "production") ? 1 : 0
name = "hologrid-data-${terraform.workspace}"
size = var.volume_size
location = var.location
format = "ext4"
}
# App server
resource "hcloud_server" "app" {
count = (terraform.workspace == "staging" || terraform.workspace == "production") ? 1 : 0
name = "hologrid-app-${terraform.workspace}"
server_type = var.app_server_type
location = var.location
image = "ubuntu-24.04"
ssh_keys = [hcloud_ssh_key.deploy.id]
firewall_ids = [hcloud_firewall.app[0].id]
user_data = templatefile("${path.module}/cloud-init/app.yml", {
ssh_public_key = var.ssh_public_key
domain = terraform.workspace == "production" ? var.domain : "staging.${var.domain}"
volume_id = hcloud_volume.data[0].id
})
labels = {
role = "app"
environment = terraform.workspace
}
# Wait for the subnet to exist before creating the server
depends_on = [hcloud_network_subnet.servers]
}
# Attach volume to the app server
resource "hcloud_volume_attachment" "data" {
count = (terraform.workspace == "staging" || terraform.workspace == "production") ? 1 : 0
volume_id = hcloud_volume.data[0].id
server_id = hcloud_server.app[0].id
automount = true
}
# Private network attachment
resource "hcloud_server_network" "app" {
count = (terraform.workspace == "staging" || terraform.workspace == "production") ? 1 : 0
server_id = hcloud_server.app[0].id
network_id = hcloud_network.internal[0].id
}

71
infra/cloud-init/app.yml Normal file
View file

@ -0,0 +1,71 @@
#cloud-config
# ──────────────────────────────────────────────────────────────
# Quark's Holo-Grid Ledger — App server cloud-init
# Provisions: Docker, deploy user, volume mount, app directory
# Stack: Caddy · Tongo Core · Web Terminal · Valkey
# ──────────────────────────────────────────────────────────────
package_update: true
package_upgrade: true
packages:
- ca-certificates
- curl
- gnupg
- lsb-release
# ── Install Docker & prepare the application environment ────
runcmd:
# Add Docker GPG key
- install -m 0755 -d /etc/apt/keyrings
- curl -fsSL https://download.docker.com/linux/ubuntu/gpg -o /etc/apt/keyrings/docker.asc
- chmod a+r /etc/apt/keyrings/docker.asc
# Add Docker apt repository
- |
echo "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.asc] https://download.docker.com/linux/ubuntu $(. /etc/os-release && echo "$VERSION_CODENAME") stable" \
> /etc/apt/sources.list.d/docker.list
- apt-get update -y
- apt-get install -y docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin
# Enable and start Docker
- systemctl enable docker
- systemctl start docker
# Create deploy user with docker group access
- useradd -m -s /bin/bash -G docker deploy
- mkdir -p /home/deploy/.ssh
- echo "${ssh_public_key}" > /home/deploy/.ssh/authorized_keys
- chmod 700 /home/deploy/.ssh
- chmod 600 /home/deploy/.ssh/authorized_keys
- chown -R deploy:deploy /home/deploy/.ssh
# Harden SSH — disable root login
- sed -i 's/^#\?PermitRootLogin.*/PermitRootLogin no/' /etc/ssh/sshd_config
- systemctl restart sshd
# Create application directory
- mkdir -p /opt/hologrid
- chown deploy:deploy /opt/hologrid
# Mount the Hetzner persistent volume at /mnt/data
# The volume is pre-formatted as ext4 by Hetzner
- mkdir -p /mnt/data
- |
VOLUME_DEV="/dev/disk/by-id/scsi-0HC_Volume_${volume_id}"
# Wait briefly for the device to appear
for i in $(seq 1 10); do [ -e "$VOLUME_DEV" ] && break; sleep 2; done
mount -o defaults "$VOLUME_DEV" /mnt/data
- |
echo "/dev/disk/by-id/scsi-0HC_Volume_${volume_id} /mnt/data ext4 defaults 0 2" >> /etc/fstab
- chown deploy:deploy /mnt/data
# ── Placeholder for the application stack ────────────────────
# The production docker-compose.prod.yml will be deployed to
# /opt/hologrid/ by the Forgejo Actions CI/CD pipeline.
# It will include:
# - Caddy (reverse proxy + automatic TLS for ${domain})
# - Tongo Core (Go backend)
# - Web Terminal (Node.js SSR frontend)
# - Valkey (cache, data dir mounted from /mnt/data)

View file

@ -0,0 +1,129 @@
#cloud-config
# ──────────────────────────────────────────────────────────────
# Quark's Holo-Grid Ledger — Forgejo server cloud-init
# Provisions: Docker, deploy user, Forgejo + Actions runner
# ──────────────────────────────────────────────────────────────
package_update: true
package_upgrade: true
packages:
- ca-certificates
- curl
- gnupg
- lsb-release
# ── Install Docker from official apt repository ─────────────
runcmd:
# Add Docker GPG key
- install -m 0755 -d /etc/apt/keyrings
- curl -fsSL https://download.docker.com/linux/ubuntu/gpg -o /etc/apt/keyrings/docker.asc
- chmod a+r /etc/apt/keyrings/docker.asc
# Add Docker apt repository
- |
echo "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.asc] https://download.docker.com/linux/ubuntu $(. /etc/os-release && echo "$VERSION_CODENAME") stable" \
> /etc/apt/sources.list.d/docker.list
- apt-get update -y
- apt-get install -y docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin
# Enable and start Docker
- systemctl enable docker
- systemctl start docker
# Create deploy user with docker group access
- useradd -m -s /bin/bash -G docker deploy
- mkdir -p /home/deploy/.ssh
- echo "${ssh_public_key}" > /home/deploy/.ssh/authorized_keys
- chmod 700 /home/deploy/.ssh
- chmod 600 /home/deploy/.ssh/authorized_keys
- chown -R deploy:deploy /home/deploy/.ssh
# Harden SSH — disable root login
- sed -i 's/^#\?PermitRootLogin.*/PermitRootLogin no/' /etc/ssh/sshd_config
- systemctl restart sshd
# Create Forgejo directory
- mkdir -p /opt/forgejo
# Start Forgejo stack (automatic HTTPS via Caddy; runner remains stopped until registered)
- cd /opt/forgejo && docker compose up -d caddy forgejo
# ── Configuration Files ─────────────────────────────────────
write_files:
# ── Caddy Proxy Rule Configuration ────────────────────────
- path: /opt/forgejo/Caddyfile
permissions: "0644"
content: |
${forgejo_domain} {
reverse_proxy forgejo:3000
}
# ── Write docker-compose.yml for Forgejo + Actions runner ───
- path: /opt/forgejo/docker-compose.yml
permissions: "0644"
content: |
# Quark's Holo-Grid Ledger — Forgejo stack
services:
caddy:
image: caddy:2-alpine
container_name: caddy
restart: unless-stopped
ports:
- "80:80"
- "443:443"
volumes:
- ./Caddyfile:/etc/caddy/Caddyfile:ro
- caddy-data:/data
- caddy-config:/config
depends_on:
- forgejo
forgejo:
image: codeberg.org/forgejo/forgejo:9
container_name: forgejo
restart: unless-stopped
expose:
- "3000"
ports:
- "2222:22"
volumes:
- forgejo-data:/data
environment:
- FORGEJO__server__ROOT_URL=https://${forgejo_domain}
- FORGEJO__actions__ENABLED=true
# Docker-in-Docker sidecar for the Actions runner
runner-dind:
image: docker:dind
container_name: runner-dind
restart: unless-stopped
privileged: true
command: dockerd -H tcp://0.0.0.0:2375 --tls=false
volumes:
- runner-dind-data:/var/lib/docker
# Forgejo Actions runner
# NOTE: After first boot, register the runner manually:
# 1. Get a registration token from Forgejo admin → Actions → Runners
# 2. docker exec -it runner forgejo-runner register --instance https://${forgejo_domain} --token <TOKEN> --name hologrid-runner
# 3. docker compose up -d runner
runner:
image: data.forgejo.org/forgejo/runner:6.2.0
container_name: runner
restart: unless-stopped
depends_on:
- runner-dind
environment:
- DOCKER_HOST=tcp://runner-dind:2375
volumes:
- runner-data:/data
command: forgejo-runner daemon --config /data/config.yml
volumes:
caddy-data:
caddy-config:
forgejo-data:
runner-dind-data:
runner-data:

25
infra/forgejo.tf Normal file
View file

@ -0,0 +1,25 @@
#
# Quark's Holo-Grid Ledger Forgejo server (git forge + CI)
#
resource "hcloud_server" "forgejo" {
count = terraform.workspace == "tooling" ? 1 : 0
name = "hologrid-forgejo"
server_type = var.forgejo_server_type
location = var.location
image = "ubuntu-24.04"
ssh_keys = [hcloud_ssh_key.deploy.id]
firewall_ids = [hcloud_firewall.forgejo[0].id]
user_data = templatefile("${path.module}/cloud-init/forgejo.yml", {
ssh_public_key = var.ssh_public_key
forgejo_domain = var.forgejo_domain
})
labels = {
role = "forgejo"
environment = terraform.workspace
}
}

104
infra/network.tf Normal file
View file

@ -0,0 +1,104 @@
#
# Quark's Holo-Grid Ledger SSH key, network & firewalls
#
# SSH key
resource "hcloud_ssh_key" "deploy" {
name = "hologrid-deploy-${terraform.workspace}"
public_key = var.ssh_public_key
}
# Private network
resource "hcloud_network" "internal" {
count = (terraform.workspace == "staging" || terraform.workspace == "production") ? 1 : 0
name = "hologrid-internal-${terraform.workspace}"
ip_range = "10.0.0.0/16"
}
resource "hcloud_network_subnet" "servers" {
count = (terraform.workspace == "staging" || terraform.workspace == "production") ? 1 : 0
network_id = hcloud_network.internal[0].id
type = "cloud"
network_zone = "eu-central"
ip_range = "10.0.1.0/24"
}
# Firewall: Forgejo server
resource "hcloud_firewall" "forgejo" {
count = terraform.workspace == "tooling" ? 1 : 0
name = "hologrid-forgejo"
# SSH (Host OS)
rule {
direction = "in"
protocol = "tcp"
port = "22"
source_ips = ["0.0.0.0/0", "::/0"]
}
# Forgejo SSH (Git pushes)
rule {
direction = "in"
protocol = "tcp"
port = "2222"
source_ips = ["0.0.0.0/0", "::/0"]
}
# HTTP
rule {
direction = "in"
protocol = "tcp"
port = "80"
source_ips = ["0.0.0.0/0", "::/0"]
}
# HTTPS
rule {
direction = "in"
protocol = "tcp"
port = "443"
source_ips = ["0.0.0.0/0", "::/0"]
}
# Forgejo Web UI
rule {
direction = "in"
protocol = "tcp"
port = "3000"
source_ips = ["0.0.0.0/0", "::/0"]
}
}
# Firewall: App server
resource "hcloud_firewall" "app" {
count = (terraform.workspace == "staging" || terraform.workspace == "production") ? 1 : 0
name = "hologrid-app-${terraform.workspace}"
# SSH
rule {
direction = "in"
protocol = "tcp"
port = "22"
source_ips = ["0.0.0.0/0", "::/0"]
}
# HTTP
rule {
direction = "in"
protocol = "tcp"
port = "80"
source_ips = ["0.0.0.0/0", "::/0"]
}
# HTTPS
rule {
direction = "in"
protocol = "tcp"
port = "443"
source_ips = ["0.0.0.0/0", "::/0"]
}
}

23
infra/outputs.tf Normal file
View file

@ -0,0 +1,23 @@
#
# Quark's Holo-Grid Ledger Outputs
#
output "forgejo_ipv4" {
description = "Public IPv4 address of the Forgejo server"
value = length(hcloud_server.forgejo) > 0 ? hcloud_server.forgejo[0].ipv4_address : null
}
output "app_ipv4" {
description = "Public IPv4 address of the App server"
value = length(hcloud_server.app) > 0 ? hcloud_server.app[0].ipv4_address : null
}
output "volume_id" {
description = "Hetzner volume ID for the persistent data volume"
value = length(hcloud_volume.data) > 0 ? hcloud_volume.data[0].id : null
}
output "private_network_id" {
description = "Hetzner private network ID"
value = length(hcloud_network.internal) > 0 ? hcloud_network.internal[0].id : null
}

54
infra/variables.tf Normal file
View file

@ -0,0 +1,54 @@
#
# Quark's Holo-Grid Ledger Input variables
#
variable "hcloud_token" {
description = "Hetzner Cloud API token (read/write)"
type = string
sensitive = true
}
variable "ssh_public_key" {
description = "SSH public key for the deploy user (e.g. ssh-ed25519 AAAA...)"
type = string
}
variable "domain" {
description = "Base domain for the application stack (e.g. hologrid.example.com)"
type = string
}
variable "forgejo_domain" {
description = "Domain for the Forgejo instance (e.g. forgejo.example.com)"
type = string
}
variable "location" {
description = "Hetzner datacenter location"
type = string
default = "nbg1"
}
variable "forgejo_server_type" {
description = "Server type for the Forgejo + CI/CD server"
type = string
default = "cx23"
}
variable "app_server_type" {
description = "Server type for the production application server"
type = string
default = "cx23"
}
variable "volume_size" {
description = "Persistent volume size in GB (for Valkey data)"
type = number
default = 20
}
variable "environment" {
description = "Deployment environment label (e.g. production, staging)"
type = string
default = "production"
}

22
infra/versions.tf Normal file
View file

@ -0,0 +1,22 @@
#
# Quark's Holo-Grid Ledger Provider & version constraints
#
terraform {
required_version = ">= 1.8.0"
required_providers {
hcloud = {
source = "hetznercloud/hcloud"
version = "~> 1.49"
}
}
}
#
# Hetzner Cloud provider authenticated via API token
#
provider "hcloud" {
token = var.hcloud_token
}

652
reference/CICD.md Normal file
View file

@ -0,0 +1,652 @@
# CI/CD Pipeline Reference: Quark's Holo-Grid Ledger
This document defines the complete CI/CD strategy for building, testing, and deploying the Quark's Holo-Grid Ledger monorepo using **Forgejo Actions**, **Hetzner Cloud**, **Docker Compose**, and **OpenTofu**.
---
## 1. Design Principles
| Principle | Rationale |
|---|---|
| **Git-ops driven** | Every infrastructure and deployment change is triggered by a commit, never by manual SSH |
| **Immutable artifacts** | Every merge to `main` produces versioned Docker images stored in the Forgejo Container Registry |
| **Infrastructure-as-Code** | All Hetzner resources (servers, firewalls, networks, volumes) are declared in OpenTofu and applied via pipeline |
| **Separation of concerns** | Infrastructure changes (`infra/`) and application changes (`tongo-core/`, `web-terminal/`) use distinct pipeline stages |
| **Zero-downtime deploy** | Rolling container restarts via `docker compose up -d` with health checks ensure uninterrupted service |
---
## 2. Architecture Overview
The platform uses three isolated OpenTofu workspaces:
1. **`tooling`**: Houses the Forgejo Git host, registry, and runner (Server 1).
2. **`staging`**: Houses the staging application deployment (Server 2 + Volume).
3. **`production`**: Houses the production application deployment (Server 2 + Volume).
This keeps CI/CD workloads isolated from both environments, and ensures staging and production networks are fully partitioned.
```mermaid
graph TB
subgraph Developer
DEV[Developer Workstation]
end
subgraph "Hetzner Cloud"
subgraph "tooling Workspace — Server 1 (cx22 · ~€5.49/mo)"
FORGEJO[Forgejo Instance]
REG[(Container Registry)]
RUNNER[Actions Runner + DinD]
end
subgraph "staging / production Workspace — Server 2 (cx22 · ~€5.49/mo)"
CADDY[Caddy Reverse Proxy]
WEB[Web Terminal Container]
TONGO[Tongo Core Container]
VALKEY[(Valkey Cache)]
VOL[(Hetzner Volume · 20 GB)]
PRIVNET{{Private Network 10.0.0.0/16}}
end
end
DEV -- git push --> FORGEJO
FORGEJO -- triggers --> RUNNER
RUNNER -- docker build + push --> REG
RUNNER -- SSH deploy --> CADDY
REG -- docker pull --> WEB
REG -- docker pull --> TONGO
CADDY --> WEB
WEB --> VALKEY
TONGO --> VALKEY
VALKEY --> VOL
CADDY -.- PRIVNET
VALKEY -.- PRIVNET
```
### Cost Summary
| Resource | Spec | Workspace | Monthly Cost |
|---|---|---|---|
| Forgejo server | CX22 · 2 vCPU · 4 GB RAM | `tooling` | ~€5.49 |
| Application server | CX22 · 2 vCPU · 4 GB RAM | `staging` / `production` | ~€5.49 each |
| Persistent volume | 20 GB ext4 | `staging` / `production` | ~€0.96 each |
| **Total (Staging + Prod + Tools)** | | | **~€18.39** |
---
## 3. Directory Layout
New directories and files added to the monorepo root:
```
quarkshologridledger/
├── .forgejo/
│ └── workflows/
│ ├── ci.yml # Lint, test, build on every push/PR
│ ├── deploy.yml # Build images, push to registry, deploy on main merge
│ └── infra.yml # OpenTofu plan/apply for infrastructure changes
├── infra/ # OpenTofu infrastructure-as-code
│ ├── versions.tf # Provider version constraints (hcloud ~> 1.49)
│ ├── variables.tf # Input variables (token, SSH key, domains)
│ ├── network.tf # Private network, subnets, firewalls, SSH keys
│ ├── forgejo.tf # Forgejo server resource + cloud-init
│ ├── app.tf # Application server + volume resources
│ ├── outputs.tf # Server IPs, volume ID, network ID
│ ├── cloud-init/
│ │ ├── forgejo.yml # Bootstrap: Docker + Forgejo + runner compose
│ │ └── app.yml # Bootstrap: Docker + volume mount
│ ├── terraform.tfvars.example # Example variable values (never commit real)
│ └── .gitignore # Ignore state files, .terraform/, tfvars
├── deploy/
│ ├── docker-compose.prod.yml # Production compose (Caddy, Tongo, Web, Valkey)
│ ├── Caddyfile # Reverse proxy + automatic HTTPS
│ └── .env.example # Template for production env vars
├── tongo-core/
│ └── Dockerfile # Multi-stage Go build
├── web-terminal/
│ └── Dockerfile # Multi-stage Node build (Vinxi/Nitro)
└── docker-compose.yml # Local dev compose (existing — unchanged)
```
---
## 4. Pipeline Stages
### 4.1 CI Pipeline — Every Push & Pull Request
**File:** `.forgejo/workflows/ci.yml`
**Trigger:** All pushes and pull requests to any branch.
**Purpose:** Fast feedback loop — lint, type-check, unit test, and build verification.
```yaml
name: CI
on:
push:
branches: ['*']
pull_request:
branches: [main]
jobs:
# ── Go Backend ──────────────────────────────────────────
tongo-core:
runs-on: self-hosted
steps:
- uses: actions/checkout@v4
- name: Setup Go
uses: actions/setup-go@v5
with:
go-version: '1.23'
- name: Lint
working-directory: tongo-core
run: |
go install github.com/golangci/golangci-lint/cmd/golangci-lint@latest
golangci-lint run ./...
- name: Test
working-directory: tongo-core
run: go test -race -coverprofile=coverage.out ./...
- name: Build
working-directory: tongo-core
run: go build -o bin/tongo-core ./cmd/tongo-core
# ── Frontend ────────────────────────────────────────────
web-terminal:
runs-on: self-hosted
steps:
- uses: actions/checkout@v4
- name: Setup Node
uses: actions/setup-node@v4
with:
node-version: '22'
- name: Install
working-directory: web-terminal
run: npm ci
- name: Lint & Format Check
working-directory: web-terminal
run: npm run format
- name: Type Check
working-directory: web-terminal
run: npx tsc --noEmit
- name: Build
working-directory: web-terminal
run: npm run build
```
> [!NOTE]
> Forgejo Actions uses the same YAML syntax as GitHub Actions. The runner executes
> jobs inside containers via Docker-in-Docker (DinD), so all standard actions are
> compatible.
---
### 4.2 Deploy Pipeline — Merge to Main
**File:** `.forgejo/workflows/deploy.yml`
**Trigger:** Push to `main` branch only.
**Purpose:** Build production Docker images, push to the Forgejo Container Registry, then SSH into the production server and pull + restart.
```yaml
name: Deploy
on:
push:
branches: [main]
env:
REGISTRY: forgejo.yourdomain.com
IMAGE_TONGO: forgejo.yourdomain.com/costap/quarkshologridledger/tongo-core
IMAGE_WEB: forgejo.yourdomain.com/costap/quarkshologridledger/web-terminal
jobs:
build-and-push:
runs-on: self-hosted
steps:
- uses: actions/checkout@v4
- name: Set image tag
run: echo "TAG=sha-$(git rev-parse --short HEAD)" >> $GITHUB_ENV
- name: Login to Forgejo Registry
run: echo "${{ secrets.REGISTRY_TOKEN }}" | docker login $REGISTRY -u ${{ secrets.REGISTRY_USER }} --password-stdin
- name: Build & Push Tongo Core
run: |
docker build -t $IMAGE_TONGO:$TAG -t $IMAGE_TONGO:latest -f tongo-core/Dockerfile .
docker push $IMAGE_TONGO:$TAG
docker push $IMAGE_TONGO:latest
- name: Build & Push Web Terminal
run: |
docker build -t $IMAGE_WEB:$TAG -t $IMAGE_WEB:latest -f web-terminal/Dockerfile .
docker push $IMAGE_WEB:$TAG
docker push $IMAGE_WEB:latest
deploy:
runs-on: self-hosted
needs: build-and-push
steps:
- uses: actions/checkout@v4
- name: Deploy to production
env:
DEPLOY_HOST: ${{ secrets.DEPLOY_HOST }}
DEPLOY_KEY: ${{ secrets.DEPLOY_SSH_KEY }}
run: |
mkdir -p ~/.ssh
echo "$DEPLOY_KEY" > ~/.ssh/deploy_key
chmod 600 ~/.ssh/deploy_key
# Copy production compose file
scp -i ~/.ssh/deploy_key -o StrictHostKeyChecking=no \
deploy/docker-compose.prod.yml \
deploy/Caddyfile \
deploy@$DEPLOY_HOST:/opt/hologrid/
# Pull new images and restart
ssh -i ~/.ssh/deploy_key -o StrictHostKeyChecking=no deploy@$DEPLOY_HOST << 'EOF'
cd /opt/hologrid
echo "${{ secrets.REGISTRY_TOKEN }}" | docker login ${{ env.REGISTRY }} -u ${{ secrets.REGISTRY_USER }} --password-stdin
docker compose -f docker-compose.prod.yml pull
docker compose -f docker-compose.prod.yml up -d --remove-orphans
docker image prune -f
EOF
```
> [!IMPORTANT]
> **Secrets required in Forgejo repository settings:**
> - `REGISTRY_TOKEN` — Personal Access Token with `packages:write` scope
> - `REGISTRY_USER` — Forgejo username
> - `DEPLOY_HOST` — Production server IP (from OpenTofu output)
> - `DEPLOY_SSH_KEY` — Private SSH key for deployment user
---
### 4.3 Infrastructure Pipeline — Changes to `infra/`
**File:** `.forgejo/workflows/infra.yml`
**Trigger:** Push to `main` with changes in `infra/**`.
**Purpose:** Plan and apply OpenTofu changes to Hetzner Cloud resources.
```yaml
name: Infrastructure
on:
push:
branches: [main]
paths: ['infra/**']
pull_request:
branches: [main]
paths: ['infra/**']
jobs:
plan:
runs-on: self-hosted
steps:
- uses: actions/checkout@v4
- name: Setup OpenTofu
uses: opentofu/setup-opentofu@v1
- name: Init
working-directory: infra
run: tofu init
- name: Plan
working-directory: infra
env:
HCLOUD_TOKEN: ${{ secrets.HCLOUD_TOKEN }}
run: tofu plan -out=tfplan
- name: Upload plan artifact
if: github.event_name == 'pull_request'
uses: actions/upload-artifact@v4
with:
name: tfplan
path: infra/tfplan
apply:
runs-on: self-hosted
needs: plan
if: github.ref == 'refs/heads/main' && github.event_name == 'push'
steps:
- uses: actions/checkout@v4
- name: Setup OpenTofu
uses: opentofu/setup-opentofu@v1
- name: Init & Apply
working-directory: infra
env:
HCLOUD_TOKEN: ${{ secrets.HCLOUD_TOKEN }}
run: |
tofu init
tofu apply -auto-approve
```
> [!WARNING]
> The `apply` job runs **only** on direct pushes to `main`, never on PRs. PRs
> will show the `plan` output for review. Consider adding a manual approval gate
> for production infrastructure changes as the project matures.
---
## 5. Infrastructure Layer (OpenTofu)
All infrastructure is declared in `infra/` and managed via the `infra.yml` pipeline.
See the scaffolded files for the full implementation.
### 5.1 Resource Map
| Resource | Type | Purpose |
|---|---|---|
| `hcloud_ssh_key.deploy` | SSH Key | Deployment access to both servers |
| `hcloud_network.internal` | Network | Private network `10.0.0.0/16` for inter-server communication |
| `hcloud_network_subnet.servers` | Subnet | `10.0.1.0/24` in `eu-central` zone |
| `hcloud_firewall.forgejo` | Firewall | Forgejo server: allow ports 22, 80, 443 inbound |
| `hcloud_firewall.app` | Firewall | App server: allow ports 22, 80, 443 inbound |
| `hcloud_server.forgejo` | Server (cx22) | Forgejo instance + Actions runner + DinD |
| `hcloud_server.app` | Server (cx22) | Production app stack (Caddy, Tongo, Web, Valkey) |
| `hcloud_volume.data` | Volume (20 GB) | Persistent Valkey data, mounted at `/mnt/data` |
| `hcloud_volume_attachment.data` | Attachment | Binds volume to app server |
| `hcloud_server_network.forgejo` | Network Attach | Forgejo server on private network |
| `hcloud_server_network.app` | Network Attach | App server on private network |
### 5.2 Server Specifications
| Parameter | Forgejo Server | Application Server |
|---|---|---|
| Role | Git forge, CI runner, container registry | Production app stack |
| Type | `cx22` (2 vCPU, 4 GB RAM) | `cx22` (2 vCPU, 4 GB RAM) |
| Image | `ubuntu-24.04` | `ubuntu-24.04` |
| Location | `nbg1` (Nuremberg) | `nbg1` (Nuremberg) |
| Volume | — | 20 GB ext4 at `/mnt/data` |
| Cloud-init | `infra/cloud-init/forgejo.yml` | `infra/cloud-init/app.yml` |
### 5.3 Cloud-Init Bootstrap
Each server is provisioned immutably via `cloud-init` templates in `infra/cloud-init/`.
**Forgejo server (`forgejo.yml`)** — on first boot:
1. Installs Docker Engine + Compose plugin
2. Creates `deploy` user with SSH key + docker group
3. Disables root SSH login
4. Writes a Docker Compose file to `/opt/forgejo/` containing:
- Forgejo v9 (ports 3000 + 2222, Actions enabled)
- Docker-in-Docker daemon for the runner
- Forgejo Actions runner (requires one-time manual registration)
5. Starts Forgejo via `docker compose up -d`
**Application server (`app.yml`)** — on first boot:
1. Installs Docker Engine + Compose plugin
2. Creates `deploy` user with SSH key + docker group
3. Disables root SSH login
4. Formats and mounts the Hetzner Volume at `/mnt/data`
5. Creates `/opt/hologrid/` ready for CI-deployed compose files
> [!NOTE]
> Cloud-init templates use OpenTofu `templatefile()` interpolation.
> Variables like `${ssh_public_key}`, `${forgejo_domain}`, and `${volume_id}`
> are injected at `tofu apply` time.
---
## 6. Production Docker Compose
### 6.1 `deploy/docker-compose.prod.yml`
```yaml
services:
caddy:
image: caddy:2-alpine
container_name: hologrid-caddy
ports:
- "80:80"
- "443:443"
volumes:
- ./Caddyfile:/etc/caddy/Caddyfile:ro
- caddy_data:/data
- caddy_config:/config
depends_on:
web-terminal:
condition: service_healthy
restart: unless-stopped
tongo-core:
image: forgejo.yourdomain.com/costap/quarkshologridledger/tongo-core:latest
container_name: hologrid-tongo
environment:
- TONGO_CACHE_ADDR=valkey:6379
- TONGO_LOG_LEVEL=info
depends_on:
valkey:
condition: service_healthy
restart: unless-stopped
web-terminal:
image: forgejo.yourdomain.com/costap/quarkshologridledger/web-terminal:latest
container_name: hologrid-web
environment:
- VALKEY_URL=redis://valkey:6379
- PORT=3000
healthcheck:
test: ["CMD", "wget", "--spider", "-q", "http://localhost:3000"]
interval: 10s
timeout: 5s
retries: 3
depends_on:
valkey:
condition: service_healthy
restart: unless-stopped
valkey:
image: valkey/valkey:8.0
container_name: hologrid-valkey
volumes:
- /mnt/data/valkey:/data
healthcheck:
test: ["CMD", "valkey-cli", "ping"]
interval: 5s
timeout: 3s
retries: 5
restart: unless-stopped
volumes:
caddy_data:
caddy_config:
```
### 6.2 `deploy/Caddyfile`
```caddyfile
hologrid.yourdomain.com {
reverse_proxy web-terminal:3000
}
```
> [!TIP]
> Caddy automatically provisions and renews Let's Encrypt TLS certificates.
> No certbot configuration required. Just point your DNS A record to the
> Hetzner server IP.
---
## 7. Dockerfiles
### 7.1 `tongo-core/Dockerfile`
```dockerfile
# ── Build Stage ───────────────────────────────────────────
FROM golang:1.23-alpine AS builder
WORKDIR /build
COPY tongo-core/go.mod tongo-core/go.sum ./
RUN go mod download
COPY tongo-core/ .
COPY data-source/ /data-source/
RUN CGO_ENABLED=0 GOOS=linux go build -ldflags="-s -w" -o /tongo-core ./cmd/tongo-core
# ── Runtime Stage ─────────────────────────────────────────
FROM alpine:3.20
RUN apk --no-cache add ca-certificates
COPY --from=builder /tongo-core /usr/local/bin/tongo-core
COPY --from=builder /data-source /data-source
ENTRYPOINT ["tongo-core"]
CMD ["serve"]
```
### 7.2 `web-terminal/Dockerfile`
```dockerfile
# ── Build Stage ───────────────────────────────────────────
FROM node:22-alpine AS builder
WORKDIR /app
COPY web-terminal/package.json web-terminal/package-lock.json ./
RUN npm ci
COPY web-terminal/ .
RUN npm run build
# ── Runtime Stage ─────────────────────────────────────────
FROM node:22-alpine
WORKDIR /app
COPY --from=builder /app/.output .output
COPY --from=builder /app/package.json .
EXPOSE 3000
CMD ["node", ".output/server/index.mjs"]
```
---
## 8. Secrets & Environment Variables
All secrets are stored in **Forgejo Repository Settings → Actions → Secrets**.
| Secret Name | Description | Used By |
|---|---|---|
| `HCLOUD_TOKEN` | Hetzner Cloud API token | `infra.yml` |
| `REGISTRY_TOKEN` | Forgejo PAT with `packages:write` | `deploy.yml` |
| `REGISTRY_USER` | Forgejo username | `deploy.yml` |
| `DEPLOY_HOST` | Production server IP address | `deploy.yml` |
| `DEPLOY_SSH_KEY` | Private SSH key for `deploy` user | `deploy.yml` |
> [!CAUTION]
> Never commit real tokens, keys, or passwords to the repository. The
> `terraform.tfvars` file is `.gitignore`-d. All sensitive values flow through
> Forgejo Secrets → pipeline environment variables → runtime containers.
---
## 9. Forgejo Runner Setup
The Forgejo Actions runner runs on the **Forgejo server** (Server 1), deployed automatically via cloud-init alongside the Forgejo instance.
The runner uses Docker-in-Docker (DinD) to execute workflow jobs in isolated containers.
### 9.1 Compose Stack (auto-deployed via cloud-init)
The `infra/cloud-init/forgejo.yml` template writes a Docker Compose file to `/opt/forgejo/docker-compose.yml`
containing three services:
| Service | Image | Purpose |
|---|---|---|
| `forgejo` | `codeberg.org/forgejo/forgejo:9` | Git forge, container registry, Actions coordinator |
| `runner-dind` | `docker:dind` | Privileged Docker daemon for runner job execution |
| `runner` | `data.forgejo.org/forgejo/runner:6.2.0` | Actions runner daemon (connects to DinD) |
### 9.2 One-Time Runner Registration
After the Forgejo server boots and the services are running, SSH in and register the runner:
```bash
# SSH into the Forgejo server
ssh deploy@<FORGEJO_IP>
# Enter the runner container
cd /opt/forgejo
docker compose exec runner forgejo-runner register \
--instance https://forgejo.yourdomain.com \
--token <REGISTRATION_TOKEN> \
--name hologrid-runner \
--labels self-hosted \
--no-interactive
# Restart to apply config
docker compose restart runner
```
> [!TIP]
> Get the registration token from Forgejo: **Site Administration → Actions → Runners → Create New Runner**.
> This is a one-time operation. The runner persists its config in a Docker volume.
---
## 10. Deployment Flow Summary
```mermaid
sequenceDiagram
participant Dev as Developer
participant Forgejo as Forgejo (Server 1)
participant Runner as CI Runner (Server 1)
participant Registry as Container Registry (Server 1)
participant App as App Server (Server 2)
Dev->>Forgejo: git push (feature branch)
Forgejo->>Runner: Trigger ci.yml
Runner->>Runner: Lint + Test + Build (Go & Node)
Runner-->>Forgejo: ✓ Status check passes
Dev->>Forgejo: Merge PR → main
Forgejo->>Runner: Trigger deploy.yml
Runner->>Runner: docker build (tongo-core, web-terminal)
Runner->>Registry: docker push :sha-abc1234 + :latest
Runner->>App: SSH → copy compose + pull + up -d
App->>Registry: docker pull latest images (via private network)
App->>App: Rolling restart containers
App-->>Runner: ✓ Deployment complete
```
---
## 11. Future Enhancements
| Enhancement | Priority | Description |
|---|---|---|
| **Staging environment** | Medium | Add a second Hetzner server for pre-production validation before promoting to prod |
| **Database migrations** | Medium | Add a `migrate` job in `deploy.yml` if persistent SQL storage is added later |
| **Smoke tests** | Medium | Add a post-deploy health check job that curls the production URL and validates HTTP 200 |
| **Notifications** | Low | Integrate Forgejo webhooks to send deployment status to a Matrix/Discord channel |
| **Horizontal scaling** | Low | Move from single server to Docker Swarm or Nomad when traffic demands it |
| **Secrets manager** | Low | Replace Forgejo Secrets with HashiCorp Vault or Infisical for rotation policies |
| **Monitoring stack** | Low | Add Prometheus + Grafana containers to the production compose for observability |
| **Automatic rollback** | Low | If the post-deploy health check fails, automatically roll back to the previous image tag |
---
## 12. Decision Log
| Decision | Chosen | Alternatives Considered | Rationale |
|---|---|---|---|
| **Forgejo hosting** | Self-hosted on Hetzner CX22 (~€5.49/mo) | Codeberg (free, FOSS only), Codey managed (~€39/mo), same-server co-hosting | Cheapest option with full control; separate server isolates CI builds from production; built-in container registry eliminates Docker Hub dependency |
| **Server topology** | Two CX22 servers + private network | Single server for everything, dedicated auction server | Isolation between CI and production workloads; either can be rebuilt independently; total cost ~€12/mo |
| CI/CD engine | Forgejo Actions | Woodpecker CI, Drone | Native integration with Forgejo, GitHub Actions syntax familiarity, built-in container registry |
| Container registry | Forgejo Packages | Harbor, Docker Hub | Zero additional infrastructure — ships with every Forgejo instance |
| IaC tool | OpenTofu | Pulumi, Ansible | Declarative HCL, excellent Hetzner provider, open-source fork of Terraform |
| Reverse proxy | Caddy | Nginx, Traefik | Automatic HTTPS with zero config, single binary, Caddyfile simplicity |
| Server provisioning | cloud-init | Ansible, Packer | No agent required, runs once at boot, perfect for immutable infrastructure |
| Deployment strategy | SSH + docker compose pull | Watchtower, Kubernetes | Simplest approach for a two-server deployment, easy to reason about |
| Persistent storage | Hetzner Volume | Local disk, NFS | Survives server rebuilds, attached to app server only |