Infra Backups

2025-05-30 12:45:19 -04:00 · 2025-05-30 12:45:19 -04:00 · 75357d31a2
parent 8a6b48bf54
commit 75357d31a2
44 changed files with 1190 additions and 0 deletions
--- a/.env.example
+++ b/.env.example
@ -0,0 +1,20 @@
+# ---- Port Bindings ----
+PROMETHEUS_PORT=9390
+ALERTMANAGER_PORT=9043
+LOKI_PORT=4100
+NODE_EXPORTER_PORT=9112
+DOCKER_EXPORTER_PORT=9487
+PIHOLE_WEB_PORT=8040
+PIHOLE_DNS_PORT=53
+PIHOLE_HTTPS_PORT=443
+
+# ---- Pihole Config ----
+PIHOLE_PASSWORD=
+TZ=America/Toronto
+WEBPASSWORD=
+
+# ---- Alertmanager ----
+DISCORD_WEBHOOK=
+
+# ---- Metadata ----
+HOSTNAME=
--- a/.gitignore
+++ b/.gitignore
@ -0,0 +1 @@
+.env
--- a/docs.md
+++ b/docs.md
@ -0,0 +1,124 @@
+# 🧠 Infrastructure Usage Guide
+
+This guide explains how to use the new Docker-based infrastructure stack deployed on **Unraid**, **Raspberry Pi**, and **Ubuntu**.
+
+---
+
+## 🚀 Starting the Monitoring Stack
+
+Each machine has a `start.sh` wrapper script that:
+- Runs `docker compose up -d`
+- Waits a few seconds for container readiness
+- Logs container IPs, ports, and web URLs
+
+To start:
+
+```bash
+chmod +x start.sh
+./start.sh
+```
+
+---
+
+## 📜 Service Overview
+
+The following services are deployed (excluding Pi-hole on Ubuntu):
+
+| Service           | Unraid | Pi | Ubuntu |
+|------------------|--------|----|-------|
+| Grafana          | ✅     | ❌ | ❌     |
+| Prometheus       | ✅     | ✅ | ✅     |
+| Loki             | ✅     | ✅ | ✅     |
+| Promtail         | ✅     | ✅ | ✅     |
+| Alertmanager     | ✅     | ✅ | ✅     |
+| Node Exporter    | ✅     | ✅ | ✅     |
+| Docker Exporter  | ✅     | ✅ | ✅     |
+| NGINX            | ✅     | ❌ | ❌     |
+| Pi-hole          | ❌     | ✅ | ❌     |
+
+---
+
+## 🌐 Defining IPs in Pi-hole
+
+Pi-hole is used for internal DNS routing of your infrastructure services.
+
+1. Open the **Pi-hole Admin Panel**
+2. Go to **Local DNS > DNS Records**
+3. Add A records like:
+
+```
+grafana.wyvern.bray.io -> 192.168.2.16
+prometheus.unraid.bray.io -> 192.168.2.28
+```
+
+> 💡 You can verify entries with `dig grafana.wyvern.bray.io`.
+
+---
+
+## 🌐 Defining IPs in NGINX
+
+NGINX is used to expose internal services via friendly URLs (e.g. `https://grafana.wyvern.bray.io`).
+
+1. Go to: `configs/nginx/conf.d/`
+2. Create or modify `.conf` files like:
+
+```nginx
+server {
+    listen 443 ssl;
+    server_name grafana.wyvern.bray.io;
+
+    ssl_certificate /etc/nginx/ssl/fullchain.pem;
+    ssl_certificate_key /etc/nginx/ssl/privkey.pem;
+
+    location / {
+        proxy_pass http://192.168.2.16:3000;
+        proxy_set_header Host $host;
+        proxy_set_header X-Real-IP $remote_addr;
+    }
+}
+```
+
+3. Reload NGINX (or restart the container):
+
+```bash
+docker restart nginx
+```
+
+---
+
+## 📦 Adding New Services
+
+1. Add a new service to `docker-compose.yml`
+2. Assign a static IP if needed
+3. Add a DNS record in Pi-hole
+4. Add a reverse proxy entry in NGINX
+
+---
+
+## 🧼 Resetting the Stack
+
+You can bring down and rebuild everything:
+
+```bash
+docker compose down
+docker compose up -d --build
+```
+
+This will recreate containers with preserved volumes. If you wipe volumes, services will reset.
+
+---
+
+## ✅ Final Tip: Keep a Network Sheet
+
+Keep a `infra-ip-map.md` or spreadsheet like:
+
+| Container      | Host    | Static IP     | Port | Domain                            |
+|----------------|---------|----------------|------|-----------------------------------|
+| Grafana        | Unraid  | 192.168.2.16   | 3000 | grafana.wyvern.bray.io            |
+| Prometheus     | Pi      | 192.168.2.28   | 9090 | prometheus.pi.bray.io             |
+| Loki           | Ubuntu  | 192.168.2.26   | 3100 | loki.ubuntu.bray.io               |
+| ...            | ...     | ...            | ...  | ...                               |
+
+---
+
+Happy hacking 🛠️
--- a/pi/Dockerfile.alertmanager
+++ b/pi/Dockerfile.alertmanager
@ -0,0 +1,19 @@
+# Stage 1: Pull original Alertmanager binary
+FROM prom/alertmanager:latest as upstream
+
+# Stage 2: Alpine + envsubst
+FROM alpine:latest
+
+# Install envsubst and ca-certificates
+RUN apk add --no-cache gettext ca-certificates
+
+# Create directories
+RUN mkdir -p /etc/alertmanager /alertmanager
+
+# Copy Alertmanager binary from upstream
+COPY --from=upstream /bin/alertmanager /bin/alertmanager
+COPY --from=upstream /etc/alertmanager /etc/alertmanager
+
+# Default config will be overwritten by volume mount
+ENTRYPOINT ["/bin/sh", "-c"]
+CMD ["envsubst < /etc/alertmanager/alertmanager.template.yml > /etc/alertmanager/alertmanager.yml && /bin/alertmanager --config.file=/etc/alertmanager/alertmanager.yml"]
--- a/pi/configs/alertmanager/alertmanager.template.yml
+++ b/pi/configs/alertmanager/alertmanager.template.yml
@ -0,0 +1,14 @@
+global:
+  resolve_timeout: 5m
+
+route:
+  receiver: 'discord'
+  group_wait: 10s
+  group_interval: 30s
+  repeat_interval: 1h
+
+receivers:
+  - name: 'discord'
+    webhook_configs:
+      - url: 'https://discord.com/api/webhooks/1367657026280226907/vRMRq22mikrAAJerUBAcxWPbRgZeY5fF3YE_3u0fZnGCEzNIPot36fBLP7yZ4i55IMSz'
+        send_resolved: true
--- a/pi/configs/alertmanager/alertmanager.yml
+++ b/pi/configs/alertmanager/alertmanager.yml
@ -0,0 +1,14 @@
+global:
+  resolve_timeout: 5m
+
+route:
+  receiver: 'discord'
+  group_wait: 10s
+  group_interval: 30s
+  repeat_interval: 1h
+
+receivers:
+  - name: 'discord'
+    webhook_configs:
+      - url: 'https://discord.com/api/webhooks/1367657026280226907/vRMRq22mikrAAJerUBAcxWPbRgZeY5fF3YE_3u0fZnGCEzNIPot36fBLP7yZ4i55IMSz'
+        send_resolved: true
--- a/pi/configs/loki/loki-config.yml
+++ b/pi/configs/loki/loki-config.yml
@ -0,0 +1,53 @@
+auth_enabled: false
+
+server:
+  http_listen_port: 3100
+  grpc_listen_port: 9095
+
+ingester:
+  lifecycler:
+    ring:
+      kvstore:
+        store: inmemory
+      replication_factor: 1
+  chunk_idle_period: 5m
+  chunk_retain_period: 30s
+  wal:
+    dir: /loki/wal
+
+limits_config:
+  reject_old_samples: true
+  reject_old_samples_max_age: 168h
+  allow_structured_metadata: false
+
+schema_config:
+  configs:
+    - from: 2024-01-01
+      store: boltdb-shipper
+      object_store: filesystem
+      schema: v12
+      index:
+        prefix: index_
+        period: 24h
+
+storage_config:
+  boltdb_shipper:
+    active_index_directory: /loki/index
+    cache_location: /loki/cache
+  filesystem:
+    directory: /loki/chunks
+
+ruler:
+  storage:
+    type: local
+    local:
+      directory: /loki/rules
+  rule_path: /loki/rules-temp
+  alertmanager_url: http://localhost:9093
+  ring:
+    kvstore:
+      store: inmemory
+  enable_api: true
+
+compactor:
+  working_directory: /loki/compactor
--- a/pi/configs/prometheus/prometheus.yml
+++ b/pi/configs/prometheus/prometheus.yml
@ -0,0 +1,19 @@
+global:
+  scrape_interval: 15s
+
+scrape_configs:
+  - job_name: 'prometheus'
+    static_configs:
+      - targets: ['localhost:9090']
+
+  - job_name: 'node_exporter'
+    static_configs:
+      - targets: ['localhost:9100']
+
+  - job_name: 'docker_exporter'
+    static_configs:
+      - targets: ['localhost:9487']
+
+  - job_name: 'pi_services'
+    static_configs:
+      - targets: ['localhost:3100', 'localhost:9093']
--- a/pi/configs/promtail/positions.yaml
+++ b/pi/configs/promtail/positions.yaml
@ -0,0 +1,9 @@
+positions:
+  /var/log/alf.log: "0"
+  /var/log/fsck_apfs.log: "23880"
+  /var/log/fsck_apfs_error.log: "676"
+  /var/log/fsck_hfs.log: "4044"
+  /var/log/install.log: "3800109"
+  /var/log/shutdown_monitor.log: "1971"
+  /var/log/system.log: "6132"
+  /var/log/wifi.log: "1107090"
--- a/pi/configs/promtail/promtail.yml
+++ b/pi/configs/promtail/promtail.yml
@ -0,0 +1,18 @@
+server:
+  http_listen_port: 9080
+  grpc_listen_port: 0
+
+positions:
+  filename: /etc/promtail/positions.yaml
+
+clients:
+  - url: http://loki:3100/loki/api/v1/push
+
+scrape_configs:
+  - job_name: system
+    static_configs:
+      - targets:
+          - localhost
+        labels:
+          job: varlogs
+          __path__: /var/log/*.log
--- a/pi/docker-compose.yml
+++ b/pi/docker-compose.yml
@ -0,0 +1,87 @@
+services:
+  prometheus:
+    image: prom/prometheus:latest
+    container_name: ${HOSTNAME}-prometheus
+    ports:
+      - "${PROMETHEUS_PORT}:9090"
+    volumes:
+      - ./configs/prometheus:/etc/prometheus
+      - prometheus-data:/prometheus
+    command:
+      - --config.file=/etc/prometheus/prometheus.yml
+    restart: unless-stopped
+
+  alertmanager:
+    build:
+      context: .
+      dockerfile: Dockerfile.alertmanager
+    container_name: ${HOSTNAME}-alertmanager
+    ports:
+      - "${ALERTMANAGER_PORT}:9093"
+    volumes:
+      - ./configs/alertmanager:/etc/alertmanager
+    environment:
+      - DISCORD_WEBHOOK_URL=${DISCORD_WEBHOOK_URL}
+    restart: unless-stopped
+
+  loki:
+    image: grafana/loki:latest
+    container_name: ${HOSTNAME}-loki
+    ports:
+      - "${LOKI_PORT}:3100"
+    volumes:
+      - ./configs/loki:/etc/loki
+      - loki-data:/loki
+    command: ["-config.file=/etc/loki/loki-config.yml", "-config.expand-env=true"]
+    restart: unless-stopped
+
+  promtail:
+    image: grafana/promtail:latest
+    container_name: ${HOSTNAME}-promtail
+    volumes:
+      - /var/log:/var/log:ro
+      - ./configs/promtail:/etc/promtail
+    command: -config.file=/etc/promtail/promtail.yml
+    restart: unless-stopped
+
+  node_exporter:
+    image: prom/node-exporter:latest
+    container_name: ${HOSTNAME}-node_exporter
+    ports:
+      - "${NODE_EXPORTER_PORT}:9100"
+    command:
+      - --web.listen-address=:${NODE_EXPORTER_PORT}
+    restart: unless-stopped
+
+  docker_exporter:
+    image: prometheusnet/docker_exporter
+    container_name: ${HOSTNAME}-docker_exporter
+    ports:
+      - "${DOCKER_EXPORTER_PORT}:9487"
+    volumes:
+      - /var/run/docker.sock:/var/run/docker.sock
+    restart: unless-stopped
+
+  pihole:
+    image: pihole/pihole:latest
+    container_name: ${HOSTNAME}-pihole
+    hostname: ${HOSTNAME}
+    ports:
+      - "${PIHOLE_WEB_PORT}:80"
+      - "${PIHOLE_HTTPS_PORT}:443"
+      - "${PIHOLE_DNS_PORT}:53/udp"
+    environment:
+      TZ: ${TZ}
+      WEBPASSWORD: ${WEBPASSWORD}
+    volumes:
+      - pihole-config:/etc/pihole
+      - dnsmasq-config:/etc/dnsmasq.d
+    cap_add:
+      - NET_ADMIN
+    restart: unless-stopped
+
+volumes:
+  prometheus-data:
+  loki-data:
+  pihole-config:
+  dnsmasq-config:
--- a/pi/stack-status.sh
+++ b/pi/stack-status.sh
@ -0,0 +1,27 @@
+#!/bin/bash
+
+# Optional base hostname (e.g., monitor.local)
+BASE_HOST="${NGINX_HOST:-localhost}"
+
+printf "%-20s | %-15s | %-25s | %-40s\n" "Service" "IP Address" "Port Bindings" "URL"
+printf "%s\n" "---------------------------------------------------------------------------------------------------------------"
+
+for container in $(docker ps --format '{{.Names}}'); do
+  ip=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' "$container")
+
+  ports=$(docker inspect -f '{{range $p, $conf := .NetworkSettings.Ports}}{{if $conf}}{{$conf  | json}}{{end}}{{end}}' "$container" \
+    | jq -r '.[] | "\(.HostPort)"' 2>/dev/null | paste -sd "," -)
+
+  if [ -z "$ports" ]; then
+    ports="(none)"
+    url="(none)"
+  else
+    # Just use the first port for URL
+    first_port=$(echo "$ports" | cut -d',' -f1)
+    scheme="http"
+    [ "$first_port" = "443" ] && scheme="https"
+    url="$scheme://$BASE_HOST:$first_port"
+  fi
+
+  printf "%-20s | %-15s | %-25s | %-40s\n" "$container" "$ip" "$ports" "$url"
+done
--- a/pi/start.sh
+++ b/pi/start.sh
@ -0,0 +1,16 @@
+#!/bin/bash
+
+echo "🚀 Starting monitoring stack..."
+docker-compose up -d
+
+if [ $? -ne 0 ]; then
+  echo "❌ Failed to start containers. Check docker-compose logs."
+  exit 1
+fi
+
+echo "✅ Containers started. Waiting a few seconds for services to boot..."
+sleep 5
+
+echo ""
+echo "🔍 Gathering service info:"
+./stack-status.sh
--- a/ubuntu-1/Dockerfile.alertmanager
+++ b/ubuntu-1/Dockerfile.alertmanager
@ -0,0 +1,19 @@
+# Stage 1: Pull original Alertmanager binary
+FROM prom/alertmanager:latest as upstream
+
+# Stage 2: Alpine + envsubst
+FROM alpine:latest
+
+# Install envsubst and ca-certificates
+RUN apk add --no-cache gettext ca-certificates
+
+# Create directories
+RUN mkdir -p /etc/alertmanager /alertmanager
+
+# Copy Alertmanager binary from upstream
+COPY --from=upstream /bin/alertmanager /bin/alertmanager
+COPY --from=upstream /etc/alertmanager /etc/alertmanager
+
+# Default config will be overwritten by volume mount
+ENTRYPOINT ["/bin/sh", "-c"]
+CMD ["envsubst < /etc/alertmanager/alertmanager.template.yml > /etc/alertmanager/alertmanager.yml && /bin/alertmanager --config.file=/etc/alertmanager/alertmanager.yml"]
--- a/ubuntu-1/configs/alertmanager/alertmanager.template.yml
+++ b/ubuntu-1/configs/alertmanager/alertmanager.template.yml
@ -0,0 +1,14 @@
+global:
+  resolve_timeout: 5m
+
+route:
+  receiver: 'discord'
+  group_wait: 10s
+  group_interval: 30s
+  repeat_interval: 1h
+
+receivers:
+  - name: 'discord'
+    webhook_configs:
+      - url: 'https://discord.com/api/webhooks/1367657026280226907/vRMRq22mikrAAJerUBAcxWPbRgZeY5fF3YE_3u0fZnGCEzNIPot36fBLP7yZ4i55IMSz'
+        send_resolved: true
--- a/ubuntu-1/configs/alertmanager/alertmanager.yml
+++ b/ubuntu-1/configs/alertmanager/alertmanager.yml
@ -0,0 +1,14 @@
+global:
+  resolve_timeout: 5m
+
+route:
+  receiver: 'discord'
+  group_wait: 10s
+  group_interval: 30s
+  repeat_interval: 1h
+
+receivers:
+  - name: 'discord'
+    webhook_configs:
+      - url: 'https://discord.com/api/webhooks/1367657026280226907/vRMRq22mikrAAJerUBAcxWPbRgZeY5fF3YE_3u0fZnGCEzNIPot36fBLP7yZ4i55IMSz'
+        send_resolved: true
--- a/ubuntu-1/configs/loki/loki-config.yml
+++ b/ubuntu-1/configs/loki/loki-config.yml
@ -0,0 +1,53 @@
+auth_enabled: false
+
+server:
+  http_listen_port: 3100
+  grpc_listen_port: 9095
+
+ingester:
+  lifecycler:
+    ring:
+      kvstore:
+        store: inmemory
+      replication_factor: 1
+  chunk_idle_period: 5m
+  chunk_retain_period: 30s
+  wal:
+    dir: /loki/wal
+
+limits_config:
+  reject_old_samples: true
+  reject_old_samples_max_age: 168h
+  allow_structured_metadata: false
+
+schema_config:
+  configs:
+    - from: 2024-01-01
+      store: boltdb-shipper
+      object_store: filesystem
+      schema: v12
+      index:
+        prefix: index_
+        period: 24h
+
+storage_config:
+  boltdb_shipper:
+    active_index_directory: /loki/index
+    cache_location: /loki/cache
+  filesystem:
+    directory: /loki/chunks
+
+ruler:
+  storage:
+    type: local
+    local:
+      directory: /loki/rules
+  rule_path: /loki/rules-temp
+  alertmanager_url: http://localhost:9093
+  ring:
+    kvstore:
+      store: inmemory
+  enable_api: true
+
+compactor:
+  working_directory: /loki/compactor
--- a/ubuntu-1/configs/prometheus/prometheus.yml
+++ b/ubuntu-1/configs/prometheus/prometheus.yml
@ -0,0 +1,19 @@
+global:
+  scrape_interval: 15s
+
+scrape_configs:
+  - job_name: 'prometheus'
+    static_configs:
+      - targets: ['localhost:9090']
+
+  - job_name: 'node_exporter'
+    static_configs:
+      - targets: ['localhost:9100']
+
+  - job_name: 'docker_exporter'
+    static_configs:
+      - targets: ['localhost:9487']
+
+  - job_name: 'pi_services'
+    static_configs:
+      - targets: ['localhost:3100', 'localhost:9093']
--- a/ubuntu-1/configs/promtail/positions.yaml
+++ b/ubuntu-1/configs/promtail/positions.yaml
@ -0,0 +1,9 @@
+positions:
+  /var/log/alf.log: "0"
+  /var/log/fsck_apfs.log: "23880"
+  /var/log/fsck_apfs_error.log: "676"
+  /var/log/fsck_hfs.log: "4044"
+  /var/log/install.log: "3800109"
+  /var/log/shutdown_monitor.log: "1971"
+  /var/log/system.log: "6132"
+  /var/log/wifi.log: "1107090"
--- a/ubuntu-1/configs/promtail/promtail.yml
+++ b/ubuntu-1/configs/promtail/promtail.yml
@ -0,0 +1,18 @@
+server:
+  http_listen_port: 9080
+  grpc_listen_port: 0
+
+positions:
+  filename: /etc/promtail/positions.yaml
+
+clients:
+  - url: http://loki:3100/loki/api/v1/push
+
+scrape_configs:
+  - job_name: system
+    static_configs:
+      - targets:
+          - localhost
+        labels:
+          job: varlogs
+          __path__: /var/log/*.log
--- a/ubuntu-1/docker-compose.yml
+++ b/ubuntu-1/docker-compose.yml
@ -0,0 +1,68 @@
+services:
+  prometheus:
+    image: prom/prometheus:latest
+    container_name: ${HOSTNAME}-prometheus
+    ports:
+      - "${PROMETHEUS_PORT}:9090"
+    volumes:
+      - ./configs/prometheus:/etc/prometheus
+      - prometheus-data:/prometheus
+    command:
+      - --config.file=/etc/prometheus/prometheus.yml
+    restart: unless-stopped
+
+  alertmanager:
+    build:
+      context: .
+      dockerfile: Dockerfile.alertmanager
+    container_name: ${HOSTNAME}-alertmanager
+    ports:
+      - "${ALERTMANAGER_PORT}:9093"
+    volumes:
+      - ./configs/alertmanager:/etc/alertmanager
+    environment:
+      - DISCORD_WEBHOOK_URL=${DISCORD_WEBHOOK_URL}
+    restart: unless-stopped
+
+  loki:
+    image: grafana/loki:latest
+    container_name: ${HOSTNAME}-loki
+    ports:
+      - "${LOKI_PORT}:3100"
+    volumes:
+      - ./configs/loki:/etc/loki
+      - loki-data:/loki
+    command: ["-config.file=/etc/loki/loki-config.yml", "-config.expand-env=true"]
+    restart: unless-stopped
+
+  promtail:
+    image: grafana/promtail:latest
+    container_name: ${HOSTNAME}-promtail
+    volumes:
+      - /var/log:/var/log:ro
+      - ./configs/promtail:/etc/promtail
+    command: -config.file=/etc/promtail/promtail.yml
+    restart: unless-stopped
+
+  node_exporter:
+    image: prom/node-exporter:latest
+    container_name: ${HOSTNAME}-node_exporter
+    ports:
+      - "${NODE_EXPORTER_PORT}:9100"
+    command:
+      - --web.listen-address=:${NODE_EXPORTER_PORT}
+    restart: unless-stopped
+
+  docker_exporter:
+    image: prometheusnet/docker_exporter
+    container_name: ${HOSTNAME}-docker_exporter
+    ports:
+      - "${DOCKER_EXPORTER_PORT}:9487"
+    volumes:
+      - /var/run/docker.sock:/var/run/docker.sock
+    restart: unless-stopped
+
+
+volumes:
+  prometheus-data:
+  loki-data:
--- a/ubuntu-1/stack-status.sh
+++ b/ubuntu-1/stack-status.sh
@ -0,0 +1,27 @@
+#!/bin/bash
+
+# Optional base hostname (e.g., monitor.local)
+BASE_HOST="${NGINX_HOST:-localhost}"
+
+printf "%-20s | %-15s | %-25s | %-40s\n" "Service" "IP Address" "Port Bindings" "URL"
+printf "%s\n" "---------------------------------------------------------------------------------------------------------------"
+
+for container in $(docker ps --format '{{.Names}}'); do
+  ip=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' "$container")
+
+  ports=$(docker inspect -f '{{range $p, $conf := .NetworkSettings.Ports}}{{if $conf}}{{$conf  | json}}{{end}}{{end}}' "$container" \
+    | jq -r '.[] | "\(.HostPort)"' 2>/dev/null | paste -sd "," -)
+
+  if [ -z "$ports" ]; then
+    ports="(none)"
+    url="(none)"
+  else
+    # Just use the first port for URL
+    first_port=$(echo "$ports" | cut -d',' -f1)
+    scheme="http"
+    [ "$first_port" = "443" ] && scheme="https"
+    url="$scheme://$BASE_HOST:$first_port"
+  fi
+
+  printf "%-20s | %-15s | %-25s | %-40s\n" "$container" "$ip" "$ports" "$url"
+done
--- a/ubuntu-1/start.sh
+++ b/ubuntu-1/start.sh
@ -0,0 +1,16 @@
+#!/bin/bash
+
+echo "🚀 Starting monitoring stack..."
+docker-compose up -d
+
+if [ $? -ne 0 ]; then
+  echo "❌ Failed to start containers. Check docker-compose logs."
+  exit 1
+fi
+
+echo "✅ Containers started. Waiting a few seconds for services to boot..."
+sleep 5
+
+echo ""
+echo "🔍 Gathering service info:"
+./stack-status.sh
--- a/ubuntu-2/Dockerfile.alertmanager
+++ b/ubuntu-2/Dockerfile.alertmanager
@ -0,0 +1,19 @@
+# Stage 1: Pull original Alertmanager binary
+FROM prom/alertmanager:latest as upstream
+
+# Stage 2: Alpine + envsubst
+FROM alpine:latest
+
+# Install envsubst and ca-certificates
+RUN apk add --no-cache gettext ca-certificates
+
+# Create directories
+RUN mkdir -p /etc/alertmanager /alertmanager
+
+# Copy Alertmanager binary from upstream
+COPY --from=upstream /bin/alertmanager /bin/alertmanager
+COPY --from=upstream /etc/alertmanager /etc/alertmanager
+
+# Default config will be overwritten by volume mount
+ENTRYPOINT ["/bin/sh", "-c"]
+CMD ["envsubst < /etc/alertmanager/alertmanager.template.yml > /etc/alertmanager/alertmanager.yml && /bin/alertmanager --config.file=/etc/alertmanager/alertmanager.yml"]
--- a/ubuntu-2/configs/alertmanager/alertmanager.template.yml
+++ b/ubuntu-2/configs/alertmanager/alertmanager.template.yml
@ -0,0 +1,14 @@
+global:
+  resolve_timeout: 5m
+
+route:
+  receiver: 'discord'
+  group_wait: 10s
+  group_interval: 30s
+  repeat_interval: 1h
+
+receivers:
+  - name: 'discord'
+    webhook_configs:
+      - url: 'https://discord.com/api/webhooks/1367657026280226907/vRMRq22mikrAAJerUBAcxWPbRgZeY5fF3YE_3u0fZnGCEzNIPot36fBLP7yZ4i55IMSz'
+        send_resolved: true
--- a/ubuntu-2/configs/alertmanager/alertmanager.yml
+++ b/ubuntu-2/configs/alertmanager/alertmanager.yml
@ -0,0 +1,14 @@
+global:
+  resolve_timeout: 5m
+
+route:
+  receiver: 'discord'
+  group_wait: 10s
+  group_interval: 30s
+  repeat_interval: 1h
+
+receivers:
+  - name: 'discord'
+    webhook_configs:
+      - url: 'https://discord.com/api/webhooks/1367657026280226907/vRMRq22mikrAAJerUBAcxWPbRgZeY5fF3YE_3u0fZnGCEzNIPot36fBLP7yZ4i55IMSz'
+        send_resolved: true
--- a/ubuntu-2/configs/loki/loki-config.yml
+++ b/ubuntu-2/configs/loki/loki-config.yml
@ -0,0 +1,53 @@
+auth_enabled: false
+
+server:
+  http_listen_port: 3100
+  grpc_listen_port: 9095
+
+ingester:
+  lifecycler:
+    ring:
+      kvstore:
+        store: inmemory
+      replication_factor: 1
+  chunk_idle_period: 5m
+  chunk_retain_period: 30s
+  wal:
+    dir: /loki/wal
+
+limits_config:
+  reject_old_samples: true
+  reject_old_samples_max_age: 168h
+  allow_structured_metadata: false
+
+schema_config:
+  configs:
+    - from: 2024-01-01
+      store: boltdb-shipper
+      object_store: filesystem
+      schema: v12
+      index:
+        prefix: index_
+        period: 24h
+
+storage_config:
+  boltdb_shipper:
+    active_index_directory: /loki/index
+    cache_location: /loki/cache
+  filesystem:
+    directory: /loki/chunks
+
+ruler:
+  storage:
+    type: local
+    local:
+      directory: /loki/rules
+  rule_path: /loki/rules-temp
+  alertmanager_url: http://localhost:9093
+  ring:
+    kvstore:
+      store: inmemory
+  enable_api: true
+
+compactor:
+  working_directory: /loki/compactor
--- a/ubuntu-2/configs/prometheus/prometheus.yml
+++ b/ubuntu-2/configs/prometheus/prometheus.yml
@ -0,0 +1,19 @@
+global:
+  scrape_interval: 15s
+
+scrape_configs:
+  - job_name: 'prometheus'
+    static_configs:
+      - targets: ['localhost:9090']
+
+  - job_name: 'node_exporter'
+    static_configs:
+      - targets: ['localhost:9100']
+
+  - job_name: 'docker_exporter'
+    static_configs:
+      - targets: ['localhost:9487']
+
+  - job_name: 'pi_services'
+    static_configs:
+      - targets: ['localhost:3100', 'localhost:9093']
--- a/ubuntu-2/configs/promtail/positions.yaml
+++ b/ubuntu-2/configs/promtail/positions.yaml
@ -0,0 +1,9 @@
+positions:
+  /var/log/alf.log: "0"
+  /var/log/fsck_apfs.log: "23880"
+  /var/log/fsck_apfs_error.log: "676"
+  /var/log/fsck_hfs.log: "4044"
+  /var/log/install.log: "3800109"
+  /var/log/shutdown_monitor.log: "1971"
+  /var/log/system.log: "6069"
+  /var/log/wifi.log: "1107090"
--- a/ubuntu-2/configs/promtail/promtail.yml
+++ b/ubuntu-2/configs/promtail/promtail.yml
@ -0,0 +1,18 @@
+server:
+  http_listen_port: 9080
+  grpc_listen_port: 0
+
+positions:
+  filename: /etc/promtail/positions.yaml
+
+clients:
+  - url: http://loki:3100/loki/api/v1/push
+
+scrape_configs:
+  - job_name: system
+    static_configs:
+      - targets:
+          - localhost
+        labels:
+          job: varlogs
+          __path__: /var/log/*.log
--- a/ubuntu-2/docker-compose.yml
+++ b/ubuntu-2/docker-compose.yml
@ -0,0 +1,68 @@
+services:
+  prometheus:
+    image: prom/prometheus:latest
+    container_name: ${HOSTNAME}-prometheus
+    ports:
+      - "${PROMETHEUS_PORT}:9090"
+    volumes:
+      - ./configs/prometheus:/etc/prometheus
+      - prometheus-data:/prometheus
+    command:
+      - --config.file=/etc/prometheus/prometheus.yml
+    restart: unless-stopped
+
+  alertmanager:
+    build:
+      context: .
+      dockerfile: Dockerfile.alertmanager
+    container_name: ${HOSTNAME}-alertmanager
+    ports:
+      - "${ALERTMANAGER_PORT}:9093"
+    volumes:
+      - ./configs/alertmanager:/etc/alertmanager
+    environment:
+      - DISCORD_WEBHOOK_URL=${DISCORD_WEBHOOK_URL}
+    restart: unless-stopped
+
+  loki:
+    image: grafana/loki:latest
+    container_name: ${HOSTNAME}-loki
+    ports:
+      - "${LOKI_PORT}:3100"
+    volumes:
+      - ./configs/loki:/etc/loki
+      - loki-data:/loki
+    command: ["-config.file=/etc/loki/loki-config.yml", "-config.expand-env=true"]
+    restart: unless-stopped
+
+  promtail:
+    image: grafana/promtail:latest
+    container_name: ${HOSTNAME}-promtail
+    volumes:
+      - /var/log:/var/log:ro
+      - ./configs/promtail:/etc/promtail
+    command: -config.file=/etc/promtail/promtail.yml
+    restart: unless-stopped
+
+  node_exporter:
+    image: prom/node-exporter:latest
+    container_name: ${HOSTNAME}-node_exporter
+    ports:
+      - "${NODE_EXPORTER_PORT}:9100"
+    command:
+      - --web.listen-address=:${NODE_EXPORTER_PORT}
+    restart: unless-stopped
+
+  docker_exporter:
+    image: prometheusnet/docker_exporter
+    container_name: ${HOSTNAME}-docker_exporter
+    ports:
+      - "${DOCKER_EXPORTER_PORT}:9487"
+    volumes:
+      - /var/run/docker.sock:/var/run/docker.sock
+    restart: unless-stopped
+
+
+volumes:
+  prometheus-data:
+  loki-data:
--- a/ubuntu-2/stack-status.sh
+++ b/ubuntu-2/stack-status.sh
@ -0,0 +1,27 @@
+#!/bin/bash
+
+# Optional base hostname (e.g., monitor.local)
+BASE_HOST="${NGINX_HOST:-localhost}"
+
+printf "%-20s | %-15s | %-25s | %-40s\n" "Service" "IP Address" "Port Bindings" "URL"
+printf "%s\n" "---------------------------------------------------------------------------------------------------------------"
+
+for container in $(docker ps --format '{{.Names}}'); do
+  ip=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' "$container")
+
+  ports=$(docker inspect -f '{{range $p, $conf := .NetworkSettings.Ports}}{{if $conf}}{{$conf  | json}}{{end}}{{end}}' "$container" \
+    | jq -r '.[] | "\(.HostPort)"' 2>/dev/null | paste -sd "," -)
+
+  if [ -z "$ports" ]; then
+    ports="(none)"
+    url="(none)"
+  else
+    # Just use the first port for URL
+    first_port=$(echo "$ports" | cut -d',' -f1)
+    scheme="http"
+    [ "$first_port" = "443" ] && scheme="https"
+    url="$scheme://$BASE_HOST:$first_port"
+  fi
+
+  printf "%-20s | %-15s | %-25s | %-40s\n" "$container" "$ip" "$ports" "$url"
+done
--- a/ubuntu-2/start.sh
+++ b/ubuntu-2/start.sh
@ -0,0 +1,16 @@
+#!/bin/bash
+
+echo "🚀 Starting monitoring stack..."
+docker-compose up -d
+
+if [ $? -ne 0 ]; then
+  echo "❌ Failed to start containers. Check docker-compose logs."
+  exit 1
+fi
+
+echo "✅ Containers started. Waiting a few seconds for services to boot..."
+sleep 5
+
+echo ""
+echo "🔍 Gathering service info:"
+./stack-status.sh
--- a/unraid/Dockerfile.alertmanager
+++ b/unraid/Dockerfile.alertmanager
@ -0,0 +1,19 @@
+# Stage 1: Pull original Alertmanager binary
+FROM prom/alertmanager:latest as upstream
+
+# Stage 2: Alpine + envsubst
+FROM alpine:latest
+
+# Install envsubst and ca-certificates
+RUN apk add --no-cache gettext ca-certificates
+
+# Create directories
+RUN mkdir -p /etc/alertmanager /alertmanager
+
+# Copy Alertmanager binary from upstream
+COPY --from=upstream /bin/alertmanager /bin/alertmanager
+COPY --from=upstream /etc/alertmanager /etc/alertmanager
+
+# Default config will be overwritten by volume mount
+ENTRYPOINT ["/bin/sh", "-c"]
+CMD ["envsubst < /etc/alertmanager/alertmanager.template.yml > /etc/alertmanager/alertmanager.yml && /bin/alertmanager --config.file=/etc/alertmanager/alertmanager.yml"]
--- a/unraid/configs/alertmanager/alertmanager.template.yml
+++ b/unraid/configs/alertmanager/alertmanager.template.yml
@ -0,0 +1,14 @@
+global:
+  resolve_timeout: 5m
+
+route:
+  receiver: 'discord'
+  group_wait: 10s
+  group_interval: 30s
+  repeat_interval: 1h
+
+receivers:
+  - name: 'discord'
+    webhook_configs:
+      - url: '${DISCORD_ALERT_WEBHOOK}'
+        send_resolved: true
--- a/unraid/configs/alertmanager/alertmanager.yml
+++ b/unraid/configs/alertmanager/alertmanager.yml
@ -0,0 +1,14 @@
+global:
+  resolve_timeout: 5m
+
+route:
+  receiver: 'discord'
+  group_wait: 10s
+  group_interval: 30s
+  repeat_interval: 1h
+
+receivers:
+  - name: 'discord'
+    webhook_configs:
+      - url: 'https://discord.com/api/webhooks/1367657026280226907/vRMRq22mikrAAJerUBAcxWPbRgZeY5fF3YE_3u0fZnGCEzNIPot36fBLP7yZ4i55IMSz'
+        send_resolved: true
--- a/unraid/configs/grafana/grafana.ini
+++ b/unraid/configs/grafana/grafana.ini
--- a/unraid/configs/loki/loki-config.yml
+++ b/unraid/configs/loki/loki-config.yml
@ -0,0 +1,53 @@
+auth_enabled: false
+
+server:
+  http_listen_port: 3100
+  grpc_listen_port: 9095
+
+ingester:
+  lifecycler:
+    ring:
+      kvstore:
+        store: inmemory
+      replication_factor: 1
+  chunk_idle_period: 5m
+  chunk_retain_period: 30s
+  wal:
+    dir: /loki/wal
+
+limits_config:
+  reject_old_samples: true
+  reject_old_samples_max_age: 168h
+  allow_structured_metadata: false
+
+schema_config:
+  configs:
+    - from: 2024-01-01
+      store: boltdb-shipper
+      object_store: filesystem
+      schema: v12
+      index:
+        prefix: index_
+        period: 24h
+
+storage_config:
+  boltdb_shipper:
+    active_index_directory: /loki/index
+    cache_location: /loki/cache
+  filesystem:
+    directory: /loki/chunks
+
+ruler:
+  storage:
+    type: local
+    local:
+      directory: /loki/rules
+  rule_path: /loki/rules-temp
+  alertmanager_url: http://localhost:9093
+  ring:
+    kvstore:
+      store: inmemory
+  enable_api: true
+
+compactor:
+  working_directory: /loki/compactor
--- a/unraid/configs/nginx/default.conf
+++ b/unraid/configs/nginx/default.conf
--- a/unraid/configs/prometheus/prometheus.yml
+++ b/unraid/configs/prometheus/prometheus.yml
--- a/unraid/configs/promtail/promtail.yml
+++ b/unraid/configs/promtail/promtail.yml
@ -0,0 +1,18 @@
+server:
+  http_listen_port: 9080
+  grpc_listen_port: 0
+
+positions:
+  filename: /tmp/positions.yaml
+
+clients:
+  - url: http://loki:3100/loki/api/v1/push
+
+scrape_configs:
+  - job_name: system
+    static_configs:
+      - targets:
+          - localhost
+        labels:
+          job: varlogs
+          __path__: /var/log/*.log
--- a/unraid/docker-compose.yml
+++ b/unraid/docker-compose.yml
@ -0,0 +1,94 @@
+version: "3.8"
+
+services:
+  alertmanager:
+    build:
+      context: .
+      dockerfile: Dockerfile.alertmanager
+    container_name: ${HOSTNAME}-alertmanager
+    ports:
+      - "9093:9093"
+    volumes:
+      - ./configs/alertmanager:/etc/alertmanager
+    environment:
+      - DISCORD_ALERT_WEBHOOK=${DISCORD_ALERT_WEBHOOK}
+    restart: unless-stopped
+  grafana:
+    image: grafana/grafana:latest
+    container_name: ${HOSTNAME}-grafana
+    ports:
+      - "${GRAFANA_PORT}:3000"
+    environment:
+      - GF_SECURITY_ADMIN_USER=${GRAFANA_USER}
+      - GF_SECURITY_ADMIN_PASSWORD=${GRAFANA_PASS}
+    volumes:
+      - ./configs/grafana:/etc/grafana
+      - grafana-data:/var/lib/grafana
+    restart: unless-stopped
+
+  prometheus:
+    image: prom/prometheus:latest
+    container_name: ${HOSTNAME}-prometheus
+    ports:
+      - "${PROMETHEUS_PORT}:9090"
+    volumes:
+      - ./configs/prometheus:/etc/prometheus
+      - prometheus-data:/prometheus
+    command:
+      - --config.file=/etc/prometheus/prometheus.yml
+    restart: unless-stopped
+
+  loki:
+    image: grafana/loki:latest
+    container_name: ${HOSTNAME}-loki
+    ports:
+      - "${LOKI_PORT}:3100"
+    volumes:
+      - ./configs/loki:/etc/loki
+      - loki-data:/loki
+    command: ["-config.file=/etc/loki/loki-config.yml", "-config.expand-env=true"]
+    restart: unless-stopped
+
+  promtail:
+    image: grafana/promtail:latest
+    container_name: ${HOSTNAME}-promtail
+    volumes:
+      - /var/log:/var/log:ro
+      - ./configs/promtail:/etc/promtail
+    command: -config.file=/etc/promtail/promtail.yml
+    restart: unless-stopped
+
+  docker_exporter:
+    image: prometheusnet/docker_exporter
+    container_name: ${HOSTNAME}-docker_exporter
+    ports:
+      - "${DOCKER_EXPORTER_PORT}:9487"
+    volumes:
+      - /var/run/docker.sock:/var/run/docker.sock
+    restart: unless-stopped
+
+  node_exporter:
+    image: prom/node-exporter:latest
+    container_name: ${HOSTNAME}-node_exporter
+    ports:
+      - "${NODE_EXPORTER_PORT}:9100"
+    command:
+      - --web.listen-address=:${NODE_EXPORTER_PORT}
+    restart: unless-stopped
+
+  nginx:
+    image: nginx:latest
+    container_name: ${HOSTNAME}-nginx
+    ports:
+      - "${NGINX_HTTP_PORT}:80"
+      - "${NGINX_HTTPS_PORT}:443"
+    volumes:
+      - ./configs/nginx:/etc/nginx/conf.d
+      - ./configs/nginx/ssl:/etc/nginx/ssl
+      - ./configs/nginx/html:/usr/share/nginx/html
+    restart: unless-stopped
+
+volumes:
+  grafana-data:
+  prometheus-data:
+  loki-data:
--- a/unraid/stack-status.sh
+++ b/unraid/stack-status.sh
@ -0,0 +1,27 @@
+#!/bin/bash
+
+# Optional base hostname (e.g., monitor.local)
+BASE_HOST="${NGINX_HOST:-localhost}"
+
+printf "%-20s | %-15s | %-25s | %-40s\n" "Service" "IP Address" "Port Bindings" "URL"
+printf "%s\n" "---------------------------------------------------------------------------------------------------------------"
+
+for container in $(docker ps --format '{{.Names}}'); do
+  ip=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' "$container")
+
+  ports=$(docker inspect -f '{{range $p, $conf := .NetworkSettings.Ports}}{{if $conf}}{{$conf  | json}}{{end}}{{end}}' "$container" \
+    | jq -r '.[] | "\(.HostPort)"' 2>/dev/null | paste -sd "," -)
+
+  if [ -z "$ports" ]; then
+    ports="(none)"
+    url="(none)"
+  else
+    # Just use the first port for URL
+    first_port=$(echo "$ports" | cut -d',' -f1)
+    scheme="http"
+    [ "$first_port" = "443" ] && scheme="https"
+    url="$scheme://$BASE_HOST:$first_port"
+  fi
+
+  printf "%-20s | %-15s | %-25s | %-40s\n" "$container" "$ip" "$ports" "$url"
+done
--- a/unraid/start.sh
+++ b/unraid/start.sh
@ -0,0 +1,16 @@
+#!/bin/bash
+
+echo "🚀 Starting monitoring stack..."
+docker-compose up -d
+
+if [ $? -ne 0 ]; then
+  echo "❌ Failed to start containers. Check docker-compose logs."
+  exit 1
+fi
+
+echo "✅ Containers started. Waiting a few seconds for services to boot..."
+sleep 5
+
+echo ""
+echo "🔍 Gathering service info:"
+./stack-status.sh