Developing for Kubernetes

Rancher Desktop VM (moby mode):
+---------------------------------------------+
|                                             |
|   dockerd (moby)         k3s (containerd)   |
|   +-- image store <----> image store --+    |
|       (shared)           (same!)       |    |
|                                             |
|   docker build -> image appears in both     |
|   NO PUSH NEEDED!                           |
+---------------------------------------------+

Rancher Desktop VM (containerd mode):
+---------------------------------------------+
|                                             |
|   nerdctl (containerd)   k3s (containerd)   |
|   +-- image store        image store --+    |
|       (separate!)        (separate!)   |    |
|                                             |
|   nerdctl build -> push -> pull -> k3s      |
|   3 steps instead of 1 = slower             |
+---------------------------------------------+

docker pull dp.apps.rancher.io/containers/bci-base:latest

# Log in to the registry (SUSE Customer Center credentials)
docker login dp.apps.rancher.io

# Verify
docker pull dp.apps.rancher.io/containers/bci-base:latest

kubectl create secret docker-registry application-collection \
  --docker-server=dp.apps.rancher.io \
  --docker-username=<USERNAME> \
  --docker-password=<PASSWORD>

brew install tilt

curl -fsSL https://raw.githubusercontent.com/tilt-dev/tilt/master/scripts/install.sh | bash

tilt version

iex ((new-object net.webclient).DownloadString('https://raw.githubusercontent.com/tilt-dev/tilt/master/scripts/install.ps1'))

tilt version

allow_k8s_contexts('rancher-desktop')

Rancher-Developer-Access-Demo/
+-- src/
|   +-- server.js               Application (API + UI + Prometheus metrics)
+-- k8s/
|   +-- appco/
|   |   +-- deployment.yaml     Pod spec with Prometheus annotations
|   |   +-- service.yaml        ClusterIP service
|   |   +-- keycloak.yaml       Keycloak Deployment + Service (Application Collection image)
|   +-- shared/
|       +-- grafana-dashboard.yaml   8-panel dashboard (auto-provisioned via sidecar)
|       +-- keycloak-realm.json      Realm config (demo user + OAuth client)
+-- scripts/
|   +-- setup-keycloak-realm.sh      Keycloak realm import via Admin REST API
+-- values_yaml/
|   +-- postgresql.yaml          Helm values for PostgreSQL
|   +-- prometheus.yaml          Helm values for Prometheus
|   +-- grafana.yaml             Helm values for Grafana
+-- Dockerfile                   Container image (Application Collection base)
+-- Tiltfile                     Inner loop config (build, deploy, sync, monitoring)
+-- package.json

{
  "name": "message-wall",
  "version": "1.0.0",
  "description": "SUSE Rancher Developer Access + Tilt: Demo",
  "main": "src/server.js",
  "scripts": {
    "start": "node src/server.js"
  },
  "dependencies": {
    "pg": "^8.13.0",
    "prom-client": "^15.1.0"
  }
}

const http = require('http');
const { Client } = require('pg');
const promClient = require('prom-client');

const PORT = 3000;

// Change this color, save, see it update!
const ACCENT_COLOR = "#747dcd";

// --- Prometheus metrics ---
// collectDefaultMetrics() automatically exposes Node.js
// metrics: CPU, heap memory, event loop lag, GC...
promClient.collectDefaultMetrics();

// Custom metrics -- prefixed "app_" for easy discovery
const httpDuration = new promClient.Histogram({
  name: 'app_http_request_duration_seconds',
  help: 'Duration of HTTP requests in seconds',
  labelNames: ['method', 'path', 'status'],
  buckets: [0.005, 0.01, 0.05, 0.1, 0.5, 1],
});

const messagesPosted = new promClient.Counter({
  name: 'app_messages_posted_total',
  help: 'Total number of messages posted',
});

const messagesDeleted = new promClient.Counter({
  name: 'app_messages_deleted_total',
  help: 'Total number of bulk deletes',
});

const messagesCurrent = new promClient.Gauge({
  name: 'app_messages_count',
  help: 'Current number of messages in the database',
});

if (req.method === 'GET' && req.url === '/metrics') {
  const metrics = await promClient.register.metrics();
  res.writeHead(200, { 'Content-Type': promClient.register.contentType });
  res.end(metrics);
  // Do not record /metrics in the histogram (noise)
  return;
}

const end = httpDuration.startTimer();
// ... request processing ...
end({ method: req.method, path: routePath, status: statusCode });

FROM dp.apps.rancher.io/containers/nodejs:24-dev
WORKDIR /app
COPY package.json ./
RUN npm install --no-package-lock
COPY . .
EXPOSE 3000
CMD ["node", "src/server.js"]

apiVersion: apps/v1
kind: Deployment
metadata:
  name: message-wall
spec:
  replicas: 1
  selector:
    matchLabels:
      app: message-wall
  template:
    metadata:
      labels:
        app: message-wall
      annotations:
        prometheus.io/scrape: "true"
        prometheus.io/port: "3000"
        prometheus.io/path: "/metrics"
    spec:
      containers:
        - name: message-wall
          image: message-wall           # Tilt replaces with local image
          ports:
            - containerPort: 3000
          env:
            - name: DB_HOST
              value: "demo-db-postgresql"
            - name: DB_PORT
              value: "5432"
            - name: DB_USER
              value: "demo"
            - name: DB_PASSWORD
              value: "demo"
            - name: DB_NAME
              value: "demo"
          livenessProbe:
            httpGet:
              path: /health
              port: 3000
            initialDelaySeconds: 5
          readinessProbe:
            httpGet:
              path: /health
              port: 3000
            initialDelaySeconds: 5

apiVersion: v1
kind: Service
metadata:
  name: message-wall
spec:
  selector:
    app: message-wall
  ports:
    - port: 3000
      targetPort: 3000

auth:
  database: demo
  postgresPassword: demo
  postgresUsername: demo
  username: demo
global:
  imagePullSecrets:
  - application-collection

alertmanager:
  service:
    type: NodePort
global:
  imagePullSecrets:
  - application-collection

adminPassword: admin
global:
  imagePullSecrets:
  - application-collection
sidecar:
  dashboards:
    enabled: true
  datasources:
    enabled: true

kubectl get pods -l app.kubernetes.io/name=grafana \
  -o jsonpath='{.items[0].spec.containers[*].name}'
# Expected: grafana grafana-sc-dashboard grafana-sc-datasources

apiVersion: apps/v1
kind: Deployment
metadata:
  name: keycloak-appco
spec:
  replicas: 1
  selector:
    matchLabels:
      app: keycloak
  template:
    metadata:
      labels:
        app: keycloak
    spec:
      volumes:
        - name: realm-config
          configMap:
            name: keycloak-realm
      containers:
        - name: keycloak
          image: dp.apps.rancher.io/containers/keycloak:26.5.4
          args: ["start-dev", "--health-enabled=true", "--import-realm"]
          volumeMounts:
            - name: realm-config
              mountPath: /opt/keycloak/data/import
          env:
            - name: KC_DB
              value: postgres
            - name: KC_DB_URL
              value: jdbc:postgresql://PLACEHOLDER_PG_SVC:5432/keycloak
            # ... KC_DB_USERNAME, KC_DB_PASSWORD, KEYCLOAK_ADMIN, etc.
          readinessProbe:
            httpGet:
              path: /health/ready
              port: 9000
            initialDelaySeconds: 30
      imagePullSecrets:
        - name: application-collection

apiVersion: v1
kind: ConfigMap
metadata:
  name: message-wall-dashboard
  labels:
    grafana_dashboard: "1"      # the sidecar detects this label
data:
  message-wall.json: |
    {
      "uid": "message-wall",
      "title": "Message Wall",
      "panels": [ ... ]
    }

# Demo Tiltfile
load('ext://restart_process', 'docker_build_with_restart')

allow_k8s_contexts('rancher-desktop')

# --- Helpers ---------------------------------------------------------
def find_service(label_selector, required=False, name='Service'):
    # Discover a Kubernetes service by label selector.
    #
    # Helm charts installed via Rancher Desktop get random release
    # names (e.g. postgresql-1772033328). Searching by label is
    # robust regardless of the release name.
    svc = str(local(
        "kubectl get svc -l " + label_selector +
        " -o jsonpath='{.items[0].metadata.name}'",
        quiet=True,
    )).strip()
    if required and svc == '':
        fail(name + ' not found. Install it via Rancher Desktop ' +
             '(Application Collection).')
    return svc

# --- Service discovery -----------------------------------------------
pg_svc = find_service(
    'app.kubernetes.io/name=postgresql',
    required=True,
    name='PostgreSQL',
)
grafana_svc = find_service('app.kubernetes.io/name=grafana')
prometheus_svc = find_service(
    'app.kubernetes.io/name=prometheus,app.kubernetes.io/component=server',
)

# --- Application -----------------------------------------------------
docker_build_with_restart(
    'message-wall',
    '.',
    entrypoint=['node', 'src/server.js'],
    only=['src/', 'package.json'],
    live_update=[
        sync('./src', '/app/src'),
        run('cd /app && npm install --no-package-lock',
            trigger=['package.json']),
    ],
)

deployment = str(read_file('k8s/appco/deployment.yaml')).replace(
    'PLACEHOLDER_PG_SVC', pg_svc)
service = str(read_file('k8s/appco/service.yaml'))
k8s_yaml([blob(deployment), blob(service), 'k8s/shared/grafana-dashboard.yaml'])

k8s_resource(
    'message-wall-appco',
    port_forwards='3000:3000',
    labels=['app'],
)

# --- Keycloak (no Helm chart — deployed as raw K8s manifest) ---------
# Ensure keycloak DB exists in PostgreSQL
pg_pod = str(local(
    "kubectl get pods -l app.kubernetes.io/name=postgresql "
    "-o jsonpath='{.items[0].metadata.name}'", quiet=True)).strip()
local("kubectl exec " + pg_pod +
    " -- env PGPASSWORD=demo psql -U demo -tc "
    "\"SELECT 1 FROM pg_database WHERE datname='keycloak'\""
    " | grep -q 1 || kubectl exec " + pg_pod +
    " -- env PGPASSWORD=demo psql -U demo -c 'CREATE DATABASE keycloak'",
    quiet=True)

# Realm ConfigMap (auto-imports realm with demo user + OAuth client)
local('kubectl create configmap keycloak-realm '
      '--from-file=message-wall.json=k8s/shared/keycloak-realm.json '
      '--dry-run=client -o yaml | kubectl apply -f -', quiet=True)

# Deploy Keycloak using the Application Collection container image
keycloak_yaml = str(read_file('k8s/appco/keycloak.yaml')).replace(
    'PLACEHOLDER_PG_SVC', pg_svc)
k8s_yaml(blob(keycloak_yaml))

k8s_resource('keycloak-appco', port_forwards='8080:8080', labels=['app'])

# Realm setup via Admin REST API (idempotent fallback)
local_resource(
    'keycloak-realm-setup',
    cmd='bash scripts/setup-keycloak-realm.sh http://localhost:8080 '
        'k8s/shared/keycloak-realm.json',
    labels=['app'],
    resource_deps=['keycloak-appco'],
)

# --- Monitoring (optional) -------------------------------------------
if grafana_svc:
    local_resource(
        'grafana',
        serve_cmd='kubectl port-forward svc/' + grafana_svc + ' 3001:80',
        labels=['monitoring'],
        allow_parallel=True,
        links=['http://localhost:3001',
               'http://localhost:3001/d/message-wall/'],
    )

if prometheus_svc:
    local_resource(
        'prometheus',
        serve_cmd='kubectl port-forward svc/' + prometheus_svc
                  + ' 9090:80',
        labels=['monitoring'],
        allow_parallel=True,
        links=['http://localhost:9090'],
    )

    datasource_cm = """apiVersion: v1
kind: ConfigMap
metadata:
  name: grafana-datasource-prometheus
  labels:
    grafana_datasource: "1"
data:
  prometheus.yaml: |
    apiVersion: 1
    datasources:
      - name: Prometheus
        type: prometheus
        uid: prometheus
        url: http://{svc}:80
        access: proxy
        isDefault: true
        editable: false
""".format(svc=prometheus_svc)

    k8s_yaml(blob(datasource_cm))

    k8s_resource(
        objects=['message-wall-dashboard:configmap',
                 'grafana-datasource-prometheus:configmap'],
        new_name='grafana-config',
        labels=['monitoring'],
        links=['http://localhost:3001/d/message-wall/'],
    )

# 1. Clone the repo
git clone https://github.com/fxHouard/Rancher-Developer-Access-Demo.git
cd Rancher-Developer-Access-Demo

# 2. Install services via Rancher Desktop (one time only):
#
#    PostgreSQL:
#      Application Collection tab -> search PostgreSQL -> Install
#      Switch to YAML mode, paste values_yaml/postgresql.yaml, Install
#
#    Prometheus:
#      Application Collection tab -> search Prometheus -> Install
#      Switch to YAML mode, paste values_yaml/prometheus.yaml, Install
#
#    Grafana:
#      Application Collection tab -> search Grafana -> Install
#      Switch to YAML mode, paste values_yaml/grafana.yaml, Install

# 3. Start the inner loop:
tilt up

# 4. Press Space to open the Tilt dashboard in your browser

# 5. From the Tilt dashboard, click the links to:
#    -> http://localhost:3000   -- The Message Wall app
#    -> http://localhost:8080   -- Keycloak (admin / admin)
#    -> http://localhost:9090   -- Prometheus (check targets)
#    -> http://localhost:3001   -- Grafana (admin / admin)

# 6. In Grafana: go to Dashboards -> "Message Wall" is already there
#    (or click the direct link in Tilt)

# 7. Post messages on localhost:3000 and watch the metrics
#    update in real time in Grafana

# 8. Change ACCENT_COLOR in src/server.js (line 9)
#    -> save -> ~2 sec -> the color changes

1. `tilt up` on the host:
   +-- Auto-detects PostgreSQL (label app.kubernetes.io/name=postgresql)
   +-- Auto-detects Prometheus and Grafana (labels app.kubernetes.io/name=...)
   +-- Creates keycloak DB in PostgreSQL if needed
   +-- docker build message-wall -> image in dockerd local store
   |   +-- k3s sees the image because same store -> imagePullPolicy: IfNotPresent
   +-- kubectl apply deployment + service -> k3s creates the app pod
   |   +-- Pod connects to detected PG service (K8s internal DNS)
   |   +-- Prometheus scrapes the pod (prometheus.io/* annotations)
   +-- Deploys Keycloak (Application Collection image, raw K8s manifest)
   |   +-- Imports realm via ConfigMap volume mount
   |   +-- Runs setup script via Admin REST API (idempotent fallback)
   +-- kubectl apply ConfigMaps (datasource + dashboard)
   |   +-- Grafana sidecars detect and load them
   +-- Port-forward 3000 -> localhost:3000 (app)
   +-- Port-forward 8080 -> Keycloak
   +-- Port-forward 3001 -> Grafana
   +-- Port-forward 9090 -> Prometheus

2. Dev modifies src/server.js:
   +-- Tilt (on host) detects the change (file watcher)
   +-- live_update syncs ./src -> /app/src in the pod (kubectl cp)
   +-- restart_process relaunches `node src/server.js` in the container
   +-- ~2 seconds later: change visible on localhost:3000

3. Observability loop (continuous):
   +-- Prometheus scrapes localhost:3000/metrics every 15-60s
   +-- Grafana queries Prometheus to display dashboards
   +-- Dev sees the impact of their changes in real time

Browser (localhost:3000)
  -> Tilt port-forward
    -> K8s Service (ClusterIP)
      -> Your app pod (:3000)
        -> Connects to PostgreSQL Service (:5432)
          -> PostgreSQL pod

postgresql://user:pass@demo-db-postgresql.default.svc.cluster.local:5432/mydb

FROM dp.apps.rancher.io/containers/nodejs:24-dev

# System tools (available in the SLE_BCI repo)
# gawk: required by VS Code Server (check-requirements.sh)
RUN zypper --non-interactive install -y git openssh make gawk \
    && zypper clean -a

{
  "name": "Message Wall - Node.js",
  "build": {
    "dockerfile": "Dockerfile"
  },
  "customizations": {
    "vscode": {
      "extensions": [
        "ms-kubernetes-tools.vscode-kubernetes-tools"
      ],
      "settings": {
        "vs-kubernetes.disable-linters": true
      }
    }
  },
  "postCreateCommand": "if [ -f package.json ]; then npm install --no-package-lock; fi"
}

resources:
  requests:
    cpu: 100m           # 0.1 vCPU -- scheduler reserves this
    memory: 128Mi       # 128 MB -- guaranteed minimum
  limits:
    cpu: 500m           # 0.5 vCPU -- ceiling, throttled beyond
    memory: 256Mi       # 256 MB -- OOMKill beyond

# macOS / Linux -- already configured if ~/.rd/bin is in PATH
export DOCKER_HOST=unix://$HOME/.rd/run/docker.sock

const { GenericContainer } = require('testcontainers');
const { Client } = require('pg');

describe('Database integration', () => {
  let container, client;

  beforeAll(async () => {
    container = await new GenericContainer('dp.apps.rancher.io/containers/postgresql:17')
      .withExposedPorts(5432)
      .withEnvironment({ POSTGRES_USER: 'test', POSTGRES_PASSWORD: 'test', POSTGRES_DB: 'test' })
      .start();

    client = new Client({
      host: container.getHost(),
      port: container.getMappedPort(5432),
      user: 'test', password: 'test', database: 'test',
    });
    await client.connect();
  });

  afterAll(async () => {
    await client.end();
    await container.stop();
  });

  test('should insert and retrieve data', async () => {
    await client.query('CREATE TABLE test (id SERIAL, name TEXT)');
    await client.query("INSERT INTO test (name) VALUES ('hello')");
    const result = await client.query('SELECT * FROM test');
    expect(result.rows).toHaveLength(1);
  });
});

trivy image dp.apps.rancher.io/containers/nodejs:24
trivy k8s --report summary cluster

cosign sign --key cosign.key myregistry/myapp:v1.0
cosign verify --key cosign.pub myregistry/myapp:v1.0