Enterprise Deployment Guide#

# Pull the latest release image
docker pull ghcr.io/zerfoo/zerfoo:latest

# Or pin to a specific version
docker pull ghcr.io/zerfoo/zerfoo:0.1.0

# CPU-only build (zero CGo, compiles everywhere)
go build -o zerfoo ./cmd/zerfoo

# Build container image
docker build -t ghcr.io/zerfoo/zerfoo:custom .

helm install zerfoo deploy/helm/zerfoo/ \
  --namespace zerfoo \
  --create-namespace \
  --set model.name="google/gemma-3-1b" \
  --set model.quantization="Q4_K_M"

# values-gpu.yaml
resources:
  requests:
    cpu: "4"
    memory: 16Gi
  limits:
    cpu: "8"
    memory: 32Gi
    nvidia.com/gpu: "1"

tolerations:
  - key: nvidia.com/gpu
    operator: Exists
    effect: NoSchedule

nodeSelector:
  nvidia.com/gpu.present: "true"

volumes:
  - name: model-storage
    persistentVolumeClaim:
      claimName: zerfoo-models

volumeMounts:
  - name: model-storage
    mountPath: /models

helm install zerfoo deploy/helm/zerfoo/ \
  --namespace zerfoo \
  --create-namespace \
  -f values-gpu.yaml \
  --set model.name="meta-llama/Llama-3-8B"

apiVersion: zerfoo.feza.ai/v1
kind: ZerfooInferenceService
metadata:
  name: llama3-8b
  namespace: zerfoo
spec:
  modelRef: "meta-llama/Llama-3-8B-Q4_K_M"
  replicas: 3
  minReplicas: 2
  maxReplicas: 10
  resources:
    cpu: "4"
    memory: "16Gi"
    gpuMemory: "24Gi"
  healthCheck:
    path: "/health"
    interval: 10s
    timeout: 5s

apiVersion: zerfoo.feza.ai/v1
kind: ZerfooInferenceService
metadata:
  name: llama3-8b
  namespace: zerfoo
spec:
  modelRef: "meta-llama/Llama-3-8B-Q4_K_M"
  replicas: 3
  minReplicas: 2
  maxReplicas: 10
  resources:
    cpu: "4"
    memory: "16Gi"
    gpuMemory: "24Gi"
  healthCheck:
    path: "/health"
    interval: 10s
    timeout: 5s
  canary:
    modelRef: "meta-llama/Llama-3-8B-Q8_0"
    weight: 10  # 10% traffic to canary

apiVersion: v1
kind: Namespace
metadata:
  name: zerfoo-system
---
apiVersion: v1
kind: ServiceAccount
metadata:
  name: zerfoo-server
  namespace: zerfoo-system
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
  name: zerfoo-server
rules:
  - apiGroups: [""]
    resources: ["configmaps", "secrets"]
    verbs: ["get", "list", "watch"]
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
  name: zerfoo-server
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: ClusterRole
  name: zerfoo-server
subjects:
  - kind: ServiceAccount
    name: zerfoo-server
    namespace: zerfoo-system

apiVersion: rbac.authorization.k8s.io/v1
kind: Role
metadata:
  name: zerfoo-operator
  namespace: zerfoo
rules:
  - apiGroups: ["apps"]
    resources: ["deployments"]
    verbs: ["get", "list", "create", "update", "delete"]
  - apiGroups: [""]
    resources: ["services"]
    verbs: ["get", "list", "create", "update"]
  - apiGroups: ["autoscaling"]
    resources: ["horizontalpodautoscalers"]
    verbs: ["get", "list", "create", "update"]
  - apiGroups: ["zerfoo.feza.ai"]
    resources: ["zerfooinferenceservices"]
    verbs: ["get", "list", "watch", "update"]

apiVersion: apps/v1
kind: Deployment
metadata:
  name: zerfoo-gemma-3-7b
  namespace: zerfoo-system
  labels:
    app: zerfoo
    model: gemma-3-7b
spec:
  replicas: 2
  selector:
    matchLabels:
      app: zerfoo
      model: gemma-3-7b
  template:
    metadata:
      labels:
        app: zerfoo
        model: gemma-3-7b
      annotations:
        prometheus.io/scrape: "true"
        prometheus.io/port: "8080"
        prometheus.io/path: "/metrics"
    spec:
      serviceAccountName: zerfoo-server
      runtimeClassName: nvidia   # omit for CPU-only
      containers:
        - name: zerfoo
          image: ghcr.io/zerfoo/zerfoo:v1.0.0
          command: ["zerfoo", "serve"]
          args:
            - "google/gemma-3-7b-it-q4_k_m"
            - "--port"
            - "8080"
            - "--cache-dir"
            - "/models"
            # For multi-GPU, add: --gpus 0,1
          ports:
            - name: http
              containerPort: 8080
              protocol: TCP
            - name: health
              containerPort: 8081
              protocol: TCP
          env:
            - name: GOMAXPROCS
              valueFrom:
                resourceFieldRef:
                  resource: limits.cpu
                  divisor: "1"
          resources:
            requests:
              cpu: "4"
              memory: "16Gi"
              nvidia.com/gpu: "1"
            limits:
              cpu: "8"
              memory: "24Gi"
              nvidia.com/gpu: "1"
          volumeMounts:
            - name: models
              mountPath: /models
              readOnly: true
            - name: tls-certs
              mountPath: /etc/zerfoo/tls
              readOnly: true
          livenessProbe:
            httpGet:
              path: /healthz
              port: health
            initialDelaySeconds: 30
            periodSeconds: 10
            failureThreshold: 3
          readinessProbe:
            httpGet:
              path: /readyz
              port: health
            initialDelaySeconds: 60
            periodSeconds: 5
            failureThreshold: 6
      volumes:
        - name: models
          persistentVolumeClaim:
            claimName: zerfoo-models
        - name: tls-certs
          secret:
            secretName: zerfoo-tls
      topologySpreadConstraints:
        - maxSkew: 1
          topologyKey: kubernetes.io/hostname
          whenUnsatisfiable: DoNotSchedule
          labelSelector:
            matchLabels:
              app: zerfoo
              model: gemma-3-7b

apiVersion: v1
kind: Service
metadata:
  name: zerfoo-gemma-3-7b
  namespace: zerfoo-system
spec:
  selector:
    app: zerfoo
    model: gemma-3-7b
  ports:
    - name: http
      port: 80
      targetPort: http
    - name: https
      port: 443
      targetPort: http
  type: ClusterIP
---
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
  name: zerfoo
  namespace: zerfoo-system
  annotations:
    nginx.ingress.kubernetes.io/proxy-read-timeout: "300"
    nginx.ingress.kubernetes.io/proxy-send-timeout: "300"
    nginx.ingress.kubernetes.io/proxy-buffering: "off"   # required for SSE streaming
    nginx.ingress.kubernetes.io/ssl-redirect: "true"
spec:
  ingressClassName: nginx
  tls:
    - hosts:
        - api.example.com
      secretName: zerfoo-tls-ingress
  rules:
    - host: api.example.com
      http:
        paths:
          - path: /
            pathType: Prefix
            backend:
              service:
                name: zerfoo-gemma-3-7b
                port:
                  name: http

# values-autoscaling.yaml
autoscaling:
  enabled: true
  minReplicas: 2
  maxReplicas: 10
  targetCPUUtilizationPercentage: 80

apiVersion: autoscaling/v2
kind: HorizontalPodAutoscaler
metadata:
  name: zerfoo
spec:
  scaleTargetRef:
    apiVersion: apps/v1
    kind: Deployment
    name: zerfoo
  minReplicas: 2
  maxReplicas: 10
  metrics:
    - type: Resource
      resource:
        name: cpu
        target:
          type: Utilization
          averageUtilization: 80

apiVersion: autoscaling/v2
kind: HorizontalPodAutoscaler
metadata:
  name: zerfoo-gemma-3-7b
  namespace: zerfoo-system
spec:
  scaleTargetRef:
    apiVersion: apps/v1
    kind: Deployment
    name: zerfoo-gemma-3-7b
  minReplicas: 2
  maxReplicas: 8
  metrics:
    - type: Pods
      pods:
        metric:
          name: tokens_per_second
        target:
          type: AverageValue
          averageValue: "150"   # scale out when avg TPS per pod drops below 150
    - type: Resource
      resource:
        name: cpu
        target:
          type: Utilization
          averageUtilization: 70

metrics:
  - type: Pods
    pods:
      metric:
        name: request_latency_ms_p99
      target:
        type: AverageValue
        averageValue: "500"

# Prefill workers (compute-optimized GPU nodes)
apiVersion: apps/v1
kind: Deployment
metadata:
  name: zerfoo-prefill
spec:
  replicas: 2
  template:
    spec:
      containers:
        - name: zerfoo
          image: ghcr.io/zerfoo/zerfoo:latest
          args: ["worker", "--role", "prefill", "--port", "50051"]
          resources:
            limits:
              nvidia.com/gpu: "1"
---
# Decode workers (memory-optimized GPU nodes)
apiVersion: apps/v1
kind: Deployment
metadata:
  name: zerfoo-decode
spec:
  replicas: 4
  template:
    spec:
      containers:
        - name: zerfoo
          image: ghcr.io/zerfoo/zerfoo:latest
          args: ["worker", "--role", "decode", "--port", "50052"]
          resources:
            limits:
              nvidia.com/gpu: "1"

# Single GPU (default -- uses GPU 0 if CUDA is available)
zerfoo serve meta-llama/llama-3-70b-q4_k_m

# Two-GPU tensor parallel (GPUs 0 and 1)
zerfoo serve meta-llama/llama-3-70b-q4_k_m --gpus 0,1

# Four-GPU for a 70B model at full precision
zerfoo serve meta-llama/llama-3-70b-q8_0 --gpus 0,1,2,3

import (
    "github.com/zerfoo/zerfoo/inference"
    "github.com/zerfoo/zerfoo/serve"
)

model, err := inference.Load("meta-llama/llama-3-70b-q4_k_m")
if err != nil {
    log.Fatal(err)
}

srv := serve.NewServer(model,
    serve.WithGPUs([]int{0, 1, 2, 3}),   // distribute across 4 GPUs
    serve.WithLogger(logger),
    serve.WithMetrics(collector),
)

resources:
  limits:
    nvidia.com/gpu: "4"   # request 4 GPUs
args:
  - "meta-llama/llama-3-70b-q4_k_m"
  - "--port"
  - "8080"
  - "--gpus"
  - "0,1,2,3"

env:
  - name: CUDA_VISIBLE_DEVICES
    value: "0,1,2,3"

# values-tls.yaml
ingress:
  enabled: true
  className: nginx
  annotations:
    cert-manager.io/cluster-issuer: letsencrypt-prod
  hosts:
    - host: inference.example.com
      paths:
        - path: /
          pathType: Prefix
  tls:
    - secretName: zerfoo-tls
      hosts:
        - inference.example.com

import (
    "crypto/tls"
    "net/http"
    "github.com/zerfoo/zerfoo/serve"
)

srv := serve.NewServer(model,
    serve.WithLogger(logger),
    serve.WithMetrics(collector),
)
httpServer := &http.Server{
    Addr:    ":8443",
    Handler: srv.Handler(),
    TLSConfig: &tls.Config{
        MinVersion: tls.VersionTLS13,
        CurvePreferences: []tls.CurveID{
            tls.X25519,
            tls.CurveP256,
        },
    },
    ReadHeaderTimeout: 10 * time.Second,
}
if err := httpServer.ListenAndServeTLS("server.crt", "server.key"); err != nil {
    log.Fatal(err)
}

import (
    "crypto/tls"
    "crypto/x509"
    "os"
)

caCert, err := os.ReadFile("/etc/zerfoo/tls/ca.crt")
if err != nil {
    log.Fatal(err)
}
caCertPool := x509.NewCertPool()
caCertPool.AppendCertsFromPEM(caCert)

cert, err := tls.LoadX509KeyPair(
    "/etc/zerfoo/tls/server.crt",
    "/etc/zerfoo/tls/server.key",
)
if err != nil {
    log.Fatal(err)
}

tlsConfig := &tls.Config{
    MinVersion:   tls.VersionTLS13,
    Certificates: []tls.Certificate{cert},
    ClientAuth:   tls.RequireAndVerifyClientCert,
    ClientCAs:    caCertPool,
}

httpServer := &http.Server{
    Addr:      ":8443",
    Handler:   srv.Handler(),
    TLSConfig: tlsConfig,
}

apiVersion: cert-manager.io/v1
kind: Certificate
metadata:
  name: zerfoo-tls
  namespace: zerfoo-system
spec:
  secretName: zerfoo-tls
  duration: 2160h    # 90 days
  renewBefore: 360h  # renew 15 days before expiry
  subject:
    organizations:
      - Feza Inc
  dnsNames:
    - api.example.com
    - zerfoo-gemma-3-7b.zerfoo-system.svc.cluster.local
  issuerRef:
    name: letsencrypt-prod
    kind: ClusterIssuer

upstream zerfoo {
    server 127.0.0.1:8080;
    keepalive 32;
}

server {
    listen 443 ssl http2;
    server_name api.example.com;

    ssl_certificate     /etc/ssl/certs/api.example.com.crt;
    ssl_certificate_key /etc/ssl/private/api.example.com.key;
    ssl_protocols       TLSv1.3;
    ssl_ciphers         HIGH:!aNULL:!MD5;

    # Required for SSE streaming (token-by-token responses)
    proxy_buffering off;
    proxy_cache off;

    # Long inference requests
    proxy_read_timeout 300s;
    proxy_send_timeout 300s;

    location / {
        proxy_pass http://zerfoo;
        proxy_http_version 1.1;
        proxy_set_header Host              $host;
        proxy_set_header X-Real-IP         $remote_addr;
        proxy_set_header X-Forwarded-For   $proxy_add_x_forwarded_for;
        proxy_set_header X-Forwarded-Proto $scheme;
        proxy_set_header Connection        "";
    }

    # Restrict metrics to internal network
    location /metrics {
        allow 10.0.0.0/8;
        allow 172.16.0.0/12;
        allow 192.168.0.0/16;
        deny all;
        proxy_pass http://zerfoo;
    }
}

scrape_configs:
  - job_name: zerfoo
    scrape_interval: 15s
    static_configs:
      - targets:
          - "zerfoo-host:8080"
    metrics_path: /metrics

scrape_configs:
  - job_name: zerfoo-pods
    scrape_interval: 15s
    kubernetes_sd_configs:
      - role: pod
        namespaces:
          names: [zerfoo-system]
    relabel_configs:
      - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_scrape]
        action: keep
        regex: "true"
      - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_path]
        action: replace
        target_label: __metrics_path__
        regex: (.+)
      - source_labels: [__address__, __meta_kubernetes_pod_annotation_prometheus_io_port]
        action: replace
        regex: ([^:]+)(?::\d+)?;(\d+)
        replacement: $1:$2
        target_label: __address__

apiVersion: monitoring.coreos.com/v1
kind: PodMonitor
metadata:
  name: zerfoo
  namespace: zerfoo
spec:
  selector:
    matchLabels:
      app.kubernetes.io/name: zerfoo
  podMetricsEndpoints:
    - port: http
      path: /metrics
      interval: 15s

groups:
  - name: zerfoo
    rules:
      - alert: ZerfooHighLatency
        expr: |
          histogram_quantile(0.99, rate(request_latency_ms_bucket[5m])) > 5000
        for: 2m
        labels:
          severity: warning
        annotations:
          summary: "p99 request latency above 5 s"
          description: "p99 latency is {{ $value }}ms"

      - alert: ZerfooLowThroughput
        expr: tokens_per_second < 10
        for: 5m
        labels:
          severity: warning
        annotations:
          summary: "Token throughput critically low"
          description: "Tokens/s: {{ $value }}"

      - alert: ZerfooDown
        expr: up{job="zerfoo-pods"} == 0
        for: 1m
        labels:
          severity: critical
        annotations:
          summary: "Zerfoo instance down"

      - alert: ZerfooNoRequests
        expr: rate(requests_total[5m]) == 0
        for: 10m
        labels:
          severity: critical
        annotations:
          summary: "Zerfoo is not processing any requests"

      - alert: ZerfooOOM
        expr: increase(requests_total{status="503"}[5m]) > 0
        for: 1m
        labels:
          severity: critical
        annotations:
          summary: "Zerfoo returning 503 (possible OOM)"

livenessProbe:
  httpGet:
    path: /health
    port: http
  initialDelaySeconds: 30
  periodSeconds: 10

readinessProbe:
  httpGet:
    path: /health
    port: http
  initialDelaySeconds: 10
  periodSeconds: 5

method=POST path=/v1/chat/completions model=gemma-3-1b prompt_tokens=0 completion_tokens=0 latency_ms=142 status_code=200

apiVersion: v1
kind: PersistentVolumeClaim
metadata:
  name: zerfoo-models
  namespace: zerfoo-system
spec:
  accessModes:
    - ReadOnlyMany           # ReadWriteOnce if single-node
  storageClassName: efs-sc   # replace with your StorageClass
  resources:
    requests:
      storage: 200Gi

volumes:
  - name: model-storage
    persistentVolumeClaim:
      claimName: zerfoo-models

volumeMounts:
  - name: model-storage
    mountPath: /models

/models/
  llama-3-7b-q4_k_m/
    model.gguf
    metadata.json
  gemma-3-7b-it-q4_k_m/
    model.gguf
    metadata.json

import "github.com/zerfoo/zerfoo/serve/repository"

repo, err := repository.NewFileSystemRepository("/models")
if err != nil {
    log.Fatal(err)
}

// Upload a model
f, _ := os.Open("gemma-3-7b-it-q4_k_m.gguf")
err = repo.Upload(repository.ModelMetadata{
    ID:      "gemma-3-7b-it-q4_k_m",
    Name:    "Gemma 3 7B IT Q4_K_M",
    Version: "v1.0",
    Format:  "gguf",
}, f)

// List models
models, err := repo.List()

// Get model metadata
meta, err := repo.Get("gemma-3-7b-it-q4_k_m")
fmt.Printf("Size: %d bytes, SHA256: %s\n", meta.Size, meta.SHA256)

// Delete a model
err = repo.Delete("gemma-3-7b-it-q4_k_m")

initContainers:
  - name: model-pull
    image: ghcr.io/zerfoo/zerfoo:v1.0.0
    command: ["zerfoo", "pull", "google/gemma-3-7b-it-q4_k_m", "--cache-dir", "/models"]
    volumeMounts:
      - name: models
        mountPath: /models

Request -> ModelManager.Get("model-id")
              |
              +-- Already loaded? -> promote to MRU, return handle
              |
              +-- Not loaded:
                    Evict LRU models until usedBytes + newSize <= MaxGPUMemoryBytes
                    Load model via ModelLoader.Load()
                    Track in LRU list

import "github.com/zerfoo/zerfoo/serve/multimodel"

manager, err := multimodel.NewModelManager(loader, multimodel.Config{
    MaxGPUMemoryBytes: 40 * 1024 * 1024 * 1024, // 40 GB VRAM budget
    PreloadModels: []string{
        "gemma-3-7b-it-q4_k_m",   // preloaded at startup
        "llama-3-1b-q4_k_m",
    },
})
if err != nil {
    log.Fatal(err)
}
defer manager.Close()

type GGUFLoader struct {
    cacheDir string
}

func (l *GGUFLoader) Load(ctx context.Context, modelID string) (io.Closer, int64, error) {
    path := filepath.Join(l.cacheDir, modelID, "model.gguf")
    info, err := os.Stat(path)
    if err != nil {
        return nil, 0, err
    }
    model, err := inference.Load(modelID, inference.WithCacheDir(l.cacheDir))
    if err != nil {
        return nil, 0, err
    }
    return model, info.Size(), nil
}

// Get a model (loads if not cached, evicts LRU if needed)
model, err := manager.Get(ctx, "deepseek-v3-q4_k_m")

// Explicit eviction
err = manager.Unload("llama-3-1b-q4_k_m")

// Inspect state
loadedIDs := manager.Loaded()
usedBytes := manager.UsedBytes()

resources:
  limits:
    nvidia.com/gpu: "2"
    memory: "128Gi"
env:
  - name: ZERFOO_MAX_GPU_MEMORY_GB
    value: "80"   # 2x A100 40GB

model:
  quantization: "Q4_K_M"

import "github.com/zerfoo/zerfoo/serve/batcher"

scheduler := batcher.New(
    16,   // maxBatchSize -- max concurrent active sequences
    func(ctx context.Context, batch *batcher.StepBatch) {
        // Run one forward pass; append one token to each Slot.GeneratedToks
        // and set Slot.Done = true when EOS or max tokens reached.
        runForwardPass(ctx, batch)
    },
    batcher.WithPollInterval(1*time.Millisecond),
)
scheduler.Start()
defer scheduler.Stop()

import "github.com/zerfoo/zerfoo/serve/adaptive"

batcher := adaptive.New(adaptive.Config{
    MinBatchSize:    1,
    MaxBatchSize:    32,
    TargetLatencyMS: 200.0,   // target p50 latency SLO in ms
    QueueTimeoutMS:  50.0,    // max wait to fill a batch before dispatching
}, handler)
batcher.Start()
defer batcher.Stop()

import "github.com/zerfoo/zerfoo/serve"

bs := serve.NewBatchScheduler(serve.BatchConfig{
    MaxBatchSize: 8,
    BatchTimeout: 10 * time.Millisecond,
    // Handler is auto-wired to model.GenerateBatch when nil
})
bs.Start()

srv := serve.NewServer(model,
    serve.WithBatchScheduler(bs),
    serve.WithGPUs([]int{0, 1}),
    serve.WithLogger(logger),
    serve.WithMetrics(collector),
)

srv := serve.NewServer(targetModel,
    serve.WithDraftModel(draftModel),
)

apiVersion: networking.k8s.io/v1
kind: NetworkPolicy
metadata:
  name: zerfoo-inference
  namespace: zerfoo-system
spec:
  podSelector:
    matchLabels:
      app: zerfoo
  policyTypes:
    - Ingress
    - Egress
  ingress:
    - from:
        - namespaceSelector:
            matchLabels:
              kubernetes.io/metadata.name: ingress-nginx
      ports:
        - port: 8080
    - from:
        - namespaceSelector:
            matchLabels:
              kubernetes.io/metadata.name: monitoring
      ports:
        - port: 8080   # metrics scrape
        - port: 8081   # health probes
  egress:
    # Allow DNS resolution
    - to:
        - namespaceSelector: {}
      ports:
        - protocol: UDP
          port: 53
    # Allow model downloads (if pulling at startup)
    - to:
        - ipBlock:
            cidr: 0.0.0.0/0
      ports:
        - protocol: TCP
          port: 443

securityContext:
  runAsNonRoot: true
  runAsUser: 1000
  runAsGroup: 1000
  readOnlyRootFilesystem: true
  allowPrivilegeEscalation: false
  capabilities:
    drop:
      - ALL

apiVersion: v1
kind: Secret
metadata:
  name: zerfoo-model-credentials
  namespace: zerfoo
type: Opaque
stringData:
  HF_TOKEN: "hf_xxxxxxxxxxxxxxxxxxxxx"

env:
  - name: HF_TOKEN
    valueFrom:
      secretKeyRef:
        name: zerfoo-model-credentials
        key: HF_TOKEN

apiVersion: policy/v1
kind: PodDisruptionBudget
metadata:
  name: zerfoo-gemma-3-7b
  namespace: zerfoo-system
spec:
  minAvailable: 1
  selector:
    matchLabels:
      app: zerfoo
      model: gemma-3-7b

proxy_buffering off;
proxy_cache off;

annotations:
  nginx.ingress.kubernetes.io/proxy-buffering: "off"

env:
  - name: ZERFOO_LOG_LEVEL
    value: "debug"

# Check pod status and events
kubectl get pods -n zerfoo -o wide
kubectl describe pod -n zerfoo <pod-name>

# Stream logs
kubectl logs -n zerfoo deploy/zerfoo -f

# Check metrics endpoint
kubectl port-forward -n zerfoo svc/zerfoo 8080:80
curl localhost:8080/metrics

# Check model info
curl localhost:8080/v1/models

# Test inference
curl localhost:8080/v1/chat/completions \
  -H "Content-Type: application/json" \
  -d '{"model": "gemma-3-1b", "messages": [{"role": "user", "content": "Hello"}]}'

# Check HPA status
kubectl get hpa -n zerfoo