Architecture

flowchart TD
    subgraph CLIENT["AGENT SIDE - any MCP client"]
        A["AI agent<br/>sees ONLY:<br/>35 tools, 10 prompts,<br/>6 resources<br/>never: kubeconfig,<br/>Secrets, exec, kubectl"]
    end

    A -- "MCP JSON-RPC 2.0<br/>over stdio" --> B

    subgraph SERVER["SERVER PROCESS - ocm-mcp-server"]
        B["FastMCP dispatch<br/>server.py<br/>argument schema validation<br/>readOnlyHint and<br/>destructiveHint per tool"]
        B --> C["traced_tool wrapper<br/>tracing.py<br/>1. optional OTel span<br/>2. classify outcome<br/>3. hash-chained audit line<br/>4. metrics.record"]
        C --> D{"toolset?"}
        D -- "reads: 29 tools,<br/>6 resources" --> E["_read wrapper<br/>server.py<br/>missing API -> UNAVAILABLE<br/>ApiException -> clear<br/>message, no stack trace"]
        D -- "writes: 6 tools" --> F["gate chain<br/>read-only backstop<br/>size cap, schema<br/>static guardrails<br/>Kyverno dry-run<br/>proposal store<br/>human token, apply"]
        E --> G["ocm.py<br/>typed OCM operations<br/>summarized CRs<br/>paged_list: limit 500,<br/>continue tokens,<br/>5000-item ceiling"]
        F --> G
        G --> H["k8s.py client factory<br/>ApiClient cache per<br/>context, TTL 600 s<br/>hub: CustomObjectsApi<br/>spokes: CoreV1, AppsV1<br/>bounded, timed reads"]
    end

    subgraph STATE["LOCAL STATE - OCM_MCP_HOME"]
        S1["proposals/uuid.json<br/>0600, fsync, flock<br/>pending -> approved -><br/>applied -> rolled_back<br/>forward-only"]
        S2["audit.jsonl<br/>hash chain<br/>audit_anchors.jsonl<br/>signed heads"]
        S3["approval_ed25519.pub<br/>verifier key ONLY<br/>server can never sign"]
        S4["used_tokens.jsonl<br/>spent jti ledger<br/>compacted at 2000"]
    end
    F -.-> S1
    C -.-> S2
    F -.-> S3
    F -.-> S4

    subgraph HUMAN["TRUSTED TERMINAL - ocm-mcp CLI"]
        T1["ocm-mcp pending / show<br/>human reviews the<br/>EXACT manifests"]
        T2["ocm-mcp approve<br/>signs claims with the<br/>Ed25519 PRIVATE key<br/>ideally off-box or KMS"]
        T3["ocm-mcp audit-anchor<br/>and audit-verify<br/>sign + verify the<br/>audit chain head"]
    end
    T1 -. reads .-> S1
    T2 -- "token handed over<br/>out-of-band" --> A
    T3 -.-> S2

    H -- "HTTPS to hub apiserver<br/>OCM_MCP_HUB_CONTEXT" --> I

    subgraph HUB["OCM HUB CLUSTER"]
        I["kube-apiserver"]
        I --> J["Kyverno admission<br/>9 ClusterPolicies<br/>match ManifestWork<br/>foreach embedded manifest<br/>label + requester scoped<br/>dry-run = layer 2 gate<br/>at propose time"]
        J --> K["RBAC rbac.yaml<br/>read OCM groups<br/>create/delete works<br/>patch clusters<br/>scoped CSR approval<br/>NO secrets, NO exec,<br/>NO arbitrary delete"]
        K --> L["OCM control plane CRs<br/>ManagedCluster, Placement<br/>PlacementDecision<br/>ManifestWork, MWRS<br/>ManagedClusterAddOn<br/>Policy, CSR"]
    end

    L -- "OCM work agents<br/>klusterlet pull model" --> M

    subgraph SPOKES["MANAGED CLUSTERS - spokes"]
        M["klusterlet applies the<br/>embedded manifests<br/>status feeds back into<br/>ManifestWork status<br/>Applied / Available"]
        N["ocm-mcp-reader SA<br/>read-only RBAC for<br/>health, events, pod logs<br/>production: cluster-proxy"]
    end
    H -- "HTTPS bounded reads<br/>limit 500, timeout 5/30 s" --> N

sequenceDiagram
    autonumber
    participant AG as Agent (MCP client)
    participant FM as FastMCP (server.py)
    participant TR as traced_tool (tracing.py)
    participant OC as ocm.py
    participant K8 as k8s.py
    participant API as hub kube-apiserver

    AG->>FM: JSON-RPC tools/call {name:"list_clusters"}
    FM->>FM: validate arguments against generated schema
    FM->>TR: list_clusters()
    TR->>TR: start OTel span "tool.list_clusters" (if OTLP endpoint set)
    TR->>OC: _read(ocm.list_managed_clusters)
    OC->>OC: paged_list(list_fn) - page loop
    OC->>K8: hub_custom() -> cached ApiClient for OCM_MCP_HUB_CONTEXT<br/>(rebuilt if older than OCM_MCP_CLIENT_TTL=600s)
    K8->>API: GET /apis/cluster.open-cluster-management.io/v1/managedclusters?limit=500
    API-->>K8: page 1 (+ metadata.continue if more)
    K8->>API: ...follow continue tokens until done or 5000 items (then explicit "truncated")
    API-->>OC: raw ManagedCluster list
    OC->>OC: summarize: name, labels, Available/Joined conditions,<br/>kubernetes version, cpu/memory capacity (nothing else)
    OC-->>TR: list[dict]
    TR->>TR: classify_outcome -> "ok"
    TR->>TR: audit append under flock: {tool,args,outcome,duration_ms,<br/>actor,ts,seq=N,prev=hash(N-1),hash=sha256(prev+canonical)} + fsync
    TR->>TR: metrics.record("list_clusters","ok",ms)
    TR-->>FM: JSON string
    FM-->>AG: JSON-RPC result (content: text)

flowchart TD
    FH0["get_fleet_health call"] --> FH1["hub conditions:<br/>every cluster, one list"]
    FH0 --> FH2["fan-out: bounded workers<br/>OCM_MCP_FANOUT_WORKERS"]
    FH2 --> FH3["spoke scan: cluster 1"]
    FH2 --> FH4["spoke scan: cluster 2"]
    FH2 --> FH5["spoke scan: cluster N<br/>error entry, not a failure"]
    FH1 --> FH6["merge + sort:<br/>unhealthy clusters first"]
    FH3 --> FH6
    FH4 --> FH6
    FH5 --> FH6

sequenceDiagram
    autonumber
    participant AG as Agent
    participant SV as server.py (write tools)
    participant GR as guardrails.py
    participant AP as approvals.py
    participant ST as $OCM_MCP_HOME
    participant HU as Human (ocm-mcp CLI)
    participant API as hub apiserver (+ Kyverno)

    Note over AG,API: PROPOSE - nothing is applied
    AG->>SV: propose_manifestwork(cluster, name, summary, manifests_json)
    SV->>SV: read-only backstop (OCM_MCP_READ_ONLY -> refuse)
    SV->>SV: raw byte cap BEFORE parsing (256 KiB) - no JSON bomb
    SV->>GR: validate_manifests(manifests)
    GR->>GR: list + count cap (max 10 manifests)
    GR->>GR: per-manifest schema check first (malformed = clean violation, never a crash)
    GR->>GR: exact (apiVersion, kind) in ALLOWED_GVK - group spoofing blocked
    GR->>GR: namespace required + protected list + kube-*/openshift-*/ocm prefixes
    GR->>GR: Restricted PSS on every container role (regular/init/ephemeral):<br/>runAsNonRoot, no runAsUser 0, allowPrivilegeEscalation=false,<br/>drop ALL, no added caps, seccomp, automountServiceAccountToken=false
    GR->>GR: volume ALLOW-list (configMap/emptyDir/downwardAPI/projected),<br/>no projected serviceAccountToken/secret sources
    GR->>GR: no env secretKeyRef / envFrom secretRef (no path to Secret contents)
    GR->>GR: image pinned - no :latest, no tagless, @sha256 digest in strict mode
    GR->>GR: Service type ClusterIP only, no externalIPs - HPA maxReplicas capped at 100
    GR-->>SV: ok, or GuardrailViolation with EVERY problem listed (agent self-corrects)
    SV->>API: create ManifestWork with dryRun=All (server-side)
    API->>API: Kyverno admission: 9 ClusterPolicies foreach embedded manifest (layer 2,<br/>independent re-check of the same invariants + org policies)
    API-->>SV: admitted (dry-run) or denied with the policy's own message
    SV->>ST: Proposal.save(): uuid4-hex id, content_hash = sha256(canonical<br/>cluster+name+manifests+kind+action+params), status=pending,<br/>atomic write + fsync + chmod 0600
    SV-->>AG: proposal_id + "ask a human to run ocm-mcp approve with this id"

    Note over HU,ST: APPROVE - happens OUTSIDE the agent's reach
    HU->>ST: ocm-mcp show id - reviews the exact manifests
    HU->>AP: mint_token(proposal, operation="apply")
    AP->>AP: claims: jti (random), iss, aud, id, hash=content_hash,<br/>op, approver, iat/nbf/exp (TTL 3600 s)
    AP->>AP: Ed25519 sign with PRIVATE key (CLI side only)
    HU-->>AG: token handed over out-of-band

    Note over AG,API: APPLY - the token is proven, then burned
    AG->>SV: apply_manifestwork(proposal_id, approval_token)
    SV->>ST: proposal_lock(id) - flock serializes concurrent applies
    SV->>SV: _valid_id (32-hex only - path traversal blocked), load, status must be pending
    SV->>SV: TOCTOU defense: re-hash stored content, re-run ALL static guardrails NOW
    SV->>AP: verify_token(proposal, token, operation="apply")
    AP->>AP: signature valid under verifier .pub (or .pub.prev during planned rotation)?
    AP->>AP: iss/aud match? now within nbf..exp? claims.id == proposal.id?
    AP->>AP: claims.hash == re-computed content hash? claims.op == "apply"?<br/>(an apply token can NEVER authorize a rollback)
    AP->>ST: jti already in used_tokens.jsonl? -> replay refused
    AP->>ST: consume: append jti (compact ledger at 2000 lines)
    SV->>API: create ManifestWork (real this time) with<br/>label app.kubernetes.io/managed-by=ocm-mcp-server
    API-->>SV: created (Kyverno + RBAC enforce again, for real)
    SV->>ST: status pending -> applied, record approver + work UID
    SV-->>AG: applied (agent verifies via get_manifestwork status feedback)

flowchart TD
    R0["propose_rollback<br/>applied proposal id"] --> R1["loads the APPLIED proposal<br/>creates a DISTINCT<br/>rollback proposal binding<br/>target work name,<br/>current UID, origin id"]
    R1 --> R2["human approves it<br/>separately: token<br/>claims op=rollback"]
    R2 --> R3["rollback_manifestwork<br/>rollback id + token"]
    R3 --> R4{"ownership checks<br/>BEFORE the token<br/>is burned"}
    R4 -- "managed-by<br/>label missing" --> RX["REJECTED<br/>never deletes a work<br/>it did not create"]
    R4 -- "UID changed<br/>since approval" --> RX2["REJECTED<br/>work was re-created,<br/>re-propose"]
    R4 -- ok --> R5["verify + consume<br/>rollback-scoped token<br/>DELETE ManifestWork<br/>rollback prop: applied<br/>origin prop: rolled_back"]

    C0["propose_cluster_action<br/>cluster, action, params"] --> C1["action allow-list:<br/>cordon, uncordon,<br/>set_label, accept,<br/>enable / disable addon<br/>nothing else proposable"]
    C1 --> C2["server-side dry-run of<br/>the exact patch/create<br/>at PROPOSE time"]
    C2 --> C3["same store, same<br/>human token,<br/>apply_cluster_action"]
    C3 --> C4["patch actions merge-patch<br/>the ManagedCluster<br/>accept approves ONLY the<br/>captured CSRs: name, uid,<br/>request-hash, CN verified<br/>addon actions create or<br/>delete ManagedClusterAddOn"]

flowchart TD
    subgraph AUD["audit.jsonl - hash chain"]
        A1["entry N-1<br/>hash H1"] --> A2["entry N<br/>prev = H1<br/>hash H2 = sha256 of<br/>H1 + canonical entry"]
        A2 --> A3["entry N+1<br/>prev = H2 ..."]
    end
    A3 --> V1["verify_audit_chain<br/>ocm-mcp audit-verify<br/>recomputes every hash<br/>catches edit, reorder,<br/>mid-log deletion"]
    V1 --> V2["blind spot of a bare chain:<br/>tail truncation,<br/>full rewrite"]
    V2 --> V3["closed by ANCHORS<br/>ocm-mcp audit-anchor<br/>trusted terminal signs<br/>seq + hash + timestamp<br/>with the PRIVATE key"]
    V3 --> V4["audit-verify also fails<br/>unless the log still<br/>EXTENDS every anchored<br/>head - verifier key only"]

    subgraph KEYS["key custody"]
        K1["PRIVATE approval key<br/>CLI, trusted terminal,<br/>or KMS<br/>signs tokens + anchors"]
        K2["PUBLIC verifier key<br/>plus .pub.prev in rotation<br/>server side, verify only<br/>compromised server<br/>cannot mint"]
    end

    subgraph FILES["OCM_MCP_HOME on disk"]
        F1["proposals/*.json<br/>0600, fsync,<br/>forward-only status"]
        F2["audit.jsonl +<br/>audit_anchors.jsonl<br/>0600, flock"]
        F3["used_tokens.jsonl<br/>replay ledger"]
    end

flowchart TD
    TC["one tool call<br/>traced_tool decorator<br/>tracing.py"]

    TC --> AUD["AUDIT LINE - always on<br/>the tamper-evident record<br/>tool, args, outcome,<br/>duration, actor, seq,<br/>prev, hash - flock + fsync"]
    AUD --> AUD2["audit.jsonl hash chain<br/>+ signed anchors<br/>answers: WHAT happened,<br/>in what order,<br/>on whose authority"]

    TC --> SPAN["OTEL SPAN - opt-in<br/>span name: tool.name<br/>attributes: arg.* truncated<br/>to 200 chars<br/>approval_token NEVER attached"]
    SPAN --> EXP["BatchSpanProcessor<br/>OTLP/HTTP exporter<br/>POST /v1/traces"]
    EXP --> BE["Jaeger all-in-one :4318<br/>or any OTel collector<br/>Tempo, vendor backend<br/>UI: Jaeger :16686<br/>service: ocm-mcp-server<br/>answers: WHERE time went"]

    TC --> MET["METRICS - opt-in<br/>metrics.py, stdlib only<br/>counters by tool + outcome<br/>+ duration sums"]
    MET --> PROM["GET /metrics<br/>Prometheus format<br/>binds 127.0.0.1 unless<br/>OCM_MCP_METRICS_HOST set<br/>answers: HOW OFTEN and<br/>HOW SLOW, for alerting"]