Skip to content

Setup Cilium Clustermesh

Overview

This guide explains how to setup cilium clustermesh on top of MetaKube installation with cilium cni.

Prerequisites

This How-To requires:

  • Access to two MetaKube clusters in the same region.
  • PodCIDR/NodeCIDR ranges in all clusters and all nodes must be non-conflicting and unique IP addresses.
  • Nodes in all clusters must have IP connectivity between each other using the configured InternalIP for each node.
  • Access to OpenStack with valid and sufficient credentials
  • OpenStack Subnet

Create 2 MetaKube clusters with non overlapping pod/node CIDRs

Cluster-1

...
   "spec": {
        "cloud": {
            "openstack":{
                "subnetCIDR":"192.168.0.0/24",
...
            }
        }
        "cniPlugin": {
            "type": "cilium"
        },
        "clusterNetwork": {
          "proxyMode":"none",
          "pods": {
            "cidrBlocks":[
              "172.0.0.0/16"
            ]
          },
        }
...
    }

Cluster-2

...
   "spec": {
        "cloud": {
            "openstack":{
                "subnetCIDR":"192.168.1.0/24",
...
            }
        }
        "cniPlugin": {
            "type": "cilium"
        },
        "clusterNetwork": {
          "proxyMode":"none",
          "pods": {
            "cidrBlocks":[
              "172.1.0.0/16"
            ]
          },
        }
...
    }

Patch cilium cluster-id

Info

If you change the cluster ID or cluster name in a cluster with running workloads, you will need to restart all workloads. The cluster ID is used to generate the security identity and it will need to be re-created to establish access across clusters. See

Cluster-1

kubectl -n kube-system patch configmap cilium-config --type=merge --patch="$(jq -n '{data: {"cluster-id": "1"}}')"

Cluster-2

kubectl -n kube-system patch configmap cilium-config --type=merge --patch="$(jq -n '{data: {"cluster-id": "2"}}')"

Connect to the clusters

Enable clustermesh

  1. on first cluster

Cluster-1

METAKUBE_CLUSTER_ID=gvb5wlmvgp
helm template cilium cilium/cilium \
--version v1.18.5 \
--namespace kube-system \
--set cluster.name=${METAKUBE_CLUSTER_ID} \
--set cluster.id=1 \
--set ipam.mode=kubernetes \
--set routingMode=native \
--set enableIPv4Masquerade=true \
--set kubeProxyReplacement=true \
--set autoDirectNodeRoutes=true \
--set bpf.masquerade=true \
--set bpf.hostLegacyRouting=false \
--set ipv4NativeRoutingCIDR=172.0.0.0/15 \
--set requireIPv4PodCIDR=false \
--set envoy.enabled=true \
--set hubble.enabled=true \
--set operator.identityHeartbeatTimeout=5m \
--set localRedirectPolicies.enabled=true \
--set clustermesh.useAPIServer=true \
--set clustermesh.config.enabled=true \
--set clustermesh.apiserver.kvstoremesh.enabled=true \
--set clustermesh.apiserver.service.type=LoadBalancer | \
yq 'select(.metadata.name | test("clustermesh|kvstoremesh|cilium-ca"))' | \
kubectl apply -f -
  1. copy cilium-ca over to the second cluster
kubectl get secret -n kube-system cilium-ca -o yaml | \
kubectl --kubeconfig kubeconfig-cluster-2  create -f -
  1. on second cluster
METAKUBE_CLUSTER_ID=9k2fgbv2km
helm template cilium cilium/cilium \
--version v1.18.5 \
--namespace kube-system \
--set cluster.name=${METAKUBE_CLUSTER_ID} \
--set cluster.id=2 \
--set ipam.mode=kubernetes \
--set routingMode=native \
--set enableIPv4Masquerade=true \
--set kubeProxyReplacement=true \
--set autoDirectNodeRoutes=true \
--set bpf.masquerade=true \
--set bpf.hostLegacyRouting=false \
--set ipv4NativeRoutingCIDR=172.0.0.0/15 \
--set requireIPv4PodCIDR=false \
--set envoy.enabled=true \
--set hubble.enabled=true \
--set operator.identityHeartbeatTimeout=5m \
--set localRedirectPolicies.enabled=true \
--set tls.ca.cert=$(kubectl -n kube-system get secret cilium-ca -o jsonpath='{.data.ca\.crt}') \
--set tls.ca.key=$(kubectl -n kube-system get secret cilium-ca -o jsonpath='{.data.ca\.key}') \
--set clustermesh.useAPIServer=true \
--set clustermesh.config.enabled=true \
--set clustermesh.apiserver.kvstoremesh.enabled=true \
--set clustermesh.apiserver.service.type=LoadBalancer | \
yq 'select(.metadata.name | test("clustermesh|kvstoremesh"))' | \
kubectl apply -f -

Connect the cluster networks

openstack network create interconn
openstack subnet create --network interconn --subnet-range 10.0.0.0/30 --gateway none interconn

cluster-1

METAKUBE_CLUSTER_ID=gvb5wlmvgp
openstack port create --network interconn --fixed-ip subnet=interconn,ip-address=10.0.0.1 ic-${METAKUBE_CLUSTER_ID}
openstack router add port metakube-${METAKUBE_CLUSTER_ID} ic-${METAKUBE_CLUSTER_ID}
openstack router add route --route destination=192.168.1.0/24,gateway=10.0.0.2 metakube-${METAKUBE_CLUSTER_ID}
openstack router add route --route destination=172.1.0.0/16,gateway=10.0.0.2 metakube-${METAKUBE_CLUSTER_ID}
openstack security group rule create --ingress --remote-ip '192.168.1.0/24' metakube-${METAKUBE_CLUSTER_ID}
openstack security group rule create --ingress --remote-ip '172.0.0.0/15' metakube-${METAKUBE_CLUSTER_ID}

cluster-2

PAIR_METAKUBE_CLUSTER_ID=9k2fgbv2km
openstack port create --network interconn --fixed-ip subnet=interconn,ip-address=10.0.0.2 ic-${METAKUBE_CLUSTER_ID}
openstack router add port metakube-${METAKUBE_CLUSTER_ID} ic-${METAKUBE_CLUSTER_ID}
openstack router add route --route destination=192.168.0.0/24,gateway=10.0.0.1 metakube-${METAKUBE_CLUSTER_ID}
openstack router add route --route destination=172.0.0.0/16,gateway=10.0.0.1 metakube-${METAKUBE_CLUSTER_ID}
openstack security group rule create --ingress --remote-ip '192.168.0.0/24' metakube-${METAKUBE_CLUSTER_ID}
openstack security group rule create --ingress --remote-ip '172.0.0.0/15' metakube-${METAKUBE_CLUSTER_ID}

Connect the clusters

cluster-1

PAIR_METAKUBE_CLUSTER_ID=gvb5wlmvgp
helm template cilium cilium/cilium \
  --version v1.18.5 \
  --namespace kube-system \
  --set cluster.name=${METAKUBE_CLUSTER_ID} \
  --set cluster.id=1 \
  --set ipam.mode=kubernetes \
  --set routingMode=native \
  --set enableIPv4Masquerade=true \
  --set kubeProxyReplacement=true \
  --set autoDirectNodeRoutes=true \
  --set bpf.masquerade=true \
  --set bpf.hostLegacyRouting=false \
  --set ipv4NativeRoutingCIDR=172.0.0.0/15 \
  --set requireIPv4PodCIDR=false \
  --set envoy.enabled=true \
  --set hubble.enabled=true \
  --set operator.identityHeartbeatTimeout=5m \
  --set localRedirectPolicies.enabled=true \
  --set tls.ca.cert=$(kubectl -n kube-system get secret cilium-ca -o jsonpath='{.data.ca\.crt}') \
  --set tls.ca.key=$(kubectl -n kube-system get secret cilium-ca -o jsonpath='{.data.ca\.key}') \
  --set clustermesh.useAPIServer=true \
  --set clustermesh.config.enabled=true \
  --set clustermesh.apiserver.kvstoremesh.enabled=true \
  --set clustermesh.apiserver.service.type=LoadBalancer \
  --set 'clustermesh.config.clusters[0].name='${PAIR_METAKUBE_CLUSTER_ID} \
  --set 'clustermesh.config.clusters[0].port=2379' \
  --set 'clustermesh.config.clusters[0].ips={'$(kubectl --kubeconfig kubeconfig-cluster-2 get -n kube-system service clustermesh-apiserver -o jsonpath='{.status.loadBalancer.ingress[0].ip}')$'}' |\
yq -cr -y 'select(.metadata.name | test("clustermesh|kvstoremesh"))' | \
kubectl apply -f -

cluster-2

PAIR_METAKUBE_CLUSTER_ID=9k2fgbv2km
helm template cilium cilium/cilium \
  --version v1.18.5 \
  --namespace kube-system \
  --set cluster.name=${METAKUBE_CLUSTER_ID} \
  --set cluster.id=2 \
  --set ipam.mode=kubernetes \
  --set routingMode=native \
  --set enableIPv4Masquerade=true \
  --set kubeProxyReplacement=true \
  --set autoDirectNodeRoutes=true \
  --set bpf.masquerade=true \
  --set bpf.hostLegacyRouting=false \
  --set ipv4NativeRoutingCIDR=172.0.0.0/15 \
  --set requireIPv4PodCIDR=false \
  --set envoy.enabled=true \
  --set hubble.enabled=true \
  --set operator.identityHeartbeatTimeout=5m \
  --set localRedirectPolicies.enabled=true \
  --set tls.ca.cert=$(kubectl -n kube-system get secret cilium-ca -o jsonpath='{.data.ca\.crt}') \
  --set tls.ca.key=$(kubectl -n kube-system get secret cilium-ca -o jsonpath='{.data.ca\.key}') \
  --set clustermesh.useAPIServer=true \
  --set clustermesh.config.enabled=true \
  --set clustermesh.apiserver.kvstoremesh.enabled=true \
  --set clustermesh.apiserver.service.type=LoadBalancer \
  --set 'clustermesh.config.clusters[0].name='${PAIR_METAKUBE_CLUSTER_ID} \
  --set 'clustermesh.config.clusters[0].port=2379' \
  --set 'clustermesh.config.clusters[0].ips={'$(kubectl --kubeconfig kubeconfig-cluster-1 get -n kube-system service clustermesh-apiserver -o jsonpath='{.status.loadBalancer.ingress[0].ip}')$'}' |\
yq -cr -y 'select(.metadata.name | test("clustermesh|kvstoremesh"))' | \
kubectl apply -f -

Troubleshooting

  1. cilium endpoint
kubectl -n kube-system exec -it daemonset/cilium -- cilium-dbg endpoint list | grep app=http-echo
622        Disabled           Disabled          66711      k8s:app=http-echo

If you don't see the expected endpoint try to roll overt the cilium DS

  1. clustermesh service
kubectl -n kube-system exec -ti deployment/clustermesh-apiserver -c apiserver -- clustermesh-apiserver shell kvstore/list | grep -E "# cilium/(cache|state)/services/"
# cilium/cache/services/v1/9k2fgbv2km/default/http-echo
# cilium/state/services/v1/gvb5wlmvgp/default/http-echo
  1. cilium agent connecting to the local kvstoremesh
kubectl -n kube-system exec -it daemonset/cilium -- cilium-dbg troubleshoot clustermesh
    Found 1 cluster configurations

    Cluster "9k2fgbv2km":
    📄 Configuration path: /var/lib/cilium/clustermesh/9k2fgbv2km

    🔌 Endpoints:
    - https://clustermesh-apiserver.kube-system.svc:2379
        ✅ Hostname resolved to: 10.240.22.125
        ✅ TCP connection successfully established to 10.240.22.125:2379
        ✅ TLS connection successfully established to 10.240.22.125:2379
        ℹī¸  Negotiated TLS version: TLS 1.3, ciphersuite TLS_AES_128_GCM_SHA256
        ℹī¸  Etcd server version: 3.6.6

    🔑 Digital certificates:
    ✅ TLS Root CA certificates:
        - Serial number:       fb:62:5c:15:b9:20:12:e7:70:3b:6d:12:ee:7d:cc:14
            Subject:             CN=Cilium CA
            Issuer:              CN=Cilium CA
            Validity:
            Not before:  2026-01-12 15:25:04 +0000 UTC
            Not after:   2029-01-11 15:25:04 +0000 UTC
    ✅ TLS client certificates:
        - Serial number:       2c:41:b7:b3:0d:94:29:67:6a:c1:80:9b:2a:ea:e7:14
            Subject:             CN=local-gvb5wlmvgp
            Issuer:              CN=Cilium CA
            Validity:
            Not before:  2026-01-12 16:03:35 +0000 UTC
            Not after:   2029-01-11 16:03:35 +0000 UTC

    ⚙ī¸ Etcd client:
    ✅ Etcd connection successfully established
    ℹī¸  Etcd cluster ID: f159592cd2603408
  1. kvstoremesh connecting to pair cluster
kubectl -n kube-system exec -ti deployment/clustermesh-apiserver -c kvstoremesh -- clustermesh-apiserver kvstoremesh-dbg troubleshoot
    Found 1 cluster configurations

    Cluster "9k2fgbv2km":
    📄 Configuration path: /var/lib/cilium/clustermesh/9k2fgbv2km

    🔌 Endpoints:
    - https://9k2fgbv2km.mesh.cilium.io:2379
        ✅ Hostname resolved to: 109.68.229.69
        ✅ TCP connection successfully established to 109.68.229.69:2379
        ✅ TLS connection successfully established to 109.68.229.69:2379
        ℹī¸  Negotiated TLS version: TLS 1.3, ciphersuite TLS_AES_128_GCM_SHA256
        ℹī¸  Etcd server version: 3.6.6

    🔑 Digital certificates:
    ✅ TLS Root CA certificates:
        - Serial number:       fb:62:5c:15:b9:20:12:e7:70:3b:6d:12:ee:7d:cc:14
            Subject:             CN=Cilium CA
            Issuer:              CN=Cilium CA
            Validity:
            Not before:  2026-01-12 15:25:04 +0000 UTC
            Not after:   2029-01-11 15:25:04 +0000 UTC
    ✅ TLS client certificates:
        - Serial number:       d2:77:54:8b:e6:b4:91:3d:cd:b5:3f:fe:ad:b3:d2:84
            Subject:             CN=remote
            Issuer:              CN=Cilium CA
            Validity:
            Not before:  2026-01-12 16:03:35 +0000 UTC
            Not after:   2029-01-11 16:03:35 +0000 UTC

    ⚙ī¸ Etcd client:
    ✅ Etcd connection successfully established
    ℹī¸  Etcd cluster ID: 7711876c9ebb329e
  1. overall cluster health
kubectl -n kube-system exec -it daemonset/cilium -- cilium-dbg status --all-clusters
    KVStore:                Disabled
    Kubernetes:             Ok         1.34 (v1.34.2) [linux/amd64]
    Kubernetes APIs:        ["EndpointSliceOrEndpoint", "cilium/v2::CiliumCIDRGroup", "cilium/v2::CiliumClusterwideNetworkPolicy", "cilium/v2::CiliumEndpoint", "cilium/v2::CiliumNetworkPolicy", "cilium/v2::CiliumNode", "core/v1::Pods", "networking.k8s.io/v1::NetworkPolicy"]
    KubeProxyReplacement:   True   [ens3   192.168.0.10 fe80::f816:3eff:fe26:8da8 (Direct Routing)]
    Host firewall:          Disabled
    SRv6:                   Disabled
    CNI Chaining:           none
    CNI Config file:        successfully wrote CNI configuration file to /host/etc/cni/net.d/05-cilium.conflist
    Cilium:                 Ok   1.18.4 (v1.18.4-afda2aa9)
    NodeMonitor:            Listening for events on 2 CPUs with 64x4096 of shared memory
    Cilium health daemon:   Ok
    IPAM:                   IPv4: 9/254 allocated from 172.0.1.0/24,
    ClusterMesh:            1/1 remote clusters ready, 1 global-services
    9k2fgbv2km: ready, 2 nodes, 19 endpoints, 0 identities, 1 services, 0 MCS-API service exports, 0 reconnections (last: never)
    └  etcd: 1/1 connected, leases=0, lock leases=0, has-quorum=true: endpoint status checks are disabled, ID: f159592cd2603408
    └  remote configuration: expected=true, retrieved=true, cluster-id=2, kvstoremesh=true, sync-canaries=true, service-exports=disabled
    └  synchronization status: nodes=true, endpoints=true, identities=true, services=true
    IPv4 BIG TCP:            Disabled
    IPv6 BIG TCP:            Disabled
    BandwidthManager:        Disabled
    Routing:                 Network: Native   Host: BPF
    Attach Mode:             TCX
    Device Mode:             veth
    Masquerading:            BPF   [ens3]   172.0.0.0/16  [IPv4: Enabled, IPv6: Disabled]
    Controller Status:       59/59 healthy
    Proxy Status:            OK, ip 172.0.1.149, 0 redirects active on ports 10000-20000, Envoy: external
    Global Identity Range:   min 65536, max 131071
    Hubble:                  Ok              Current/Max Flows: 4095/4095 (100.00%), Flows/s: 36.02   Metrics: Ok
    Encryption:              Disabled
    Cluster health:          4/4 reachable   (2026-01-13T09:02:48Z)   (Probe interval: 2m11.83347464s)
    Name                     IP              Node                     Endpoints
    Modules Health:          Stopped(16) Degraded(3) OK(84)