OPNSense Bridge

  1. Interfaces: Assignments » add igb[n]
  2. Interfaces: Other Types: Bridge » create Bridge
  3. Interfaces: [OPT(n)] » enable Device
  4. Interfaces: Other Types: Bridge » add member Device(s)
  5. Interfaces: Assignments » change LAN device to Bridge
  6. (add old LAN Interface to Bridge)

install Kubernetes on Debian 12 Bookworm

Sources:
https://kubernetes.io/docs/setup/
https://www.server-world.info/en/note?os=Debian_12&p=kubernetes&f=1

  • prepare Hosts [all nodes]
    • disable swap
      edit /etc/fstab && swapoff -a
    • add cluster DNS to /etc/hosts
      • e.g. onprem.cloud
        echo “127.0.0.1 onprem.cloud” >> /etc/hosts
apt install -y curl gpg
# edit /etc/sysctl.conf
# net.ipv4.ip_forward=1
# net.bridge.bridge-nf-call-iptables=1
# net.bridge.bridge-nf-call-ip6tables=1
sysctl -w net.ipv4.ip_forward=1
sysctl -w net.bridge.bridge-nf-call-iptables=1
sysctl -w net.bridge.bridge-nf-call-ip6tables=1
# add kernel modules
echo -e "overlay\nbr_netfilter" | sudo tee -a /etc/modules
modprobe br_netfilter
export KUBECONFIG=/etc/kubernetes/admin.conf
echo "export KUBECONFIG=/etc/kubernetes/admin.conf" >> /root/.bashrc
  • add K8s Repo [all nodes]
curl -fsSL https://pkgs.k8s.io/core:/stable:/v1.31/deb/Release.key | sudo gpg --dearmor -o /etc/apt/keyrings/kubernetes-apt-keyring.gpg

echo "deb [signed-by=/etc/apt/keyrings/kubernetes-apt-keyring.gpg] https://pkgs.k8s.io/core:/stable:/v1.31/deb/ /" | sudo tee /etc/apt/sources.list.d/kubernetes.list
  • add CRI-O Repo [all nodes]
curl -fsSL https://pkgs.k8s.io/addons:/cri-o:/stable:/v1.31/deb/Release.key | sudo gpg --dearmor -o /etc/apt/keyrings/cri-o-apt-keyring.gpg

echo "deb [signed-by=/etc/apt/keyrings/cri-o-apt-keyring.gpg] https://pkgs.k8s.io/addons:/cri-o:/stable:/v1.31/deb/ /" | sudo tee /etc/apt/sources.list.d/cri-o.list 
  • install K8s (https://kubernetes.io/docs/setup/production-environment/tools/kubeadm/install-kubeadm/) and Container Runtime CRI-O (https://cri-o.io/) [all nodes]
sudo apt update
sudo apt install -y cri-o kubelet kubeadm kubectl

sudo systemctl start crio.service
sudo systemctl enable crio.service

sudo apt-mark hold cri-o kubelet kubeadm kubectl

install CNI Cillium (https://docs.cilium.io/en/stable/gettingstarted/k8s-install-default/) [all nodes]

# CILIUM_CLI_VERSION=$(curl -s https://raw.githubusercontent.com/cilium/cilium-cli/main/stable.txt)
CILIUM_CLI_VERSION=v0.16.18
curl -L --fail --remote-name-all https://github.com/cilium/cilium-cli/releases/download/${CILIUM_CLI_VERSION}/cilium-linux-amd64.tar.gz{,.sha256sum}
sha256sum --check cilium-linux-amd64.tar.gz.sha256sum
sudo tar xzvfC cilium-linux-amd64.tar.gz /usr/local/bin
rm cilium-linux-amd64.tar.gz{,.sha256sum}
# HUBBLE_VERSION=$(curl -s https://raw.githubusercontent.com/cilium/hubble/master/stable.txt)
HUBBLE_VERSION=v1.16.1
curl -L --fail --remote-name-all https://github.com/cilium/hubble/releases/download/$HUBBLE_VERSION/hubble-linux-amd64.tar.gz{,.sha256sum}
sha256sum --check hubble-linux-amd64.tar.gz.sha256sum
sudo tar xzvfC hubble-linux-amd64.tar.gz /usr/local/bin
rm hubble-linux-amd64.tar.gz{,.sha256sum}
  • Create Cluster [control plane]
sudo kubeadm config images pull
sudo kubeadm init --control-plane-endpoint=onprem.cloud --v=5 --pod-network-cidr=10.244.0.0/16
# wait for node to get 'ready'
watch --color 'sudo kubectl get nodes'

sudo cilium install
sudo cilium status
  • Join another control plane [control plane]
kubeadm token create --print-join-command
sudo kubeadm join --control-plane

sudo cilium install
sudo cilium status
  • Join Nodes [worker nodes]
kubeadm token create --print-join-command
  • reset K8s Node
kubeadm reset
curl https://baltocdn.com/helm/signing.asc | gpg --dearmor | sudo tee /usr/share/keyrings/helm.gpg > /dev/null

echo "deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/helm.gpg] https://baltocdn.com/helm/stable/debian/ all main" | sudo tee /etc/apt/sources.list.d/helm-stable-debian.list


sudo apt update
sudo apt install helm
helm repo add longhorn https://charts.longhorn.io
helm repo update
helm install longhorn longhorn/longhorn --namespace longhorn-system --create-namespace --version 1.7.1

kubectl -n longhorn-system get pod
  • test Cluster
kubectl create namespace test
kubectl config set-context --current --namespace=test
kubectl create deployment whoami --image=traefik/whoami:latest
#kubectl expose deployment whoami --name whoami-service --port 80 --target-port=30080 --type NodePort
kubectl create service nodeport whoami --node-port=30080 --tcp=80:80

https://kubernetes.io/de/docs/reference/kubectl/cheatsheet/

https://kubernetes.io/docs/tasks/access-application-cluster/web-ui-dashboard

helm repo add kubernetes-dashboard https://kubernetes.github.io/dashboard/
helm upgrade --install kubernetes-dashboard kubernetes-dashboard/kubernetes-dashboard --create-namespace --namespace kubernetes-dashboard
kubectl -n kubernetes-dashboard port-forward svc/kubernetes-dashboard-kong-proxy 8443:443
kubectl create serviceaccount admin-user -n kubernetes-dashboard
kubectl create clusterrolebinding admin-user --clusterrole=cluster-admin --serviceaccount=kubernetes-dashboard:admin-user
kubectl -n kubernetes-dashboard create token admin-user

https://github.com/kubernetes/dashboard/blob/master/docs/user/access-control/creating-sample-user.md

use physical Port in KVM

https://wiki.debian.org/BridgeNetworkConnections

apt install bridge-utils

ip link show
brctl addbr br42
brctl addif br42 enp5s2
ip link set enp5s2 up
ip link set enp5s2 master br42
ip link set br42 up
ip link show

cat /etc/network/interfaces

iface eno2 inet static
        address 192.168.168.123
        broadcast 192.168.168.255
        netmask 255.255.255.0
        gateway 192.168.168.254

iface enp0s25 inet manual

auto br42
iface br42 inet dhcp
        bridge_ports enp0s25
        bridge_stp off       # disable Spanning Tree Protocol
        bridge_waitport 0    # no delay before a port becomes available
        bridge_fd 0          # no forwarding delay

cat bridged-network.xml

<network>
<name>bridged-network</name>
<forward mode="bridge" />
<bridge name="br42" />
</network>

virsh net-define bridged-network.xml
virsh net-start bridged-network
virsh net-autostart bridged-network

systemctl restart networking.service


virsh net-list –all
virsh net-autostart default

install Kubernetes on Debian 11 Bullseye

  • prepare Hosts [all nodes]
    • disable swap
      swapoff -a
  • add cluster DNS to /etc/hosts
    • e.g. onprem.cloud
      echo “127.0.0.1 onprem.cloud” >> /etc/hosts
apt install -y curl gpg
# edit /etc/sysctl.conf
# net.ipv4.ip_forward=1
# net.bridge.bridge-nf-call-iptables=1
sysctl -w net.ipv4.ip_forward=1
sysctl -w net.bridge.bridge-nf-call-iptables=1
# edit /etc/modules
# br_netfilter
modprobe br_netfilter
export KUBECONFIG=/etc/kubernetes/admin.conf
echo "export KUBECONFIG=/etc/kubernetes/admin.conf" >> /root/.bashrc
  • install Container Runtime CRI-O (https://cri-o.io/) [all nodes]
export OS=Debian_11
export VERSION=1.25
echo "deb https://download.opensuse.org/repositories/devel:/kubic:/libcontainers:/stable/$OS/ /" > /etc/apt/sources.list.d/devel:kubic:libcontainers:stable.list
echo "deb http://download.opensuse.org/repositories/devel:/kubic:/libcontainers:/stable:/cri-o:/$VERSION/$OS/ /" > /etc/apt/sources.list.d/devel:kubic:libcontainers:stable:cri-o:$VERSION.list

curl -L https://download.opensuse.org/repositories/devel:/kubic:/libcontainers:/stable:/cri-o:/$VERSION/$OS/Release.key | apt-key add -
curl -L https://download.opensuse.org/repositories/devel:/kubic:/libcontainers:/stable/$OS/Release.key | apt-key add -

apt update
apt install -y cri-o cri-o-runc
systemctl enable crio.service
systemctl start crio.service
  • install K8s (https://kubernetes.io/docs/setup/production-environment/tools/kubeadm/install-kubeadm/) [all nodes]
apt install -y apt-transport-https ca-certificates curl
mkdir /etc/apt/keyrings
curl -fsSL https://packages.cloud.google.com/apt/doc/apt-key.gpg | gpg --dearmor | dd status=none of=/etc/apt/keyrings/kubernetes-archive-keyring.gpg
echo "deb [signed-by=/etc/apt/keyrings/kubernetes-archive-keyring.gpg] https://apt.kubernetes.io/ kubernetes-xenial main" | tee /etc/apt/sources.list.d/kubernetes.list
apt update
apt install -y kubelet kubeadm kubectl
apt-mark hold kubelet kubeadm kubectl
  • install CNI Cillium (https://docs.cilium.io/en/stable/gettingstarted/k8s-install-default/) [all nodes]
CILIUM_CLI_VERSION=$(curl -s https://raw.githubusercontent.com/cilium/cilium-cli/master/stable.txt)
CLI_ARCH=amd64
if [ "$(uname -m)" = "aarch64" ]; then CLI_ARCH=arm64; fi
curl -L --fail --remote-name-all https://github.com/cilium/cilium-cli/releases/download/${CILIUM_CLI_VERSION}/cilium-linux-${CLI_ARCH}.tar.gz{,.sha256sum}
sha256sum --check cilium-linux-${CLI_ARCH}.tar.gz.sha256sum
tar xzvfC cilium-linux-${CLI_ARCH}.tar.gz /usr/local/bin
rm cilium-linux-${CLI_ARCH}.tar.gz{,.sha256sum}

Create Cluster [control plane]

kubeadm init --control-plane-endpoint=onprem.cloud --v=5 --pod-network-cidr=10.244.0.0/16
# wait for node to get 'ready'
watch --color 'kubectl get nodes'

cilium install
cilium status
  • Join Nodes [worker nodes]
kubeadm token create --print-join-command
  • reset K8s Node
kubeadm reset
  • expose K8s Port
iptables -t nat -A PREROUTING -d 10.10.10.172/24 -i eno1 -p tcp --dport 6443 -j DNAT --to-destination 192.168.122.234:6443
iptables -I FORWARD -p tcp -d 192.168.122.234/24 --dport 6443 -m state --state NEW,RELATED,ESTABLISHED -j ACCEPT
  • test Cluster
kubectl create namespace test
kubectl config set-context --current --namespace=test
kubectl create deployment whoami --image=traefik/whoami:latest
#kubectl expose deployment whoami --name whoami-service --port 80 --target-port=30080 --type NodePort
kubectl create service nodeport whoami --node-port=30080 --tcp=80:80

https://kubernetes.io/de/docs/reference/kubectl/cheatsheet/