refresh previous cluster
Table of Content
refresh previous cluster¶
scripts used to re-install containerd, cni, runc, and kube¶
Confirm the latest version at the time.
# reset k8s node
sudo kubeadm reset
sudo iptables -F && iptables -t nat -F && iptables -t mangle -F && iptables -X
sudo ipvsadm -C
sudo rm -rf /etc/cni
rm -rf ~/.kube # applicable to control plane nodes
# working directory
mkdir ~/dnld
cd ~/dnld
# backup runc
mv runc.amd64 runc.amd64.bak
# see the list of installed packages
sudo apt list '?narrow(?installed, ?not(?origin(Debian)))'
sudo apt-mark showhold
sudo apt-mark unhold kubeadm kubectl kubelet
sudo rm -rf /opt/cni/bin
sudo apt remove cri-tools kubernetes-cni kubeadm kubectl kubelet
# and also remove anything you saw in the previous step
sudo apt autoremove
curl -s https://api.github.com/repos/containernetworking/plugins/releases/latest \
| grep browser_download | grep linux-amd64 | head -1 \
| cut -d : -f 2,3 | tr -d \" \
| wget -qi -
sudo mkdir -p /opt/cni/bin
sudo tar Cxzvf /opt/cni/bin cni-plugins-linux-amd64-v1.4.1.tgz
curl -s https://api.github.com/repos/containerd/containerd/releases/latest \
| grep browser_download | grep linux-amd64 | head -1 \
| cut -d : -f 2,3 | tr -d \" \
| wget -qi -
sudo tar Cxzvf /usr/local containerd-1.7.15-linux-amd64.tar.gz
curl -s https://api.github.com/repos/opencontainers/runc/releases/latest \
| grep browser_download | grep runc.amd64 | head -1 \
| cut -d : -f 2,3 | tr -d \" \
| wget -qi -
sudo install -m 755 runc.amd64 /usr/local/sbin/runc
sudo rm /etc/apt/sources.list.d/docker.list /etc/apt/sources.list.d/kubernetes.list
curl -fsSL https://pkgs.k8s.io/core:/stable:/v1.30/deb/Release.key | sudo gpg --dearmor -o /etc/apt/keyrings/kubernetes-apt-keyring.gpg
echo 'deb [signed-by=/etc/apt/keyrings/kubernetes-apt-keyring.gpg] https://pkgs.k8s.io/core:/stable:/v1.30/deb/ /' | sudo tee /etc/apt/sources.list.d/kubernetes.list
sudo apt-get update
sudo apt-get install -y kubelet kubeadm kubectl
sudo apt-mark hold kubelet kubeadm kubectl
sudo systemctl enable --now kubelet
sudo systemctl enable --now containerd
systemctl status kubelet
systemctl status containerd
sudo reboot
build cluster and join second control plane node¶
https://kubernetes.io/docs/setup/production-environment/tools/kubeadm/create-cluster-kubeadm/
--control-plane-endpoint--control-plane-endpoint=kube-endpoint.blink-1x52.net
--pod-network-cidr--pod-network-cidr=10.244.0.0/16
--dry-run--service-dns-domain--service-dns-domain=hlv2.blink-1x52.net
# on the first control plane node
# dry run
sudo kubeadm init --dry-run \
--control-plane-endpoint=kube-endpoint.blink-1x52.net \
--pod-network-cidr=10.244.0.0/16 \
--service-dns-domain=hlv2.blink-1x52.net
# pull images
sudo kubeadm config images pull
# initilize the cluster
sudo kubeadm init \
--control-plane-endpoint=kube-endpoint.blink-1x52.net \
--pod-network-cidr=10.244.0.0/16 \
--service-dns-domain=hlv2.blink-1x52.net
# routine
mkdir -p $HOME/.kube
sudo cp -i /etc/kubernetes/admin.conf $HOME/.kube/config
sudo chown $(id -u):$(id -g) $HOME/.kube/config
# copy pki files for the other control plane nodes to fetch
sudo cp -r /etc/kubernetes/pki ~/pki
sudo chown -R $USER:$USER ~/pki
# now on the second control plane node
rm -rf pki
scp -r rpi4bp:/home/osho/pki .
# get the pki files ready under /etc/kubernetes
sudo mkdir -p /etc/kubernetes/pki/etcd
sudo cp pki/ca.crt /etc/kubernetes/pki/ca.crt
sudo cp pki/sa.key /etc/kubernetes/pki/sa.key
sudo cp pki/sa.pub /etc/kubernetes/pki/sa.pub
sudo cp pki/front-proxy-ca.crt /etc/kubernetes/pki/front-proxy-ca.crt
sudo cp pki/etcd/ca.crt /etc/kubernetes/pki/etcd/ca.crt
sudo cp pki/ca.key /etc/kubernetes/pki/ca.key
sudo cp pki/front-proxy-ca.key /etc/kubernetes/pki/front-proxy-ca.key
sudo cp pki/etcd/ca.key /etc/kubernetes/pki/etcd/ca.key
# and then run the kubeadm join command
# watch for /etc/hosts
# you may need to edit entry for kube-endpoint.blink-1x52.net
# and point to the primary control plane node when joining,
# and revert the settings once init/join is done on the secondary control plane nodes
# and then move on with the rest of the worker nodes and run kubeadm join for workers
# and finally get the copy of ~/.kube/config on other admin machine to run kubectl
latest kubectl on remote management machine¶
other tools¶
flux¶
https://fluxcd.io/flux/installation/
sops¶
https://github.com/getsops/sops/releases
# Download the binary
curl -LO https://github.com/getsops/sops/releases/download/v3.8.1/sops-v3.8.1.linux.amd64
# Move the binary in to your PATH
mv sops-v3.8.1.linux.amd64 /usr/local/bin/sops
# Make the binary executable
chmod +x /usr/local/bin/sops
# confirm
sops --version
helm¶
install calico¶
# confirm the latest version on cli
curl -sfL "api.github.com/repos/projectcalico/calico/releases/latest" | grep "tag_name"
# download v3.27.3
curl -LO https://raw.githubusercontent.com/projectcalico/calico/v3.27.3/manifests/tigera-operator.yaml
curl -LO https://raw.githubusercontent.com/projectcalico/calico/v3.27.3/manifests/custom-resources.yaml
# modify cidr in the custom-resources.yaml
# >> 10.244.0.0/16 to match what's specified in kubeadm init
# and then create
kubectl create -f tigera-operator.yaml
kubectl create -f custom-resources.yaml
# watch and wait for everything to be in RUNNING state
watch kubectl get pods -n calico-system -o wide
# allow control plane to work
kubectl taint nodes --all node-role.kubernetes.io/control-plane-
# confirm
kubectl get nodes -o wide
setup gitops¶
gitops repository access token¶
- create or go to the gitops repository on gitlab
- create a project access token
- name: hlv2
- expiration date: default (1 month?)
- role: owner
- scope: api
flux bootstrap¶
export GITLAB_TOKEN={access_token_string_here}
export GITLAB_SERVER=cp.blink-1x52.net
flux bootstrap gitlab \
--deploy-token-auth \
--hostname="$GITLAB_SERVER" \
--owner=gitops \
--repository=homelab \
--path=./clusters/hlv2 \
--cluster-domain=hlv2.blink-1x52.net \
--branch=main
setup sops¶
preparing gpg key
$ export KEY_NAME="hlv2.blink-1x52.net"
$ export KEY_COMMENT="flux sops secrets"
$ gpg --batch --full-generate-key <<EOF
%no-protection
Key-Type: 1
Key-Length: 4096
Subkey-Type: 1
Subkey-Length: 4096
Expire-Date: 0
Name-Comment: ${KEY_COMMENT}
Name-Real: ${KEY_NAME}
EOF
gpg: key D29EC441F18C4B0A marked as ultimately trusted
gpg: revocation certificate stored as '/home/user/.gnupg/openpgp-revocs.d/23C3E8985B27FF45689B2D8ED29EC441F18C4B0A.rev'
$ gpg --list-secret-keys "${KEY_NAME}"
gpg: checking the trustdb
gpg: marginals needed: 3 completes needed: 1 trust model: pgp
gpg: depth: 0 valid: 6 signed: 0 trust: 0-, 0q, 0n, 0m, 0f, 6u
sec rsa4096 2024-04-24 [SCEA]
23C3E8985B27FF45689B2D8ED29EC441F18C4B0A
uid [ultimate] hlv2.blink-1x52.net (flux sops secrets)
ssb rsa4096 2024-04-24 [SEA]
$ export KEY_FP=23C3E8985B27FF45689B2D8ED29EC441F18C4B0A
$ gpg --export-secret-keys --armor "${KEY_FP}" |
kubectl create secret generic sops-gpg \
--namespace=flux-system \
--from-file=sops.asc=/dev/stdin
secret/sops-gpg created
Now, go to the gitops repository and configure sops.
# gitops repository
cd {gitops repo}
mkdir -p sops/hlv2
cd sops/hlv2
# add sops configuration
cat <<EOF > .sops.yaml
creation_rules:
- path_regex: .*.yaml
encrypted_regex: ^(data|stringData)$
pgp: ${KEY_FP}
EOF
# place public key so that others can import and use to encrypt
gpg --export --armor "${KEY_FP}" > .sops.pub.asc
# gpg --import .sops.pub.asc
# git commit and push
And finally add flux kustomization at ./clusters/hlv2/sops.yaml for this sops directory, ./sops/hlv2.
./clusters/hlv2/sops.yaml
---
apiVersion: kustomize.toolkit.fluxcd.io/v1
kind: Kustomization
metadata:
name: hlv2-sops
namespace: flux-system
spec:
decryption:
provider: sops
secretRef:
name: sops-gpg
interval: 1m0s
path: ./sops/hlv2
prune: true
sourceRef:
kind: GitRepository
name: flux-system
testing sops¶
# still on the gitops repo, ./sops/hlv2 directory
kubectl -n default create secret generic basic-auth \
--from-literal=user=admin \
--from-literal=password=change-me \
--dry-run=client \
-o yaml > basic-auth.yaml
# confirm that the data are base64 encoded
cat basic-auth.yaml
# encrypt
sops -i --encrypt basic-auth.yaml
# confirm that the data are encrypted
cat basic-auth.yaml
# git commit and push
# confirm
kubectl -n default get secret basic-auth -o yaml
setup rook¶
https://rook.io/docs/rook/latest-release/Getting-Started/intro/
requirements¶
RDB¶
https://rook.io/docs/rook/latest-release/Getting-Started/Prerequisites/prerequisites/#rbd
RDB on Raspberry Pi¶
rpi4bp does not have it.
https://gist.github.com/devantler/7dfcf13f12cb68b65eaa0e6f93dc2b19
https://www.raspberrypi.com/documentation/computers/linux_kernel.html
# build requirements
sudo apt install git bc bison flex libssl-dev make
# clone source
mkdir ~/clones
cd ~/clones
git clone --depth=1 https://github.com/raspberrypi/linux
# apply default configuration
cd linux
KERNEL=kernel8
make bcm2711_defconfig
# open .config and set followings
# CONFIG_BLK_DEV_DRBD=m
# CONFIG_BLK_DEV_RBD=m
# CONFIG_BLK_DEV_NDB=m
# make
make -j4 Image.gz modules dtbs
sudo make modules_install
sudo cp arch/arm64/boot/dts/broadcom/*.dtb /boot/
sudo cp arch/arm64/boot/dts/overlays/*.dtb* /boot/overlays/
sudo cp arch/arm64/boot/dts/overlays/README /boot/overlays/
sudo cp arch/arm64/boot/Image /boot/$KERNEL.img
# wait for 4 hours
# reboot
# check
sudo modprobe rbd
lsmod | grep rbd
CEPH¶
https://rook.io/docs/rook/latest-release/Getting-Started/Prerequisites/prerequisites/#cephfs
CephFS should be on a system with kernel version >= 4.17.