Home K8S setup

step2/001-prepare-ansible.sh

@@ -0,0 +1,7 @@
+#!/bin/bash
+
+AVAILABLE_HOSTS=$(grep "\." pi-hosts.txt | awk '{print $2}' | cut -d "=" -f2)
+for HOST in $AVAILABLE_HOSTS; do
+  echo "Working on host $HOST"
+  ssh -t root@$HOST 'setup-ntp -c chrony; setup-apkrepos -f; apk update; apk add python3'
+done

step2/Makefile

@@ -0,0 +1,4 @@
+
+all:
+	./001-prepare-ansible.sh
+	ansible-playbook rpi.yaml -f 10

step2/ansible.cfg

@@ -0,0 +1,8 @@
+[defaults]
+host_key_checking = False
+remote_user = root
+inventory = ./pi-hosts.txt
+interpreter_python = auto_silent
+
+[ssh_connection]
+pipelining = True

step2/pi-hosts.txt

@@ -0,0 +1,9 @@
+[masters]
+pi0 ansible_host=192.168.50.132 # Pi0
+
+[workers]
+pi1 ansible_host=192.168.50.135 # Pi1
+pi3 ansible_host=192.168.50.60  # Pi3
+pi4 ansible_host=192.168.50.36  # Pi4
+pi2 ansible_host=192.168.50.85  # Pi2
+pi5 ansible_host=192.168.50.230 # Pi5

step2/roles/basics/tasks/k8s-general-setup.yml

@@ -0,0 +1,16 @@
+---
+- name: Taint nodes
+  ansible.builtin.shell:
+    cmd: |
+      kubectl taint nodes --all node-role.kubernetes.io/master:NoSchedule- || true
+
+- name: Install py3-pip
+  ansible.builtin.package:
+    name:
+      - py3-pip
+      - helm
+    state: present
+
+- name: Install yaml package
+  pip:
+    name: PyYAML

step2/roles/basics/tasks/k8s-github.yml

@@ -0,0 +1,5 @@
+---
+# - name: Github authentication
+#   ansible.builtin.shell:
+#     cmd: |
+#       kubectl create secret docker-registry docker-ghcr --docker-server=https://ghcr.io --docker-username=lukaszraczylo --docker-password=$GITHUB_TOKEN [email protected]

step2/roles/basics/tasks/main.yml

@@ -0,0 +1,3 @@
+---
+# - include: k8s-general-setup.yml
+- include: k8s-github.yml

step2/roles/common/tasks/filesystem.yml

@@ -0,0 +1,94 @@
+---
+- name: Install packages - filesystem
+  ansible.builtin.package:
+    name:
+      - e2fsprogs
+      - parted
+    state: present
+
+- name: Create partition for perm data
+  community.general.parted:
+    device: /dev/mmcblk0
+    number: 2
+    fs_type: ext4
+    part_start: 1GiB
+    part_end: 100%
+    flags: [ boot ]
+    state: present
+
+- name: Fomat perm data partition
+  community.general.filesystem:
+    dev: /dev/mmcblk0p2
+    fstype: ext4
+  ignore_errors: yes
+
+- name: Re-mount old data partition
+  ansible.posix.mount:
+    path: /media/mmcblk0p1
+    src: /dev/mmcblk0p1
+    opts: rw
+    state: remounted
+
+- name: Mount perm data partition
+  ansible.posix.mount:
+    path: /mnt
+    src: /dev/mmcblk0p2
+    opts: rw
+    fstype: ext4
+    state: mounted
+
+- name: Overwrite setup-disk
+  ansible.builtin.copy:
+    src: static/setup-disk
+    dest: /sbin/setup-disk
+    owner: root
+    group: root
+    mode: '0744'
+    backup: yes
+
+- name: Setup new disk as system
+  ansible.builtin.command:
+    cmd: setup-disk -m sys /mnt
+    creates: /mnt/bin/date
+
+- name: Housekeeping
+  ansible.builtin.shell:
+    cmd: |
+      rm -f /media/mmcblk0p1/boot/*
+      cd /mnt
+      rm boot/boot
+      mv boot/* /media/mmcblk0p1/boot/
+      rm -Rf boot
+      mkdir media/mmcblk0p1
+      ln -s media/mmcblk0p1/boot boot
+    removes: /mnt/boot/boot
+
+- name: Add mountpoints to fstab
+  ansible.builtin.shell:
+    cmd: |
+      echo "/dev/mmcblk0p1 /media/mmcblk0p1 vfat defaults 0 0" >> etc/fstab
+      echo "/dev/mmcblk0p1 /media/mmcblk0p1 vfat defaults 0 0" >> /etc/fstab
+      sed -i '/cdrom/d' etc/fstab
+      sed -i '/floppy/d' etc/fstab
+      sed -i '/usbdisk/d' etc/fstab
+      touch /mnt/.nvm_mountpoints
+    chdir: /mnt
+    creates: /mnt/.nvm_mountpoints
+
+- name: Enable repositories and cgroups
+  ansible.builtin.shell:
+    cmd: |
+      sed -i '/edge/s/^#//' /mnt/etc/apk/repositories
+      sed -i 's/^/root=\/dev\/mmcblk0p2 /' /media/mmcblk0p1/cmdline.txt
+      echo "cgroup_enable=cpuset cgroup_enable=memory cgroup_memory=1" >> /media/mmcblk0p1/cmdline.txt
+      sed -i ':a;N;$!ba;s/\n/ /g' /media/mmcblk0p1/cmdline.txt
+      touch /mnt/.nvm_cgroups
+    creates: /mnt/.nvm_cgroups
+
+- name: Commit changes
+  ansible.builtin.shell:
+    chdir: /media/mmcblk0p1
+    cmd: lbu_commit -d || true
+
+- name: Reboot
+  reboot:

step2/roles/common/tasks/kubernetes.yml

@@ -0,0 +1,11 @@
+---
+- name: Pull basic images
+  ansible.builtin.shell:
+    cmd: kubeadm config images pull
+
+- name: Install helm
+  ansible.builtin.shell:
+    cmd: |
+      curl https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3 | bash
+    creates: /etc/kubernetes/kubelet.conf
+  register: k8sinit_output

step2/roles/common/tasks/main.yml

@@ -0,0 +1,9 @@
+---
+- name: Check for base system config lockfile
+  stat: path="/mnt/.nvm_mountpoints"
+  register: basic_config_lock
+- include: filesystem.yml
+  when: basic_config_lock.stat.exists == False
+- include: networking.yml
+- include: kubernetes.yml
+- include: nfs.yml

step2/roles/common/tasks/networking.yml

@@ -0,0 +1,68 @@
+---
+- name: Set hostname
+  ansible.builtin.hostname:
+    name: '{{ inventory_hostname }}'
+
+- name: "Build hosts file"
+  lineinfile:
+    dest: /etc/hosts
+    regexp: '.*{{ item }}$'
+    line: "{{hostvars[item]['ansible_default_ipv4']['address']}} {{item}} {{item}}.local"
+    state: present
+  register: etchostsupdate
+  when: hostvars[item]['ansible_facts']['default_ipv4'] is defined
+  with_items:
+    - "{{ groups['all'] }}"
+
+- name: Install packages for networking
+  ansible.builtin.package:
+    name:
+      - dbus
+      - avahi
+      - kubernetes
+      - kubectl
+      - docker
+      - cni-plugins
+      - kubelet
+      - kubeadm
+      - sudo
+      - curl
+    state: present
+
+- name: Enable service dbus
+  ansible.builtin.service:
+    name: dbus
+    state: started
+    enabled: yes
+
+- name: Overwrite avahi config
+  ansible.builtin.copy:
+    src: static/avahi-config
+    dest: /etc/avahi/avahi-daemon.conf
+    owner: root
+    group: root
+    mode: '0644'
+    backup: yes
+
+- name: Enable service avahi
+  ansible.builtin.service:
+    name: avahi-daemon
+    state: restarted
+    enabled: yes
+
+- name: Enable service docker
+  ansible.builtin.service:
+    name: docker
+    state: started
+    enabled: yes
+
+- name: Enable service kubelet
+  ansible.builtin.service:
+    name: kubelet
+    state: started
+    enabled: yes
+
+- name: Enable promisc mode for ARP requests
+  ansible.builtin.shell:
+    cmd: |
+      ip link set eth0 promisc on

step2/roles/common/tasks/nfs.yml

@@ -0,0 +1,34 @@
+---
+- name: Install nfsprogs
+  ansible.builtin.package:
+    name:
+      - nfs-utils
+
+- name: Prepare nfs mountpoint
+  ansible.builtin.shell:
+    cmd: |
+      mkdir -p /media/nfs
+
+- name: Mount NFS resources
+  ansible.posix.mount:
+    src: "{{settings.nfs_host}}:{{settings.nfs_multi_share_path}}"
+    path: /media/nfs
+    fstype: nfs
+    opts: nofail,noatime,nolock,tcp,actimeo=1800
+    state: mounted
+
+- name: Mount NFS torrent
+  ansible.posix.mount:
+    src: "{{settings.nfs_host}}:{{settings.nfs_torrent_path}}"
+    path: /mnt/drobo-storage/docker-volumes/torrent
+    fstype: nfs
+    opts: nofail,noatime,nolock,tcp,actimeo=1800
+    state: mounted
+
+- name: Mount NFS adguard
+  ansible.posix.mount:
+    src: "{{settings.nfs_host}}:{{settings.nfs_adguard_path}}"
+    path: /mnt/drobo-storage/docker-volumes/adguard
+    fstype: nfs
+    opts: nofail,noatime,nolock,tcp,actimeo=1800
+    state: mounted

step2/roles/kubemaster/tasks/main.yml

@@ -0,0 +1,2 @@
+---
+- include: master-setup.yml

step2/roles/kubemaster/tasks/master-setup.yml

@@ -0,0 +1,35 @@
+---
+- name: Initialize cluster
+  ansible.builtin.shell:
+    cmd: |
+      kubeadm reset -f
+      kubeadm init --pod-network-cidr=10.244.0.0/16
+    creates: /etc/kubernetes/kubelet.conf
+  register: k8sinit_output
+
+- name: Storing Logs and Generated token for nodes config
+  ansible.builtin.copy:
+    content: '{{ k8sinit_output.stdout }}'
+    dest: cluster_join_file
+  when: '"token" in k8sinit_output.stdout'
+
+- name: Pulling the token file for setup purposes
+  ansible.builtin.fetch:
+    flat: yes
+    src: cluster_join_file
+    dest: static/token_file
+  when: '"token" in k8sinit_output.stdout'
+
+- name: Copying kubeconfig file
+  ansible.builtin.shell:
+    cmd: |
+      mkdir -p $HOME/.kube
+      cp -f /etc/kubernetes/admin.conf $HOME/.kube/config
+      chown $(id -u):$(id -g) $HOME/.kube/config
+
+- name: Save kubeconfig locally
+  ansible.builtin.fetch:
+    flat: yes
+    src: /etc/kubernetes/admin.conf
+    dest: static/kubectl.conf
+    fail_on_missing: yes

step2/roles/kubeworkers/tasks/main.yml

@@ -0,0 +1,2 @@
+---
+- include: worker-setup.yml

step2/roles/kubeworkers/tasks/worker-setup.yml

@@ -0,0 +1,31 @@
+---
+- name: Copy token file to workers
+  ansible.builtin.copy:
+    src: static/token_file
+    dest: /root/cluster_token_file
+    owner: root
+    group: root
+    mode: '0644'
+
+- name: Joining worker nodes with kubernetes master
+  ansible.builtin.shell:
+    cmd: |
+      kubeadm reset -f
+      cat /root/cluster_token_file | tail -2 > out.sh
+      sh out.sh
+    creates: out.sh
+
+- name: Copy kubectl config to workers
+  ansible.builtin.copy:
+    src: static/kubectl.conf
+    dest: /root/kubectl-config
+    owner: root
+    group: root
+    mode: '0644'
+
+- name: Copying kubeconfig file
+  ansible.builtin.shell:
+    cmd: |
+      mkdir -p $HOME/.kube
+      cp -f /root/kubectl-config $HOME/.kube/config
+      chown $(id -u):$(id -g) $HOME/.kube/config

step2/rpi.yaml

@@ -0,0 +1,22 @@
+---
+- hosts: all
+  vars:
+    settings:
+      nfs_host: 192.168.50.5
+      nfs_torrent_path: "/mnt/drobo-storage/docker-volumes/torrent"
+      nfs_adguard_path: "/mnt/drobo-storage/docker-volumes/adguard"
+      nfs_multi_share_path: "/mnt/drobo-storage/docker-volumes/nfs-share"
+  roles:
+    - common
+
+- hosts: masters
+  roles:
+    - kubemaster
+
+- hosts: workers
+  roles:
+    - kubeworkers
+
+- hosts: masters
+  roles:
+    - basics

step2/static/avahi-config

@@ -0,0 +1,49 @@
+[server]
+#host-name=foo
+domain-name=local
+#browse-domains=0pointer.de, zeroconf.org
+use-ipv4=yes
+use-ipv6=yes
+#allow-interfaces=eth0
+#deny-interfaces=eth1
+#check-response-ttl=no
+#use-iff-running=no
+#enable-dbus=yes
+#disallow-other-stacks=no
+#allow-point-to-point=no
+#cache-entries-max=4096
+#clients-max=4096
+#objects-per-client-max=1024
+#entries-per-entry-group-max=32
+ratelimit-interval-usec=1000000
+ratelimit-burst=1000
+
+[wide-area]
+enable-wide-area=yes
+
+[publish]
+#disable-publishing=no
+#disable-user-service-publishing=no
+#add-service-cookie=no
+#publish-addresses=yes
+publish-hinfo=no
+publish-workstation=no
+#publish-domain=yes
+#publish-dns-servers=192.168.50.1, 192.168.50.2
+#publish-resolv-conf-dns-servers=yes
+#publish-aaaa-on-ipv4=yes
+#publish-a-on-ipv6=no
+
+[reflector]
+#enable-reflector=no
+#reflect-ipv=no
+#reflect-filters=_airplay._tcp.local,_raop._tcp.local
+
+[rlimits]
+#rlimit-as=
+#rlimit-core=0
+#rlimit-data=8388608
+#rlimit-fsize=0
+#rlimit-nofile=768
+#rlimit-stack=8388608
+#rlimit-nproc=3