commit f311e2ac00b16fda76ec3f5d7ff8a7b8acf52123 Author: Michael Skrynski Date: Wed Oct 22 08:20:53 2025 +0200 initial commit diff --git a/README.md b/README.md new file mode 100644 index 0000000..1bd128d --- /dev/null +++ b/README.md @@ -0,0 +1,343 @@ +# K3s Ansible Deployment for Raspberry Pi CM4/CM5 + +Ansible playbook to deploy a k3s Kubernetes cluster on Raspberry Pi Compute Module 4 and 5 devices. + +## Prerequisites + +- Raspberry Pi CM4/CM5 modules running Raspberry Pi OS (64-bit recommended) +- SSH access to all nodes +- Ansible installed on your control machine +- SSH key-based authentication configured + +## Project Structure + +``` +k3s-ansible/ +├── ansible.cfg # Ansible configuration +├── site.yml # Main playbook +├── inventory/ +│ └── hosts.ini # Inventory file +├── manifests/ +│ └── nginx-test-deployment.yaml # Test nginx deployment +└── roles/ + ├── prereq/ # Prerequisites role + │ └── tasks/ + │ └── main.yml + ├── k3s-server/ # K3s master/server role + │ └── tasks/ + │ └── main.yml + ├── k3s-agent/ # K3s worker/agent role + │ └── tasks/ + │ └── main.yml + └── k3s-deploy-test/ # Test deployment role + └── tasks/ + └── main.yml +``` + +## Configuration + +### 1. Update Inventory + +Edit `inventory/hosts.ini` and add your Raspberry Pi nodes: + +```ini +[master] +pi-master ansible_host=192.168.1.100 ansible_user=pi + +[worker] +pi-worker-1 ansible_host=192.168.1.101 ansible_user=pi +pi-worker-2 ansible_host=192.168.1.102 ansible_user=pi +pi-worker-3 ansible_host=192.168.1.103 ansible_user=pi +``` + +### 2. Configure Variables + +In `inventory/hosts.ini`, you can customize: + +- `k3s_version`: K3s version to install (default: v1.28.3+k3s1) +- `extra_server_args`: Additional arguments for k3s server +- `extra_agent_args`: Additional arguments for k3s agent + +## Usage + +### Test Connectivity + +```bash +ansible all -m ping +``` + +### Deploy K3s Cluster + +```bash +ansible-playbook site.yml +``` + +This will deploy the full k3s cluster with the test nginx application. + +### Deploy Without Test Application + +To skip the test deployment: + +```bash +ansible-playbook site.yml --skip-tags test +``` + +### Deploy Only the Test Application + +If the cluster is already running and you just want to deploy the test app: + +```bash +ansible-playbook site.yml --tags deploy-test +``` + +### Deploy Only Prerequisites + +```bash +ansible-playbook site.yml --tags prereq +``` + +## What the Playbook Does + +### Prerequisites Role (`prereq`) +- Sets hostname on each node +- Updates and upgrades system packages +- Installs required packages (curl, wget, git, iptables, etc.) +- Enables cgroup memory and swap in boot config +- Configures legacy iptables (required for k3s on ARM) +- Disables swap +- Reboots if necessary + +### K3s Server Role (`k3s-server`) +- Installs k3s in server mode on master node(s) +- Configures k3s with Flannel VXLAN backend (optimized for ARM) +- Retrieves and stores the node token for workers +- Copies kubeconfig to master node user +- Fetches kubeconfig to local machine for kubectl access + +### K3s Agent Role (`k3s-agent`) +- Installs k3s in agent mode on worker nodes +- Joins workers to the cluster using the master's token +- Configures agents to connect to the master + +### K3s Deploy Test Role (`k3s-deploy-test`) +- Waits for all cluster nodes to be ready +- Deploys the nginx test application with 5 replicas +- Verifies deployment is successful +- Displays pod distribution across nodes + +## Post-Installation + +After successful deployment: + +1. The kubeconfig file will be saved to `./kubeconfig` +2. Use it with kubectl: + +```bash +export KUBECONFIG=$(pwd)/kubeconfig +kubectl get nodes +``` + +You should see all your nodes in Ready state: + +``` +NAME STATUS ROLES AGE VERSION +pi-master Ready control-plane,master 5m v1.28.3+k3s1 +pi-worker-1 Ready 3m v1.28.3+k3s1 +pi-worker-2 Ready 3m v1.28.3+k3s1 +``` + +## Accessing the Cluster + +### From Master Node + +SSH into the master node and use kubectl: + +```bash +ssh pi@pi-master +kubectl get nodes +``` + +### From Your Local Machine + +Use the fetched kubeconfig: + +```bash +export KUBECONFIG=/path/to/k3s-ansible/kubeconfig +kubectl get nodes +kubectl get pods --all-namespaces +``` + +## Testing the Cluster + +A sample nginx deployment with 5 replicas is provided to test your cluster. + +### Automated Deployment (via Ansible) + +The test application is automatically deployed when you run the full playbook: + +```bash +ansible-playbook site.yml +``` + +Or deploy it separately after the cluster is up: + +```bash +ansible-playbook site.yml --tags deploy-test +``` + +The Ansible role will: +- Wait for all nodes to be ready +- Deploy the nginx application +- Wait for all pods to be running +- Show you the deployment status and pod distribution + +### Manual Deployment (via kubectl) + +Alternatively, deploy manually using kubectl: + +```bash +export KUBECONFIG=$(pwd)/kubeconfig +kubectl apply -f manifests/nginx-test-deployment.yaml +``` + +### Verify the Deployment + +Check that all 5 replicas are running: + +```bash +kubectl get deployments +kubectl get pods -o wide +``` + +You should see output similar to: + +``` +NAME READY UP-TO-DATE AVAILABLE AGE +nginx-test 5/5 5 5 1m + +NAME READY STATUS RESTARTS AGE NODE +nginx-test-7d8f4c9b6d-2xk4p 1/1 Running 0 1m pi-worker-1 +nginx-test-7d8f4c9b6d-4mz9r 1/1 Running 0 1m pi-worker-2 +nginx-test-7d8f4c9b6d-7w3qs 1/1 Running 0 1m pi-worker-3 +nginx-test-7d8f4c9b6d-9k2ln 1/1 Running 0 1m pi-worker-1 +nginx-test-7d8f4c9b6d-xr5wp 1/1 Running 0 1m pi-worker-2 +``` + +### Access the Service + +K3s includes a built-in load balancer (Klipper). Get the external IP: + +```bash +kubectl get service nginx-test +``` + +If you see an external IP assigned, you can access nginx: + +```bash +curl http:// +``` + +Or from any node in the cluster: + +```bash +curl http://nginx-test.default.svc.cluster.local +``` + +### Scale the Deployment + +Test scaling: + +```bash +# Scale up to 10 replicas +kubectl scale deployment nginx-test --replicas=10 + +# Scale down to 3 replicas +kubectl scale deployment nginx-test --replicas=3 + +# Watch the pods being created/terminated +kubectl get pods -w +``` + +### Clean Up Test Deployment + +When you're done testing: + +```bash +kubectl delete -f manifests/nginx-test-deployment.yaml +``` + +## Troubleshooting + +### Check k3s service status + +On master: +```bash +sudo systemctl status k3s +sudo journalctl -u k3s -f +``` + +On workers: +```bash +sudo systemctl status k3s-agent +sudo journalctl -u k3s-agent -f +``` + +### Reset a node + +If you need to reset a node and start over: + +```bash +# On the node +/usr/local/bin/k3s-uninstall.sh # For server +/usr/local/bin/k3s-agent-uninstall.sh # For agent +``` + +### Common Issues + +1. **Nodes not joining**: Check firewall rules. K3s requires port 6443 open on the master. +2. **Memory issues**: Ensure cgroup memory is enabled (the playbook handles this). +3. **Network issues**: The playbook uses VXLAN backend which works better on ARM devices. + +## Customization + +### Add More Master Nodes (HA Setup) + +For a high-availability setup, you can add more master nodes: + +```ini +[master] +pi-master-1 ansible_host=192.168.1.100 ansible_user=pi +pi-master-2 ansible_host=192.168.1.101 ansible_user=pi +pi-master-3 ansible_host=192.168.1.102 ansible_user=pi +``` + +You'll need to configure an external database (etcd or PostgreSQL) for HA. + +### Custom K3s Arguments + +Modify `extra_server_args` or `extra_agent_args` in the inventory: + +```ini +[k3s_cluster:vars] +extra_server_args="--flannel-backend=vxlan --disable traefik --disable servicelb" +extra_agent_args="--node-label foo=bar" +``` + +## Uninstall + +To completely remove k3s from all nodes: + +```bash +# Create an uninstall playbook or run manually on each node +ansible all -m shell -a "/usr/local/bin/k3s-uninstall.sh" --become +ansible workers -m shell -a "/usr/local/bin/k3s-agent-uninstall.sh" --become +``` + +## License + +MIT + +## References + +- [K3s Documentation](https://docs.k3s.io/) +- [K3s on Raspberry Pi](https://docs.k3s.io/installation/requirements) diff --git a/ansible.cfg b/ansible.cfg new file mode 100644 index 0000000..4dbd9f6 --- /dev/null +++ b/ansible.cfg @@ -0,0 +1,17 @@ +[defaults] +inventory = inventory/hosts.ini +host_key_checking = False +retry_files_enabled = False +roles_path = roles +interpreter_python = auto_silent +deprecation_warnings = False + +[privilege_escalation] +become = True +become_method = sudo +become_user = root +become_ask_pass = False + +[ssh_connection] +ssh_args = -o ControlMaster=auto -o ControlPersist=60s -o StrictHostKeyChecking=no +pipelining = True diff --git a/inventory/hosts.ini b/inventory/hosts.ini new file mode 100644 index 0000000..485c3d1 --- /dev/null +++ b/inventory/hosts.ini @@ -0,0 +1,28 @@ +[master] +# Add your k3s master/server nodes here +# Example: pi-master ansible_host=192.168.1.100 ansible_user=pi + +[worker] +# Add your k3s worker/agent nodes here +# Example: pi-worker-1 ansible_host=192.168.1.101 ansible_user=pi +# Example: pi-worker-2 ansible_host=192.168.1.102 ansible_user=pi + +[k3s_cluster:children] +master +worker + +[k3s_cluster:vars] +# K3s version to install +k3s_version=v1.28.3+k3s1 + +# Network settings +ansible_user=pi +ansible_python_interpreter=/usr/bin/python3 + +# K3s configuration +k3s_server_location=/var/lib/rancher/k3s +systemd_dir=/etc/systemd/system + +# Flannel backend (vxlan is recommended for ARM) +extra_server_args="--flannel-backend=vxlan" +extra_agent_args="" diff --git a/manifests/nginx-test-deployment.yaml b/manifests/nginx-test-deployment.yaml new file mode 100644 index 0000000..b3981f3 --- /dev/null +++ b/manifests/nginx-test-deployment.yaml @@ -0,0 +1,60 @@ +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: nginx-test + namespace: default + labels: + app: nginx-test +spec: + replicas: 5 + selector: + matchLabels: + app: nginx-test + template: + metadata: + labels: + app: nginx-test + spec: + containers: + - name: nginx + image: nginx:alpine + ports: + - containerPort: 80 + name: http + resources: + requests: + memory: "64Mi" + cpu: "100m" + limits: + memory: "128Mi" + cpu: "200m" + livenessProbe: + httpGet: + path: / + port: 80 + initialDelaySeconds: 10 + periodSeconds: 10 + readinessProbe: + httpGet: + path: / + port: 80 + initialDelaySeconds: 5 + periodSeconds: 5 +--- +apiVersion: v1 +kind: Service +metadata: + name: nginx-test + namespace: default + labels: + app: nginx-test +spec: + type: LoadBalancer + ports: + - port: 80 + targetPort: 80 + protocol: TCP + name: http + selector: + app: nginx-test diff --git a/roles/k3s-agent/tasks/main.yml b/roles/k3s-agent/tasks/main.yml new file mode 100644 index 0000000..082af88 --- /dev/null +++ b/roles/k3s-agent/tasks/main.yml @@ -0,0 +1,36 @@ +--- +- name: Check if k3s agent is already installed + stat: + path: /usr/local/bin/k3s + register: k3s_binary + +- name: Get master node token + set_fact: + k3s_url: "https://{{ hostvars[groups['master'][0]]['ansible_host'] }}:6443" + k3s_token: "{{ hostvars['k3s_token_holder']['token'] }}" + +- name: Download k3s installation script + get_url: + url: https://get.k3s.io + dest: /tmp/k3s-install.sh + mode: '0755' + when: not k3s_binary.stat.exists + +- name: Install k3s agent + shell: | + INSTALL_K3S_VERSION="{{ k3s_version }}" \ + K3S_URL="{{ k3s_url }}" \ + K3S_TOKEN="{{ k3s_token }}" \ + INSTALL_K3S_EXEC="agent {{ extra_agent_args }}" \ + sh /tmp/k3s-install.sh + when: not k3s_binary.stat.exists + +- name: Wait for k3s agent to be ready + wait_for: + path: /var/lib/rancher/k3s/agent/kubelet.kubeconfig + state: present + timeout: 300 + +- name: Display success message + debug: + msg: "K3s agent installed successfully and joined to cluster" diff --git a/roles/k3s-deploy-test/tasks/main.yml b/roles/k3s-deploy-test/tasks/main.yml new file mode 100644 index 0000000..b6c990c --- /dev/null +++ b/roles/k3s-deploy-test/tasks/main.yml @@ -0,0 +1,125 @@ +--- +- name: Wait for k3s to be fully ready + wait_for: + port: 6443 + host: "{{ hostvars[groups['master'][0]]['ansible_host'] }}" + delay: 5 + timeout: 300 + delegate_to: localhost + run_once: true + +- name: Check if kubectl is available locally + command: which kubectl + register: kubectl_check + delegate_to: localhost + run_once: true + failed_when: false + changed_when: false + +- name: Install kubectl locally if not present + get_url: + url: "https://dl.k8s.io/release/{{ kubectl_version | default('v1.28.3') }}/bin/{{ ansible_system | lower }}/{{ 'arm64' if ansible_architecture == 'aarch64' else 'amd64' }}/kubectl" + dest: /tmp/kubectl + mode: '0755' + delegate_to: localhost + run_once: true + when: kubectl_check.rc != 0 + +- name: Set kubectl path + set_fact: + kubectl_bin: "{{ '/tmp/kubectl' if kubectl_check.rc != 0 else 'kubectl' }}" + delegate_to: localhost + run_once: true + +- name: Verify kubeconfig exists + stat: + path: "{{ playbook_dir }}/kubeconfig" + register: kubeconfig_file + delegate_to: localhost + run_once: true + +- name: Fail if kubeconfig not found + fail: + msg: "Kubeconfig not found at {{ playbook_dir }}/kubeconfig. Please run the k3s-server role first." + when: not kubeconfig_file.stat.exists + delegate_to: localhost + run_once: true + +- name: Wait for all nodes to be ready + shell: | + {{ kubectl_bin }} --kubeconfig={{ playbook_dir }}/kubeconfig get nodes --no-headers | grep -v Ready | wc -l + register: nodes_not_ready + until: nodes_not_ready.stdout | int == 0 + retries: 30 + delay: 10 + delegate_to: localhost + run_once: true + changed_when: false + +- name: Deploy nginx test application + shell: | + {{ kubectl_bin }} --kubeconfig={{ playbook_dir }}/kubeconfig apply -f {{ playbook_dir }}/manifests/nginx-test-deployment.yaml + register: deploy_result + delegate_to: localhost + run_once: true + changed_when: "'created' in deploy_result.stdout or 'configured' in deploy_result.stdout" + +- name: Wait for nginx deployment to be ready + shell: | + {{ kubectl_bin }} --kubeconfig={{ playbook_dir }}/kubeconfig wait --for=condition=available --timeout=300s deployment/nginx-test -n default + register: deployment_ready + delegate_to: localhost + run_once: true + changed_when: false + +- name: Get deployment status + shell: | + {{ kubectl_bin }} --kubeconfig={{ playbook_dir }}/kubeconfig get deployment nginx-test -n default -o wide + register: deployment_status + delegate_to: localhost + run_once: true + changed_when: false + +- name: Get pods status + shell: | + {{ kubectl_bin }} --kubeconfig={{ playbook_dir }}/kubeconfig get pods -l app=nginx-test -n default -o wide + register: pods_status + delegate_to: localhost + run_once: true + changed_when: false + +- name: Get service details + shell: | + {{ kubectl_bin }} --kubeconfig={{ playbook_dir }}/kubeconfig get service nginx-test -n default + register: service_status + delegate_to: localhost + run_once: true + changed_when: false + +- name: Display deployment information + debug: + msg: | + ==================================== + NGINX Test Deployment Successful! + ==================================== + + Deployment Status: + {{ deployment_status.stdout }} + + Pods Status: + {{ pods_status.stdout }} + + Service: + {{ service_status.stdout }} + + To access the service: + - export KUBECONFIG={{ playbook_dir }}/kubeconfig + - kubectl get svc nginx-test + + To scale the deployment: + - kubectl scale deployment nginx-test --replicas=10 + + To delete the test deployment: + - kubectl delete -f {{ playbook_dir }}/manifests/nginx-test-deployment.yaml + delegate_to: localhost + run_once: true diff --git a/roles/k3s-server/tasks/main.yml b/roles/k3s-server/tasks/main.yml new file mode 100644 index 0000000..c8dc12d --- /dev/null +++ b/roles/k3s-server/tasks/main.yml @@ -0,0 +1,78 @@ +--- +- name: Check if k3s is already installed + stat: + path: /usr/local/bin/k3s + register: k3s_binary + +- name: Download k3s installation script + get_url: + url: https://get.k3s.io + dest: /tmp/k3s-install.sh + mode: '0755' + when: not k3s_binary.stat.exists + +- name: Install k3s server + shell: | + INSTALL_K3S_VERSION="{{ k3s_version }}" \ + INSTALL_K3S_EXEC="server {{ extra_server_args }}" \ + sh /tmp/k3s-install.sh + when: not k3s_binary.stat.exists + +- name: Wait for k3s to be ready + wait_for: + port: 6443 + delay: 10 + timeout: 300 + +- name: Wait for node-token file to be created + wait_for: + path: /var/lib/rancher/k3s/server/node-token + state: present + timeout: 300 + +- name: Read node token + slurp: + src: /var/lib/rancher/k3s/server/node-token + register: node_token + +- name: Store master node token + set_fact: + k3s_node_token: "{{ node_token.content | b64decode | trim }}" + +- name: Add node token to dummy host + add_host: + name: "k3s_token_holder" + token: "{{ k3s_node_token }}" + +- name: Create .kube directory for user + file: + path: "/home/{{ ansible_user }}/.kube" + state: directory + owner: "{{ ansible_user }}" + group: "{{ ansible_user }}" + mode: '0755' + +- name: Copy k3s kubeconfig to user home + copy: + src: /etc/rancher/k3s/k3s.yaml + dest: "/home/{{ ansible_user }}/.kube/config" + owner: "{{ ansible_user }}" + group: "{{ ansible_user }}" + mode: '0600' + remote_src: yes + +- name: Replace localhost with master IP in kubeconfig + replace: + path: "/home/{{ ansible_user }}/.kube/config" + regexp: '127.0.0.1' + replace: "{{ ansible_host }}" + +- name: Fetch kubeconfig to local machine + fetch: + src: "/home/{{ ansible_user }}/.kube/config" + dest: "{{ playbook_dir }}/kubeconfig" + flat: yes + +- name: Display success message + debug: + msg: "K3s server installed successfully. Kubeconfig saved to {{ playbook_dir }}/kubeconfig" diff --git a/roles/prereq/tasks/main.yml b/roles/prereq/tasks/main.yml new file mode 100644 index 0000000..fb5c45e --- /dev/null +++ b/roles/prereq/tasks/main.yml @@ -0,0 +1,67 @@ +--- +- name: Set hostname + hostname: + name: "{{ inventory_hostname }}" + +- name: Update apt cache + apt: + update_cache: yes + cache_valid_time: 3600 + +- name: Upgrade all packages + apt: + upgrade: dist + autoremove: yes + autoclean: yes + +- name: Install required packages + apt: + name: + - curl + - wget + - git + - python3-pip + - iptables + - conntrack + - apparmor + - apparmor-utils + state: present + +- name: Enable cgroup memory and swap + lineinfile: + path: /boot/firmware/cmdline.txt + backrefs: yes + regexp: '^(.*rootwait.*)$' + line: '\1 cgroup_enable=cpuset cgroup_memory=1 cgroup_enable=memory' + register: cmdline + +- name: Enable legacy iptables (required for k3s on Raspberry Pi) + alternatives: + name: iptables + path: /usr/sbin/iptables-legacy + +- name: Enable legacy ip6tables + alternatives: + name: ip6tables + path: /usr/sbin/ip6tables-legacy + +- name: Disable swap + command: swapoff -a + when: ansible_swaptotal_mb > 0 + +- name: Remove swap from /etc/fstab + lineinfile: + path: /etc/fstab + regexp: '^.*swap.*$' + state: absent + +- name: Reboot if cmdline was changed + reboot: + reboot_timeout: 300 + when: cmdline.changed + +- name: Wait for system to become reachable + wait_for_connection: + delay: 30 + timeout: 300 + when: cmdline.changed diff --git a/site.yml b/site.yml new file mode 100644 index 0000000..c90ca51 --- /dev/null +++ b/site.yml @@ -0,0 +1,29 @@ +--- +- name: Prepare all nodes + hosts: k3s_cluster + gather_facts: yes + become: yes + roles: + - role: prereq + +- name: Setup k3s server + hosts: master + become: yes + roles: + - role: k3s-server + +- name: Setup k3s agents + hosts: worker + become: yes + roles: + - role: k3s-agent + +- name: Deploy test applications + hosts: master + gather_facts: yes + become: no + roles: + - role: k3s-deploy-test + tags: + - test + - deploy-test