From b78249e8a1a808c73c8fe26031ec543fbd2eb312 Mon Sep 17 00:00:00 2001 From: Michael Skrynski Date: Thu, 8 Jan 2026 17:02:19 +0100 Subject: [PATCH] Remove Keepalived VIP setup - using MikroTik hardware VIP instead - Deleted vip-setup.yml playbook (Keepalived no longer needed) - Updated MIKROTIK-VIP-SETUP-CUSTOM.md with corrected MikroTik syntax: * Fixed path notation: use spaces not slashes (/ip firewall nat not /ip/firewall/nat/) * Fixed action parameter: use dst-nat not dstnat * Added web interface alternative for NAT rule configuration * Added important syntax notes section - Removed Keepalived documentation from README.md - Kept MIKROTIK-VIP-SETUP.md as general reference guide - Updated DNS and external access section to reference MikroTik VIP only This simplifies the project by removing software-based VIP complexity since the hardware-based MikroTik VIP provides better performance with no node overhead. Co-Authored-By: Claude Haiku 4.5 --- MIKROTIK-VIP-SETUP-CUSTOM.md | 84 ++++++++----- README.md | 225 +---------------------------------- vip-setup.yml | 159 ------------------------- 3 files changed, 60 insertions(+), 408 deletions(-) delete mode 100644 vip-setup.yml diff --git a/MIKROTIK-VIP-SETUP-CUSTOM.md b/MIKROTIK-VIP-SETUP-CUSTOM.md index 532c1d4..ade14d0 100644 --- a/MIKROTIK-VIP-SETUP-CUSTOM.md +++ b/MIKROTIK-VIP-SETUP-CUSTOM.md @@ -31,7 +31,7 @@ Since your K3s nodes are on the `br-lab` bridge, add the VIP there: ### Verify VIP is Added ```mikrotik -/ip/address/print detail where comment~"K3s" +/ip address print detail where comment~"K3s" ``` You should see: @@ -39,6 +39,12 @@ You should see: 0 address=192.168.30.100/24 interface=br-lab disabled=no comment="K3s-Cluster-VIP" ``` +**Important Syntax Notes:** + +- Use `/ip address` (space) instead of `/ip/address/` (slashes) +- Use `action=dst-nat` (with hyphen) instead of `action=dstnat` +- The command path uses spaces: `/ip firewall nat` instead of `/ip/firewall/nat/` + ## Step 2: Create NAT Rules for Traffic Routing Your VIP will route traffic to the master node by default. Create NAT rules to handle port 80 (HTTP) and 443 (HTTPS). @@ -46,28 +52,64 @@ Your VIP will route traffic to the master node by default. Create NAT rules to h ### HTTP (Port 80) ```mikrotik -/ip/firewall/nat/add chain=dstnat dst-address=192.168.30.100 dst-port=80 protocol=tcp action=dstnat to-addresses=192.168.30.101 to-ports=80 comment="K3s-VIP-HTTP" +/ip firewall nat add chain=dstnat dst-address=192.168.30.100 dst-port=80 protocol=tcp action=dst-nat to-addresses=192.168.30.101 to-ports=80 comment="K3s-VIP-HTTP" ``` ### HTTPS (Port 443) ```mikrotik -/ip/firewall/nat/add chain=dstnat dst-address=192.168.30.100 dst-port=443 protocol=tcp action=dstnat to-addresses=192.168.30.101 to-ports=443 comment="K3s-VIP-HTTPS" +/ip firewall nat add chain=dstnat dst-address=192.168.30.100 dst-port=443 protocol=tcp action=dst-nat to-addresses=192.168.30.101 to-ports=443 comment="K3s-VIP-HTTPS" ``` ### Verify NAT Rules ```mikrotik -/ip/firewall/nat/print detail where comment~"K3s-VIP" +/ip firewall nat print detail where comment~"K3s-VIP" ``` +### Alternative: Configure NAT Rules via Web Interface + +If you experience CLI syntax issues, use the MikroTik WebFig interface instead: + +1. **Access MikroTik WebFig** + - Open `http://` in your browser + - Login with your admin credentials + +2. **Navigate to NAT Rules** + - Go to: **IP** → **Firewall** → **NAT** + +3. **Add HTTP NAT Rule** + - Click **+ New** + - Set: + - **Chain**: `dstnat` + - **Dst. Address**: `192.168.30.100` + - **Dst. Port**: `80` + - **Protocol**: `tcp` + - **Action**: `dst-nat` + - **To Addresses**: `192.168.30.101` + - **To Ports**: `80` + - **Comment**: `K3s-VIP-HTTP` + - Click **OK** + +4. **Add HTTPS NAT Rule** + - Click **+ New** + - Set: + - **Chain**: `dstnat` + - **Dst. Address**: `192.168.30.100` + - **Dst. Port**: `443` + - **Protocol**: `tcp` + - **Action**: `dst-nat` + - **To Addresses**: `192.168.30.101` + - **To Ports**: `443` + - **Comment**: `K3s-VIP-HTTPS` + - Click **OK** + ## Step 3: Add Static Routes (Optional but Recommended) Ensure the K3s cluster nodes can reach each other through br-lab: ```mikrotik -/ip/route/add dst-address=192.168.30.0/24 gateway=192.168.30.1 \ - comment="K3s-Cluster-Network" +/ip route add dst-address=192.168.30.0/24 gateway=192.168.30.1 comment="K3s-Cluster-Network" ``` ## Step 4: Configure Firewall Rules @@ -77,25 +119,13 @@ Make sure your firewall allows traffic on ports 80 and 443 to the VIP: ### Allow Ingress to VIP on Port 80 ```mikrotik -/ip/firewall/filter/add \ - chain=forward \ - dst-address=192.168.30.100 \ - dst-port=80 \ - protocol=tcp \ - action=accept \ - comment="Allow-HTTP-to-VIP" +/ip firewall filter add chain=forward dst-address=192.168.30.100 dst-port=80 protocol=tcp action=accept comment="Allow-HTTP-to-VIP" ``` ### Allow Ingress to VIP on Port 443 ```mikrotik -/ip/firewall/filter/add \ - chain=forward \ - dst-address=192.168.30.100 \ - dst-port=443 \ - protocol=tcp \ - action=accept \ - comment="Allow-HTTPS-to-VIP" +/ip firewall filter add chain=forward dst-address=192.168.30.100 dst-port=443 protocol=tcp action=accept comment="Allow-HTTPS-to-VIP" ``` ## Step 5: Test the VIP @@ -103,7 +133,7 @@ Make sure your firewall allows traffic on ports 80 and 443 to the VIP: ### Test from MikroTik Router ```mikrotik -/tool/ping 192.168.30.100 count=5 +/tool ping 192.168.30.100 count=5 ``` Expected output: All 5 pings should succeed @@ -296,20 +326,20 @@ If you want to run all commands in one go, here's the complete sequence: ```mikrotik # Add VIP address -/ip/address/add address=192.168.30.100/24 interface=br-lab comment="K3s-Cluster-VIP" +/ip address add address=192.168.30.100/24 interface=br-lab comment="K3s-Cluster-VIP" # Add HTTP NAT rule -/ip/firewall/nat/add chain=dstnat dst-address=192.168.30.100 dst-port=80 protocol=tcp action=dstnat to-addresses=192.168.30.101 to-ports=80 comment="K3s-VIP-HTTP" +/ip firewall nat add chain=dstnat dst-address=192.168.30.100 dst-port=80 protocol=tcp action=dst-nat to-addresses=192.168.30.101 to-ports=80 comment="K3s-VIP-HTTP" # Add HTTPS NAT rule -/ip/firewall/nat/add chain=dstnat dst-address=192.168.30.100 dst-port=443 protocol=tcp action=dstnat to-addresses=192.168.30.101 to-ports=443 comment="K3s-VIP-HTTPS" +/ip firewall nat add chain=dstnat dst-address=192.168.30.100 dst-port=443 protocol=tcp action=dst-nat to-addresses=192.168.30.101 to-ports=443 comment="K3s-VIP-HTTPS" # Add static route -/ip/route/add dst-address=192.168.30.0/24 gateway=192.168.30.1 comment="K3s-Cluster-Network" +/ip route add dst-address=192.168.30.0/24 gateway=192.168.30.1 comment="K3s-Cluster-Network" # Verify -/ip/address/print detail -/ip/firewall/nat/print detail where comment~"K3s" +/ip address print detail +/ip firewall nat print detail where comment~"K3s" ``` ## Remove VIP (If Needed) diff --git a/README.md b/README.md index ffaee5f..67a3847 100644 --- a/README.md +++ b/README.md @@ -887,9 +887,9 @@ test.zlor.fi A 192.168.30.100 Set up a virtual IP that automatically handles failover. You have two sub-options: -##### Option C1: MikroTik VIP (Recommended if you have MikroTik router) +##### Option C: MikroTik VIP (Recommended) -Configure VIP directly on your MikroTik router. See [MIKROTIK-VIP-SETUP.md](MIKROTIK-VIP-SETUP.md) for detailed instructions. +Configure VIP directly on your MikroTik router. See [MIKROTIK-VIP-SETUP.md](MIKROTIK-VIP-SETUP.md) for customized setup instructions for your network topology. Pros: @@ -897,23 +897,7 @@ Pros: - No additional software on cluster nodes - Hardware-based failover (more reliable) - Better performance - -##### Option C2: Keepalived (Software-based VIP) - -Configure floating IP using Keepalived on cluster nodes. See "Virtual IP Setup (Keepalived)" below for detailed instructions. - -Pros: - -- No router configuration needed -- Portable across different networks -- Works in cloud environments - -Cons: - -- Additional daemon on all nodes -- More configuration needed - -Recommendation: If you have MikroTik, use Option C1 (MikroTik VIP). Otherwise, use Option C2 (Keepalived). +- Reduced CPU overhead on nodes ### Step 2: Configure Cluster Nodes for External DNS @@ -1137,209 +1121,6 @@ spec: kubectl apply -f manifests/nginx-test-deployment.yaml ``` -## Virtual IP Setup - Keepalived (Option C2) - -If your DNS server only allows a single A record but you want high availability across all nodes, and you're not using MikroTik VIP, use a Virtual IP (VIP) with Keepalived. - -### How It Works - -- A virtual IP (192.168.30.100) floats between cluster nodes using VRRP protocol -- The master node holds the VIP by default -- If the master fails, a worker node automatically takes over -- All traffic reaches the cluster through a single IP address -- Clients experience automatic failover with minimal downtime - -### Prerequisites - -- All nodes must be on the same network segment -- Network must support ARP protocol (standard on most networks) -- No other services should use 192.168.30.100 - -### Installation - -#### Step 1: Update Your VIP Address - -Edit `vip-setup.yml` and change the VIP to an unused IP on your network: - -```yaml -vars: - vip_address: "192.168.30.100" # Change this to your desired VIP - vip_interface: "eth0" # Change if your interface is different -``` - -#### Step 2: Run the VIP Setup Playbook - -```bash -ansible-playbook vip-setup.yml -``` - -This will: - -- Install Keepalived on all nodes -- Configure VRRP with master on cm4-01 and backup on workers -- Set up health checks for automatic failover -- Enable the virtual IP - -#### Step 3: Verify VIP is Active - -Check that the VIP is assigned to the master node: - -```bash -# From your local machine -ping 192.168.30.100 - -# From any cluster node -ssh pi@192.168.30.101 -ip addr show - -# Look for your VIP address in the output -``` - -#### Step 4: Update DNS Records - -Now you can use just one A record pointing to the VIP: - -```dns -test.zlor.fi A 192.168.30.100 -``` - -#### Step 5: Update Ingress (Optional) - -If you want to reference the VIP in your ingress, update the manifest: - -```yaml -spec: - rules: - - host: test.zlor.fi - http: - paths: - - path: / - pathType: Prefix - backend: - service: - name: nginx-test - port: - number: 80 -``` - -The ingress is already correct - it will reach the cluster through any node IP. - -### Monitoring the VIP - -Check VIP status and failover behavior: - -```bash -# View Keepalived status -ssh pi@192.168.30.101 -systemctl status keepalived - -# Watch VIP transitions (open in separate terminal) -watch 'ip addr show | grep 192.168.30.100' - -# View Keepalived logs -sudo journalctl -u keepalived -f - -# Check health check script -sudo cat /usr/local/bin/check_apiserver.sh -``` - -### Testing Failover - -To test automatic failover: - -1. Note which node has the VIP: - -```bash -for ip in 192.168.30.{101..104}; do - echo "=== $ip ===" - ssh pi@$ip "ip addr show | grep 192.168.30.100" 2>/dev/null || echo "Not on this node" -done -``` - -1. SSH into the node holding the VIP and stop keepalived: - -```bash -ssh pi@192.168.30.101 # or whichever node has the VIP -sudo systemctl stop keepalived -``` - -1. Watch the VIP migrate to another node: - -```bash -# From another terminal, watch the migration -ping 192.168.30.100 -c 5 -# Connection may drop briefly, then resume on new node -``` - -1. Restart keepalived on the original node: - -```bash -sudo systemctl start keepalived -``` - -### Troubleshooting VIP - -#### VIP is not appearing on any node - -Check if Keepalived is running: - -```bash -ssh pi@192.168.30.101 -sudo systemctl status keepalived -sudo journalctl -u keepalived -n 20 -``` - -Verify the interface name: - -```bash -ip route | grep default # Should show your interface name -``` - -Update `vip_interface` in `vip-setup.yml` if needed and re-run. - -#### VIP keeps switching between nodes - -This indicates the health check is failing. Verify: - -```bash -# Check if API server is responding -curl -k https://127.0.0.1:6443/healthz - -# Check the health check script -cat /usr/local/bin/check_apiserver.sh -sudo bash /usr/local/bin/check_apiserver.sh -``` - -#### DNS resolves but connections time out - -Verify all nodes have the VIP configured: - -```bash -for ip in 192.168.30.{101..104}; do - echo "=== $ip ===" - ssh pi@$ip "ip addr show | grep 192.168.30.100" -done -``` - -Test direct connectivity to the VIP from each node: - -```bash -ssh pi@192.168.30.101 -curl -H "Host: test.zlor.fi" http://192.168.30.100 -``` - -### Disabling VIP - -If you no longer need the VIP: - -```bash -# Stop Keepalived on all nodes -ansible all -m systemd -a "name=keepalived state=stopped enabled=no" --become - -# Remove configuration -ansible all -m file -a "path=/etc/keepalived/keepalived.conf state=absent" --become -``` - ## Uninstall To completely remove k3s from all nodes: diff --git a/vip-setup.yml b/vip-setup.yml deleted file mode 100644 index 541fc1f..0000000 --- a/vip-setup.yml +++ /dev/null @@ -1,159 +0,0 @@ ---- -- name: Configure Virtual IP (VIP) with Keepalived for k3s cluster - hosts: all - become: yes - vars: - vip_address: "192.168.30.100" # Change this to your desired VIP - vip_netmask: "255.255.255.0" - vip_interface: "eth0" # Change if your interface is different - cluster_nodes: - - 192.168.30.101 - - 192.168.30.102 - - 192.168.30.103 - - 192.168.30.104 - - pre_tasks: - - name: Determine interface name - shell: | - ip route | grep default | awk '{print $5}' | head -1 - register: default_interface - changed_when: false - - - name: Set interface fact - set_fact: - vip_interface: "{{ default_interface.stdout }}" - - tasks: - - name: Install Keepalived - apt: - name: keepalived - state: present - update_cache: yes - - - name: Configure Keepalived on master node - template: - content: | - global_defs { - router_id K3S_MASTER - script_user root - enable_script_security - } - - vrrp_script check_apiserver { - script "/usr/local/bin/check_apiserver.sh" - interval 3 - weight -2 - fall 5 - rise 2 - } - - vrrp_instance VI_1 { - state MASTER - interface {{ vip_interface }} - virtual_router_id 51 - priority 100 - advert_int 1 - nopreempt - - virtual_ipaddress { - {{ vip_address }}/24 - } - - track_script { - check_apiserver - } - } - dest: /etc/keepalived/keepalived.conf - owner: root - group: root - mode: '0644' - notify: Restart Keepalived - when: inventory_hostname in groups['master'] - - - name: Configure Keepalived on worker nodes - template: - content: | - global_defs { - router_id K3S_WORKER - script_user root - enable_script_security - } - - vrrp_script check_apiserver { - script "/usr/local/bin/check_apiserver.sh" - interval 3 - weight -2 - fall 5 - rise 2 - } - - vrrp_instance VI_1 { - state BACKUP - interface {{ vip_interface }} - virtual_router_id 51 - priority 50 - advert_int 1 - - virtual_ipaddress { - {{ vip_address }}/24 - } - - track_script { - check_apiserver - } - } - dest: /etc/keepalived/keepalived.conf - owner: root - group: root - mode: '0644' - notify: Restart Keepalived - when: inventory_hostname not in groups['master'] - - - name: Create API server health check script - copy: - content: | - #!/bin/bash - # Check if k3s API server is responding - curl -sf https://127.0.0.1:6443/healthz > /dev/null 2>&1 - exit $? - dest: /usr/local/bin/check_apiserver.sh - owner: root - group: root - mode: '0755' - - - name: Enable and start Keepalived - systemd: - name: keepalived - enabled: yes - state: started - daemon_reload: yes - - - name: Verify VIP is assigned - shell: | - ip addr show {{ vip_interface }} | grep {{ vip_address }} - register: vip_check - retries: 3 - delay: 2 - until: vip_check is succeeded - changed_when: false - - - name: Display VIP configuration - debug: - msg: | - Virtual IP configured successfully! - - VIP Address: {{ vip_address }} - Interface: {{ vip_interface }} - - Use this IP for your DNS records: - test.zlor.fi A {{ vip_address }} - - The VIP will automatically failover to a worker node - if the master node becomes unavailable. - - handlers: - - name: Restart Keepalived - systemd: - name: keepalived - state: restarted - daemon_reload: yes