updates

k0sctl.yaml

@@ -11,24 +11,76 @@ spec:
       port: 22
       keyPath: ~/.ssh/id_ed25519
     role: controller
+    installFlags:
+      - --enable-metrics-scraper
+    files:
+      - name: prometheus-config
+        src: manifests/prometheues-service-monitor.yaml
+        dstDir: /var/lib/k0s/manifests/prometheus/
+        perm: 0644
+      - name: selinux-stuff
+        src: selinux.conf
+        dstDir: /etc/containerd.d/selinux-containers.conf
+        perm: 0644
+      - name: seaweedfs-crds
+        src: 
   - ssh:
       address: k2.lab.smig.tech
       user: smig
       port: 22
       keyPath: ~/.ssh/id_ed25519
     role: worker
+    files:
+      - name: selinux-script
+        src: selinux-script
+        dstDir: /home/smig/selinux-script.sh
+        perm: 0700
+        user: smig
+        group: smig
+    hooks:
+      apply:
+        after:
+          - date > k0s-selinux.log
+          - echo "Starting SELinux Script" >> k0s-selinux.log
+          - bash /home/smig/script.sh &>> k0s-selinux.log
   - ssh:
       address: k3.lab.smig.tech
       user: smig
       port: 22
       keyPath: ~/.ssh/id_ed25519
     role: worker
+    files:
+      - name: selinux-script
+        src: selinux-script
+        dstDir: /home/smig/selinux-script.sh
+        perm: 0700
+        user: smig
+        group: smig
+    hooks:
+      apply:
+        after:
+          - date > k0s-selinux.log
+          - echo "Starting SELinux Script" >> k0s-selinux.log
+          - bash /home/smig/script.sh &>> k0s-selinux.log
   - ssh:
       address: k4.lab.smig.tech
       user: smig
       port: 22
       keyPath: ~/.ssh/id_ed25519
     role: worker
+    files:
+      - name: selinux-script
+        src: selinux-script
+        dstDir: /home/smig/selinux-script.sh
+        perm: 0700
+        user: smig
+        group: smig
+    hooks:
+      apply:
+        after:
+          - date > k0s-selinux.log
+          - echo "Starting SELinux Script" >> k0s-selinux.log
+          - bash /home/smig/script.sh &>> k0s-selinux.log
   k0s:
     config:
       apiVersion: k0s.k0sproject.io/v1beta1
@@ -70,29 +122,104 @@ spec:
         extensions:
           helm:
             repositories:
+              - name: prometheus
+                url: https://prometheus-community.github.io/helm-charts
               - name: cilium
                 url: https://helm.cilium.io/
               - name: cert-manager
                 url: https://charts.jetstack.io
+              - name: openebs-internal
+                url: https://openebs.github.io/charts
+              - name: seaweedfs-operator
+                url: https://seaweedfs.github.io/seaweedfs-operator/helm
+
             charts:
+              - name: seaweedfs-operator
+                chartname: seaweedfs-operator/seaweedfs-operator
+                version: "0.1.1"
+                order: 2
+                values: |
+                  image:
+                    registry: git.thecodedom.com
+                    repository: smig/seaweedfs-operator
+                    tag: 0.1.0
+
+
+              - name: openebs
+                chartname: openebs-internal/openebs
+                version: "3.9.0"
+                namespace: openebs
+                order: 2
+                values: |
+                  localprovisioner:
+                    hostpathClass:
+                      enabled: true
+                      isDefaultClass: false
+
+              - name: prometheus
+                chartName: prometheus/kube-prometheus-stack
+                version: "75.1.0"
+                namespace: monitoring
+                order: 3
+                values: |
+                  prometheus:
+                    prometheusSpec:
+                      maximumStartupDurationSeconds: null
+                  additionalPrometheusRulesMap:
+                    k0s-control-plane-alerts:
+                      groups:
+                      - name: control-plane-activity
+                        rules:
+                        - alert: KubeSchedulerDown
+                          expr: absent(apiserver_audit_event_total{job="kube-scheduler"})
+                          for: 15m
+                          labels:
+                            severity: critical
+
+                        - alert: KubeControllerManagerDown
+                          expr: absent(apiserver_audit_event_total{job="kube-controller-manager"})
+                          for: 15m
+                          labels:
+                            severity: critical
+                  alertmanager:
+                    alertmanagerSpec:
+                      replicas: 1
+                  grafana:
+                    initChownData:
+                      enabled: false
+                        
+                    persistence: 
+                      enabled: true
+                      storageClassName: openebs-hostpath 
+                    ingress:
+                      enabled: true
+                      ingressClassName: cilium
+                      hosts:
+                        - grafana-k0s.lab.smig.tech
+
               - name: cert-manager
                 chartName: cert-manager/cert-manager
                 version: "v1.17.2"
+                order: 2
                 namespace: cert-manager
                 values: |
                   crds:
                     enabled: true
+
               - name: cilium
                 chartName: cilium/cilium
                 namespace: kube-system
                 version: "1.18.0-pre.3"
+                order: 1
                 values: |
+                  hubble:
+                    enabled: true
                   cluster:
                     name: k0s-cluster
                   envoy:
                     enabled: true
                     image:
-                      digest: sha256:e265e4b2e10eaa19c4a5a305086f81bbe0f9f5f41fff60ab0ec6effdb21e2a79
+                      digest: sha256:bb73643e4b8c95d852bf25fc0e2f44e6d77617a809b63b119aba9edc001f4ea4
                       repository: git.thecodedom.com/smig/cilium-envoy
                       tag: latest
                   k8sServiceHost: k1.lab.smig.tech
@@ -109,6 +236,8 @@ spec:
                     enabled: true
                   ingressController:
                     enabled: true
+                    enforceHttps: false
+                    loadbalancerMode: shared
                     service:
                       externalTrafficPolicy: Cluster
   options:

manifests/prometheues-service-monitor.yaml

@@ -0,0 +1,14 @@
+apiVersion: monitoring.coreos.com/v1
+kind: ServiceMonitor
+metadata:
+  name: k0s
+  namespace: k0s-system
+spec:
+  endpoints:
+  - port: http
+    honorLabels: true
+  selector:
+    matchLabels:
+      app: k0s-observability
+      component: pushgateway
+      k0s.k0sproject.io/stack: metrics

selinux-script.sh

@@ -0,0 +1,182 @@
+#!/bin/bash
+
+set -euo pipefail
+
+# Configuration
+DATA_DIR="/var/lib/k0s"
+SCRIPT_NAME="$(basename "$0")"
+
+# Logging function
+log() {
+    echo "[$(date '+%Y-%m-%d %H:%M:%S')] $*" >&2
+}
+
+# Error handling function
+error_exit() {
+    log "ERROR: $1"
+    exit 1
+}
+
+# Check if running as root or with sudo
+check_privileges() {
+    if [[ $EUID -ne 0 ]] && ! sudo -n true 2>/dev/null; then
+        error_exit "This script requires root privileges or passwordless sudo access"
+    fi
+}
+
+# Check if SELinux is enabled
+check_selinux() {
+    if ! command -v getenforce >/dev/null 2>&1; then
+        error_exit "SELinux tools not found. Is SELinux installed?"
+    fi
+    
+    local selinux_status
+    selinux_status=$(getenforce 2>/dev/null || echo "Disabled")
+    
+    if [[ "$selinux_status" == "Disabled" ]]; then
+        error_exit "SELinux is disabled. This script requires SELinux to be enabled."
+    fi
+    
+    log "SELinux status: $selinux_status"
+}
+
+# Check if required tools are available
+check_tools() {
+    local missing_tools=()
+    
+    for tool in semanage restorecon; do
+        if ! command -v "$tool" >/dev/null 2>&1; then
+            missing_tools+=("$tool")
+        fi
+    done
+    
+    if [[ ${#missing_tools[@]} -gt 0 ]]; then
+        error_exit "Missing required tools: ${missing_tools[*]}. Please install policycoreutils-python-utils or equivalent package."
+    fi
+}
+
+# Check if data directory exists
+check_data_dir() {
+    if [[ ! -d "$DATA_DIR" ]]; then
+        error_exit "Data directory $DATA_DIR does not exist"
+    fi
+    log "Data directory $DATA_DIR exists"
+}
+
+# Check if fcontext rule already exists
+fcontext_exists() {
+    local pattern="$1"
+    local context="$2"
+    
+    sudo semanage fcontext -l | grep -q "^${pattern}.*${context}" 2>/dev/null
+}
+
+# Add SELinux file context rule (idempotent)
+add_fcontext() {
+    local pattern="$1"
+    local context="$2"
+    local description="$3"
+    
+    if fcontext_exists "$pattern" "$context"; then
+        log "SKIP: $description - fcontext rule already exists"
+        return 0
+    fi
+    
+    log "ADDING: $description"
+    if sudo semanage fcontext -a -t "$context" "$pattern"; then
+        log "SUCCESS: Added fcontext rule for $pattern"
+        return 0
+    else
+        error_exit "Failed to add fcontext rule for $pattern"
+    fi
+}
+
+# Restore SELinux contexts
+restore_contexts() {
+    local path="$1"
+    local description="$2"
+    
+    if [[ ! -e "$path" ]]; then
+        log "SKIP: $description - path $path does not exist"
+        return 0
+    fi
+    
+    log "RESTORING: $description"
+    if sudo restorecon -R -v "$path" 2>&1 | while read -r line; do
+        [[ -n "$line" ]] && log "  $line"
+    done; then
+        log "SUCCESS: Restored contexts for $path"
+        return 0
+    else
+        error_exit "Failed to restore contexts for $path"
+    fi
+}
+
+# Verify that contexts were applied correctly
+verify_contexts() {
+    local path="$1"
+    local expected_pattern="$2"
+    local description="$3"
+    
+    if [[ ! -e "$path" ]]; then
+        log "SKIP VERIFY: $description - path $path does not exist"
+        return 0
+    fi
+    
+    log "VERIFYING: $description"
+    
+    # Get the actual context
+    local actual_contexts
+    actual_contexts=$(find "$path" -exec ls -Z {} \; 2>/dev/null | awk '{print $1}' | sort -u)
+    
+    if echo "$actual_contexts" | grep -q "$expected_pattern"; then
+        log "SUCCESS: Verified contexts for $path contain $expected_pattern"
+        return 0
+    else
+        log "WARNING: Expected pattern $expected_pattern not found in contexts for $path"
+        log "Actual contexts found: $actual_contexts"
+        return 1
+    fi
+}
+
+# Main execution
+main() {
+    log "Starting $SCRIPT_NAME"
+    
+    # Pre-flight checks
+    check_privileges
+    check_selinux
+    check_tools
+    check_data_dir
+    
+    # Add file context rules (idempotent)
+    add_fcontext "${DATA_DIR}/bin/containerd.*" "container_runtime_exec_t" "containerd executables"
+    add_fcontext "${DATA_DIR}/bin/runc" "container_runtime_exec_t" "runc executable"
+    add_fcontext "${DATA_DIR}/containerd(/.*)?", "container_var_lib_t" "containerd directory"
+    add_fcontext "${DATA_DIR}/containerd/io.containerd.snapshotter.*/snapshots(/.*)?" "container_ro_file_t" "containerd snapshots"
+    
+    # Restore contexts
+    restore_contexts "${DATA_DIR}/bin" "k0s binaries"
+    restore_contexts "${DATA_DIR}/containerd" "containerd directory"
+    
+    # Verify contexts were applied (optional verification)
+    local verification_failed=0
+    
+    if [[ -d "${DATA_DIR}/bin" ]]; then
+        verify_contexts "${DATA_DIR}/bin" "container_runtime_exec_t" "k0s binaries" || verification_failed=1
+    fi
+    
+    if [[ -d "${DATA_DIR}/containerd" ]]; then
+        verify_contexts "${DATA_DIR}/containerd" "container_" "containerd directory" || verification_failed=1
+    fi
+    
+    if [[ $verification_failed -eq 1 ]]; then
+        log "WARNING: Some context verifications failed. Check the logs above."
+        log "You may want to run 'sudo restorecon -R -v $DATA_DIR' manually."
+    fi
+    
+    log "Completed $SCRIPT_NAME successfully"
+}
+
+# Run main function
+main "$@"

selinux.conf

@@ -0,0 +1,2 @@
+[plugins."io.containerd.grpc.v1.cri"]
+  enable_selinux = true