From 724d415fc81dfd1587473ef663b9d59000f0a295 Mon Sep 17 00:00:00 2001 From: mike Date: Wed, 18 Jun 2025 19:27:48 -0400 Subject: [PATCH] updates --- k0sctl.yaml | 131 ++++++++++++++- manifests/prometheues-service-monitor.yaml | 14 ++ selinux-script.sh | 182 +++++++++++++++++++++ selinux.conf | 2 + 4 files changed, 328 insertions(+), 1 deletion(-) create mode 100644 manifests/prometheues-service-monitor.yaml create mode 100644 selinux-script.sh create mode 100644 selinux.conf diff --git a/k0sctl.yaml b/k0sctl.yaml index ba06039..891b953 100644 --- a/k0sctl.yaml +++ b/k0sctl.yaml @@ -11,24 +11,76 @@ spec: port: 22 keyPath: ~/.ssh/id_ed25519 role: controller + installFlags: + - --enable-metrics-scraper + files: + - name: prometheus-config + src: manifests/prometheues-service-monitor.yaml + dstDir: /var/lib/k0s/manifests/prometheus/ + perm: 0644 + - name: selinux-stuff + src: selinux.conf + dstDir: /etc/containerd.d/selinux-containers.conf + perm: 0644 + - name: seaweedfs-crds + src: - ssh: address: k2.lab.smig.tech user: smig port: 22 keyPath: ~/.ssh/id_ed25519 role: worker + files: + - name: selinux-script + src: selinux-script + dstDir: /home/smig/selinux-script.sh + perm: 0700 + user: smig + group: smig + hooks: + apply: + after: + - date > k0s-selinux.log + - echo "Starting SELinux Script" >> k0s-selinux.log + - bash /home/smig/script.sh &>> k0s-selinux.log - ssh: address: k3.lab.smig.tech user: smig port: 22 keyPath: ~/.ssh/id_ed25519 role: worker + files: + - name: selinux-script + src: selinux-script + dstDir: /home/smig/selinux-script.sh + perm: 0700 + user: smig + group: smig + hooks: + apply: + after: + - date > k0s-selinux.log + - echo "Starting SELinux Script" >> k0s-selinux.log + - bash /home/smig/script.sh &>> k0s-selinux.log - ssh: address: k4.lab.smig.tech user: smig port: 22 keyPath: ~/.ssh/id_ed25519 role: worker + files: + - name: selinux-script + src: selinux-script + dstDir: /home/smig/selinux-script.sh + perm: 0700 + user: smig + group: smig + hooks: + apply: + after: + - date > k0s-selinux.log + - echo "Starting SELinux Script" >> k0s-selinux.log + - bash /home/smig/script.sh &>> k0s-selinux.log k0s: config: apiVersion: k0s.k0sproject.io/v1beta1 @@ -70,29 +122,104 @@ spec: extensions: helm: repositories: + - name: prometheus + url: https://prometheus-community.github.io/helm-charts - name: cilium url: https://helm.cilium.io/ - name: cert-manager url: https://charts.jetstack.io + - name: openebs-internal + url: https://openebs.github.io/charts + - name: seaweedfs-operator + url: https://seaweedfs.github.io/seaweedfs-operator/helm + charts: + - name: seaweedfs-operator + chartname: seaweedfs-operator/seaweedfs-operator + version: "0.1.1" + order: 2 + values: | + image: + registry: git.thecodedom.com + repository: smig/seaweedfs-operator + tag: 0.1.0 + + + - name: openebs + chartname: openebs-internal/openebs + version: "3.9.0" + namespace: openebs + order: 2 + values: | + localprovisioner: + hostpathClass: + enabled: true + isDefaultClass: false + + - name: prometheus + chartName: prometheus/kube-prometheus-stack + version: "75.1.0" + namespace: monitoring + order: 3 + values: | + prometheus: + prometheusSpec: + maximumStartupDurationSeconds: null + additionalPrometheusRulesMap: + k0s-control-plane-alerts: + groups: + - name: control-plane-activity + rules: + - alert: KubeSchedulerDown + expr: absent(apiserver_audit_event_total{job="kube-scheduler"}) + for: 15m + labels: + severity: critical + + - alert: KubeControllerManagerDown + expr: absent(apiserver_audit_event_total{job="kube-controller-manager"}) + for: 15m + labels: + severity: critical + alertmanager: + alertmanagerSpec: + replicas: 1 + grafana: + initChownData: + enabled: false + + persistence: + enabled: true + storageClassName: openebs-hostpath + ingress: + enabled: true + ingressClassName: cilium + hosts: + - grafana-k0s.lab.smig.tech + - name: cert-manager chartName: cert-manager/cert-manager version: "v1.17.2" + order: 2 namespace: cert-manager values: | crds: enabled: true + - name: cilium chartName: cilium/cilium namespace: kube-system version: "1.18.0-pre.3" + order: 1 values: | + hubble: + enabled: true cluster: name: k0s-cluster envoy: enabled: true image: - digest: sha256:e265e4b2e10eaa19c4a5a305086f81bbe0f9f5f41fff60ab0ec6effdb21e2a79 + digest: sha256:bb73643e4b8c95d852bf25fc0e2f44e6d77617a809b63b119aba9edc001f4ea4 repository: git.thecodedom.com/smig/cilium-envoy tag: latest k8sServiceHost: k1.lab.smig.tech @@ -109,6 +236,8 @@ spec: enabled: true ingressController: enabled: true + enforceHttps: false + loadbalancerMode: shared service: externalTrafficPolicy: Cluster options: diff --git a/manifests/prometheues-service-monitor.yaml b/manifests/prometheues-service-monitor.yaml new file mode 100644 index 0000000..cd6aca8 --- /dev/null +++ b/manifests/prometheues-service-monitor.yaml @@ -0,0 +1,14 @@ +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: k0s + namespace: k0s-system +spec: + endpoints: + - port: http + honorLabels: true + selector: + matchLabels: + app: k0s-observability + component: pushgateway + k0s.k0sproject.io/stack: metrics diff --git a/selinux-script.sh b/selinux-script.sh new file mode 100644 index 0000000..83e0759 --- /dev/null +++ b/selinux-script.sh @@ -0,0 +1,182 @@ +#!/bin/bash + +set -euo pipefail + +# Configuration +DATA_DIR="/var/lib/k0s" +SCRIPT_NAME="$(basename "$0")" + +# Logging function +log() { + echo "[$(date '+%Y-%m-%d %H:%M:%S')] $*" >&2 +} + +# Error handling function +error_exit() { + log "ERROR: $1" + exit 1 +} + +# Check if running as root or with sudo +check_privileges() { + if [[ $EUID -ne 0 ]] && ! sudo -n true 2>/dev/null; then + error_exit "This script requires root privileges or passwordless sudo access" + fi +} + +# Check if SELinux is enabled +check_selinux() { + if ! command -v getenforce >/dev/null 2>&1; then + error_exit "SELinux tools not found. Is SELinux installed?" + fi + + local selinux_status + selinux_status=$(getenforce 2>/dev/null || echo "Disabled") + + if [[ "$selinux_status" == "Disabled" ]]; then + error_exit "SELinux is disabled. This script requires SELinux to be enabled." + fi + + log "SELinux status: $selinux_status" +} + +# Check if required tools are available +check_tools() { + local missing_tools=() + + for tool in semanage restorecon; do + if ! command -v "$tool" >/dev/null 2>&1; then + missing_tools+=("$tool") + fi + done + + if [[ ${#missing_tools[@]} -gt 0 ]]; then + error_exit "Missing required tools: ${missing_tools[*]}. Please install policycoreutils-python-utils or equivalent package." + fi +} + +# Check if data directory exists +check_data_dir() { + if [[ ! -d "$DATA_DIR" ]]; then + error_exit "Data directory $DATA_DIR does not exist" + fi + log "Data directory $DATA_DIR exists" +} + +# Check if fcontext rule already exists +fcontext_exists() { + local pattern="$1" + local context="$2" + + sudo semanage fcontext -l | grep -q "^${pattern}.*${context}" 2>/dev/null +} + +# Add SELinux file context rule (idempotent) +add_fcontext() { + local pattern="$1" + local context="$2" + local description="$3" + + if fcontext_exists "$pattern" "$context"; then + log "SKIP: $description - fcontext rule already exists" + return 0 + fi + + log "ADDING: $description" + if sudo semanage fcontext -a -t "$context" "$pattern"; then + log "SUCCESS: Added fcontext rule for $pattern" + return 0 + else + error_exit "Failed to add fcontext rule for $pattern" + fi +} + +# Restore SELinux contexts +restore_contexts() { + local path="$1" + local description="$2" + + if [[ ! -e "$path" ]]; then + log "SKIP: $description - path $path does not exist" + return 0 + fi + + log "RESTORING: $description" + if sudo restorecon -R -v "$path" 2>&1 | while read -r line; do + [[ -n "$line" ]] && log " $line" + done; then + log "SUCCESS: Restored contexts for $path" + return 0 + else + error_exit "Failed to restore contexts for $path" + fi +} + +# Verify that contexts were applied correctly +verify_contexts() { + local path="$1" + local expected_pattern="$2" + local description="$3" + + if [[ ! -e "$path" ]]; then + log "SKIP VERIFY: $description - path $path does not exist" + return 0 + fi + + log "VERIFYING: $description" + + # Get the actual context + local actual_contexts + actual_contexts=$(find "$path" -exec ls -Z {} \; 2>/dev/null | awk '{print $1}' | sort -u) + + if echo "$actual_contexts" | grep -q "$expected_pattern"; then + log "SUCCESS: Verified contexts for $path contain $expected_pattern" + return 0 + else + log "WARNING: Expected pattern $expected_pattern not found in contexts for $path" + log "Actual contexts found: $actual_contexts" + return 1 + fi +} + +# Main execution +main() { + log "Starting $SCRIPT_NAME" + + # Pre-flight checks + check_privileges + check_selinux + check_tools + check_data_dir + + # Add file context rules (idempotent) + add_fcontext "${DATA_DIR}/bin/containerd.*" "container_runtime_exec_t" "containerd executables" + add_fcontext "${DATA_DIR}/bin/runc" "container_runtime_exec_t" "runc executable" + add_fcontext "${DATA_DIR}/containerd(/.*)?", "container_var_lib_t" "containerd directory" + add_fcontext "${DATA_DIR}/containerd/io.containerd.snapshotter.*/snapshots(/.*)?" "container_ro_file_t" "containerd snapshots" + + # Restore contexts + restore_contexts "${DATA_DIR}/bin" "k0s binaries" + restore_contexts "${DATA_DIR}/containerd" "containerd directory" + + # Verify contexts were applied (optional verification) + local verification_failed=0 + + if [[ -d "${DATA_DIR}/bin" ]]; then + verify_contexts "${DATA_DIR}/bin" "container_runtime_exec_t" "k0s binaries" || verification_failed=1 + fi + + if [[ -d "${DATA_DIR}/containerd" ]]; then + verify_contexts "${DATA_DIR}/containerd" "container_" "containerd directory" || verification_failed=1 + fi + + if [[ $verification_failed -eq 1 ]]; then + log "WARNING: Some context verifications failed. Check the logs above." + log "You may want to run 'sudo restorecon -R -v $DATA_DIR' manually." + fi + + log "Completed $SCRIPT_NAME successfully" +} + +# Run main function +main "$@" diff --git a/selinux.conf b/selinux.conf new file mode 100644 index 0000000..33cff41 --- /dev/null +++ b/selinux.conf @@ -0,0 +1,2 @@ +[plugins."io.containerd.grpc.v1.cri"] + enable_selinux = true