apiVersion: k0sctl.k0sproject.io/v1beta1
kind: Cluster
metadata:
  name: k0s-cluster
  user: admin
spec:
  hosts:
  - ssh:
      address: k1.lab.smig.tech
      user: smig
      port: 22
      keyPath: ~/.ssh/id_ed25519
    role: controller
    installFlags:
      - --enable-metrics-scraper
    files:
      - name: prometheus-config
        src: manifests/prometheues-service-monitor.yaml
        dstDir: /var/lib/k0s/manifests/prometheus/
        perm: 0644
      - name: selinux-stuff
        src: selinux.conf
        dstDir: /etc/containerd.d/selinux-containers.conf
        perm: 0644
  - ssh:
      address: k2.lab.smig.tech
      user: smig
      port: 22
      keyPath: ~/.ssh/id_ed25519
    role: worker
    files:
      - name: selinux-script
        src: ./selinux-script.sh
        dstDir: /home/smig/selinux-script.sh
        perm: 0700
        user: smig
        group: smig
    hooks:
      apply:
        after:
          - date > k0s-selinux.log
          - echo "Starting SELinux Script" >> k0s-selinux.log
          - bash /home/smig/selinux-script.sh &>> k0s-selinux.log
  - ssh:
      address: k3.lab.smig.tech
      user: smig
      port: 22
      keyPath: ~/.ssh/id_ed25519
    role: worker
    files:
      - name: selinux-script
        src: ./selinux-script.sh
        dstDir: /home/smig/selinux-script.sh
        perm: 0700
        user: smig
        group: smig
    hooks:
      apply:
        after:
          - date > k0s-selinux.log
          - echo "Starting SELinux Script" >> k0s-selinux.log
          - bash /home/smig/selinux-script.sh &>> k0s-selinux.log
  - ssh:
      address: k4.lab.smig.tech
      user: smig
      port: 22
      keyPath: ~/.ssh/id_ed25519
    role: worker
    files:
      - name: selinux-script
        src: ./selinux-script.sh
        dstDir: /home/smig/selinux-script.sh
        perm: 0700
        user: smig
        group: smig
    hooks:
      apply:
        after:
          - date > k0s-selinux.log
          - echo "Starting SELinux Script" >> k0s-selinux.log
          - bash /home/smig/selinux-script.sh &>> k0s-selinux.log
  k0s:
    config:
      apiVersion: k0s.k0sproject.io/v1beta1
      kind: Cluster
      metadata:
        name: k0s
      spec:
        api:
          k0sApiPort: 9443
          port: 6443
        installConfig:
          users:
            etcdUser: etcd
            kineUser: kube-apiserver
            konnectivityUser: konnectivity-server
            kubeAPIserverUser: kube-apiserver
            kubeSchedulerUser: kube-scheduler
        konnectivity:
          adminPort: 8133
          agentPort: 8132
        network:
          kubeProxy:
            disabled: true
            # mode: iptables
          kuberouter:
            autoMTU: true
            mtu: 0
            peerRouterASNs: ""
            peerRouterIPs: ""
          podCIDR: 10.244.0.0/16
          provider: custom
          serviceCIDR: 10.96.0.0/12
        podSecurityPolicy:
          defaultPolicy: 00-k0s-privileged
        storage:
          type: etcd
        telemetry:
          enabled: false
        extensions:
          helm:
            repositories:
              - name: prometheus
                url: https://prometheus-community.github.io/helm-charts
              - name: cilium
                url: https://helm.cilium.io/
              - name: cert-manager
                url: https://charts.jetstack.io
              - name: openebs-internal
                url: https://openebs.github.io/charts
              - name: seaweedfs-operator
                url: https://seaweedfs.github.io/seaweedfs-operator/helm

            charts:
              - name: seaweedfs-operator
                chartname: seaweedfs-operator/seaweedfs-operator
                version: "0.1.1"
                order: 2
                namespace: seaweefs-operator-system
                values: |
                  image:
                    registry: git.thecodedom.com
                    repository: smig/seaweedfs-operator
                    tag: 0.1.0


              - name: openebs
                chartname: openebs-internal/openebs
                version: "3.9.0"
                namespace: openebs
                order: 2
                values: |
                  localprovisioner:
                    hostpathClass:
                      enabled: true
                      isDefaultClass: false

              - name: prometheus
                chartName: prometheus/kube-prometheus-stack
                version: "75.1.0"
                namespace: monitoring
                order: 3
                values: |
                  prometheus:
                    prometheusSpec:
                      maximumStartupDurationSeconds: null
                  additionalPrometheusRulesMap:
                    k0s-control-plane-alerts:
                      groups:
                      - name: control-plane-activity
                        rules:
                        - alert: KubeSchedulerDown
                          expr: absent(apiserver_audit_event_total{job="kube-scheduler"})
                          for: 15m
                          labels:
                            severity: critical

                        - alert: KubeControllerManagerDown
                          expr: absent(apiserver_audit_event_total{job="kube-controller-manager"})
                          for: 15m
                          labels:
                            severity: critical
                  alertmanager:
                    alertmanagerSpec:
                      replicas: 1
                  grafana:
                    initChownData:
                      enabled: false
                        
                    persistence: 
                      enabled: true
                      storageClassName: openebs-hostpath 
                    ingress:
                      enabled: true
                      ingressClassName: cilium
                      hosts:
                        - grafana-k0s.lab.smig.tech

              - name: cert-manager
                chartName: cert-manager/cert-manager
                version: "v1.17.2"
                order: 2
                namespace: cert-manager
                values: |
                  crds:
                    enabled: true

              - name: cilium
                chartName: cilium/cilium
                namespace: kube-system
                version: "1.18.0-pre.3"
                order: 1
                values: |
                  hubble:
                    enabled: true
                  cluster:
                    name: k0s-cluster
                  envoy:
                    enabled: true
                    image:
                      digest: sha256:bb73643e4b8c95d852bf25fc0e2f44e6d77617a809b63b119aba9edc001f4ea4
                      repository: git.thecodedom.com/smig/cilium-envoy
                      tag: latest
                  k8sServiceHost: k1.lab.smig.tech
                  k8sServicePort: 6443
                  kubeProxyReplacement: true
                  operator:
                    replicas: 1
                  routingMode: tunnel
                  tunnelProtocol: vxlan
                  nodeIPAM:
                    enabled: true
                  defaultLBServiceIPAM: nodeipam
                  gatewayAPI:
                    enabled: true
                  ingressController:
                    enabled: true
                    enforceHttps: false
                    loadbalancerMode: shared
                    service:
                      externalTrafficPolicy: Cluster
  options:
    wait:
      enabled: true
    drain:
      enabled: true
      gracePeriod: 2m0s
      timeout: 5m0s
      force: true
      ignoreDaemonSets: true
      deleteEmptyDirData: true
      podSelector: ""
      skipWaitForDeleteTimeout: 0s
    concurrency:
      limit: 30
      workerDisruptionPercent: 10
      uploads: 5
    evictTaint:
      enabled: false
      taint: k0sctl.k0sproject.io/evict=true
      effect: NoExecute
      controllerWorkers: false