feat: Custom init package for Nutanix CSI driver (#42)

feat: Replace ceph with nutanix-csi for persistent storage

README.md

@@ -102,13 +102,23 @@ These are the default bucket names. Gitlab allows you to add a suffix in your `u
 
 > NOTE: All database and object storage credentials must be provided via username and password in the uds-config.
 
+**Storage**
+
+This bundle utilizes the Nutanix CSI Helm chart for persistent storage. Before the bundle can be deployed the following needs to be configured:
+* Prism Element user and password for the CSI provider to connect to Prism Element. Username, password, and Prism Element IP/Hostname will need passed to uds-config.yaml.
+* Nutanix Storage Container for RWO persistent volumes. Can either be a new container configured specifically for cluster storage, or an existing container depending on your needs/desires. Storage container name will need passed to uds-config.yaml.
+* Nutanix File Server configured to use for RWX persistent volumes. Make sure to configure the DNS records that it asks you to make. File Server name as it appears in Prism Element will need passed to uds-config.yaml.
+
+> NOTE: User/password and Nutanix File server must be configured in Prism Element not Prism Central.
+
 ### Configuration
 Deployment configuration is managed via a `uds-config.yaml` file in the deployment directory. Some values in the configuration will be sensitive, **we do not recommend checking this into source control in its entierty**. Best practice would involve either storing the configuration in an external secrets manager (like Vault), or managing deployments via CD and generating the config file dynamically at deploy time using CD managed secrets.
 
 For demonstration purposes, you can setup a local configfile as follows:
 * Copy an example configuration from [config/uds-config.yaml](config/uds-config.yaml) to your working directory
 * Update the config according to your environment taking care to set:
   * domain variables
+  * init variables for Nutanix csi
   * certificate values
   * bucket names and credentials
   * database names and credentials

bundles/uds-core-swf/uds-bundle.yaml

@@ -11,20 +11,10 @@ metadata:
 packages:
   # Zarf init
   - name: init
-    repository: ghcr.io/defenseunicorns/uds-capability/rook-ceph/init
+    path: ../../build
     optionalComponents:
       - git-server
-    ref: v0.33.0-0.2.7
-    overrides:
-      rook-ceph-cluster:
-        rook-ceph-cluster:
-          variables:
-            - path: cephClusterSpec.resources.osd.requests.memory
-              name: CEPH_OSD_MEM_REQUESTS
-            - path: cephClusterSpec.resources.osd.limits.memory
-              name: CEPH_OSD_MEM_LIMITS
-            - path: toolbox.enabled
-              name: ENABLE_CEPH_TOOLBOX
+    ref: v0.33.0
 
   # Namespace pre-reqs for swf capabilities
   - name: software-factory-namespaces
@@ -135,7 +125,7 @@ packages:
             - path: "persistence.accessMode"
               value: "ReadWriteMany"
             - path: "persistence.storageClassName"
-              value: "ceph-filesystem"
+              value: "nutanix-dynamicfile"
           variables:
             - name: KEYCLOAK_DB_USERNAME
               description: "keycloak database username"

config/uds-config.yaml

@@ -7,9 +7,11 @@ shared:
 variables:
   init:
     REGISTRY_HPA_ENABLE: false
-    CEPH_OSD_MEM_REQUESTS: "4Gi"
-    CEPH_OSD_MEM_LIMITS: "4Gi"
-    ENABLE_CEPH_TOOLBOX: "true"
+    PRISM_ENDPOINT: "PRISM element IP address"
+    PRISM_USERNAME: "csi-user-prism-element-user"
+    PRISM_PASSWORD: "csi-user-passoword"
+    STORAGE_CONTAINER: "nutanix-storage-container"
+    DYNAMIC_FILE_STORE_NAME: "nutanix-file-server-name"
   metallb:
     # Replace with a valid IP address range
     IP_ADDRESS_POOL: "10.0.0.10-10.0.0.20"
@@ -108,7 +110,6 @@ variables:
     JIRA_DB_USERNAME: "postgres"
     JIRA_LOCAL_HOME_ENABLED: "true"
     JIRA_LOCAL_HOME_SIZE: "128Gi"
-    JIRA_RWO_STORAGE_CLASS: "ceph-block"
     JIRA_DB_ENDPOINT: "jira-pg.replace.with.db.url"
   confluence-database-secret:
     CONFLUENCE_DB_PASSWORD: "replace-me-db-passwords"
@@ -117,7 +118,6 @@ variables:
     CONFLUENCE_DB_USERNAME: "postgres"
     CONFLUENCE_LOCAL_HOME_ENABLED: "true"
     CONFLUENCE_LOCAL_HOME_SIZE: "128Gi"
-    CONFLUENCE_RWO_STORAGE_CLASS: "ceph-block"
     CONFLUENCE_DB_ENDPOINT: "confluence-pg.replace.with.db.url"
   mattermost:
     ACCESS_KEY: "replace-me-object-store-access-key"

packages/additional-manifests/pepr-policy-exemptions/nutanix-csi-exemptions.yaml

@@ -0,0 +1,26 @@
+apiVersion: uds.dev/v1alpha1
+kind: Exemption
+metadata:
+  name: nutanix-csi
+  namespace: uds-policy-exemptions
+spec:
+  exemptions:
+    - policies:
+        - DisallowHostNamespaces
+        - DisallowPrivileged
+        - DisallowSELinuxOptions
+        - DropAllCapabilities
+        - RequireNonRootUser
+        - RestrictCapabilities
+        - RestrictHostPathWrite
+        - RestrictHostPorts
+        - RestrictProcMount
+        - RestrictSeccomp
+        - RestrictSELinuxType
+        - RestrictVolumeTypes
+      matcher:
+        namespace: ntnx-system
+        name: "^nutanix-csi-*"
+        kind: pod
+      title: "nutanix-csi exemptions"
+      description: "Nutanix CSI needs exemptions"

packages/additional-manifests/zarf.yaml

@@ -29,7 +29,7 @@ components:
     manifests:
       - name: pepr-policy-exemptions
         files:
-          - pepr-policy-exemptions/rook-ceph-exemptions.yaml
+          - pepr-policy-exemptions/nutanix-csi-exemptions.yaml
           - pepr-policy-exemptions/metallb-exemptions.yaml
   - name: mattermost-ca-secret
     required: true

packages/init/nutanix-dynamicfile.yaml

@@ -0,0 +1,22 @@
+allowVolumeExpansion: true
+apiVersion: storage.k8s.io/v1
+kind: StorageClass
+metadata:
+  name: nutanix-dynamicfile
+parameters:
+  csi.storage.k8s.io/controller-expand-secret-name: ntnx-secret
+  csi.storage.k8s.io/controller-expand-secret-namespace: ntnx-system
+  csi.storage.k8s.io/node-publish-secret-name: ntnx-secret
+  csi.storage.k8s.io/node-publish-secret-namespace: ntnx-system
+  csi.storage.k8s.io/provisioner-secret-name: ntnx-secret
+  csi.storage.k8s.io/provisioner-secret-namespace: ntnx-system
+  description: nutanix-dynamicfile
+  dynamicProv: ENABLED
+  nfsServerName: dev-fs
+  squashType: root-squash
+  storageType: NutanixFiles
+provisioner: csi.nutanix.com
+reclaimPolicy: Delete
+volumeBindingMode: Immediate
+mountOptions:
+  - nfsvers=4.1

packages/init/values/namespaces.yaml

@@ -0,0 +1,2 @@
+namespaces:
+  - name: ntnx-system

packages/init/values/nutanix-snapshot-values.yaml

@@ -0,0 +1,33 @@
+# Default values for nutanix-csi-snapshot.
+# This is a YAML-formatted file.
+# Declare variables to be passed into your templates.
+
+# Global Settings for all pods
+
+nodeSelector: {}
+tolerations: []
+imagePullPolicy: IfNotPresent
+
+controller:
+  replicas: 2
+  image: ###ZARF_REGISTRY###/sig-storage/snapshot-controller
+  nodeSelector: {}
+  tolerations: []
+
+validationWebHook:
+  replicas: 2
+  timeout: 2
+  failurePolicy: Fail
+  image: ###ZARF_REGISTRY###/sig-storage/snapshot-validation-webhook
+  nodeSelector: {}
+  tolerations: []
+
+tls:
+  # Where to get the cert for the webhook. - "generate, secret"
+  source: generate
+  # Allow to renew self-signed generated certificate
+  renew: false
+  # Name of the secret where certificate are stored
+  secretName: "csi-snapshot-validation-webhook-cert"
+  # Validity of certificate when generated by Helm
+  validityDuration: 3650

packages/init/values/nutanix-storage-values.yaml

@@ -0,0 +1,161 @@
+# Default values for nutanix-csi-storage.
+# This is a YAML-formatted file.
+# Declare variables to be passed into your templates.
+
+# parameters
+
+# Legacy mode
+#
+# if legacy set to true we keep the old reverse domain notation for CSI driver name (com.nutanix.csi).
+# need to be set to true only if upgrade and initialy installed with helm package before 2.2.x
+legacy: false
+
+# Openshift settings
+#
+
+openshift:
+  scc: false
+  masterIscsiConfig: false
+  workerIscsiConfig: false
+
+# kubeletDir allows overriding the host location of kubelet's internal state.
+kubeletDir: "/var/lib/kubelet"
+
+# Global Settings for all pods
+
+nodeSelector: {}
+tolerations: []
+imagePullPolicy: IfNotPresent
+
+# Storage Class settings
+#
+# choose for which mode (Volume, File, Dynamic File) storageclass need to be created
+volumeClass: true
+volumeClassName: "nutanix-volume"
+volumeClassRetention: "Delete"
+# volumeClassDescription: ""
+# volumeClassAnnotations: {}
+# volumeClassLabels: {}
+
+volumeSnapshotClassName: "nutanix-snapshot-class"
+# volumeSnapshotClassAnnotations: {}
+# volumeSnapshotClassLabels: {}
+
+fileClass: false
+fileClassName: "nutanix-file"
+fileClassRetention: "Delete"
+# fileClassAnnotations: {}
+# fileClassLabels: {}
+
+dynamicFileClass: false
+dynamicFileClassName: "nutanix-dynamicfile"
+dynamicFileClassRetention: "Delete"
+# dynamicFileClassDescription: ""
+# dynamicFileClassAnnotations: {}
+# dynamicFileClassLabels: {}
+
+# Default Storage Class settings
+#
+# Decide wich storageclass will be the default
+# value are: none, volume, file, dynfile
+defaultStorageClass: volume
+
+# Nutanix Prism Elements settings
+#
+# Allow dynamic creation of Volumes and Fileshare
+# needed if volumeClass or dynamicFileClass is set to true
+
+## fully qualified domain name (FQDN) or the cluster virtual IP address (if one is not configured, use the virtual IP address of any Controller VM in the cluster).
+prismEndPoint: ###ZARF_VAR_PRISM_ENDPOINT###
+prismPort: 9440
+
+## username of the Prism Element (PE) cluster admin
+username: ###ZARF_VAR_PRISM_USERNAME###
+
+## password for the PE cluster admin.
+password: ###ZARF_VAR_PRISM_PASSWORD###
+
+## secret name that stores Nutanix cluster credentials
+secretName: ntnx-secret
+
+## Nutanix Prism Elements Existing Secret
+#
+
+# if set to false a new secret will not be created
+createSecret: true
+
+## Volumes Settings
+#
+
+## name of the Nutanix storage container
+storageContainer: ###ZARF_VAR_STORAGE_CONTAINER###
+
+## Filesystem used in volume PV
+fsType: xfs
+
+lvmVolume: false
+lvmDisks: 4
+
+networkSegmentation: false
+
+# Files Settings
+#
+
+## NFS server fully qualified domain name (FQDN) or IP address
+# fileHost:
+
+## path for the NFS share
+# filePath:
+
+# Dynamic Files Settings
+#
+
+## name of the file server. (As seen in the Prism Interface)
+fileServerName: ###ZARF_VAR_DYNAMIC_FILE_STORE_NAME###
+
+# Squash-type for dynamic files.
+# Values are: none, root-squash, all-squash. Default is root-squash
+dynamicFileSquashType: root-squash
+
+# Volume metrics and CSI operations metrics configuration
+#
+
+servicemonitor:
+  enabled: false
+  labels:
+    # This should match the serviceMonitorSelector logic configured
+    # on the prometheus.
+    k8s-app: csi-driver
+
+# Pod pecific Settings
+#
+
+controller:
+  replicas: 2
+  image: ###ZARF_REGISTRY###/karbon/ntnx-csi:v2.6.6
+  nodeSelector: {}
+  tolerations: []
+
+node:
+  image: ###ZARF_REGISTRY###/karbon/ntnx-csi:v2.6.6
+  nodeSelector: {}
+  tolerations: []
+
+sidecars:
+  registrar:
+    image: ###ZARF_REGISTRY###/sig-storage/csi-node-driver-registrar:v2.9.1
+  provisioner:
+    image: ###ZARF_REGISTRY###/sig-storage/csi-provisioner:v3.6.2
+    imageLegacy: ###ZARF_REGISTRY###/sig-storage/csi-provisioner:v2.2.2
+  snapshotter:
+    image: ###ZARF_REGISTRY###/sig-storage/csi-snapshotter:v6.3.2
+    imageBeta: ###ZARF_REGISTRY###/sig-storage/csi-snapshotter:v3.0.3
+  resizer:
+    image: ###ZARF_REGISTRY###/sig-storage/csi-resizer:v1.9.2
+  livenessprobe:
+    image: ###ZARF_REGISTRY###/sig-storage/livenessprobe:v2.11.0
+
+# Used for deployment test in kind cluster
+#
+
+kindtest: false

packages/init/values/registry-values.yaml

@@ -0,0 +1,8 @@
+persistence:
+  enabled: ###ZARF_VAR_UPGRADE_PERSISTENCE###
+
+autoscaling:
+  enabled: false
+
+extraEnvVars:
+  ###ZARF_VAR_UPGRADE_ENV_VARS###