fix: slim-dev monitoring handling (#383)

## Description

Fixes a problem where missing CRDs on slim-dev would throw any `Package`
CRs into a pending state without clear errors:
- Adds CRDs to slim-dev
- Add better logging around servicemonitors/errors
- Add CI to test slim-dev

## Type of change

- [x] Bug fix (non-breaking change which fixes an issue)
- [ ] New feature (non-breaking change which adds functionality)
- [ ] Other (security config, docs update, etc)

## Checklist before merging

- [ ] Test, docs, adr added or updated as needed
- [x] [Contributor Guide
Steps](https://github.com/defenseunicorns/uds-template-capability/blob/main/CONTRIBUTING.md)(https://github.com/defenseunicorns/uds-template-capability/blob/main/CONTRIBUTING.md#submitting-a-pull-request)
followed

.github/workflows/commitlint.yaml

@@ -4,7 +4,7 @@ on:
   pull_request:
     branches: [main]
     # milestoned is added here as a workaround for release-please not triggering PR workflows (PRs should be added to a milestone to trigger the workflow).
-    types: [milestoned, opened, reopened, synchronize]
+    types: [milestoned, opened, reopened, edited, synchronize]
 
 jobs:
   title_check:

.github/workflows/slim-dev-test.yaml

@@ -0,0 +1,52 @@
+name: Slim Dev
+
+# This workflow is triggered on pull requests
+on:
+  pull_request:
+    # milestoned is added here as a workaround for release-please not triggering PR workflows (PRs should be added to a milestone to trigger the workflow).
+    types: [milestoned, opened, reopened, synchronize]
+    paths:
+      - src/pepr/*
+      - src/keycloak/*
+      - src/istio/*
+      - src/prometheus-stack/*
+      - packages/slim-dev/*
+      - bundles/core-slim-dev/*
+      - .github/workflows/slim-dev*
+
+# Permissions for the GITHUB_TOKEN used by the workflow.
+permissions:
+  id-token: write # Needed for OIDC-related operations.
+  contents: read # Allows reading the content of the repository.
+  pull-requests: read # Allows reading pull request metadata.
+
+# Default settings for all run commands in the workflow jobs.
+defaults:
+  run:
+    shell: bash -e -o pipefail {0} # Ensures that scripts fail on error and pipefail is set.
+
+# Abort prior jobs in the same workflow / PR
+concurrency:
+  group: test-slim-dev-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  # This job runs the slim-dev bundle create/deploy process.
+  test:
+    name: Test
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout the code
+        uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
+      - name: Environment setup
+        uses: ./.github/actions/setup
+      - name: Deploy Slim Dev Bundle
+        run: uds run slim-dev
+      - name: Debug Output
+        if: ${{ always() }}
+        uses: ./.github/actions/debug-output
+      - name: Save logs
+        if: always()
+        uses: ./.github/actions/save-logs
+        with:
+          suffix: -slim-dev

packages/slim-dev/zarf.yaml

@@ -8,6 +8,12 @@ metadata:
   # x-release-please-end
 
 components:
+  # CRDs
+  - name: prometheus-operator-crds
+    required: true
+    import:
+      path: ../../src/prometheus-stack
+
   # Istio
   - name: istio-controlplane
     required: true

src/keycloak/common/zarf.yaml

@@ -11,3 +11,14 @@ components:
         namespace: keycloak
         version: 24.0.3
         localPath: ../chart
+    actions:
+      onDeploy:
+        after:
+          - description: Validate Keycloak Package
+            maxTotalSeconds: 300
+            wait:
+              cluster:
+                kind: Packages
+                name: keycloak
+                namespace: keycloak
+                condition: "'{.status.phase}'=Ready"

src/pepr/operator/controllers/monitoring/service-monitor.ts

@@ -14,36 +14,46 @@ export async function serviceMonitor(pkg: UDSPackage, namespace: string) {
   const pkgName = pkg.metadata!.name!;
   const generation = (pkg.metadata?.generation ?? 0).toString();
 
+  Log.debug(`Reconciling ServiceMonitors for ${pkgName}`);
+
   // Get the list of monitored services
   const monitorList = pkg.spec?.monitor ?? [];
 
   // Create a list of generated ServiceMonitors
   const payloads: Prometheus.ServiceMonitor[] = [];
 
-  for (const monitor of monitorList) {
-    const payload = generateServiceMonitor(pkg, monitor, namespace, pkgName, generation);
-
-    // Apply the VirtualService and force overwrite any existing policy
-    await K8s(Prometheus.ServiceMonitor).Apply(payload, { force: true });
-
-    payloads.push(payload);
-  }
-
-  // Get all related ServiceMonitors in the namespace
-  const serviceMonitors = await K8s(Prometheus.ServiceMonitor)
-    .InNamespace(namespace)
-    .WithLabel("uds/package", pkgName)
-    .Get();
-
-  // Find any orphaned VirtualServices (not matching the current generation)
-  const orphanedSM = serviceMonitors.items.filter(
-    sm => sm.metadata?.labels?.["uds/generation"] !== generation,
-  );
-
-  // Delete any orphaned VirtualServices
-  for (const sm of orphanedSM) {
-    Log.debug(sm, `Deleting orphaned ServiceMonitor ${sm.metadata!.name}`);
-    await K8s(Prometheus.ServiceMonitor).Delete(sm);
+  try {
+    for (const monitor of monitorList) {
+      const payload = generateServiceMonitor(pkg, monitor, namespace, pkgName, generation);
+
+      Log.debug(payload, `Applying ServiceMonitor ${payload.metadata?.name}`);
+
+      // Apply the ServiceMonitor and force overwrite any existing policy
+      await K8s(Prometheus.ServiceMonitor).Apply(payload, { force: true });
+
+      payloads.push(payload);
+    }
+
+    // Get all related ServiceMonitors in the namespace
+    const serviceMonitors = await K8s(Prometheus.ServiceMonitor)
+      .InNamespace(namespace)
+      .WithLabel("uds/package", pkgName)
+      .Get();
+
+    // Find any orphaned ServiceMonitors (not matching the current generation)
+    const orphanedSM = serviceMonitors.items.filter(
+      sm => sm.metadata?.labels?.["uds/generation"] !== generation,
+    );
+
+    // Delete any orphaned ServiceMonitors
+    for (const sm of orphanedSM) {
+      Log.debug(sm, `Deleting orphaned ServiceMonitor ${sm.metadata!.name}`);
+      await K8s(Prometheus.ServiceMonitor).Delete(sm);
+    }
+  } catch (err) {
+    throw new Error(
+      `Failed to process ServiceMonitors for ${pkgName}, cause: ${JSON.stringify(err)}`,
+    );
   }
 
   // Return the list of monitor names