RemoveItem is too fast for store - policies.Store.removeItem(p);

[cmwylie19]

Environment

uds-core/pepr.ts

Line 35 in 7cf9c4c

policies.Store.removeItem(p);

Using the store in an atomic way is creating 422 errors when removing items. We should consider Store.clear() or something less transactional

reported by @zack-is-cool who also said everything seemed to be working fine but was noticing the 422 in the logs.

[zack-is-cool]

New deployment of uds-core:0.22.1-registry1 on a fresh EKS cluster

k logs -n pepr-system pepr-uds-core-watcher-55f6974d99-vv4lm:

{"level":30,"time":1717786578291,"pid":1,"hostname":"pepr-uds-core-watcher-55f6974d99-vv4lm","msg":"Pepr Controller (v0.0.0-development)"}
{"level":30,"time":1717786578354,"pid":1,"hostname":"pepr-uds-core-watcher-55f6974d99-vv4lm","msg":"Applying the Pepr Store CRD if it doesn't exist"}
{"level":30,"time":1717786578503,"pid":1,"hostname":"pepr-uds-core-watcher-55f6974d99-vv4lm","msg":"Loading module /app/load/module-37b63af8e3363f0cbaf741d871cec5371c02cd285632b085a9aca57c6955f9ad.js.gz"}
{"level":30,"time":1717786578506,"pid":1,"hostname":"pepr-uds-core-watcher-55f6974d99-vv4lm","msg":"File hash matches, running module"}
{"level":30,"time":1717786581173,"pid":17,"hostname":"pepr-uds-core-watcher-55f6974d99-vv4lm","msg":"Capability istio registered"}
{"level":20,"time":1717786581174,"pid":17,"hostname":"pepr-uds-core-watcher-55f6974d99-vv4lm","name":"istio","description":"UDS Core Capability for Istio service mesh."}
{"level":20,"time":1717786581175,"pid":17,"hostname":"pepr-uds-core-watcher-55f6974d99-vv4lm","msg":"Add label filter batch.kubernetes.io/job-name="}
{"level":20,"time":1717786581175,"pid":17,"hostname":"pepr-uds-core-watcher-55f6974d99-vv4lm","msg":"Add label filter service.istio.io/canonical-name="}
{"level":30,"time":1717786581175,"pid":17,"hostname":"pepr-uds-core-watcher-55f6974d99-vv4lm","msg":"Watch Action configured for CREATEORUPDATE"}
{"level":30,"time":1717786581175,"pid":17,"hostname":"pepr-uds-core-watcher-55f6974d99-vv4lm","labels":{"batch.kubernetes.io/job-name":"","service.istio.io/canonical-name":""},"msg":"istio: V1Pod"}
{"level":20,"time":1717786581175,"pid":17,"hostname":"pepr-uds-core-watcher-55f6974d99-vv4lm","msg":"async e=>{if(T.Log.info(e,`Processing Pod ${e.metadata?.namespace}/${e.metadata?.name} for istio job termination`),!e.metadata?.name||!e.metadata.namespace){T.Log.error(e,\"Invalid Pod definition\");return}let{name:t,namespace:o}=e.metadata,r=`${o}/${t}`;if(!Se.has(r)&&e.status?.phase==\"Running\"){if(!e.status.containerStatuses){T.Log.error(e,\"Invalid container status in Pod\");return}if(e.status.containerStatuses.filter(n=>n.name!=\"istio-proxy\").every(n=>n.state?.terminated&&n.state.terminated.exitCode==0)){Se.add(r),T.Log.info(`Attempting to terminate sidecar for ${r}`);try{let n=new ee.KubeConfig;n.loadFromDefault(),await new ee.Exec(n).exec(o,t,\"istio-proxy\",[\"pilot-agent\",\"request\",\"POST\",\"/quitquitquit\"],null,null,process.stdin,!0),T.Log.info(`Terminated sidecar for ${r}`)}catch(n){T.Log.error({err:n},`Failed to terminate the sidecar for ${r}`),Se.delete(r)}}}}"}
{"level":30,"time":1717786581175,"pid":17,"hostname":"pepr-uds-core-watcher-55f6974d99-vv4lm","msg":"Capability uds-core-operator registered"}
{"level":20,"time":1717786581175,"pid":17,"hostname":"pepr-uds-core-watcher-55f6974d99-vv4lm","name":"uds-core-operator","description":"The UDS Operator is responsible for managing the lifecycle of UDS resources"}
{"level":30,"time":1717786581175,"pid":17,"hostname":"pepr-uds-core-watcher-55f6974d99-vv4lm","domain":"uds.dev","isSingleTest":false,"allowAllNSExemptions":false,"msg":"Loaded UDS Config"}
{"level":20,"time":1717786581185,"pid":17,"hostname":"pepr-uds-core-watcher-55f6974d99-vv4lm","msg":"Add namespaces filter default"}
{"level":20,"time":1717786581186,"pid":17,"hostname":"pepr-uds-core-watcher-55f6974d99-vv4lm","msg":"Add name filter kubernetes"}
{"level":30,"time":1717786581186,"pid":17,"hostname":"pepr-uds-core-watcher-55f6974d99-vv4lm","msg":"Reconcile Action configured for CREATEORUPDATE"}
{"level":30,"time":1717786581186,"pid":17,"hostname":"pepr-uds-core-watcher-55f6974d99-vv4lm","name":"kubernetes","namespaces":["default"],"msg":"uds-core-operator: V1EndpointSlice"}
{"level":20,"time":1717786581186,"pid":17,"hostname":"pepr-uds-core-watcher-55f6974d99-vv4lm","msg":"async function et(e){try{g.Log.debug(\"Processing watch for endpointslices, getting k8s service for updating API server CIDR\");let t=await(0,g.K8s)(g.kind.Service).InNamespace(\"default\").Get(\"kubernetes\");await Ce(e,t)}catch(t){let o=\"Failed to update network policies from endpoint slice watch\";g.Log.error({err:t},o)}}"}
{"level":20,"time":1717786581186,"pid":17,"hostname":"pepr-uds-core-watcher-55f6974d99-vv4lm","msg":"Add namespaces filter default"}
{"level":20,"time":1717786581186,"pid":17,"hostname":"pepr-uds-core-watcher-55f6974d99-vv4lm","msg":"Add name filter kubernetes"}
{"level":30,"time":1717786581186,"pid":17,"hostname":"pepr-uds-core-watcher-55f6974d99-vv4lm","msg":"Reconcile Action configured for CREATEORUPDATE"}
{"level":30,"time":1717786581186,"pid":17,"hostname":"pepr-uds-core-watcher-55f6974d99-vv4lm","name":"kubernetes","namespaces":["default"],"msg":"uds-core-operator: V1Service"}
{"level":20,"time":1717786581186,"pid":17,"hostname":"pepr-uds-core-watcher-55f6974d99-vv4lm","msg":"async function tt(e){try{g.Log.debug(\"Processing watch for api service, getting endpoint slices for updating API server CIDR\");let t=await(0,g.K8s)(g.kind.EndpointSlice).InNamespace(\"default\").Get(\"kubernetes\");await Ce(t,e)}catch(t){let o=\"Failed to update network policies from api service watch\";g.Log.error({err:t},o)}}"}
{"level":30,"time":1717786581186,"pid":17,"hostname":"pepr-uds-core-watcher-55f6974d99-vv4lm","msg":"Watch Action configured for DELETE"}
{"level":30,"time":1717786581186,"pid":17,"hostname":"pepr-uds-core-watcher-55f6974d99-vv4lm","msg":"uds-core-operator: Package"}
{"level":20,"time":1717786581186,"pid":17,"hostname":"pepr-uds-core-watcher-55f6974d99-vv4lm","msg":"async e=>{await Ke(e),await De(e,[])}"}
{"level":30,"time":1717786581186,"pid":17,"hostname":"pepr-uds-core-watcher-55f6974d99-vv4lm","msg":"Reconcile Action configured for CREATEORUPDATE"}
{"level":30,"time":1717786581186,"pid":17,"hostname":"pepr-uds-core-watcher-55f6974d99-vv4lm","msg":"uds-core-operator: Package"}
{"level":20,"time":1717786581186,"pid":17,"hostname":"pepr-uds-core-watcher-55f6974d99-vv4lm","msg":"async function xt(e){let t=e.metadata,{namespace:o,name:r}=t;if(fe.Log.info(e,`Processing Package ${o}/${r}`),ct(e)){fe.Log.info(e,`Skipping Package ${o}/${r}`);return}ue(e);try{await ge(e,{phase:\"Pending\"});let i=await ht(e,o),n=[];await Fe(e),n=await dt(e,o);let s=[];R.isSingleTest?fe.Log.warn(`Running in single test mode, skipping ${r} ServiceMonitors.`):s=await mt(e,o);let l=await We(e);await ge(e,{phase:\"Ready\",ssoClients:l,endpoints:n,monitors:s,networkPolicyCount:i.length,observedGeneration:t.generation,retryAttempt:0}),Ie.add(e.metadata.uid)}catch(i){pt(i,e)}}"}
{"level":30,"time":1717786581187,"pid":17,"hostname":"pepr-uds-core-watcher-55f6974d99-vv4lm","msg":"Capability uds-core-policies registered"}
{"level":20,"time":1717786581187,"pid":17,"hostname":"pepr-uds-core-watcher-55f6974d99-vv4lm","name":"uds-core-policies","description":"Collection of core validation policies for Pods, ConfigMaps, and other Kubernetes resources."}
{"level":30,"time":1717786581187,"pid":17,"hostname":"pepr-uds-core-watcher-55f6974d99-vv4lm","msg":"Capability prometheus registered"}
{"level":20,"time":1717786581187,"pid":17,"hostname":"pepr-uds-core-watcher-55f6974d99-vv4lm","name":"prometheus","description":"UDS Core Capability for the Prometheus stack."}
{"level":30,"time":1717786581459,"pid":17,"hostname":"pepr-uds-core-watcher-55f6974d99-vv4lm","msg":"Package CRD registered"}
{"level":30,"time":1717786581462,"pid":17,"hostname":"pepr-uds-core-watcher-55f6974d99-vv4lm","msg":"Registering store for uds-core-operator"}
{"level":30,"time":1717786581463,"pid":17,"hostname":"pepr-uds-core-watcher-55f6974d99-vv4lm","msg":"Registering store for uds-core-policies"}
{"level":30,"time":1717786581463,"pid":17,"hostname":"pepr-uds-core-watcher-55f6974d99-vv4lm","msg":"Registering store for istio"}
{"level":30,"time":1717786581463,"pid":17,"hostname":"pepr-uds-core-watcher-55f6974d99-vv4lm","msg":"Registering store for prometheus"}
{"level":20,"time":1717786581471,"pid":17,"hostname":"pepr-uds-core-watcher-55f6974d99-vv4lm","msg":"Clearing legacy pepr store exemption entries..."}
{"level":30,"time":1717786581471,"pid":17,"hostname":"pepr-uds-core-watcher-55f6974d99-vv4lm","msg":"Server listening on port 3000"}
{"level":30,"time":1717786582128,"pid":17,"hostname":"pepr-uds-core-watcher-55f6974d99-vv4lm","msg":"Pepr store not found, creating..."}
{"level":20,"time":1717786582129,"pid":17,"hostname":"pepr-uds-core-watcher-55f6974d99-vv4lm","data":{"kind":"Status","apiVersion":"v1","metadata":{},"status":"Failure","message":"peprstores.pepr.dev \"pepr-uds-core-store\" not found","reason":"NotFound","details":{"name":"pepr-uds-core-store","group":"pepr.dev","kind":"peprstores"},"code":404},"ok":false,"status":404,"statusText":"Not Found"}
{"level":20,"time":1717786582171,"pid":17,"hostname":"pepr-uds-core-watcher-55f6974d99-vv4lm","apiVersion":"pepr.dev/v1","data":{"__pepr_do_not_delete__":"k-thx-bye"},"kind":"PeprStore","metadata":{"creationTimestamp":"2024-06-07T18:56:22Z","generation":1,"managedFields":[{"apiVersion":"pepr.dev/v1","fieldsType":"FieldsV1","fieldsV1":{"f:data":{"f:__pepr_do_not_delete__":{}}},"manager":"pepr","operation":"Apply","time":"2024-06-07T18:56:22Z"}],"name":"pepr-uds-core-store","namespace":"pepr-system","resourceVersion":"9265","uid":"f473b7cd-00e7-42cf-bb84-044986513b6c"},"msg":"Pepr Store update"}
{"level":20,"time":1717786582175,"pid":17,"hostname":"pepr-uds-core-watcher-55f6974d99-vv4lm","msg":"Pepr store data received"}
{"level":20,"time":1717786582175,"pid":17,"hostname":"pepr-uds-core-watcher-55f6974d99-vv4lm","msg":"Pepr store data received"}
{"level":20,"time":1717786582176,"pid":17,"hostname":"pepr-uds-core-watcher-55f6974d99-vv4lm","msg":"Pepr store data received"}
{"level":20,"time":1717786582176,"pid":17,"hostname":"pepr-uds-core-watcher-55f6974d99-vv4lm","msg":"Pepr store data received"}
{"level":20,"time":1717786582177,"pid":17,"hostname":"pepr-uds-core-watcher-55f6974d99-vv4lm","watchCfg":{"retryMax":5,"retryDelaySec":5,"resyncIntervalSec":300},"msg":"Effective WatchConfig"}
{"level":20,"time":1717786582177,"pid":17,"hostname":"pepr-uds-core-watcher-55f6974d99-vv4lm","watchCfg":{"retryMax":5,"retryDelaySec":5,"resyncIntervalSec":300,"relistIntervalSec":1800},"msg":"Effective WatchConfig"}
{"level":20,"time":1717786582178,"pid":17,"hostname":"pepr-uds-core-watcher-55f6974d99-vv4lm","watchCfg":{"retryMax":5,"retryDelaySec":5,"resyncIntervalSec":300,"relistIntervalSec":1800},"msg":"Effective WatchConfig"}
{"level":20,"time":1717786582178,"pid":17,"hostname":"pepr-uds-core-watcher-55f6974d99-vv4lm","watchCfg":{"retryMax":5,"retryDelaySec":5,"resyncIntervalSec":300,"relistIntervalSec":1800},"msg":"Effective WatchConfig"}
{"level":20,"time":1717786582179,"pid":17,"hostname":"pepr-uds-core-watcher-55f6974d99-vv4lm","watchCfg":{"retryMax":5,"retryDelaySec":5,"resyncIntervalSec":300,"relistIntervalSec":1800},"msg":"Effective WatchConfig"}
{"level":30,"time":1717786582179,"pid":17,"hostname":"pepr-uds-core-watcher-55f6974d99-vv4lm","msg":"✅ Controller startup complete"}
{"level":20,"time":1717786582203,"pid":17,"hostname":"pepr-uds-core-watcher-55f6974d99-vv4lm","msg":"Watch event list received. {\n  \"kind\": \"EndpointSliceList\",\n  \"apiVersion\": \"discovery.k8s.io/v1\",\n  \"metadata\": {\n    \"resourceVersion\": \"9265\"\n  },\n  \"items\": [\n    {\n      \"metadata\": {\n        \"name\": \"kubernetes\",\n        \"namespace\": \"default\",\n        \"uid\": \"cffba2fb-484f-4c85-af21-30cd5ca2e23b\",\n        \"resourceVersion\": \"2064\",\n        \"generation\": 2,\n        \"creationTimestamp\": \"2024-06-07T18:28:17Z\",\n        \"labels\": {\n          \"kubernetes.io/service-name\": \"kubernetes\"\n        },\n        \"managedFields\": [\n          {\n            \"manager\": \"kube-apiserver\",\n            \"operation\": \"Update\",\n            \"apiVersion\": \"discovery.k8s.io/v1\",\n            \"time\": \"2024-06-07T18:33:49Z\",\n            \"fieldsType\": \"FieldsV1\",\n            \"fieldsV1\": {\n              \"f:addressType\": {},\n              \"f:endpoints\": {},\n              \"f:metadata\": {\n                \"f:labels\": {\n                  \".\": {},\n                  \"f:kubernetes.io/service-name\": {}\n                }\n              },\n              \"f:ports\": {}\n            }\n          }\n        ]\n      },\n      \"addressType\": \"IPv4\",\n      \"endpoints\": [\n        {\n          \"addresses\": [\n            \"10.200.14.3\"\n          ],\n          \"conditions\": {\n            \"ready\": true\n          }\n        },\n        {\n          \"addresses\": [\n            \"10.200.23.200\"\n          ],\n          \"conditions\": {\n            \"ready\": true\n          }\n        }\n      ],\n      \"ports\": [\n        {\n          \"name\": \"https\",\n          \"protocol\": \"TCP\",\n          \"port\": 443\n        }\n      ]\n    }\n  ]\n}."}
{"level":20,"time":1717786582203,"pid":17,"hostname":"pepr-uds-core-watcher-55f6974d99-vv4lm","metadata":{"name":"kubernetes","namespace":"default","uid":"cffba2fb-484f-4c85-af21-30cd5ca2e23b","resourceVersion":"2064","generation":2,"creationTimestamp":"2024-06-07T18:28:17Z","labels":{"kubernetes.io/service-name":"kubernetes"},"managedFields":[{"manager":"kube-apiserver","operation":"Update","apiVersion":"discovery.k8s.io/v1","time":"2024-06-07T18:33:49Z","fieldsType":"FieldsV1","fieldsV1":{"f:addressType":{},"f:endpoints":{},"f:metadata":{"f:labels":{".":{},"f:kubernetes.io/service-name":{}}},"f:ports":{}}}]},"addressType":"IPv4","endpoints":[{"addresses":["10.200.14.3"],"conditions":{"ready":true}},{"addresses":["10.200.23.200"],"conditions":{"ready":true}}],"ports":[{"name":"https","protocol":"TCP","port":443}],"msg":"Watch event ADDED received"}
{"level":20,"time":1717786582204,"pid":17,"hostname":"pepr-uds-core-watcher-55f6974d99-vv4lm","msg":"Enqueueing default/kubernetes"}
{"level":20,"time":1717786582204,"pid":17,"hostname":"pepr-uds-core-watcher-55f6974d99-vv4lm","msg":"Reconciling kubernetes"}
{"level":20,"time":1717786582204,"pid":17,"hostname":"pepr-uds-core-watcher-55f6974d99-vv4lm","msg":"Processing watch for endpointslices, getting k8s service for updating API server CIDR"}
{"level":20,"time":1717786582211,"pid":17,"hostname":"pepr-uds-core-watcher-55f6974d99-vv4lm","msg":"Watch event list received. {\n  \"apiVersion\": \"uds.dev/v1alpha1\",\n  \"items\": [],\n  \"kind\": \"PackageList\",\n  \"metadata\": {\n    \"continue\": \"\",\n    \"resourceVersion\": \"9265\"\n  }\n}."}
{"level":20,"time":1717786582260,"pid":17,"hostname":"pepr-uds-core-watcher-55f6974d99-vv4lm","msg":"Watch event list received. {\n  \"kind\": \"PodList\",\n  \"apiVersion\": \"v1\",\n  \"metadata\": {\n    \"resourceVersion\": \"9265\"\n  },\n  \"items\": []\n}."}
{"level":20,"time":1717786582265,"pid":17,"hostname":"pepr-uds-core-watcher-55f6974d99-vv4lm","msg":"Watch event list received. {\n  \"kind\": \"ServiceList\",\n  \"apiVersion\": \"v1\",\n  \"metadata\": {\n    \"resourceVersion\": \"9266\"\n  },\n  \"items\": [\n    {\n      \"metadata\": {\n        \"name\": \"kubernetes\",\n        \"namespace\": \"default\",\n        \"uid\": \"ebf0c4ab-c1b9-4a6c-9386-f64b71318c74\",\n        \"resourceVersion\": \"191\",\n        \"creationTimestamp\": \"2024-06-07T18:28:17Z\",\n        \"labels\": {\n          \"component\": \"apiserver\",\n          \"provider\": \"kubernetes\"\n        },\n        \"managedFields\": [\n          {\n            \"manager\": \"kube-apiserver\",\n            \"operation\": \"Update\",\n            \"apiVersion\": \"v1\",\n            \"time\": \"2024-06-07T18:28:17Z\",\n            \"fieldsType\": \"FieldsV1\",\n            \"fieldsV1\": {\n              \"f:metadata\": {\n                \"f:labels\": {\n                  \".\": {},\n                  \"f:component\": {},\n                  \"f:provider\": {}\n                }\n              },\n              \"f:spec\": {\n                \"f:clusterIP\": {},\n                \"f:internalTrafficPolicy\": {},\n                \"f:ipFamilyPolicy\": {},\n                \"f:ports\": {\n                  \".\": {},\n                  \"k:{\\\"port\\\":443,\\\"protocol\\\":\\\"TCP\\\"}\": {\n                    \".\": {},\n                    \"f:name\": {},\n                    \"f:port\": {},\n                    \"f:protocol\": {},\n                    \"f:targetPort\": {}\n                  }\n                },\n                \"f:sessionAffinity\": {},\n                \"f:type\": {}\n              }\n            }\n          }\n        ]\n      },\n      \"spec\": {\n        \"ports\": [\n          {\n            \"name\": \"https\",\n            \"protocol\": \"TCP\",\n            \"port\": 443,\n            \"targetPort\": 443\n          }\n        ],\n        \"clusterIP\": \"172.20.0.1\",\n        \"clusterIPs\": [\n          \"172.20.0.1\"\n        ],\n        \"type\": \"ClusterIP\",\n        \"sessionAffinity\": \"None\",\n        \"ipFamilies\": [\n          \"IPv4\"\n        ],\n        \"ipFamilyPolicy\": \"SingleStack\",\n        \"internalTrafficPolicy\": \"Cluster\"\n      },\n      \"status\": {\n        \"loadBalancer\": {}\n      }\n    }\n  ]\n}."}
{"level":20,"time":1717786582266,"pid":17,"hostname":"pepr-uds-core-watcher-55f6974d99-vv4lm","metadata":{"name":"kubernetes","namespace":"default","uid":"ebf0c4ab-c1b9-4a6c-9386-f64b71318c74","resourceVersion":"191","creationTimestamp":"2024-06-07T18:28:17Z","labels":{"component":"apiserver","provider":"kubernetes"},"managedFields":[{"manager":"kube-apiserver","operation":"Update","apiVersion":"v1","time":"2024-06-07T18:28:17Z","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:labels":{".":{},"f:component":{},"f:provider":{}}},"f:spec":{"f:clusterIP":{},"f:internalTrafficPolicy":{},"f:ipFamilyPolicy":{},"f:ports":{".":{},"k:{\"port\":443,\"protocol\":\"TCP\"}":{".":{},"f:name":{},"f:port":{},"f:protocol":{},"f:targetPort":{}}},"f:sessionAffinity":{},"f:type":{}}}}]},"spec":{"ports":[{"name":"https","protocol":"TCP","port":443,"targetPort":443}],"clusterIP":"172.20.0.1","clusterIPs":["172.20.0.1"],"type":"ClusterIP","sessionAffinity":"None","ipFamilies":["IPv4"],"ipFamilyPolicy":"SingleStack","internalTrafficPolicy":"Cluster"},"status":{"loadBalancer":{}},"msg":"Watch event ADDED received"}
{"level":20,"time":1717786582266,"pid":17,"hostname":"pepr-uds-core-watcher-55f6974d99-vv4lm","msg":"Enqueueing default/kubernetes"}
{"level":20,"time":1717786582266,"pid":17,"hostname":"pepr-uds-core-watcher-55f6974d99-vv4lm","msg":"Reconciling kubernetes"}
{"level":20,"time":1717786582266,"pid":17,"hostname":"pepr-uds-core-watcher-55f6974d99-vv4lm","msg":"Processing watch for api service, getting endpoint slices for updating API server CIDR"}
{"level":20,"time":1717786582272,"pid":17,"hostname":"pepr-uds-core-watcher-55f6974d99-vv4lm","msg":"Watch event list received. {\n  \"apiVersion\": \"uds.dev/v1alpha1\",\n  \"items\": [],\n  \"kind\": \"PackageList\",\n  \"metadata\": {\n    \"continue\": \"\",\n    \"resourceVersion\": \"9266\"\n  }\n}."}
{"level":20,"time":1717786582286,"pid":17,"hostname":"pepr-uds-core-watcher-55f6974d99-vv4lm","msg":"Watch event connect received. /apis/discovery.k8s.io/v1/endpointslices."}
{"level":20,"time":1717786582287,"pid":17,"hostname":"pepr-uds-core-watcher-55f6974d99-vv4lm","msg":"Watch event connect received. /apis/uds.dev/v1alpha1/packages."}
{"level":20,"time":1717786582294,"pid":17,"hostname":"pepr-uds-core-watcher-55f6974d99-vv4lm","msg":"Watch event connect received. /api/v1/pods."}
{"level":20,"time":1717786582297,"pid":17,"hostname":"pepr-uds-core-watcher-55f6974d99-vv4lm","msg":"Watch event connect received. /apis/uds.dev/v1alpha1/packages."}
{"level":20,"time":1717786582297,"pid":17,"hostname":"pepr-uds-core-watcher-55f6974d99-vv4lm","msg":"Watch event connect received. /api/v1/services."}
{"level":20,"time":1717786582358,"pid":17,"hostname":"pepr-uds-core-watcher-55f6974d99-vv4lm","msg":"Resetting pending promise and dequeuing"}
{"level":20,"time":1717786582358,"pid":17,"hostname":"pepr-uds-core-watcher-55f6974d99-vv4lm","msg":"No element, not dequeuing"}
{"level":20,"time":1717786582371,"pid":17,"hostname":"pepr-uds-core-watcher-55f6974d99-vv4lm","msg":"Resetting pending promise and dequeuing"}
{"level":20,"time":1717786582371,"pid":17,"hostname":"pepr-uds-core-watcher-55f6974d99-vv4lm","msg":"No element, not dequeuing"}
{"level":30,"time":1717786584147,"pid":17,"hostname":"pepr-uds-core-watcher-55f6974d99-vv4lm","msg":"Pepr store not found, creating..."}
{"level":20,"time":1717786584147,"pid":17,"hostname":"pepr-uds-core-watcher-55f6974d99-vv4lm","data":{"kind":"Status","apiVersion":"v1","metadata":{},"status":"Failure","message":"peprstores.pepr.dev \"pepr-uds-core-schedule\" not found","reason":"NotFound","details":{"name":"pepr-uds-core-schedule","group":"pepr.dev","kind":"peprstores"},"code":404},"ok":false,"status":404,"statusText":"Not Found"}
{"level":20,"time":1717786584181,"pid":17,"hostname":"pepr-uds-core-watcher-55f6974d99-vv4lm","apiVersion":"pepr.dev/v1","data":{"__pepr_do_not_delete__":"k-thx-bye"},"kind":"PeprStore","metadata":{"creationTimestamp":"2024-06-07T18:56:24Z","generation":1,"managedFields":[{"apiVersion":"pepr.dev/v1","fieldsType":"FieldsV1","fieldsV1":{"f:data":{"f:__pepr_do_not_delete__":{}}},"manager":"pepr","operation":"Apply","time":"2024-06-07T18:56:24Z"}],"name":"pepr-uds-core-schedule","namespace":"pepr-system","resourceVersion":"9280","uid":"768bd455-5c7f-4903-acb0-241315b813f3"},"msg":"Pepr Store update"}
{"level":30,"time":1717786584184,"pid":17,"hostname":"pepr-uds-core-watcher-55f6974d99-vv4lm","msg":"✅ Scheduling processed"}
{"level":20,"time":1717786586464,"pid":17,"hostname":"pepr-uds-core-watcher-55f6974d99-vv4lm","remove:/data/uds-core-policies-DisallowHostNamespaces":{"op":"remove","path":"/data/uds-core-policies-DisallowHostNamespaces"},"remove:/data/uds-core-policies-DisallowNodePortServices":{"op":"remove","path":"/data/uds-core-policies-DisallowNodePortServices"},"remove:/data/uds-core-policies-DisallowPrivileged":{"op":"remove","path":"/data/uds-core-policies-DisallowPrivileged"},"remove:/data/uds-core-policies-DisallowSELinuxOptions":{"op":"remove","path":"/data/uds-core-policies-DisallowSELinuxOptions"},"remove:/data/uds-core-policies-DropAllCapabilities":{"op":"remove","path":"/data/uds-core-policies-DropAllCapabilities"},"remove:/data/uds-core-policies-RequireNonRootUser":{"op":"remove","path":"/data/uds-core-policies-RequireNonRootUser"},"remove:/data/uds-core-policies-RestrictCapabilities":{"op":"remove","path":"/data/uds-core-policies-RestrictCapabilities"},"remove:/data/uds-core-policies-RestrictExternalNames":{"op":"remove","path":"/data/uds-core-policies-RestrictExternalNames"},"remove:/data/uds-core-policies-RestrictHostPathWrite":{"op":"remove","path":"/data/uds-core-policies-RestrictHostPathWrite"},"remove:/data/uds-core-policies-RestrictHostPorts":{"op":"remove","path":"/data/uds-core-policies-RestrictHostPorts"},"remove:/data/uds-core-policies-RestrictProcMount":{"op":"remove","path":"/data/uds-core-policies-RestrictProcMount"},"remove:/data/uds-core-policies-RestrictSELinuxType":{"op":"remove","path":"/data/uds-core-policies-RestrictSELinuxType"},"remove:/data/uds-core-policies-RestrictSeccomp":{"op":"remove","path":"/data/uds-core-policies-RestrictSeccomp"},"remove:/data/uds-core-policies-RestrictVolumeTypes":{"op":"remove","path":"/data/uds-core-policies-RestrictVolumeTypes"},"msg":"Sending updates to Pepr store"}
{"level":50,"time":1717786586480,"pid":17,"hostname":"pepr-uds-core-watcher-55f6974d99-vv4lm","data":{"kind":"Status","apiVersion":"v1","metadata":{},"status":"Failure","message":"the server rejected our request due to an error in our request","reason":"Invalid","details":{},"code":422},"ok":false,"status":422,"statusText":"Unprocessable Entity","msg":"Pepr store update failure"}
{"level":30,"time":1717786587418,"pid":17,"hostname":"pepr-uds-core-watcher-55f6974d99-vv4lm","method":"GET","url":"/healthz","status":200,"duration":"5 ms"}
{"level":30,"time":1717786587420,"pid":17,"hostname":"pepr-uds-core-watcher-55f6974d99-vv4lm","method":"GET","url":"/healthz","status":200,"duration":"1 ms"}
{"level":20,"time":1717786591466,"pid":17,"hostname":"pepr-uds-core-watcher-55f6974d99-vv4lm","msg":"Sending updates to Pepr store"}
{"level":50,"time":1717786591487,"pid":17,"hostname":"pepr-uds-core-watcher-55f6974d99-vv4lm","data":{"kind":"Status","apiVersion":"v1","metadata":{},"status":"Failure","message":"the server rejected our request due to an error in our request","reason":"Invalid","details":{},"code":422},"ok":false,"status":422,"statusText":"Unprocessable Entity","msg":"Pepr store update failure"}
{"level":20,"time":1717786596467,"pid":17,"hostname":"pepr-uds-core-watcher-55f6974d99-vv4lm","msg":"Sending updates to Pepr store"}
{"level":50,"time":1717786596482,"pid":17,"hostname":"pepr-uds-core-watcher-55f6974d99-vv4lm","data":{"kind":"Status","apiVersion":"v1","metadata":{},"status":"Failure","message":"the server rejected our request due to an error in our request","reason":"Invalid","details":{},"code":422},"ok":false,"status":422,"statusText":"Unprocessable Entity","msg":"Pepr store update failure"}
{"level":30,"time":1717786597408,"pid":17,"hostname":"pepr-uds-core-watcher-55f6974d99-vv4lm","method":"GET","url":"/healthz","status":200,"duration":"1 ms"}
{"level":30,"time":1717786597410,"pid":17,"hostname":"pepr-uds-core-watcher-55f6974d99-vv4lm","method":"GET","url":"/healthz","status":200,"duration":"1 ms"}
{"level":20,"time":1717786601468,"pid":17,"hostname":"pepr-uds-core-watcher-55f6974d99-vv4lm","msg":"Sending updates to Pepr store"}
{"level":50,"time":1717786601483,"pid":17,"hostname":"pepr-uds-core-watcher-55f6974d99-vv4lm","data":{"kind":"Status","apiVersion":"v1","metadata":{},"status":"Failure","message":"the server rejected our request due to an error in our request","reason":"Invalid","details":{},"code":422},"ok":false,"status":422,"statusText":"Unprocessable Entity","msg":"Pepr store update failure"}
{"level":20,"time":1717786606468,"pid":17,"hostname":"pepr-uds-core-watcher-55f6974d99-vv4lm","msg":"Sending updates to Pepr store"}
{"level":50,"time":1717786606484,"pid":17,"hostname":"pepr-uds-core-watcher-55f6974d99-vv4lm","data":{"kind":"Status","apiVersion":"v1","metadata":{},"status":"Failure","message":"the server rejected our request due to an error in our request","reason":"Invalid","details":{},"code":422},"ok":false,"status":422,"statusText":"Unprocessable Entity","msg":"Pepr store update failure"}

[mjnagel]

Just to follow up here, the issue appears to actually be due to attempting a delete on a key that doesn't exist (as is the case on a fresh install). This is a related pepr issue - defenseunicorns/pepr#865 - for now we can just conditionally do the removes based on existence of the keys (as is done in the linked PR).