feat(alertmanager-dropevents-config): dropped events treatment config

README.md

@@ -262,6 +262,8 @@ alertmanager:
   # extralabels: "" # comma separated list of labels composed of a ':' separated name and value that is added to the Alerts. Example: my_label_1:my_value_1, my_label_1:my_value_2
   # extraannotations: "" # comma separated list of annotations composed of a ':' separated name and value that is added to the Alerts. Example: my_annotation_1:my_value_1, my_annotation_1:my_value_2
   # customseveritymap: "" # comma separated list of tuple composed of a ':' separated Falco priority and Alertmanager severity that is used to override the severity label associated to the priority level of falco event. Example: debug:value_1,critical:value2.  Default mapping (priority:severity): emergency:critical,alert:critical,critical:critical,error:warning,warning:warning,notice:information,informational:information,debug:information
+  # dropeventdefaultpriority: "" # default priority of dropped events, values are emergency|alert|critical|error|warning|notice|informational|debug (default: "critical")
+  # dropeventthresholds: # comma separated list of priority re-evaluation thresholds of dropped events composed of a ':' separated integer threshold and string priority. Example: `10000:critical, 100:warning, 1:informational` (default: `"10000:critical, 1000:critical, 100:critical, 10:warning, 1:warning"`)
 
 elasticsearch:
   # hostport: "" # http://{domain or ip}:{port}, if not empty, Elasticsearch output is enabled
@@ -770,6 +772,9 @@ care of lower/uppercases**) : `yaml: a.b --> envvar: A_B` :
 - **ALERTMANAGER_CUSTOMSEVERITYMAP** : comma separated list of tuple composed of a ':' separated Falco priority and
   Alertmanager severity that is used to override the severity label associated to the priority level of falco event.
   Example: `debug:value_1,critical:value2`. Default mapping (priority:severity): `emergency:critical,alert:critical,critical:critical,error:warning,warning:warning,notice:information,informational:information,debug:information` (default: `""`)
+- **ALERTMANAGER_DROPEVENTDEFAULTPRIORITY** : default priority of dropped events, values are emergency|alert|critical|error|warning|notice|informational|debug (default: `"critical"`)
+- **ALERTMANAGER_DROPEVENTTHRESHOLDS** : comma separated list of priority re-evaluation thresholds of dropped events composed of a ':' separated integer threshold and
+  string priority. Example: `10000:critical, 100:warning, 1:informational` (default: `"10000:critical, 1000:critical, 100:critical, 10:warning, 1:warning"`)
 - **ELASTICSEARCH_HOSTPORT** : Elasticsearch http://host:port, if not `empty`,
   Elasticsearch is _enabled_
 - **ELASTICSEARCH_INDEX** : Elasticsearch index (default: falco)

config.go

@@ -8,6 +8,8 @@ import (
 	"path"
 	"path/filepath"
 	"regexp"
+	"sort"
+	"strconv"
 	"strings"
 	"text/template"
 
@@ -113,6 +115,8 @@ func getConfig() *types.Configuration {
 	v.SetDefault("Alertmanager.CheckCert", true)
 	v.SetDefault("Alertmanager.Endpoint", "/api/v1/alerts")
 	v.SetDefault("Alertmanager.ExpiresAfter", 0)
+	v.SetDefault("Alertmanager.DropEventDefaultPriority", "critical")
+	v.SetDefault("Alertmanager.DropEventThresholds", "10000:critical, 1000:critical, 100:critical, 10:warning, 1:warning")
 
 	v.SetDefault("Elasticsearch.HostPort", "")
 	v.SetDefault("Elasticsearch.Index", "falco")
@@ -587,6 +591,10 @@ func getConfig() *types.Configuration {
 		}
 	}
 
+	if value, present := os.LookupEnv("ALERTMANAGER_DROPEVENTTHRESHOLDS"); present {
+		c.Alertmanager.DropEventThresholds = value
+	}
+
 	if value, present := os.LookupEnv("GCP_PUBSUB_CUSTOMATTRIBUTES"); present {
 		customattributes := strings.Split(value, ",")
 		for _, label := range customattributes {
@@ -620,12 +628,44 @@ func getConfig() *types.Configuration {
 		c.Prometheus.ExtraLabelsList = strings.Split(strings.ReplaceAll(c.Prometheus.ExtraLabels, " ", ""), ",")
 	}
 
+	if c.Alertmanager.DropEventThresholds != "" {
+		c.Alertmanager.DropEventThresholdsList = make([]types.ThresholdConfig, 0)
+		thresholds := strings.Split(strings.ReplaceAll(c.Alertmanager.DropEventThresholds, " ", ""), ",")
+		for _, threshold := range thresholds {
+			values := strings.SplitN(threshold, ":", 2)
+			if len(values) != 2 {
+				log.Printf("[ERROR] : AlertManager - Fail to parse threshold - No priority given for threshold %v", threshold)
+				continue
+			}
+			valueString := strings.TrimSpace(values[0])
+			valueInt, err := strconv.ParseInt(valueString, 10, 64)
+			if len(values) != 2 || err != nil {
+				log.Printf("[ERROR] : AlertManager - Fail to parse threshold - Atoi fail %v", threshold)
+				continue
+			}
+			priority := types.Priority(strings.TrimSpace(values[1]))
+			if priority == types.Default {
+				log.Printf("[ERROR] : AlertManager - Priority '%v' is not a valid falco priority level", priority.String())
+				continue
+			}
+			c.Alertmanager.DropEventThresholdsList = append(c.Alertmanager.DropEventThresholdsList, types.ThresholdConfig{Priority: priority, Value: valueInt})
+		}
+	}
+
+	if len(c.Alertmanager.DropEventThresholdsList) > 0 {
+		sort.Slice(c.Alertmanager.DropEventThresholdsList, func(i, j int) bool {
+			// The `>` is used to sort in descending order. If you want to sort in ascending order, use `<`.
+			return c.Alertmanager.DropEventThresholdsList[i].Value > c.Alertmanager.DropEventThresholdsList[j].Value
+		})
+	}
+
 	c.Slack.MinimumPriority = checkPriority(c.Slack.MinimumPriority)
 	c.Rocketchat.MinimumPriority = checkPriority(c.Rocketchat.MinimumPriority)
 	c.Mattermost.MinimumPriority = checkPriority(c.Mattermost.MinimumPriority)
 	c.Teams.MinimumPriority = checkPriority(c.Teams.MinimumPriority)
 	c.Datadog.MinimumPriority = checkPriority(c.Datadog.MinimumPriority)
 	c.Alertmanager.MinimumPriority = checkPriority(c.Alertmanager.MinimumPriority)
+	c.Alertmanager.DropEventDefaultPriority = checkPriority(c.Alertmanager.DropEventDefaultPriority)
 	c.Elasticsearch.MinimumPriority = checkPriority(c.Elasticsearch.MinimumPriority)
 	c.Influxdb.MinimumPriority = checkPriority(c.Influxdb.MinimumPriority)
 	c.Loki.MinimumPriority = checkPriority(c.Loki.MinimumPriority)

config_example.yaml

@@ -70,9 +70,11 @@ alertmanager:
   # checkcert: true # check if ssl certificate of the output is valid (default: true)
   # endpoint: "" # alertmanager endpoint for posting alerts: "/api/v1/alerts" or "/api/v2/alerts" (default: "/api/v1/alerts")
   # expiresafter: "" if set to a non-zero value, alert expires after that time in seconds (default: 0)
-  # extralabels: "" # comma separated list of labels composed of a ':' separated name and value that is added to the Alerts. Example: my_label_1:my_value_1, my_label_1:my_value_2 (default: "")
-  # extraannotations: "" # comma separated list of annotations composed of a ':' separated name and value that is added to the Alerts. Example: my_annotation_1:my_value_1, my_annotation_1:my_value_2 (default: "")
+  # extralabels: "" # comma separated list of labels composed of a ':' separated name and value that is added to the Alerts. Example: my_label_1:my_value_1, my_label_1:my_value_2
+  # extraannotations: "" # comma separated list of annotations composed of a ':' separated name and value that is added to the Alerts. Example: my_annotation_1:my_value_1, my_annotation_1:my_value_2
   # customseveritymap: "" # comma separated list of tuple composed of a ':' separated Falco priority and Alertmanager severity that is used to override the severity label associated to the priority level of falco event. Example: debug:value_1,critical:value2. Default mapping: emergency:critical,alert:critical,critical:critical,error:warning,warning:warning,notice:information,informational:information,debug:information. (default: "")
+  # dropeventdefaultpriority: "" # default priority of dropped events, values are emergency|alert|critical|error|warning|notice|informational|debug (default: "critical")
+  # dropeventthresholds: # comma separated list of priority re-evaluation thresholds of dropped events composed of a ':' separated integer threshold and string priority. Example: `10000:critical, 100:warning, 1:informational` (default: `"10000:critical, 1000:critical, 100:critical, 10:warning, 1:warning"`)
 
 elasticsearch:
   # hostport: "" # http://{domain or ip}:{port}, if not empty, Elasticsearch output is enabled

outputs/alertmanager.go

@@ -43,30 +43,28 @@ func newAlertmanagerPayload(falcopayload types.FalcoPayload, config *types.Confi
 			d, err := strconv.ParseInt(j.(string), 10, 64)
 			if err == nil {
 				var jj string
-				switch {
-				case d == 0:
+				if d == 0 {
+					if falcopayload.Priority < types.Warning {
+						falcopayload.Priority = types.Warning
+					}
 					jj = "0"
-					falcopayload.Priority = types.Warning
-				case d < 10:
-					jj = "<10"
-					falcopayload.Priority = types.Warning
-				case d > 10000:
-					jj = ">10000"
-					falcopayload.Priority = types.Critical
-				case d > 1000:
-					jj = ">1000"
-					falcopayload.Priority = types.Critical
-				case d > 100:
-					jj = ">100"
-					falcopayload.Priority = types.Critical
-				case d > 10:
-					jj = ">10"
-					falcopayload.Priority = types.Warning
-				default:
+				} else {
+					for _, threshold := range config.Alertmanager.DropEventThresholdsList {
+						if d > threshold.Value {
+							jj = ">" + strconv.FormatInt(threshold.Value, 10)
+							if falcopayload.Priority < threshold.Priority {
+								falcopayload.Priority = threshold.Priority
+							}
+							break
+						}
+					}
+				}
+				if jj == "" {
 					jj = j.(string)
-					falcopayload.Priority = types.Critical
+					if prio := types.Priority(config.Alertmanager.DropEventDefaultPriority); falcopayload.Priority < prio {
+						falcopayload.Priority = prio
+					}
 				}
-
 				amPayload.Labels[i] = jj
 			}
 			continue

outputs/alertmanager_test.go

@@ -10,6 +10,8 @@ import (
 	"github.com/falcosecurity/falcosidekick/types"
 )
 
+const defaultThresholds = `[{"priority":"critical", "value":10000}, {"priority":"critical", "value":1000}, {"priority":"critical", "value":100} ,{"priority":"warning", "value":10}, {"priority":"warning", "value":1}]`
+
 func TestNewAlertmanagerPayloadO(t *testing.T) {
 	expectedOutput := `[{"labels":{"proc_name":"falcosidekick","priority":"Debug","severity": "information","proc_tty":"1234","eventsource":"syscalls","hostname":"test-host","rule":"Test rule","source":"falco","tags":"test,example"},"annotations":{"info":"This is a test from falcosidekick","description":"This is a test from falcosidekick","summary":"Test rule"}}]`
 	var f types.FalcoPayload
@@ -19,8 +21,33 @@ func TestNewAlertmanagerPayloadO(t *testing.T) {
 	require.Nil(t, err)
 
 	config := &types.Configuration{
-		Alertmanager: types.AlertmanagerOutputConfig{},
+		Alertmanager: types.AlertmanagerOutputConfig{DropEventDefaultPriority: Critical},
+	}
+	json.Unmarshal([]byte(defaultThresholds), &config.Alertmanager.DropEventThresholdsList)
+
+	s, err := json.Marshal(newAlertmanagerPayload(f, config))
+	require.Nil(t, err)
+
+	var o1, o2 []alertmanagerPayload
+	require.Nil(t, json.Unmarshal([]byte(expectedOutput), &o1))
+	require.Nil(t, json.Unmarshal(s, &o2))
+
+	require.Equal(t, o1, o2)
+}
+
+func TestNewAlertmanagerPayloadDropEvent(t *testing.T) {
+	input := `{"hostname":"host","output":"Falco internal: syscall event drop. 815508 system calls dropped in last second.","output_fields":{"ebpf_enabled":"1","n_drops":"815508","n_drops_buffer_clone_fork_enter":"0","n_drops_buffer_clone_fork_exit":"0","n_drops_buffer_connect_enter":"0","n_drops_buffer_connect_exit":"0","n_drops_buffer_dir_file_enter":"803","n_drops_buffer_dir_file_exit":"804","n_drops_buffer_execve_enter":"0","n_drops_buffer_execve_exit":"0","n_drops_buffer_open_enter":"798","n_drops_buffer_open_exit":"798","n_drops_buffer_other_interest_enter":"0","n_drops_buffer_other_interest_exit":"0","n_drops_buffer_total":"815508","n_drops_bug":"0","n_drops_page_faults":"0","n_drops_scratch_map":"0","n_evts":"2270350"},"priority":"Debug","rule":"Falco internal: syscall event drop","time":"2023-03-03T03:03:03.000000003Z"}`
+	expectedOutput := `[{"labels":{"ebpf_enabled":"1","eventsource":"","hostname":"host","n_drops":">10000","n_drops_buffer_clone_fork_enter":"0","n_drops_buffer_clone_fork_exit":"0","n_drops_buffer_connect_enter":"0","n_drops_buffer_connect_exit":"0","n_drops_buffer_dir_file_enter":">100","n_drops_buffer_dir_file_exit":">100","n_drops_buffer_execve_enter":"0","n_drops_buffer_execve_exit":"0","n_drops_buffer_open_enter":">100","n_drops_buffer_open_exit":">100","n_drops_buffer_other_interest_enter":"0","n_drops_buffer_other_interest_exit":"0","n_drops_buffer_total":">10000","n_drops_bug":"0","n_drops_page_faults":"0","n_drops_scratch_map":"0","priority":"Critical","rule":"Falco internal: syscall event drop","severity":"critical","source":"falco"},"annotations":{"description":"Falco internal: syscall event drop. 815508 system calls dropped in last second.","info":"Falco internal: syscall event drop. 815508 system calls dropped in last second.","summary":"Falco internal: syscall event drop"},"endsAt":"0001-01-01T00:00:00Z"}]`
+	var f types.FalcoPayload
+	d := json.NewDecoder(strings.NewReader(input))
+	d.UseNumber()
+	err := d.Decode(&f) //have to decode it the way newFalcoPayload does
+	require.Nil(t, err)
+
+	config := &types.Configuration{
+		Alertmanager: types.AlertmanagerOutputConfig{DropEventDefaultPriority: Critical},
 	}
+	json.Unmarshal([]byte(defaultThresholds), &config.Alertmanager.DropEventThresholdsList)
 
 	s, err := json.Marshal(newAlertmanagerPayload(f, config))
 	require.Nil(t, err)

types/types.go

@@ -216,16 +216,24 @@ type DiscordOutputConfig struct {
 	MutualTLS       bool
 }
 
+type ThresholdConfig struct {
+	Value    int64        `json:"value" yaml:"value"`
+	Priority PriorityType `json:"priority" yaml:"priority"`
+}
+
 type AlertmanagerOutputConfig struct {
-	HostPort          string
-	MinimumPriority   string
-	CheckCert         bool
-	MutualTLS         bool
-	Endpoint          string
-	ExpiresAfter      int
-	ExtraLabels       map[string]string
-	ExtraAnnotations  map[string]string
-	CustomSeverityMap map[PriorityType]string
+	HostPort                 string
+	MinimumPriority          string
+	CheckCert                bool
+	MutualTLS                bool
+	Endpoint                 string
+	ExpiresAfter             int
+	ExtraLabels              map[string]string
+	ExtraAnnotations         map[string]string
+	CustomSeverityMap        map[PriorityType]string
+	DropEventThresholds      string
+	DropEventThresholdsList  []ThresholdConfig
+	DropEventDefaultPriority string
 }
 
 type ElasticsearchOutputConfig struct {