Merge pull request #4 from sva-mk/main

 Add Whitelisting Features to allow some external conntections

README.md

@@ -1,7 +1,7 @@
 # misp-guard
 `misp-guard` is a [mitmproxy](https://mitmproxy.org/) addon that inspects the synchronization traffic (via `PUSH` or `PULL`) between different MISP instances and applies a set of customizable rules defined in a JSON file.
 
-> **NOTE: By default this addon will block all outgoing HTTP requests that are not required during a MISP server sync.**
+> **NOTE: By default this addon will block all outgoing HTTP requests that are not required during a MISP server sync. However, individual URLs or domains can be allowed if necessary.**
 
 ## PUSH
 ```mermaid
@@ -94,6 +94,12 @@ sequenceDiagram
 * `blocked_attribute_categories`: Blocks if the event contains an attribute matching one of this categories.
 * `blocked_object_types`: Blocks if the event contains an object matching one of this types.
 
+**Allowlist**
+
+* To allow individual URLs or domains, simply add them as a JSON array under the `allowlist` element.
+  * `urls` The entire URL is checked and only exact calls are allowed.
+  * `domains` In contrast, only the domain is checked and any website behind the domain can be queried. Should only be used if adding exact URLs is not possible.
+
 See sample config [here](src/test/test_config.json).
 
 ## Instructions

src/config.schema.json

@@ -2,6 +2,25 @@
     "$schema": "http://json-schema.org/draft-07/schema",
     "type": "object",
     "properties": {
+        "allowlist": {
+            "type": "object",
+            "properties": {
+                "urls": {
+                    "type": "array",
+                    "items": {
+                        "type": "string",
+                        "format": "url"
+                    }
+                },
+                "domains": {
+                    "type": "array",
+                    "items": {
+                        "type": "string",
+                        "format": "domain"
+                    }
+                }
+            }
+        },
         "compartments_rules": {
             "type": "object",
             "properties": {

src/mispguard.py

@@ -128,32 +128,69 @@ def load(self, loader):
         loader.add_option("config", str, "", "MISP Guard configuration file")
 
     def request(self, flow: http.HTTPFlow) -> None:
-        try:
-            flow = self.enrich_flow(flow)
-            if self.can_reach_compartment(flow) and self.is_allowed_endpoint(flow.request.method, flow.request.path):
-                return self.process_request(flow)
-        except ForbiddenException as ex:
-            logger.error(ex)
-            return self.forbidden(flow, str(ex))
-        except Exception as ex:
-            logger.error(ex)
-            return self.forbidden(flow, "unexpected error, rejecting request")
+        if (not (self.url_is_allowed(flow) or self.domain_is_allowed(flow))) :
+            try:  
+                flow = self.enrich_flow(flow)
+                if self.can_reach_compartment(flow) and self.is_allowed_endpoint(flow.request.method, flow.request.path):
+                    return self.process_request(flow)
+            except ForbiddenException as ex:
+                logger.error(ex)
+                return self.forbidden(flow, str(ex))
+            except Exception as ex:
+                logger.error(ex)
+                return self.forbidden(flow, "unexpected error, rejecting request")
 
-        # filter out requests to the allowed endpoints
-        logger.error("rejecting non allowed request to %s" % flow.request.path)
-        return self.forbidden(flow)
+            # filter out requests to the allowed endpoints
+            logger.error("rejecting non allowed request to %s" % flow.request.path)
+            return self.forbidden(flow)
+        else:
+            try:  
+                if self.src_instance_is_allowed(flow):
+                    logger.info("request from allowed url - skipping further processing")
+            except ForbiddenException as ex:
+                logger.error(ex)
+                return self.forbidden(flow, str(ex))
+            except Exception as ex:
+                logger.error(ex)
+                return self.forbidden(flow, "unexpected error, rejecting request")
+
 
     def response(self,  flow: http.HTTPFlow) -> None:
-        try:
-            flow = self.enrich_flow(flow)
-            if self.can_reach_compartment(flow):
-                return self.process_response(flow)
-        except ForbiddenException as ex:
-            logger.error(ex)
-            return self.forbidden(flow, str(ex))
-        except Exception as ex:
-            logger.error(ex)
-            return self.forbidden(flow, "unexpected error, rejecting response")
+        if (not (self.url_is_allowed(flow) or self.domain_is_allowed(flow))):
+            try:
+                flow = self.enrich_flow(flow)
+                if self.can_reach_compartment(flow):
+                    return self.process_response(flow)
+            except ForbiddenException as ex:
+                logger.error(ex)
+                return self.forbidden(flow, str(ex))
+            except Exception as ex:
+                logger.error(ex)
+                return self.forbidden(flow, "unexpected error, rejecting response")
+        else:
+            try:
+                if  self.src_instance_is_allowed(flow):
+                    logger.info("response from allowed url - skipping further processing")
+            except ForbiddenException as ex:
+                logger.error(ex)
+                return self.forbidden(flow, str(ex))
+            except Exception as ex:
+                logger.error(ex)
+                return self.forbidden(flow, "unexpected error, rejecting response")
+
+
+    def url_is_allowed(self, flow: http.HTTPFlow) -> bool:
+        if flow.request.url in self.config["allowlist"]["urls"]:
+            return True
+        else:
+            return False
+
+    def domain_is_allowed(self, flow: http.HTTPFlow) -> bool:
+        if flow.request.host in self.config["allowlist"]["domains"]:
+            return True
+        else:
+            return False
+
 
     def enrich_flow(self, flow: http.HTTPFlow) -> MISPHTTPFlow:
         logger.debug("enriching http flow")
@@ -395,6 +432,12 @@ def get_src_instance_id(self, flow: http.HTTPFlow) -> str:
             raise ForbiddenException("source host does not exist in instances hosts mapping")
 
         return self.config["instances_host_mapping"][flow.client_conn.peername[0]]
+
+    def src_instance_is_allowed(self, flow: http.HTTPFlow) -> bool:
+        if flow.client_conn.peername[0] not in self.config["instances_host_mapping"]:
+            raise ForbiddenException("source host does not exist in instances hosts mapping")
+
+        return True
 
     def get_dst_instance_id(self, flow: http.HTTPFlow) -> str:
         if flow.request.host not in self.config["instances_host_mapping"]:

src/test/test_config.json

@@ -1,4 +1,12 @@
 {
+    "allowlist": {
+        "urls":  [
+                "http://www.dan.me.uk:443/torlist/?exit"
+        ],
+        "domains": [
+            "snort-org-site.s3.amazonaws.com"
+        ]
+    },
     "compartments_rules": {
         "can_reach": {
             "compartment_1": [

src/test/test_misp_guard.py

@@ -77,6 +77,105 @@ async def test_non_allowed_endpoint_is_blocked(self, caplog):
         assert "request blocked: [GET]/users - endpoint not allowed" in caplog.text
         assert flow.response.status_code == 403
 
+    @pytest.mark.asyncio
+    async def test_allowed_domain_from_unknown_src_is_blocked(self, caplog):
+        caplog.set_level("INFO")
+        mispguard = self.load_mispguard()
+        test_path="/torlist/?exit"
+
+        event_view_req = tutils.treq(
+            port=443,
+            host="snort-org-site.s3.amazonaws.com",
+            path=test_path,
+            method=b"GET",
+        )
+
+        flow = tflow.tflow(req=event_view_req)
+        flow.client_conn.peername = ("123.123.123.123", "123")
+        mispguard.request(flow)
+
+        assert "MispGuard initialized" in caplog.text
+        assert "source host does not exist in instances hosts mapping" in caplog.text
+        assert "request blocked: [GET]" + test_path +  " - source host does not exist in instances hosts mapping" in caplog.text
+        assert flow.response.status_code == 403
+
+
+    @pytest.mark.asyncio
+    async def test_allowed_domain_from_known_src_is_allowed(self, caplog):
+        caplog.set_level("INFO")
+        mispguard = self.load_mispguard()
+
+        event_view_req = tutils.treq(
+            port=443,
+            host="snort-org-site.s3.amazonaws.com",
+            path="/test.txt",
+            method=b"GET",
+        )
+
+        event_view_resp = tutils.tresp(
+            status_code=200
+        )
+
+        flow = tflow.tflow(req=event_view_req, resp=event_view_resp)
+        flow.client_conn.peername = ("20.0.0.2", "22")
+        mispguard.request(flow)
+        mispguard.response(flow)
+
+        assert "MispGuard initialized" in caplog.text
+        assert "request from allowed url - skipping further processing" in caplog.text
+        assert "response from allowed url - skipping further processing" in caplog.text
+        assert flow.response.status_code == 200
+
+
+    @pytest.mark.asyncio
+    async def test_allowed_url_from_unknown_src_is_blocked(self, caplog):
+        caplog.set_level("INFO")
+        mispguard = self.load_mispguard()
+        test_path="/torlist/?exit"
+
+        event_view_req = tutils.treq(
+            port=443,
+            host="www.dan.me.uk",
+            path=test_path,
+            method=b"GET",
+        )
+
+        flow = tflow.tflow(req=event_view_req)
+        flow.client_conn.peername = ("123.123.123.123", "123")
+        mispguard.request(flow)
+
+        assert "MispGuard initialized" in caplog.text
+        assert "source host does not exist in instances hosts mapping" in caplog.text
+        assert "request blocked: [GET]" + test_path +  " - source host does not exist in instances hosts mapping" in caplog.text
+        assert flow.response.status_code == 403
+
+
+    @pytest.mark.asyncio
+    async def test_allowed_url_from_known_src_is_allowed(self, caplog):
+        caplog.set_level("INFO")
+        mispguard = self.load_mispguard()
+
+        event_view_req = tutils.treq(
+            port=443,
+            host="www.dan.me.uk",
+            path="/torlist/?exit",
+            method=b"GET",
+        )
+
+        event_view_resp = tutils.tresp(
+            status_code=200
+        )
+
+        flow = tflow.tflow(req=event_view_req, resp=event_view_resp)
+        flow.client_conn.peername = ("20.0.0.2", "22")
+        mispguard.request(flow)
+        mispguard.response(flow)
+
+        assert "MispGuard initialized" in caplog.text
+        assert "request from allowed url - skipping further processing" in caplog.text
+        assert "response from allowed url - skipping further processing" in caplog.text
+        assert flow.response.status_code == 200
+
     @pytest.mark.asyncio
     async def test_pull_event_head_passthrough(self):
         mispguard = self.load_mispguard()