From 4d4aada35a7d03443e79505c0ba41def201fb74c Mon Sep 17 00:00:00 2001 From: AJ Bahnken Date: Fri, 7 Dec 2018 10:52:01 -0500 Subject: [PATCH 1/2] Version 0.1.5 - Adds support for a "dont block" mode. Fixes #16 - Adds injecting headers by default. Fixes #10 - Makes cache size configurable. Fixes #2 - Adds support for an IP whitelist. Fixes #15 - Makes statsd flush timer configurable. Fixes #14 --- README.md | 33 ++++++++++++------ dist.ini | 4 +-- etc/conf.d/server.conf | 3 ++ lib/resty/iprepd.lua | 76 +++++++++++++++++++++++++++++------------- 4 files changed, 80 insertions(+), 36 deletions(-) diff --git a/README.md b/README.md index a3f6dcf..024a4fe 100644 --- a/README.md +++ b/README.md @@ -33,6 +33,9 @@ init_by_lua_block { statsd_host = os.getenv("STATSD_HOST") or nil, statsd_port = tonumber(os.getenv("STATSD_PORT")) or 8125, statsd_max_buffer_count = tonumber(os.getenv("STATSD_MAX_BUFFER_COUNT")) or 100, + statsd_flush_timer = tonumber(os.getenv("STATSD_FLUSH_TIMER")) or 5, + dont_block = tonumber(os.getenv("DONT_BLOCK")) or 0, + whitelist = {}, }) } @@ -101,8 +104,9 @@ violations for your environment. -- -- Optional parameters: -- url - The base URL to iprepd (defaults to "http://localhost:8080/") --- cache_ttl - The iprepd response cache ttl in seconds (defaults to 30) -- timeout - The timeout for making requests to iprepd in milliseconds (defaults to 10) +-- cache_ttl - The iprepd response cache ttl in seconds (defaults to 30) +-- cache_buffer_count - Max number of entries allowed in the cache. (defaults to 200) -- cache_errors - Enables (1) or disables (0) caching errors. Caching errors is a good -- idea in production, as it can reduce the average additional latency -- caused by this module if anything goes wrong with the underlying @@ -111,17 +115,24 @@ violations for your environment. -- statsd_port - Port of statsd collector. (defaults to 8125) -- statsd_max_buffer_count - Max number of metrics in buffer before metrics should be submitted -- to statsd (defaults to 100) +-- statsd_flush_timer - Interval for attempting to flush the stats in seconds. (defaults to 5) +-- dont_block - Enables (1) or disables (0) not blocking within nginx by returning a 403. (defaults to disabled) +-- whitelist - List of whitelisted IP's and IP CIDR's. (defaults to empty) -- client = require("resty.iprepd").new({ - url = "http://127.0.0.1:8080", api_key = os.getenv("IPREPD_API_KEY"), threshold = 50, - cache_ttl = 30, + url = "http://127.0.0.1:8080", timeout = 10, + cache_ttl = 30, + cache_buffer_count = 1000, cache_errors = 1, statsd_host = "127.0.0.1", statsd_port = 8125, statsd_max_buffer_count = 100, + statsd_flush_timer = 10, + dont_block = 0, + whitelist = {"127.0.0.1", "10.10.10.0/24", "192.168.0.0/16"} }) ``` @@ -137,7 +148,7 @@ $ make run_dev Then you will be able to hit this proxy with: `curl http://localhost:80` -### Environment Variables +### Environment Variables for Dev #### Note: @@ -153,10 +164,12 @@ IPREPD_REPUTATION_THRESHOLD=50 # iprepd reputation threshold, block all IP's wi # # optional # -IPREPD_TIMEOUT=10 # iprepd client timeout in milliseconds (default is 10ms) -IPREPD_CACHE_TTL=60 # iprepd response cache ttl in seconds (default is 30s) -IPREPD_CACHE_ERRORS=1 # enables caching iprepd non-200 responses (1 enables, 0 disables, default is 0) -STATSD_HOST=127.0.0.1 # statsd host, setting this will also enable statsd metrics collection. -STATSD_PORT=8125 # statsd port (default is 8125) -STATSD_MAX_BUFFER_COUNT=200 # statsd max number of buffer items before submitting (default is 100) +IPREPD_TIMEOUT=10 +IPREPD_CACHE_TTL=30 +IPREPD_CACHE_ERRORS=0 +STATSD_HOST=127.0.0.1 +STATSD_PORT=8125 +STATSD_MAX_BUFFER_COUNT=200 +STATSD_FLUSH_TIMER=2 +DONT_BLOCK=0 ``` diff --git a/dist.ini b/dist.ini index f5c71a5..3803c21 100644 --- a/dist.ini +++ b/dist.ini @@ -1,11 +1,11 @@ name = iprepd-nginx abstract = iprepd openresty module author = AJ Bahnken (ajvb) -version = 0.1.4 +version = 0.1.5 is_original = yes license = mozilla2 lib_dir = lib doc_dir = lib repo_link = https://github.com/mozilla-services/iprepd-nginx main_module = lib/resty/iprepd.lua -requires = openresty/lua-resty-lrucache, pintsized/lua-resty-http +requires = openresty/lua-resty-lrucache, pintsized/lua-resty-http, hamishforbes/lua-resty-iputils diff --git a/etc/conf.d/server.conf b/etc/conf.d/server.conf index ec6757c..da6cf76 100644 --- a/etc/conf.d/server.conf +++ b/etc/conf.d/server.conf @@ -9,6 +9,9 @@ init_by_lua_block { statsd_host = os.getenv("STATSD_HOST") or nil, statsd_port = tonumber(os.getenv("STATSD_PORT")) or 8125, statsd_max_buffer_count = tonumber(os.getenv("STATSD_MAX_BUFFER_COUNT")) or 100, + statsd_flush_timer = tonumber(os.getenv("STATSD_FLUSH_TIMER")) or 5, + dont_block = tonumber(os.getenv("DONT_BLOCK")) or 0, + whitelist = {}, }) } diff --git a/lib/resty/iprepd.lua b/lib/resty/iprepd.lua index 8b36cc5..b2861ff 100644 --- a/lib/resty/iprepd.lua +++ b/lib/resty/iprepd.lua @@ -1,5 +1,6 @@ local cjson = require('cjson') local http = require('resty.http') +local iputils = require('resty.iputils') local lrucache = require('resty.lrucache') local statsd = require('resty.statsd') @@ -18,13 +19,12 @@ function _M.new(options) iprepd_url = iprepd_url:sub(1, -2) end - local cache_ttl = options.cache_ttl or 30 + local cache_buffer_count = options.cache_buffer_count or 200 local iprepd_threshold = options.threshold or fatal_error('Need to pass in a threshold') local iprepd_api_key = options.api_key or fatal_error('Need to pass in an api_key') - -- TODO: Make configurable? - local cache, err = lrucache.new(200) + local cache, err = lrucache.new(cache_buffer_count) if not cache then fatal_error('failed to create the cache: ' .. (err or 'unknown')) end @@ -34,33 +34,73 @@ function _M.new(options) statsd_client = statsd end + local whitelist = nil + local whitelist_list = options.whitelist or nil + if whitelist_list then + whitelist = iputils.parse_cidrs(whitelist_list) + end + local self = { url = iprepd_url, + timeout = options.timeout or 10, threshold = iprepd_threshold, api_key_hdr = { ['Authorization'] = 'APIKey ' .. iprepd_api_key, }, - cache_ttl = cache_ttl, - timeout = options.timeout or 10, cache = cache, + cache_ttl = options.cache_ttl or 30, cache_errors = options.cache_errors or 0, statsd = statsd_client, statsd_host = options.statsd_host, statsd_port = options.statsd_port or 8125, statsd_max_buffer_count = options.statsd_max_buffer_count or 100, + statsd_flush_timer = options.statsd_flush_timer or 5, + dont_block = options.dont_block or 0, + whitelist = whitelist, } return setmetatable(self, mt) end function _M.check(self, ip) - local httpc = http.new() - -- set timeout in ms - httpc:set_timeout(self.timeout) + if self.whitelist then + if iputils.ip_in_cidrs(ip, self.whitelist) then + return + end + end + + local reputation = self:get_reputation(ip) + if reputation then + ngx.req.set_header('X-Foxsec-IP-Reputation', tostring(reputation)) + if reputation <= self.threshold then + ngx.req.set_header('X-Foxsec-IP-Reputation-Below-Threshold', 'true') + ngx.req.set_header('X-Foxsec-Block', 'true') + if self.statsd then + self.statsd.incr("iprepd.status.rejected") + end - -- Get reputation for ip + if self.dont_block == 1 then + ngx.log(ngx.ERR, '[logonly] ' .. ip .. ' rejected with a reputation of ' .. reputation) + else + ngx.log(ngx.ERR, ip .. ' rejected with a reputation of ' .. reputation) + ngx.exit(ngx.HTTP_FORBIDDEN) + end + end + end + + ngx.req.set_header('X-Foxsec-IP-Reputation-Below-Threshold', 'false') + ngx.req.set_header('X-Foxsec-Block', 'false') + if self.statsd then + self.statsd.incr("iprepd.status.accepted") + end +end + +function _M.get_reputation(self, ip) local reputation = self.cache:get(ip) + if not reputation then + local httpc = http.new() + httpc:set_timeout(self.timeout) local resp, err = httpc:request_uri(self.url .. '/' .. ip, { method = "GET", headers = self.api_key_hdr, @@ -70,7 +110,7 @@ function _M.check(self, ip) self.statsd.incr("iprepd.err.timeout") end ngx.log(ngx.ERR, 'Error with request to iprepd: ' .. err) - return + return nil end -- If the IP was found @@ -92,19 +132,7 @@ function _M.check(self, ip) end end - -- check reputation against threshold - if reputation and reputation <= self.threshold then - -- return 403 and log rejections - ngx.log(ngx.ERR, ip .. ' rejected with a reputation of ' .. reputation) - if self.statsd then - self.statsd.incr("iprepd.status.rejected") - end - ngx.exit(ngx.HTTP_FORBIDDEN) - else - if self.statsd then - self.statsd.incr("iprepd.status.accepted") - end - end + return reputation end function _M.flush_stats(self) @@ -120,7 +148,7 @@ function _M.async_flush_stats(premature, self) end function _M.config_flush_timer(self) - ngx.timer.every(5, self.async_flush_stats, self) + ngx.timer.every(self.statsd_flush_timer, self.async_flush_stats, self) end return _M From 806033ec28274ff953d768919ce1498135933987 Mon Sep 17 00:00:00 2001 From: AJ Bahnken Date: Thu, 13 Dec 2018 09:24:33 -0800 Subject: [PATCH 2/2] Fix headers+stats for acceptance case --- lib/resty/iprepd.lua | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/lib/resty/iprepd.lua b/lib/resty/iprepd.lua index b2861ff..bbd7067 100644 --- a/lib/resty/iprepd.lua +++ b/lib/resty/iprepd.lua @@ -63,6 +63,8 @@ function _M.new(options) end function _M.check(self, ip) + ngx.req.set_header('X-Foxsec-IP-Reputation-Below-Threshold', 'false') + ngx.req.set_header('X-Foxsec-Block', 'false') if self.whitelist then if iputils.ip_in_cidrs(ip, self.whitelist) then return @@ -76,20 +78,27 @@ function _M.check(self, ip) ngx.req.set_header('X-Foxsec-IP-Reputation-Below-Threshold', 'true') ngx.req.set_header('X-Foxsec-Block', 'true') if self.statsd then - self.statsd.incr("iprepd.status.rejected") + self.statsd.incr("iprepd.status.below_threshold") end if self.dont_block == 1 then - ngx.log(ngx.ERR, '[logonly] ' .. ip .. ' rejected with a reputation of ' .. reputation) + ngx.log(ngx.ERR, ip .. ' is below threshold with a reputation of ' .. reputation) else ngx.log(ngx.ERR, ip .. ' rejected with a reputation of ' .. reputation) + if self.statsd then + self.statsd.incr("iprepd.status.rejected") + end ngx.exit(ngx.HTTP_FORBIDDEN) end + else + if self.statsd then + self.statsd.incr("iprepd.status.accepted") + end end + + return end - ngx.req.set_header('X-Foxsec-IP-Reputation-Below-Threshold', 'false') - ngx.req.set_header('X-Foxsec-Block', 'false') if self.statsd then self.statsd.incr("iprepd.status.accepted") end @@ -125,6 +134,9 @@ function _M.get_reputation(self, ip) self.cache:set(ip, 100, self.cache_ttl) else ngx.log(ngx.ERR, 'iprepd responded with a ' .. resp.status .. ' http status code') + if self.statsd then + self.statsd.incr("iprepd.err." .. resp.status) + end if self.cache_errors == 1 then ngx.log(ngx.ERR, 'cache_errors is enabled, setting reputation of ' .. ip .. ' to 100 within the cache') self.cache:set(ip, 100, self.cache_ttl)