From 432c294a6ce1e5b943657a191f4642fc03bfacb4 Mon Sep 17 00:00:00 2001 From: AJ Bahnken Date: Fri, 14 Dec 2018 11:08:41 -0800 Subject: [PATCH 1/3] v0.1.6 * verbose logging option * return reputation scores in iprepd error cases or when the IP is not found. --- README.md | 8 +++++++- dist.ini | 2 +- etc/conf.d/server.conf | 1 + lib/resty/iprepd.lua | 27 +++++++++++++++++++++------ 4 files changed, 30 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index 024a4fe..ef1fc81 100644 --- a/README.md +++ b/README.md @@ -35,6 +35,7 @@ init_by_lua_block { statsd_max_buffer_count = tonumber(os.getenv("STATSD_MAX_BUFFER_COUNT")) or 100, statsd_flush_timer = tonumber(os.getenv("STATSD_FLUSH_TIMER")) or 5, dont_block = tonumber(os.getenv("DONT_BLOCK")) or 0, + verbose = tonumber(os.getenv("VERBOSE")) or 0, whitelist = {}, }) } @@ -116,7 +117,11 @@ violations for your environment. -- statsd_max_buffer_count - Max number of metrics in buffer before metrics should be submitted -- to statsd (defaults to 100) -- statsd_flush_timer - Interval for attempting to flush the stats in seconds. (defaults to 5) --- dont_block - Enables (1) or disables (0) not blocking within nginx by returning a 403. (defaults to disabled) +-- dont_block - Enables (1) or disables (0) not blocking within nginx by returning +-- a 403. (defaults to disabled) +-- verbose - Enables (1) or disables (0) verbose logging. Messages are logged with a +-- severity of "ERROR" so that nginx log levels do not need to be changed. (defaults +-- to disabled) -- whitelist - List of whitelisted IP's and IP CIDR's. (defaults to empty) -- client = require("resty.iprepd").new({ @@ -132,6 +137,7 @@ client = require("resty.iprepd").new({ statsd_max_buffer_count = 100, statsd_flush_timer = 10, dont_block = 0, + verbose = 0, whitelist = {"127.0.0.1", "10.10.10.0/24", "192.168.0.0/16"} }) ``` diff --git a/dist.ini b/dist.ini index 3803c21..0035fc1 100644 --- a/dist.ini +++ b/dist.ini @@ -1,7 +1,7 @@ name = iprepd-nginx abstract = iprepd openresty module author = AJ Bahnken (ajvb) -version = 0.1.5 +version = 0.1.6 is_original = yes license = mozilla2 lib_dir = lib diff --git a/etc/conf.d/server.conf b/etc/conf.d/server.conf index da6cf76..8e66aa1 100644 --- a/etc/conf.d/server.conf +++ b/etc/conf.d/server.conf @@ -11,6 +11,7 @@ init_by_lua_block { statsd_max_buffer_count = tonumber(os.getenv("STATSD_MAX_BUFFER_COUNT")) or 100, statsd_flush_timer = tonumber(os.getenv("STATSD_FLUSH_TIMER")) or 5, dont_block = tonumber(os.getenv("DONT_BLOCK")) or 0, + verbose = tonumber(os.getenv("VERBOSE")) or 0, whitelist = {}, }) } diff --git a/lib/resty/iprepd.lua b/lib/resty/iprepd.lua index bbd7067..6e17437 100644 --- a/lib/resty/iprepd.lua +++ b/lib/resty/iprepd.lua @@ -56,6 +56,7 @@ function _M.new(options) statsd_max_buffer_count = options.statsd_max_buffer_count or 100, statsd_flush_timer = options.statsd_flush_timer or 5, dont_block = options.dont_block or 0, + verbose = options.verbose or 0, whitelist = whitelist, } @@ -63,16 +64,20 @@ function _M.new(options) end function _M.check(self, ip) + self:debug_log("Checking " .. ip) ngx.req.set_header('X-Foxsec-IP-Reputation-Below-Threshold', 'false') ngx.req.set_header('X-Foxsec-Block', 'false') if self.whitelist then if iputils.ip_in_cidrs(ip, self.whitelist) then + self:debug_log(ip .. " in whitelist") return end end + local reputation = self:get_reputation(ip) if reputation then + self:debug_log("Got reputation of " .. reputation .. " for " .. ip) ngx.req.set_header('X-Foxsec-IP-Reputation', tostring(reputation)) if reputation <= self.threshold then ngx.req.set_header('X-Foxsec-IP-Reputation-Below-Threshold', 'true') @@ -91,6 +96,7 @@ function _M.check(self, ip) ngx.exit(ngx.HTTP_FORBIDDEN) end else + self:debug_log(ip .. " accepted") if self.statsd then self.statsd.incr("iprepd.status.accepted") end @@ -99,6 +105,7 @@ function _M.check(self, ip) return end + self:debug_log(ip .. " accepted") if self.statsd then self.statsd.incr("iprepd.status.accepted") end @@ -125,25 +132,27 @@ function _M.get_reputation(self, ip) -- If the IP was found if resp.status == 200 then reputation = cjson.decode(resp.body)['reputation'] - if reputation and reputation >= 0 and reputation <= 100 then - self.cache:set(ip, reputation, self.cache_ttl) - else + if not reputation then ngx.log(ngx.ERR, 'Unable to parse `reputation` value from response body') end elseif resp.status == 404 then - self.cache:set(ip, 100, self.cache_ttl) + reputation = 100 else ngx.log(ngx.ERR, 'iprepd responded with a ' .. resp.status .. ' http status code') if self.statsd then self.statsd.incr("iprepd.err." .. resp.status) end if self.cache_errors == 1 then - ngx.log(ngx.ERR, 'cache_errors is enabled, setting reputation of ' .. ip .. ' to 100 within the cache') - self.cache:set(ip, 100, self.cache_ttl) + reputation = 100 + self:debug_log('cache_errors is enabled, setting reputation of ' .. ip .. ' to 100 within the cache') end end end + if reputation and reputation >= 0 and reputation <= 100 then + self.cache:set(ip, reputation, self.cache_ttl) + end + return reputation end @@ -163,4 +172,10 @@ function _M.config_flush_timer(self) ngx.timer.every(self.statsd_flush_timer, self.async_flush_stats, self) end +function _M.debug_log(self, msg) + if self.verbose == 1 then + ngx.log(ngx.ERR, "[verbose] " .. msg) + end +end + return _M From 5d5ff9097bdc32eb2289661ecdf690e5e246e31b Mon Sep 17 00:00:00 2001 From: AJ Bahnken Date: Fri, 14 Dec 2018 13:26:37 -0800 Subject: [PATCH 2/3] Submit metrics for all connection error cases --- lib/resty/iprepd.lua | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/resty/iprepd.lua b/lib/resty/iprepd.lua index 6e17437..2aa0968 100644 --- a/lib/resty/iprepd.lua +++ b/lib/resty/iprepd.lua @@ -122,8 +122,8 @@ function _M.get_reputation(self, ip) headers = self.api_key_hdr, }) if err then - if self.statsd and err == "timeout" then - self.statsd.incr("iprepd.err.timeout") + if self.statsd then + self.statsd.incr("iprepd.err." .. err) end ngx.log(ngx.ERR, 'Error with request to iprepd: ' .. err) return nil From e03fcba66414a38ff50df867a55ed5aa9f95575d Mon Sep 17 00:00:00 2001 From: AJ Bahnken Date: Mon, 17 Dec 2018 09:56:38 -0800 Subject: [PATCH 3/3] changed from concating strings to string.format --- lib/resty/iprepd.lua | 35 +++++++++++++++-------------------- 1 file changed, 15 insertions(+), 20 deletions(-) diff --git a/lib/resty/iprepd.lua b/lib/resty/iprepd.lua index 2aa0968..b05778a 100644 --- a/lib/resty/iprepd.lua +++ b/lib/resty/iprepd.lua @@ -26,7 +26,7 @@ function _M.new(options) local cache, err = lrucache.new(cache_buffer_count) if not cache then - fatal_error('failed to create the cache: ' .. (err or 'unknown')) + fatal_error(string.format('failed to create the cache: %s', (err or 'unknown'))) end local statsd_client = nil @@ -45,7 +45,7 @@ function _M.new(options) timeout = options.timeout or 10, threshold = iprepd_threshold, api_key_hdr = { - ['Authorization'] = 'APIKey ' .. iprepd_api_key, + ['Authorization'] = string.format('APIKey %s', iprepd_api_key), }, cache = cache, cache_ttl = options.cache_ttl or 30, @@ -64,12 +64,12 @@ function _M.new(options) end function _M.check(self, ip) - self:debug_log("Checking " .. ip) + self:debug_log(string.format("Checking %s", ip)) ngx.req.set_header('X-Foxsec-IP-Reputation-Below-Threshold', 'false') ngx.req.set_header('X-Foxsec-Block', 'false') if self.whitelist then if iputils.ip_in_cidrs(ip, self.whitelist) then - self:debug_log(ip .. " in whitelist") + self:debug_log(string.format("%s in whitelist", ip)) return end end @@ -77,7 +77,7 @@ function _M.check(self, ip) local reputation = self:get_reputation(ip) if reputation then - self:debug_log("Got reputation of " .. reputation .. " for " .. ip) + self:debug_log(string.format("Got reputation of %d for %s", reputation, ip)) ngx.req.set_header('X-Foxsec-IP-Reputation', tostring(reputation)) if reputation <= self.threshold then ngx.req.set_header('X-Foxsec-IP-Reputation-Below-Threshold', 'true') @@ -87,25 +87,20 @@ function _M.check(self, ip) end if self.dont_block == 1 then - ngx.log(ngx.ERR, ip .. ' is below threshold with a reputation of ' .. reputation) + ngx.log(ngx.ERR, string.format("%s is below threshold with a reputation of %d", ip, reputation)) else - ngx.log(ngx.ERR, ip .. ' rejected with a reputation of ' .. reputation) + ngx.log(ngx.ERR, string.format("%s rejected with a reputation of %d", ip, reputation)) if self.statsd then self.statsd.incr("iprepd.status.rejected") end ngx.exit(ngx.HTTP_FORBIDDEN) end - else - self:debug_log(ip .. " accepted") - if self.statsd then - self.statsd.incr("iprepd.status.accepted") - end - end - return + return + end end - self:debug_log(ip .. " accepted") + self:debug_log(string.format("%s accepted", ip)) if self.statsd then self.statsd.incr("iprepd.status.accepted") end @@ -117,7 +112,7 @@ function _M.get_reputation(self, ip) if not reputation then local httpc = http.new() httpc:set_timeout(self.timeout) - local resp, err = httpc:request_uri(self.url .. '/' .. ip, { + local resp, err = httpc:request_uri(string.format("%s/%s", self.url, ip), { method = "GET", headers = self.api_key_hdr, }) @@ -125,7 +120,7 @@ function _M.get_reputation(self, ip) if self.statsd then self.statsd.incr("iprepd.err." .. err) end - ngx.log(ngx.ERR, 'Error with request to iprepd: ' .. err) + ngx.log(ngx.ERR, string.format("Error with request to iprepd: %s", err)) return nil end @@ -138,13 +133,13 @@ function _M.get_reputation(self, ip) elseif resp.status == 404 then reputation = 100 else - ngx.log(ngx.ERR, 'iprepd responded with a ' .. resp.status .. ' http status code') + ngx.log(ngx.ERR, string.format("iprepd responded with a %d http status code", resp.status)) if self.statsd then self.statsd.incr("iprepd.err." .. resp.status) end if self.cache_errors == 1 then reputation = 100 - self:debug_log('cache_errors is enabled, setting reputation of ' .. ip .. ' to 100 within the cache') + self:debug_log(string.format("cache_errors is enabled, setting reputation of %s to 100 within the cache", ip)) end end end @@ -174,7 +169,7 @@ end function _M.debug_log(self, msg) if self.verbose == 1 then - ngx.log(ngx.ERR, "[verbose] " .. msg) + ngx.log(ngx.ERR, string.format("[verbose] %s", msg)) end end