fix: netbios panic on malformed packet, use latest libprotoident and …

…added install script, updated containers, logging improvements, chore
dreadl0ck · Jan 30, 2022 · 39a85c6 · 39a85c6
1 parent 26b1129
commit 39a85c6
Show file tree

Hide file tree

Showing 12 changed files with 84 additions and 26 deletions.
diff --git a/cmd/capture/flags.go b/cmd/capture/flags.go
@@ -103,7 +103,7 @@ var (
 	flagJa3DB         = fs.Bool("ja3DB", true, "use ja3 database for device profiling")
 	flagServiceDB     = fs.Bool("serviceDB", true, "use serviceDB for device profiling")
 	flagGeolocationDB = fs.Bool("geoDB", false, "use geolocation for device profiling")
-	flagDPI           = fs.Bool("dpi", false, "use DPI for device profiling")
+	flagDPI           = fs.Bool("dpi", false, "use DPI libs to enrich IPProfile audit records")
 
 	flagFreeOSMemory          = fs.Int("free-os-mem", 0, "free OS memory every X minutes, disabled if set to 0")
 	flagReassembleConnections = fs.Bool("reassemble-connections", true, "reassemble TCP connections")

diff --git a/collector/init.go b/collector/init.go
@@ -87,6 +87,7 @@ func (c *Collector) Init() (err error) {
 
 	// init deep packet inspection
 	if c.config.DPI {
+		c.printlnStdOut("initializing dpi libs")
 		dpi.Init()
 	}
 

diff --git a/docker/alpine-nodpi/Dockerfile b/docker/alpine-nodpi/Dockerfile
@@ -1,4 +1,4 @@
-FROM golang:1.15.1-alpine as builder
+FROM golang:1.17.6-alpine as builder
 RUN apk update
 RUN apk add --no-cache gcc libpcap-dev libnetfilter_queue-dev linux-headers musl-utils musl-dev git vim autoconf automake libtool make g++ bison flex cmake build-base abuild binutils binutils-doc gcc-doc cmake-doc extra-cmake-modules extra-cmake-modules-doc
 

diff --git a/docker/alpine/Dockerfile b/docker/alpine/Dockerfile
@@ -1,22 +1,22 @@
-FROM golang:1.15.1-alpine as builder
+FROM golang:1.17.6-alpine as builder
 RUN apk update
 RUN apk add --no-cache gcc libpcap-dev libnetfilter_queue-dev linux-headers musl-utils musl-dev git vim autoconf automake libtool make g++ bison flex cmake build-base abuild binutils binutils-doc gcc-doc cmake-doc extra-cmake-modules extra-cmake-modules-doc
 
-RUN wget https://github.com/wanduow/wandio/archive/4.2.2-1.tar.gz
-RUN tar xfz 4.2.2-1.tar.gz
-RUN cd wandio-4.2.2-1 && ./bootstrap.sh && ./configure && make && make install
+RUN wget https://github.com/wanduow/wandio/archive/4.2.3-1.tar.gz
+RUN tar xfz 4.2.3-1.tar.gz
+RUN cd wandio-4.2.3-1 && ./bootstrap.sh && ./configure && make && make install
 
-RUN wget https://github.com/LibtraceTeam/libtrace/archive/4.0.11-1.tar.gz
-RUN tar xfz 4.0.11-1.tar.gz
-RUN cd libtrace-4.0.11-1 && ./bootstrap.sh && ./configure && make && make install
+RUN wget https://github.com/LibtraceTeam/libtrace/archive/4.0.17-1.tar.gz
+RUN tar xfz 4.0.17-1.tar.gz
+RUN cd libtrace-4.0.17-1 && ./bootstrap.sh && ./configure && make && make install
 
 RUN wget https://github.com/wanduow/libflowmanager/archive/3.0.0.tar.gz
 RUN tar xfz 3.0.0.tar.gz
 RUN cd libflowmanager-3.0.0 && ./bootstrap.sh && ./configure && make && make install
 
-RUN wget https://github.com/wanduow/libprotoident/archive/2.0.14-1.tar.gz
-RUN tar xfz 2.0.14-1.tar.gz
-RUN cd libprotoident-2.0.14-1 && ./bootstrap.sh && ./configure && make && make install
+RUN wget https://github.com/wanduow/libprotoident/archive/2.0.15-1.tar.gz
+RUN tar xfz 2.0.15-1.tar.gz
+RUN cd libprotoident-2.0.15-1 && ./bootstrap.sh && ./configure && make && make install
 
 # debug linker search path: ld -llinear --verbose
 #RUN git clone https://github.com/cjlin1/liblinear.git

diff --git a/docker/ubuntu/Dockerfile b/docker/ubuntu/Dockerfile
@@ -2,28 +2,26 @@ FROM ubuntu:18.04 as builder
 
 RUN apt-get clean
 RUN apt-get update
-RUN apt-get install -y software-properties-common
+RUN apt-get install -y software-properties-common net-tools
 RUN add-apt-repository ppa:longsleep/golang-backports
 RUN apt-get update
 RUN apt-get install -y golang-go
 
 RUN apt-get install -y apt-transport-https curl lsb-release wget autogen autoconf libtool gcc libpcap-dev linux-headers-generic git vim
-RUN apt-get install apt-transport-https curl
 RUN curl -1sLf 'https://dl.cloudsmith.io/public/wand/libwandio/cfg/setup/bash.deb.sh' | bash
 RUN curl -1sLf 'https://dl.cloudsmith.io/public/wand/libwandder/cfg/setup/bash.deb.sh' | bash
 RUN curl -1sLf 'https://dl.cloudsmith.io/public/wand/libtrace/cfg/setup/bash.deb.sh' | bash
 RUN curl -1sLf 'https://dl.cloudsmith.io/public/wand/libflowmanager/cfg/setup/bash.deb.sh' | bash
 RUN curl -1sLf 'https://dl.cloudsmith.io/public/wand/libprotoident/cfg/setup/bash.deb.sh' | bash
 RUN apt-get update
+RUN apt install -y liblinear-dev libprotoident libprotoident-dev libprotoident-tools libtrace4-dev libtrace4-tools
 
 # nDPI
 RUN apt-get install -y libjson-c-dev
 RUN wget https://github.com/ntop/nDPI/archive/4.0.tar.gz
 RUN tar xfz 4.0.tar.gz
 RUN cd nDPI-4.0 && ./autogen.sh && ./configure && make && make install
 
-RUN apt install -y liblinear-dev libprotoident libprotoident-dev libprotoident-tools libtrace4-dev libtrace4-tools
-
 WORKDIR /src
 
 COPY go.mod go.sum ./
@@ -63,5 +61,6 @@ COPY --from=builder /usr/lib/libflow* /usr/lib/
 COPY --from=builder /usr/lib/libproto* /usr/lib/
 COPY --from=builder /usr/lib/x86_64-linux-gnu /usr/lib/x86_64-linux-gnu
 COPY --from=builder /usr/local/lib/* /usr/lib/
+COPY --from=builder /usr/lib/libndpi* /usr/lib/
 
 CMD ["/bin/sh"]
diff --git a/dpi/dpi.go b/dpi/dpi.go
@@ -54,7 +54,7 @@ func Init() {
 	godpi.SetModules([]Module{wm, goDPI})
 
 	if err := godpi.Initialize(); err != nil {
-		log.Fatal("goDPI initialization returned error: ", err)
+		log.Fatal("goDPI initialization returned an error: ", err)
 	}
 }
 
@@ -77,13 +77,14 @@ func GetProtocols(packet gopacket.Packet) map[string]ClassificationResult {
 		return protocols
 	}
 
-	// start := time.Now()
-	// fmt.Println("DPI", packet.NetworkLayer().NetworkFlow(), packet.TransportLayer().TransportFlow())
+	//start := time.Now()
+	//fmt.Println("DPI", packet.NetworkLayer().NetworkFlow(), packet.TransportLayer().TransportFlow())
 
 	flow, _ := godpi.GetPacketFlow(packet)
 	results := godpi.ClassifyFlowAllModules(flow)
 
-	// fmt.Println(packet.NetworkLayer().NetworkFlow(), packet.TransportLayer().TransportFlow(), "complete", time.Since(start))
+	//fmt.Println(packet.NetworkLayer().NetworkFlow(), packet.TransportLayer().TransportFlow(), "complete", time.Since(start))
+	//spew.Dump(results)
 
 	// when using all modules we might receive duplicate classifications
 	// so they will be deduplicated by protocol name before counting them later

diff --git a/go.mod b/go.mod
@@ -14,7 +14,7 @@ require (
 	github.com/denisbrodbeck/machineid v1.0.1
 	github.com/dlclark/regexp2 v1.4.0
 	github.com/dreadl0ck/cryptoutils v0.0.0-20200425144202-4608665a89a4
-	github.com/dreadl0ck/go-dpi v1.0.1
+	github.com/dreadl0ck/go-dpi v1.0.2
 	github.com/dreadl0ck/gopacket v1.1.16-0.20201228223815-140074f06498
 	github.com/dreadl0ck/ja3 v1.0.1-dreadl0ck-gopacket.0.20200917082239-a29743a537b2
 	github.com/dreadl0ck/maltego v0.0.2

diff --git a/go.sum b/go.sum
@@ -122,6 +122,8 @@ github.com/dreadl0ck/go-dpi v0.0.0-20200912122706-f830a777c45f h1:Lthma7ytUyGiaC
 github.com/dreadl0ck/go-dpi v0.0.0-20200912122706-f830a777c45f/go.mod h1:ShxssMM64chljQ3QxF9QQY8FSUoQMvMVA5NSmB1Xx10=
 github.com/dreadl0ck/go-dpi v1.0.1 h1:INQbp9vxl9Hlsf7Z87TDpnZmoY62BdgzpfS91IyBLb8=
 github.com/dreadl0ck/go-dpi v1.0.1/go.mod h1:ShxssMM64chljQ3QxF9QQY8FSUoQMvMVA5NSmB1Xx10=
+github.com/dreadl0ck/go-dpi v1.0.2 h1:rVe5/tyV6ZW9aIl1yn9t8kysCQ7QzitFvgkVpsiNha0=
+github.com/dreadl0ck/go-dpi v1.0.2/go.mod h1:ShxssMM64chljQ3QxF9QQY8FSUoQMvMVA5NSmB1Xx10=
 github.com/dreadl0ck/gopacket v1.1.16-0.20200114112008-4960f4b77557/go.mod h1:d7HEeaw/pAxzNTUprrDDpb7RxPsWA9i3NFp1ZfBNl50=
 github.com/dreadl0ck/gopacket v1.1.16-0.20200322190608-ceb6f481d0e7/go.mod h1:G6cJViboLOuUCto9bY0+lSiMRkFcJPVITDqPM5/ShkI=
 github.com/dreadl0ck/gopacket v1.1.16-0.20200831153559-a0d2e73e902d/go.mod h1:AO4gQoj71eHM7uHvvmIi0V4/vM8LJ1nuGPq9PVNHCrQ=

diff --git a/scripts/install-libprotoident.sh b/scripts/install-libprotoident.sh
@@ -0,0 +1,55 @@
+#!/bin/bash
+
+# =============================================================================== #
+# Basic libprotoident install script for linux systems and macOS                  #
+# Library GitHub: https://github.com/wanduow/libprotoident                        #
+# Motivation: the packages distributed via apt or homebrew are often outdated,    #
+# this script allows an easy setup of the latest version and its dependencies.    #
+# =============================================================================== #
+
+# TODO: test on a fresh install
+# - test on debian linux
+# - test on alpine linux
+# - test on M1 mac
+# - test on intel mac
+
+# Libprotoident version to install
+VERSION=2.0.15
+
+echo "installing libprotoident v$VERSION"
+
+if [[ $OSTYPE == 'darwin'* ]]; then
+  echo 'macOS'
+elif [[ $OSTYPE == 'linux-gnu'* ]]; then
+  echo 'linux'
+
+  # debian
+  #apt-get install -y apt-transport-https curl lsb-release wget autogen autoconf libtool gcc libpcap-dev linux-headers-generic git vim
+
+  # alpine linux
+  #apk add --no-cache gcc libpcap-dev libnetfilter_queue-dev linux-headers musl-utils musl-dev git vim autoconf automake libtool make g++ bison flex cmake build-base abuild binutils binutils-doc gcc-doc cmake-doc extra-cmake-modules extra-cmake-modules-doc
+fi
+
+# move into a new directory
+mkdir -p libprotoident
+cd libprotoident || return
+
+# wandio
+wget https://github.com/wanduow/wandio/archive/4.2.3-1.tar.gz
+tar xfz 4.2.3-1.tar.gz
+cd wandio-4.2.3-1 && ./bootstrap.sh && ./configure && make && make install
+
+# libtrace
+wget https://github.com/LibtraceTeam/libtrace/archive/4.0.17-1.tar.gz
+tar xfz 4.0.17-1.tar.gz
+cd libtrace-4.0.17-1 && ./bootstrap.sh && ./configure && make && make install
+
+# libflowmanager
+wget https://github.com/wanduow/libflowmanager/archive/3.0.0.tar.gz
+tar xfz 3.0.0.tar.gz
+cd libflowmanager-3.0.0 && ./bootstrap.sh && ./configure && make && make install
+
+# libprotoident
+wget https://github.com/wanduow/libprotoident/archive/${VERSION}-1.tar.gz
+tar xfz ${VERSION}-1.tar.gz
+cd libprotoident-${VERSION}-1 && ./bootstrap.sh && ./configure && make && make install
diff --git a/version.go b/version.go
@@ -2,7 +2,7 @@
 package netcap
 
 // Version is the current version identifier for netcap.
-var Version = "v0.6.10"
+var Version = "v0.6.11"
 
 // Commit is the git commit id of the current version.
-var Commit = "395160b4c6bae7cd027c3152947910dbd2e7d5b6"
+var Commit = "26b112978015edbfac3bcb4918e6a532c146cfdf"
diff --git a/zeus/commands.yml b/zeus/commands.yml
@@ -13,7 +13,7 @@ language: bash
 # globals for all commands
 globals:
     # NETCAP version
-    VERSION: 0.6.10
+    VERSION: 0.6.11
 
 # command data
 commands:

diff --git a/zeus/scripts/build-linux-nodpi.sh b/zeus/scripts/build-linux-nodpi.sh
@@ -1,7 +1,7 @@
 #!/bin/bash
 
 # musl
-NODPI=true ARGS="--no-cache --build-arg TAGS=-tags=nodpi" VERSION=nodpi_v${VERSION} zeus/scripts/build-alpine-docker.sh
+NODPI=true ARGS="--no-cache --build-arg TAGS=-tags=nodpi" VERSION=nodpi-v${VERSION} zeus/scripts/build-alpine-docker.sh
 
 # glibc
-NODPI=true ARGS="--no-cache --build-arg TAGS=-tags=nodpi" VERSION=nodpi_v${VERSION} zeus/scripts/build-ubuntu-docker.sh
+NODPI=true ARGS="--no-cache --build-arg TAGS=-tags=nodpi" VERSION=nodpi-v${VERSION} zeus/scripts/build-ubuntu-docker.sh