feat(docker): rework Dockerfile

Except the base image and env DEEPDETECT_ARCH=, the build is the same for cpu and gpu

New docker features are used to speedup the build. (packages and ccache
directory are cached by docker)

The local directory is used to build the image, no need to redownload
the whole deepdetect code. And this allow to build image from other git
reference not only master.

docker/README.md

@@ -77,7 +77,7 @@ Try the `POST` call twice: first time loads the net so it takes slightly below a
 
 ### Access to server logs
 
-To look at server logs, use 
+To look at server logs, use
 ```
 docker logs -f <container name>
 ```
@@ -100,7 +100,7 @@ d9944734d5d6        jolibrain/deepdetect_cpu   "/bin/sh -c './dede -"   17 secon
 
 - access server logs:
 ```
-> docker logs -f loving_shaw 
+> docker logs -f loving_shaw
 
 DeepDetect [ commit 4e2c9f4cbd55eeba3a93fae71d9d62377e91ffa5 ]
 Running DeepDetect HTTP server on 0.0.0.0:8080
@@ -121,9 +121,8 @@ We choose to prefix Dockerfiles with target architecture :
 * cpu.Dockerfile
 * cpu-armv7.Dockerfile
 * gpu.Dockerfile
-* gpu-xenial.Dockerfile
 
-### Build script 
+### Build script
 
 Build script is available in docker path : build/build.sh
 
@@ -186,32 +185,39 @@ Params usage: ./build.sh [options...]
 
 Example with CPU image:
 ```
-# Build with default cmake 
+# Build with default cmake
+export DOCKER_BUILDKIT=1
 docker build -t jolibrain/deepdetect_cpu --no-cache -f docker/cpu.Dockerfile .
 
 # Build with default cmake and without default models
+export DOCKER_BUILDKIT=1
 docker build --build-arg DEEPDETECT_DEFAULT_MODELS=false -t jolibrain/deepdetect_cpu --no-cache -f cpu.Dockerfile .
 
 # Build with custom cmake
+export DOCKER_BUILDKIT=1
 docker build --build-arg DEEPDETECT_BUILD=caffe-tf -t jolibrain/deepdetect_cpu --no-cache -f docker/cpu.Dockerfile .
 
 ```
 
 Example with CPU (armv7) image:
 ```
-# Build with default cmake 
+# Build with default cmake
+export DOCKER_BUILDKIT=1
 docker build -t jolibrain/deepdetect_cpu:armv7 --no-cache -f docker/cpu-armv7.Dockerfile .
 
 ```
 
 Example with GPU image:
 ```
-# Build with default cmake 
+# Build with default cmake
+export DOCKER_BUILDKIT=1
 docker build -t jolibrain/deepdetect_gpu --no-cache -f docker/gpu.Dockerfile .
 
 # Build with default cmake and without default models
+export DOCKER_BUILDKIT=1
 docker build --build-arg DEEPDETECT_DEFAULT_MODELS=false -t jolibrain/deepdetect_gpu --no-cache -f docker/gpu.Dockerfile .
 
 # Build with custom cmake
+export DOCKER_BUILDKIT=1
 docker build --build-arg DEEPDETECT_BUILD=caffe-tf -t jolibrain/deepdetect_gpu --no-cache -f docker/gpu.Dockerfile .
-```
+```

docker/cpu.Dockerfile

@@ -1,17 +1,35 @@
+# syntax = docker/dockerfile:1.0-experimental
 FROM ubuntu:18.04 AS build
 
 ARG DEEPDETECT_ARCH=cpu
 ARG DEEPDETECT_BUILD=default
+ARG DEEPDETECT_DEFAULT_MODELS=true
 
 # Install build dependencies
-RUN export DEBIAN_FRONTEND=noninteractive && \
-    apt-get update -y && \
-    apt-get install -y git \
+RUN rm -f /etc/apt/apt.conf.d/docker-clean; echo 'Binary::apt::APT::Keep-Downloaded-Packages "true";' > /etc/apt/apt.conf.d/keep-cache
+
+RUN --mount=type=cache,target=/var/cache/apt --mount=type=cache,target=/var/lib/apt \
+    export DEBIAN_FRONTEND=noninteractive && \
+    apt-get update -y && apt-get install -y python-dev apt-transport-https ca-certificates gnupg software-properties-common wget curl
+
+# CMake
+RUN curl https://apt.kitware.com/keys/kitware-archive-latest.asc | apt-key add -
+RUN apt-add-repository 'deb https://apt.kitware.com/ubuntu/ bionic main'
+
+# python2 pycompile + docker-buildkit is a bit buggy, it's slow as hell, so disable it for dev
+# bug closed as won't fix as python2 is eol: https://github.com/docker/for-linux/issues/502
+RUN cp /bin/true /usr/bin/pycompile
+
+RUN --mount=type=cache,target=/var/cache/apt --mount=type=cache,target=/var/lib/apt \
+    export DEBIAN_FRONTEND=noninteractive && \
+    apt-get update -y && apt-get install -y \
+    git \
+    ccache \
     automake \
-    cmake \
     build-essential \
     openjdk-8-jdk \
     pkg-config \
+    cmake \
     zip \
     g++ \
     gcc-7 g++-7 \
@@ -39,30 +57,56 @@ RUN export DEBIAN_FRONTEND=noninteractive && \
     liblmdb-dev \
     libutfcpp-dev \
     rapidjson-dev \
-    wget \
+    libmapbox-variant-dev \
     autoconf \
     libtool-bin \
     python-numpy \
     python-future \
+    python-yaml \
+    python-typing \
     swig \
     curl \
     unzip \
     libspdlog-dev \
     python-setuptools \
     python-dev \
     python-wheel \
+    python-pip \
+    python-six \
+    python-enum34 \
     unzip \
     libgoogle-perftools-dev \
+    curl \
+    libspdlog-dev \
     libarchive-dev \
-    libmapbox-variant-dev \
-    bash-completion && \
-    wget -O /tmp/bazel.deb https://github.com/bazelbuild/bazel/releases/download/0.24.1/bazel_0.24.1-linux-x86_64.deb && \
-    dpkg -i /tmp/bazel.deb && \
-    apt-get clean && \
-    rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
+    bash-completion
+
+## Need recent cmake version for cuda 10
+#RUN mkdir /tmp/cmake && cd /tmp/cmake && \
+#    apt remove cmake && \
+#    wget https://cmake.org/files/v3.14/cmake-3.14.0.tar.gz && \
+#    tar xf cmake-3.14.0.tar.gz && \
+#    cd cmake-3.14.0 && \
+#    ./configure && \
+#    make install && \
+#    rm -rf /tmp/cmake
+
+RUN wget -O /tmp/bazel.deb https://github.com/bazelbuild/bazel/releases/download/0.24.1/bazel_0.24.1-linux-x86_64.deb && \
+    dpkg -i /tmp/bazel.deb && rm -rf /tmp/bazel.deb
+
+# Fix "ImportError: No module named builtins"
+# RUN pip install future pyyaml typing
+
+ADD . /opt/deepdetect
+WORKDIR /opt/deepdetect/
+
+ENV CCACHE_DIR=/ccache
+ENV PATH=/usr/lib/ccache:/usr/local/nvidia/bin:/usr/local/cuda/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
 
 # Build cpp-netlib
-RUN wget https://github.com/cpp-netlib/cpp-netlib/archive/cpp-netlib-0.11.2-final.tar.gz && \
+# TODO(sileht): move to a package version
+RUN --mount=type=cache,target=/ccache/ \
+    wget https://github.com/cpp-netlib/cpp-netlib/archive/cpp-netlib-0.11.2-final.tar.gz && \
     tar xvzf cpp-netlib-0.11.2-final.tar.gz && \
     cd cpp-netlib-cpp-netlib-0.11.2-final && \
     mkdir build && \
@@ -71,36 +115,28 @@ RUN wget https://github.com/cpp-netlib/cpp-netlib/archive/cpp-netlib-0.11.2-fina
     make -j && \
     make install
 
-# Copy Deepdetect sources files
-WORKDIR /opt
-RUN git clone https://github.com/jolibrain/deepdetect.git /opt/deepdetect
-WORKDIR /opt/deepdetect/
-
 # Build Deepdetect
-RUN mkdir build && \
-    cd build && \
-    cp -a ../build.sh . && \
-    ./build.sh
+ENV TERM=xterm
+RUN --mount=type=cache,target=/ccache/ mkdir build && cd build && ../build.sh
 
 # Copy libs to /tmp/libs for next build stage
 RUN ./docker/get_libs.sh
 
 # Build final Docker image
 FROM ubuntu:18.04
-
-# Download default Deepdetect models
-ARG DEEPDETECT_DEFAULT_MODELS=true
+ARG DEEPDETECT_ARCH=cpu
 
 # Copy Deepdetect binaries from previous step
 COPY --from=build /opt/deepdetect/build/main /opt/deepdetect/build/main
 COPY --from=build /opt/deepdetect/get_models.sh /opt/deepdetect/get_models.sh
 
+LABEL description="DeepDetect deep learning server & API / ${DEEPDETECT_ARCH} version"
 LABEL maintainer="[email protected]"
-LABEL description="DeepDetect deep learning server & API / CPU version"
 
 # Install tools and dependencies
-RUN apt-get update && \
-    apt-get install -y wget \
+RUN --mount=type=cache,target=/var/cache/apt --mount=type=cache,target=/var/lib/apt \
+    apt-get update -y && \ apt-get install -y \
+    wget \
 	libopenblas-base \
 	liblmdb0 \
 	libleveldb1v5 \
@@ -117,7 +153,6 @@ RUN apt-get update && \
     libboost-regex1.65.1 \
 	libarchive13 \
 	libprotobuf10 && \
-    rm -rf /var/lib/apt/lists/*
 
 # Fix permissions
 RUN ln -sf /dev/stdout /var/log/deepdetect.log && \