Move ML4CVD over (#1)

add ml4cvd files

.gitattributes

@@ -0,0 +1,5 @@
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.ipynb filter=nbstripout
+*.ipynb diff=ipynb

.gitignore

@@ -0,0 +1,9 @@
+.DS_Store
+.vscode
+.idea/*
+*.linux
+*.log
+__pycache__/*
+trained_models/*
+recipes_output/*
+.ipynb_checkpoints

docker/Dockerfile

@@ -0,0 +1,35 @@
+# The suggested base images are:
+#   - ufoym/deepo:all-py36-jupyter for GPU-enabled machines
+#   - ufoym/deepo:all-py36-jupyter-cpu for CPU-only (non-GPU-enabled) machines
+# BASE_IMAGE can be specified at build time by adding the following argument:
+#   --build_arg BASE_IMAGE="some_other_image"
+
+ARG BASE_IMAGE
+FROM ${BASE_IMAGE}
+
+LABEL maintainer="James Pirruccello <[email protected]>"
+
+# Setup time zone (or else docker build hangs)
+ENV TZ=America/New_York
+RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone
+
+COPY ./config/* /app/
+
+WORKDIR /app
+
+# Note that some layers are kept separate to encourage layer re-use and to try
+# to minimize full recompilation where possible.
+
+# Basic setup
+RUN ./ubuntu.sh
+
+# FastAI. See the Developer Install under https://github.com/fastai/fastai/ to
+# understand this odd sequence of installing then uninstalling fastai before 
+# installing it from github. (Basically, to get its deps.)
+# RUN pip3 install -r fastai-requirements.txt
+# RUN pip3 uninstall -y fastai
+# RUN ./fastai.sh
+
+RUN apt-get install python3-tk -y
+# Requirements for the tensorflow project
+RUN pip3 install -r tensorflow-requirements.txt

docker/build.sh

@@ -0,0 +1,118 @@
+#!/usr/bin/env bash
+
+# This script can be used to build and tag a 'ml4cvd' image, optionally push it to Google Container Registry,
+# and again optionally, tag the image also as 'latest_<gpu|cpu>'.
+#
+# It assumes 'gcloud' has been installed and Docker has been configured to use 'gcloud' as a credential helper
+# by running 'gcloud auth configure-docker'.
+
+# Stop the execution if any of the commands fails
+set -e
+
+################### VARIABLES ############################################
+
+REPO="gcr.io/broad-ml4cvd/deeplearning"
+TAG=$( git rev-parse --short HEAD )
+CONTEXT="docker/"
+CPU_ONLY="false"
+PUSH_TO_GCR="false"
+
+BASE_IMAGE_GPU="ufoym/deepo:all-py36-jupyter"
+BASE_IMAGE_CPU="ufoym/deepo:all-py36-jupyter-cpu"
+
+LATEST_TAG_GPU="latest-gpu"
+LATEST_TAG_CPU="latest-cpu"
+
+SCRIPT_NAME=$( echo $0 | sed 's#.*/##g' )
+
+RED='\033[0;31m'
+BLUE='\033[0;34m'
+NC='\033[0m' # No colour
+
+################### HELPER FUNCTIONS ############################################
+
+usage()
+{
+    cat <<USAGE_MESSAGE
+
+    This script can be used to build and tag a 'ml4cvd' image, optionally push it to Google Container Registry,
+    and again optionally, tag the image also as 'latest_<gpu|cpu>'.
+
+    It assumes 'gcloud' has been installed and Docker has been configured to use 'gcloud' as a credential helper
+    by running 'gcloud auth configure-docker'.
+
+    Usage: ${SCRIPT_NAME} [-d <path>] [-t <tag>] [-chp]
+
+    Example: ./${SCRIPT_NAME} -d /Users/kyuksel/github/ml4cvd/jamesp/docker/deeplearning -cp
+
+        -d      <path>      Path to directory where Dockerfile is located. Default: '${CONTEXT}'
+
+        -t      <tag>       String used to tag the Docker image. Default: short version of the latest commit hash
+
+        -c                  Build off of the cpu-only base image and tag image also as '${LATEST_TAG_CPU}'.
+                            Default: Build image to run on GPU-enabled machines and tag image also as '${LATEST_TAG_GPU}'.
+
+        -p                  Push to Google Container Register
+
+        -h                  Print this help text
+
+USAGE_MESSAGE
+}
+
+################### OPTION PARSING #######################################
+
+while getopts ":d:t:chp" opt ; do
+    case ${opt} in
+        h)
+            usage
+            exit 1
+            ;;
+        d)
+            CONTEXT=$OPTARG
+            ;;
+        t)
+            TAG=$OPTARG
+            ;;
+        c)
+            CPU_ONLY="true"
+            ;;
+        p)
+            PUSH_TO_GCR="true"
+            ;;
+        :)
+            echo -e "${RED}ERROR: Option -${OPTARG} requires an argument.${NC}" 1>&2
+            usage
+            exit 1
+            ;;
+        *)
+            echo -e  "${RED}ERROR: Invalid option: -${OPTARG}${NC}" 1>&2
+            usage
+            exit 1
+            ;;
+    esac
+done
+shift $((OPTIND - 1))
+
+################### SCRIPT BODY ##########################################
+
+if [[ ${CPU_ONLY} == "true" ]]; then
+    BASE_IMAGE=${BASE_IMAGE_CPU}
+    LATEST_TAG=${LATEST_TAG_CPU}
+else
+    BASE_IMAGE=${BASE_IMAGE_GPU}
+    LATEST_TAG=${LATEST_TAG_GPU}
+fi
+
+echo -e "${BLUE}Building Docker image '${REPO}:${TAG}' from base image '${BASE_IMAGE}', and also tagging it as '${LATEST_TAG}'...${NC}"
+# --network host allows for the container's network stack to use the Docker host's network
+docker build ${CONTEXT} \
+    --build-arg BASE_IMAGE=${BASE_IMAGE} \
+    --tag "${REPO}:${TAG}" \
+    --tag "${REPO}:${LATEST_TAG}" \
+    --network host \
+
+if [[ ${PUSH_TO_GCR} == "true" ]]; then
+    echo -e "${BLUE}Pushing the image '${REPO}' to Google Container Registry with tags '${TAG}' and '${LATEST_TAG}'...${NC}"
+    docker push ${REPO}:${TAG}
+    docker push ${REPO}:${LATEST_TAG}
+fi

docker/config/fastai-requirements.txt

@@ -0,0 +1,3 @@
+--find-links https://download.pytorch.org/whl/nightly/cu92/torch_nightly.html
+torch_nightly
+fastai

docker/config/fastai.sh

@@ -0,0 +1,12 @@
+#!/bin/bash
+
+# Install the github repo version
+git clone https://github.com/fastai/fastai
+cd fastai
+
+# Peg our version to a known-working SHA, since they make 
+# post-1.0 breaking changes literally every day...
+git reset --hard 14868ca69483afbaa8e28d4e281c148d1dad1c89
+
+tools/run-after-git-clone
+pip install -e .[dev]

docker/config/pyukbb.sh

@@ -0,0 +1,9 @@
+#!/bin/bash
+
+# Fetch the latest available version
+wget https://storage.googleapis.com/ml4cvd/ml4cvd-master.zip
+unzip ml4cvd-master.zip
+cd ml4cvd-master/pyukbb
+
+tools/run-after-git-clone
+pip install -e .[dev]

docker/config/tensorflow-requirements.txt

@@ -0,0 +1,6 @@
+h5py
+pydot
+keras==2.2.4
+keras-tqdm
+pydicom
+hyperopt

docker/config/ubuntu.sh

@@ -0,0 +1,5 @@
+#!/bin/bash
+
+# Other necessities
+apt-get update
+apt-get install -y wget unzip curl python-pydot python-pydot-ng graphviz