cp4cds-for-magic.tex

%% LaTeX Beamer presentation template (requires beamer package)
%% see http://bitbucket.org/rivanvx/beamer/wiki/Home
%% idea contributed by H. Turgut Uyar
%% template based on a template by Till Tantau
%% this template is still evolving - it might differ in future releases!

\documentclass{beamer}

\mode<presentation>
{
\usetheme{Warsaw}
% \usetheme{Copenhagen}
\usecolortheme{beaver}

\setbeamercovered{transparent}
}

\usepackage[english]{babel}
\usepackage[latin1]{inputenc}

% font definitions, try \usepackage{ae} instead of the following
% three lines if you don't like this look
\usepackage{mathptmx}
\usepackage[scaled=.90]{helvet}
\usepackage{courier}

\usepackage[T1]{fontenc}

% User packages
\usepackage[absolute,overlay]{textpos}
\usepackage{tikz}
\usepackage{listings}

% colors
\usepackage{color}
\definecolor{gray}{rgb}{0.4,0.4,0.4}
\definecolor{darkblue}{rgb}{0.0,0.0,0.6}
\definecolor{cyan}{rgb}{0.0,0.6,0.6}

% XML listings
\lstset{
  basicstyle=\ttfamily,
  columns=fullflexible,
  showstringspaces=false,
  commentstyle=\color{gray}\upshape,
  escapechar=* % <=  to escape to LaTeX
}

\lstdefinelanguage{XML}
{
  morestring=[b]",
  morestring=[s]{>}{<},
  morecomment=[s]{<?}{?>},
  stringstyle=\color{black},
  identifierstyle=\color{darkblue},
  keywordstyle=\color{cyan},
  morekeywords={xmlns,version,type}% list your attributes here
}

% JSON listings
\usepackage{xcolor}

\colorlet{punct}{red!60!black}
\definecolor{background}{HTML}{EEEEEE}
\definecolor{delim}{RGB}{20,105,176}
\colorlet{numb}{magenta!60!black}

\lstdefinelanguage{JSON}{
  basicstyle=\normalfont\ttfamily,
  showstringspaces=false,
  breaklines=true,
  %frame=lines,
  %backgroundcolor=\color{background},
}

%%% Global Settings %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\graphicspath{{images/}{draw.io/}} % Root directory of the pictures

%%% Title %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\title{CP4CDS}

\subtitle{Climate Model Data and Compute for Copernicus Climate Data Store}

% - Use the \inst{?} command only if the authors have different
%   affiliation.
\author{\vspace{2.3cm}\\
C. Ehbrecht\inst{1}
\and R. Petrie\inst{2}
\and A. Stephens\inst{2}
\and S. Denvil\inst{3}
}
%\author{\inst{1}}

% - Use the \inst command only if there are several affiliations.
% - Keep it simple, no one is interested in your street address.
\institute[Institute]
{
\inst{1}%
DKRZ - German Climate Compute Center
\and
\inst{2}%
STFC - Science and Technology Facilities Council
\and
\inst{3}%
IPSL - Institute Pierre Simon Laplace
}

\date{\footnotesize{$1^{st}$ of February 2018 / MAGIC Workshop at KNMI}}


% This is only inserted into the PDF information catalog. Can be left
% out.
\subject{Talks}



% If you have a file called "university-logo-filename.xxx", where xxx
% is a graphic format that can be processed by latex or pdflatex,
% resp., then you can add a logo as follows:

% \pgfdeclareimage[height=0.5cm]{university-logo}{university-logo-filename}
% \logo{\pgfuseimage{university-logo}}



% Delete this, if you do not want the table of contents to pop up at
% the beginning of each subsection:
\AtBeginSubsection[]
{
\begin{frame}<beamer>
\frametitle{Outline}
\tableofcontents[currentsection,currentsubsection]
\end{frame}
}

% Section title slides
\AtBeginSection[]{
  \begin{frame}
  \vfill
  \centering
  \begin{beamercolorbox}[sep=8pt,center,shadow=true,rounded=true]{title}
    \usebeamerfont{title}\insertsectionhead\par%
  \end{beamercolorbox}
  \vfill
  \end{frame}
}


% If you wish to uncover everything in a step-wise fashion, uncomment
% the following command:

%\beamerdefaultoverlayspecification{<+->}

\begin{document}

\begin{frame}
   \tikz [remember picture,overlay]
    \node at
        ([yshift=4.8cm]current page.south)
        %or: (current page.center)
        {\includegraphics[height=1cm]{copernicus-logo} \includegraphics[height=1cm]{c3s-logo}};
   \titlepage
\end{frame}


\begin{frame}
\frametitle{Outline}
\tableofcontents
% You might wish to add the option [pausesections]
\end{frame}


% %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{Overview}

% -----------------------------------------------
\begin{frame}
\frametitle<presentation>{CP4CDS Services: Data + Compute}

  \begin{columns}[c]
    \begin{column}{.5\textwidth}
      \centering
      \includegraphics[width=\textwidth]{cp4cds}
    \end{column}
    \begin{column}{.5\textwidth}
      \begin{itemize}
        \item An extended ESGF sub-system populated and configured to support the Copernicus Climate Data Store.
        \item A quality controlled subset of CMIP5 data to the CDS.
        \item Deploy data and compute services across three sites: CEDA, DKRZ and IPSL.
      \end{itemize}
    \end{column}
  \end{columns}

  \vfill

  \centering
  \footnotesize{\url{https://github.com/cp4cds}}

\end{frame}

% -----------------------------------------------
\begin{frame}
\frametitle<presentation>{CP4CDS Service Interfaces}

  \begin{figure}[ht]
    \centering
    \includegraphics[height=4cm]{cp4cds-interfaces.png}
  \end{figure}

  Other protocols like {\bf Web Map Service} and {\bf Web Coverage Service}
  are possible for data access and WPS outputs.

\end{frame}

% -----------------------------------------------
\begin{frame}
\frametitle<presentation>{CP4CDS - Load Balancing}
  \begin{columns}[c]
    \begin{column}{.5\textwidth}
      \centering
      \includegraphics[width=\textwidth]{cp4cds-federated.png}
    \end{column}
    \begin{column}{.5\textwidth}
      \begin{itemize}
        \item Goals: scale to meet demand and to be operationally resilient.
        \item Geographic load balancing across three centres.
        \item Load balancing within a centre.
      \end{itemize}
    \end{column}
  \end{columns}
  \vfill
  Synchronisation between centres is critical for delivery of load-balanced solution \ldots

\end{frame}

% -----------------------------------------------
\begin{frame}
\frametitle<presentation>{CP4CDS Access Control}

  \begin{columns}[c]
    \begin{column}{.5\textwidth}
      \centering
      \includegraphics[width=\textwidth]{cp4cds-access-control}
    \end{column}
    \begin{column}{.5\textwidth}
      \begin{itemize}
        \item May be required to address potential data licensing.
        \item To address resource usage by individuals.
        \item Data access currently with X.509 certificates.
        \item OAuth2 token access possible.
      \end{itemize}
    \end{column}
  \end{columns}

\end{frame}

% %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{Data Service}

% -----------------------------------------------
\begin{frame}
\frametitle<presentation>{Climate Model Data}

  \begin{itemize}
    \item Provide CMIP5 subset selected for Copernicus.
    \item Quality-checked data.
    \item CORDEX is on its way ... another Copernicus project.
  \end{itemize}

\end{frame}

% -----------------------------------------------
\begin{frame}
\frametitle<presentation>{ESGF Data Node}

  \begin{itemize}
    \item Using vanilla ESGF data nodes.
    \item Federated between CEDA, IPSL and DKRZ.
  \end{itemize}

  \begin{figure}[ht]
    \centering
    \includegraphics[height=4cm]{cp4cds-data-node-at-ceda}
  \end{figure}

  \centering
  \footnotesize{\url{https://cp4cds-index1.ceda.ac.uk/}}

\end{frame}

% -----------------------------------------------
\begin{frame}
\frametitle<presentation>{Data Management - Web App}
  \begin{itemize}
    \item Interrogate CP4CDS data web-app
    \item Data availability matrix
  \end{itemize}

  \begin{figure}[ht]
    \centering
    \includegraphics[height=4cm]{qcapp.png}
  \end{figure}

  \centering
  \footnotesize{\url{https://cp4cds-qcapp.ceda.ac.uk}}


\end{frame}

% -----------------------------------------------
\begin{frame}
\frametitle<presentation>{Data Management - QC}
  Quality control information will become available:
  \begin{itemize}
    \item Aggregated QC information on models or variables
    \item Detailed file QC information
  \end{itemize}

  \begin{figure}[ht]
    \centering
    \includegraphics[height=4cm]{qc.png}
  \end{figure}

\end{frame}

% -----------------------------------------------
\begin{frame}
\frametitle<presentation>{Data Management - QC Tools}
  \begin{itemize}
    \item CEDA-CC:
    {\footnotesize{\url{https://github.com/cedadev/ceda-cc}}}
    \item CF-Checker:
    {\footnotesize{\url{http://cfconventions.org/compliance-checker.html}}}
    \item Time-Checker (\emph{new}):
    \footnotesize{\url{https://github.com/cedadev/time-checks}}
  \end{itemize}

\end{frame}

% -----------------------------------------------
\begin{frame}
\frametitle<presentation>{Data Management - Data Replication}
  \begin{itemize}
    \item Using {\bf Synda} data replication tool
    \item Data is replicated accross CEDA, IPSL and DKRZ
  \end{itemize}

  \centering
  \footnotesize{\url{https://github.com/Prodiguer/synda}}

\end{frame}


% %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{Compute Service}

% +++++++++++++++++++++++++++++++++++++++++++++++
\subsection{Architecture}

% -----------------------------------------------
\begin{frame}
\frametitle<presentation>{CP4CDS Compute Service}


  \begin{columns}[c]
    \begin{column}{.5\textwidth}
      \centering
      \includegraphics[width=\textwidth]{cp4cds-wps}
    \end{column}
    \begin{column}{.5\textwidth}
      \begin{itemize}
        \item A WPS request form WPS client.
        \item Nginx/Gunicorn delegates request to PyWPS WSGI app.
        \item Processing Backend has read-only access to data pool.
        \item Using PyWPS scheduler extension.
      \end{itemize}
    \end{column}
  \end{columns}
  \vfill
  \centering
  \footnotesize{\url{https://github.com/cp4cds/copernicus-wps-demo}}

\end{frame}

% -----------------------------------------------
\begin{frame}
\frametitle<presentation>{Climate Analysis: C3S MAGIC Toolbox}

  \begin{figure}[ht]
    \centering
    \includegraphics[height=5cm]{c3s-magic}
  \end{figure}
  Used by CP4CDS {\bf Processing Backend} for CMIP5 climate model projections.

  \centering
  \footnotesize{\url{https://github.com/c3s-magic}}

\end{frame}

% +++++++++++++++++++++++++++++++++++++++++++++++
\subsection{Deployment and Software}

% -----------------------------------------------
\begin{frame}
\frametitle<presentation>{SDDS - Software Dependency and Deployment Solution}

  \begin{figure}[ht]
    \centering
    \includegraphics[height=6cm]{cp4cds-toolbox}
  \end{figure}

  \centering
  \footnotesize{\url{https://github.com/cp4cds/cp4cds-wps-template}}

\end{frame}

% -----------------------------------------------
\begin{frame}
\frametitle<presentation>{Deployment Tools: Ansible}
  \begin{columns}[c]
    \begin{column}{.2\textwidth}
      \includegraphics[width=0.6\textwidth]{ansible}
    \end{column}
    \begin{column}{.8\textwidth}
      \begin{itemize}
        \item Ansible: Python library, no services, only SSH for remote deployment.
        \item Using Ansible to deploy CP4CDS WPS service.
        \item Deployment scenarios: single host, cluster, docker orchestration.
        \item Separate system installation (Nginx) and application deployment (PyWPS + MAGIC).
        \item Ansible will replace the current Buildout deployment solution (application level, single host only).
      \end{itemize}
    \end{column}
  \end{columns}

  \vfill
  \centering
  \footnotesize{\url{https://github.com/bird-house/birdhouse-ansible/tree/master/demo}}
\end{frame}

% -----------------------------------------------
\begin{frame}
\frametitle<presentation>{Deployment Tools: Conda}

  \begin{columns}[c]
    \begin{column}{.2\textwidth}
      \includegraphics[width=\textwidth]{conda_logo}
    \end{column}
    \begin{column}{.8\textwidth}
      \begin{itemize}
        \item All Application and Diagnostics software dependencies are managed with Conda.
        \item \footnotesize{\url{https://conda.io/docs/index.html}}
      \end{itemize}
    \end{column}
  \end{columns}
  % next row
  \vfill
  \begin{columns}[c]
    \begin{column}{.2\textwidth}
      \includegraphics[width=1.3\textwidth]{anaconda-cloud}
    \end{column}
    \begin{column}{.8\textwidth}
      \begin{itemize}
        \item Using Anaconda Cloud to share Conda packages.
        \item Production-ready packages are maintained on conda-forge community channel.
        \item \footnotesize{\url{https://conda-forge.org/}}
      \end{itemize}
    \end{column}
  \end{columns}
  % next row
  \vfill
  \centering
  \footnotesize{Need to work on stable Conda environments!}

\end{frame}

% -----------------------------------------------
\begin{frame}
\frametitle<presentation>{Deployment Tools: Docker}

  \begin{columns}[c]
    \begin{column}{.2\textwidth}
      \includegraphics[width=\textwidth]{docker}
    \end{column}
    \begin{column}{.8\textwidth}
      \begin{itemize}
        \item Kubernetes for Docker container orchestration
        \item Planned: PyWPS Docker Extension to launch WPS Jobs as Container
      \end{itemize}
    \end{column}
  \end{columns}
  % next row
  \vrule
  \begin{columns}[c]
    \begin{column}{.2\textwidth}
      \includegraphics[width=1.2\textwidth]{docker-cloud}
    \end{column}
    \begin{column}{.8\textwidth}
      \begin{itemize}
        \item Docker containers available on Docker Cloud ... for development and testing.
        \item We might have our own Docker repository for production.
        \item Previous versions of diagnostics can be pulled from Docker repository.
      \end{itemize}
    \end{column}
  \end{columns}

\end{frame}

% -----------------------------------------------
\begin{frame}
\frametitle<presentation>{PyWPS}

  \begin{figure}[ht]
    \centering
    \includegraphics[height=2cm]{pywps}
  \end{figure}

  \begin{itemize}
    \item An implementation of the OGC Web Processing Service standard
    \item Implements WPS 1.0.0 standard (WPS 2.0.0 in progress)
    \item Coded in the Python language (researcher friendly)
    \item Easy to hack (developer friendly)
    \item Relevant contributions by over a dozen individuals
    \item OSGeo accreditation around the corner \ldots
  \end{itemize}

  \vspace{0.2cm}
  \centering
  \footnotesize{\url{http://pywps.org}}

\end{frame}

% -----------------------------------------------
\begin{frame}
\frametitle<presentation>{PyWPS Scheduler Extension}

  \begin{figure}[ht]
    \centering
    \includegraphics[height=4cm]{pywps-scheduler-extension}
  \end{figure}

  \centering
  \footnotesize{\url{https://github.com/geopython/pywps/blob/develop/docs/extensions.rst}}

\end{frame}

% +++++++++++++++++++++++++++++++++++++++++++++++
\subsection{WPS Metadata}

% -----------------------------------------------
\begin{frame}
\frametitle<presentation>{WPS Meta: Providing Optional Metadata}

  \begin{columns}[c]
    \begin{column}{.5\textwidth}
      \centering
      \includegraphics[width=\textwidth]{client-dynamic-input.png}
    \end{column}
    \begin{column}{.5\textwidth}
      \begin{itemize}
        \item {\bf DescribeProcess} request provides infos about inputs and outputs.
        \item Static values can be provided directly.
        \item But ... we need {\bf dynamic values} (dependencies between inputs etc.)
        \item Solution: using {\bf ows:Metadata} element and WPS {\bf meta-helper service}.
      \end{itemize}
    \end{column}
  \end{columns}

\end{frame}

% -----------------------------------------------
\begin{frame}[fragile]
\frametitle<presentation>{WPS Meta: OWS Metadata Element}
  \lstset{language=XML}
  {\scriptsize
  \begin{lstlisting}
<wps:Input>
  <ows:Title>Experiment</ows:Title>
  <ows:Identifier>experiment</ows:Identifier>
  <ows:Metadata
    xlink:href="http://some.host/meta-helper/wps"
    xlink:role="http://www.opengis.net/wps/input/dynamic/meta_esgf"/>
  <LiteralData>
    <ows:DataType>string</ows:DataType>
  </LiteralData>
</wps:Input>
  \end{lstlisting}}

\end{frame}

% -----------------------------------------------
\begin{frame}[fragile]
\frametitle<presentation>{WPS Meta: meta-helper service}

  \lstset{language=XML}
  \begin{lstlisting}
  http://some.host/meta_helper/wps?
    Request=Execute &
    Identifier=meta_esgf&
    DataInputs=
      selection="{'model':'MPI-ESM-LR',
                    'variable':'tas'}";
      wanted='experiment'
  \end{lstlisting}

\end{frame}

% -----------------------------------------------
\begin{frame}[fragile]
\frametitle<presentation>{WPS Meta: JSON Response}
  \lstset{language=JSON}
  \begin{lstlisting}
  {
    "parameter": "experiment",
    "values":
      ["historical",
       "rcp26",
       "rcp45",
       "rcp85"]
  }
  \end{lstlisting}

\end{frame}

% +++++++++++++++++++++++++++++++++++++++++++++++
\subsection{WPS Clients}

% -----------------------------------------------
\begin{frame}
\frametitle<presentation>{WPS Clients: MAGIC Demo Portal}

  \begin{figure}[ht]
    \centering
    \includegraphics[height=5cm]{magic-portal}
  \end{figure}

  \centering
  \footnotesize{\url{http://portal.c3s-magic.eu/}}

\end{frame}

% -----------------------------------------------
\begin{frame}
\frametitle<presentation>{WPS Clients: Generic WPS App}

  \begin{figure}[ht]
    \centering
    \includegraphics[height=5cm]{phoenix-demo}
  \end{figure}

  \centering
  \footnotesize{\url{https://bovec.dkrz.de/processes/list?wps=copernicus}}

\end{frame}

% -----------------------------------------------
\begin{frame}
\frametitle<presentation>{WPS Clients: Command Line}
  \begin{figure}[ht]
    %\centering
    \includegraphics[height=7cm]{birdy-terminal}
  \end{figure}
  %\centering
  %\footnotesize{\url{http://birdy.readthedocs.io/en/latest/}}
\end{frame}

% -----------------------------------------------
\begin{frame}
\frametitle<presentation>{WPS Clients: Browser or wget}

  \begin{figure}[ht]
    %\centering
    \includegraphics[height=7cm]{rest-client-post}
  \end{figure}

  %\centering
  %\footnotesize{\url{http://birdhouse-workshop.readthedocs.io/en/latest/pywps/testing.html}}

\end{frame}

% %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{Summary}

\begin{frame}
\frametitle<presentation>{Summary}

  \begin{itemize}
    \item QC Web App for available datasets is online.
    \item CMIP5 data replication to all three data centres started.
    \item Initial Demo WPS compute service with MAGIC toolbox is online.
    \item Roll-out of compute service to all three data centres in 2018.
    \item Initial deployment solution (SDDS) available.
    \item Working on Kubernetes for Docker container orchestration.
    \item Working on WPS meta-helper service with MAGIC.
  \end{itemize}
\end{frame}

% %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\appendix

\section{Appendix}

% -----------------------------------------------
\begin{frame}
\frametitle<presentation>{Maybe slides from Phil?}

  \begin{itemize}
    \item Load-balancing.
    \item Kubernetes for ESGF data node.
  \end{itemize}
\end{frame}

\end{document}