% $Header: /cvsroot/latex-241/latex-beamer/solutions/generic-talks/generic-ornate-15min-45min.en.tex,v 1.4 2004/10/07 20:53:08 tantau Exp $

\documentclass{beamer}
\mode<presentation>

\usepackage{pgf,pgfarrows,pgfnodes,pgfautomata,pgfheaps,pgfshade}
\usepackage{amsmath}
\usepackage{amsfonts}
\usepackage{amssymb}
\usepackage{colortbl}
\usepackage{xcolor}
\usepackage{times}
\usepackage{psfrag}
\usepackage{dsfont}
%\usepackage{multimedia}
\usepackage{movie15}
\usepackage[english]{babel}
\usepackage[latin1]{inputenc}
\usepackage[T1]{fontenc}
\usepackage{algorithmic}
\usepackage{algorithm}

\newtheorem{assum}{Assumption}[section]
\newtheorem{rem}{Remark}[section]
\newtheorem{prop}{Proposition}[section]

\newcommand\gbfont[1]{{\fontfamily{phv}\selectfont #1}}
\DeclareMathOperator*{\argmin}{argmin}
\DeclareMathOperator*{\argminloc}{argminloc}
\DeclareMathOperator*{\diam}{diam}
\DeclareMathOperator*{\dist}{dist}
\DeclareMathOperator*{\supp}{supp}
\newcommand{\Ind}{\mathds{1}}
\newcommand{\Hrond}{\mathcal{H}}
\newcommand{\FrondT}{\mathcal{F}_T}

\newcommand{\Hrondi}{{\Hrond_i}}
\newcommand{\real}{\mathbb{R}}
\newcommand{\esp}{\mathbb{E}}
\newcommand{\prob}{\mathbb{P}}
\newcommand{\var}{\mathbb{V}}
\newcommand{\vertbar}{\left\vert\vphantom{\frac{1}{1}}\right.}
\newcommand{\proofend}{\begin{flushright} $\square$ \end{flushright}}
\newcommand{\wch}{\check{w}}
\newcommand{\Gcal}{\mathcal{G}}
\newcommand{\Dcal}{\mathcal{D}}
\newcommand{\rdkap}{\left(\real^d\right)^\kappa}
\definecolor{forestgreen}{rgb}{0.13,0.54,0.13}
\definecolor{dockerblue}{rgb}{0.11,0.56,0.98}

\newcommand{\green}{\textcolor{forestgreen}}
\newcommand{\zbf}{{\bf z}}
\newcommand{\blue}{\textcolor{blue}}
\newcommand{\dockerblue}{\textcolor{dockerblue}}
\newcommand{\purple}{\textcolor{purple}}
\newcommand{\magenta}{\textcolor{magenta}}

\mode<presentation>
{
  \usetheme{CambridgeUS}
  % or ...
\beamertemplatenavigationsymbolsempty

  \setbeamercovered{transparent}
  % or whatever (possibly just delete it)
}

\title % (optional, use only with long paper titles)
[]{Clustering algorithms distributed over a Cloud Computing Platform.}

\author[Matthieu Durut] % (optional, use only with lots of authors)
{}
% - Use the \inst{?} command only if the authors have different
%   affiliation.
\institute[Telecom/Lokad] % (optional, but mostly needed)
{}

\date[] % (optional)
{\sc September 28 th 2012}

\subject{}
\AtBeginSection[]
{
  \begin{frame}<beamer>
    \frametitle{Outline}
    \tableofcontents[currentsection,currentsubsection]
  \end{frame}
}

\begin{document}

\begin{frame}
\vspace{1cm}
  \titlepage
  \begin{center}
  {\footnotesize
  Ph. D. thesis supervised by Pr. Fabrice Rossi.
}
\end{center}

\begin{figure}[p]
$\begin{array}{cc}
\includegraphics[scale = 0.12]{../../Pictures/Logos/telecom} &
\includegraphics[scale = 0.45]{lokad-logo-fws-2009-300x63.jpg}\\
\end{array}$
\end{figure}
\end{frame}


\begin{frame}
\frametitle{Outline.}
\tableofcontents
\end{frame}

\section{Introduction to Cloud Computing}

\begin{frame}
    \frametitle{What is Cloud Computing ?}
\begin{block}{Some Features}
\begin{enumerate}
\item \alert{Abstraction of commodity hardware} that can be rent on-demand on a hourly basis.
\item Quasi-infinite hardware \blue{scale-up}.
\item \blue{Virtualization}, that makes web-applications maintenance easier.
\end{enumerate}
\end{block}
\begin{block}{Grid vs Cloud}
\begin{itemize}
\item Ownership.
\item Intensive use of \blue{Virtual Machines} (VM).
\item Elasticity.
\item Hardware administration and maintenance.
\end{itemize}
\end{block}
\end{frame}

\begin{frame}
\begin{block}{Everything a as Service}
\begin{enumerate}
\item Software as a Service (SaaS) : Gmail, Salesforce, Lokad API, etc.
\item \alert{Platform as a Service} (PaaS) : Azure, Amazon S3, etc,
\item Infrastructure as a Service (IaaS) : Amazon EC2, etc.
\end{enumerate}
\end{block}
\begin{block}{Stack of Azure}
\begin{itemize}
\item Storage Level : BlobStorage, TableStorage, QueueStorage, SQLAzure.
\item Execution Level : Dryad.
\item Domain Specific Language Level : DryadLinq, Scope.
\end{itemize}
\end{block}
\end{frame}

\begin{frame}
\begin{figure}
\centering
\includegraphics[scale=0.50]{../../Pictures/Cloud/msparallel.jpg}
\caption{Illustration of the Google, Hadoop and Microsoft technology stacks for cloud applications building.}
\end{figure}
\end{frame}

\begin{frame}
\frametitle{MapReduce}
\end{frame}

\begin{frame}
\frametitle{The Windows Azure Storage (WAS)}
\begin{block}{BlobStorage}
\begin{itemize}
\item \blue{Key-value pair} (blobname/blob) storage.
\item \alert{No more ACID}.
\item But \blue{atomicity, strong persistency and strong consistency} per blob.
\item Optimistic Read-Modify-Write primitive (RMW).
\end{itemize}
\end{block}
\begin{block}{QueueStorage}
\begin{itemize}
\item Set of scalable queues.
\item Asynchronous Message Delivery mechanism.
\item Approximately FIFO.
\item Messages returned at least once => \blue{Idempotency}.
\end{itemize}
\end{block}
\end{frame}

\begin{frame}
\frametitle{Elements of Azure applications architecture}
\begin{itemize}
\item No communication framework such as \blue{MPI}.
\item WAS used as a \alert{shared memory abstraction}.
\item No \blue{affinity} between storage and processing units.
\item Task agnosticity of workers (at least in the beginning).
\item \blue{Idempotence}.

\end{itemize}
\end{frame}

\section{Context}

\begin{frame}
\begin{block}{Why clustering?}
\begin{itemize}
\item One of the \alert{Lokad}'s abilities is to deal with \alert{large scale} data.
\item Need to group client data (\alert{clustering}) to extract information from complex objects (e.g. \blue{time series} seasonality).
\end{itemize}
\end{block}

\begin{block}{Problem Set-up}
\begin{itemize}
\item Data set is composed of $N$ points $\{\zbf_{t}\}_{t=1}^N$ in $\real^{\kappa}$.
\item Clustering POV: find a \blue{simplified representation} with $\kappa$ vectors of $\real^d$.
\item These vectors will be called \alert{prototypes/centroids} and gathered in a quantization scheme $\purple{w}=\left(\purple{w_1}, \ldots,\purple{w_\kappa}\right) \in  \rdkap$.
\end{itemize}
\end{block}
\end{frame}

\begin{frame}
\begin{block}{Objective}
Clustering challenge can be expressed as a \alert{minimization} of the \alert{empirical distortion} $C_{N}$, where
\begin{equation*}
\label{eq:EmpiricalDistortion}
C_{N}(w) = \sum_{t=1}^N{\min_{\ell = 1, \ldots, \kappa} \left\|\zbf_t  - w_\ell\right\|^2}, \quad \mbox{$w \in \left(\real^d\right)^{\kappa}$}.
\end{equation*}
\end{block}

\begin{block}{Initial challenge}
Exact minimization is computationnaly intractable.
\end{block}
\vspace{0.1cm}
\begin{block}{Some approximative algorithms}
\begin{itemize}
\item Batch K-Means
\item Vector Quantization (Online K-Means)
\item Neural Gas
\item Kohonen Maps
\end{itemize}
\end{block}
\end{frame}

\begin{frame}{}
\frametitle{Architecture Context}
\begin{block}{Why distributed?}
\begin{itemize}
\item A suitable way to allow more computing resources. Faster \blue{serial computers}: increasingly expensive + physical limits.
\vspace{0.2cm}
\item Cloud computing: adopted by \alert{Lokad} (\blue{MS Azure}). Early 2012, all apps on Cloud and scale-up $\sim$ 300VMs.
\vspace{0.2cm}
\item Consequences: \blue{communication delays} and the lack of \blue{efficient shared memory} $\rightarrow$ \alert{asynchronous} schemes.
\end{itemize}
\end{block}
\end{frame}

\section{Distributed Batch K-Means}

\begin{frame}
    \frametitle{Sequencial Batch K-Means}
%\begin{minipage}[b]{0.45\linewidth}

\begin{algorithm}[H]
    \caption{Sequential Batch K-Means}
    \begin{algorithmic}
    \label{algo_SequentialKMeans}
        \STATE Select $\kappa$ initial prototypes $(w_{k})_{k=1}^{\kappa}$
        \REPEAT
        \FOR{$t = 1$ to $N$}
        \FOR{$k=1$ to $\kappa$}
        \STATE compute $||\zbf_{t}-w_{k}||_{2}^{2}$
        \ENDFOR
        \STATE find the closest centroid $w_{k^{*}(t)}$ from $\zbf_{t}$;
        \ENDFOR
        \FOR{$k=1$ to $\kappa$}
        \STATE $w_{k} = \frac{1}{\# \{ t, k^{*}(t)=k \}} \displaystyle\sum_{\{ t, k^{*}(t)=k \}} \zbf_{t}$
        \ENDFOR
        \UNTIL {the stopping criterion is met}
    \end{algorithmic}
\end{algorithm}
%\end{minipage}
\begin{block}{}
\end{block}
\end{frame}

\begin{frame}
\begin{block}{Characteristics}
\begin{itemize}
\item Relatively \alert{fast} : $Batch_{seq}^{Wall time} = (3N\kappa d+N\kappa+Nd + \kappa d)IT^{flop}$, where $I$ refers to the number of iterations and $T^{flop}$ refers to the time for a floating point operation to be evaluated.
\item \blue{Determinist}.
\item Easy to set-up.
\item Results \blue{stationary} from a certain iteration.
\end{itemize}
\end{block}

\begin{block}{Suited for parallelization ?}
\begin{itemize}
\item Obvious \blue{data-level Parallelism}.
\item Same result than sequential.
\item Excellent \blue{speed-up efficiency} already achieved.
\end{itemize}
\end{block}
\end{frame}

\begin{frame}
\begin{block}{Distribution Scheme}
\begin{itemize}
\item Data-level parallelism suggests \alert{iterated Map-Reduce} distribution.
\item Data set $\{\zbf_{t}\}_{t=1}^N$ is homogeneously split into $M$ chunks (one per processing unit): $S^{i}, i \in \{1..M\}$.
\item The processing unit $i$ computes the distance $||\zbf^{i}_{t}-w_{k}||_{2}^{2}$ for $\zbf^{i}_{t} \in S^{i}$ and $k \in \{1..\kappa\}$ (\blue{Map phase}).
\item Then the new prototypes version is recomputed by one or several machines (\blue{Reduce phase}).
\end{itemize}
\end{block}
\end{frame}

\begin{frame}
\begin{center}
Batch K-Means distributed over a DMM architecture
\end{center}
\end{frame}

\begin{frame}
\begin{block}{Wall Time}
\begin{equation*}
Batch_{DMM}^{Wall Time} = T_{M}^{comp} + T_{M}^{comm},
\end{equation*}
where \alert{$T_{M}^{comp}$} refers to the \blue{wall time} of the \blue{assignment phase} and \alert{$T_{M}^{comm}$} refers to the wall time of the \blue{recalculation phase} (mostly spent in communications).
\end{block}
\vspace{0.3cm}
\begin{block}{Assignment phase}
\begin{equation*}
T_{M}^{comp} = \displaystyle \frac{3IN\kappa dT^{flop}}{M}.
\end{equation*}
\end{block}
\end{frame}

\begin{frame}
\begin{block}{Recalculation phase - DMM architecture with MPI}
\begin{equation*}
T_{M}^{comm}=\lceil log_{2}(M)\rceil \frac{I\kappa dS}{B},
\end{equation*}
where \alert{$S$} refers to the size of a double in memory (8 bytes in the following) and \alert{$B$} refers to the communication bandwidth per machine.
\end{block}

\vspace{0.3cm}

\begin{block}{Wall time - DMM architecture with MPI}
\begin{equation*}
Batch_{DMM}^{Wall Time} = \displaystyle \frac{3IN\kappa dT^{flop}}{M} + \lceil log_{2}(M)\rceil \frac{I\kappa dS}{B}.
\end{equation*}
\end{block}
\end{frame}

\begin{frame}
\begin{block}{Speed-up - DMM architecture with MPI}
\begin{align*}
SpeedUp_{DMM}(M,N) = \frac{3NT^{flop}}{\frac{3NT^{flop}}{M} +  \frac{S}{B} \lceil log_{2}(M) \rceil}.
\end{align*}
\end{block}

\begin{block}{Optimal number of processing units}
\begin{align*}
M_{DMM}^{*}  =  \frac{3NT^{flop}B}{S}.
\end{align*}
\end{block}
\end{frame}

\begin{frame}
\begin{center}
Batch K-Means distributed over Azure
\end{center}
\end{frame}

\begin{frame}
\begin{figure}
\centering
\includegraphics[scale=0.40]{../../Pictures/MapReduce/DistributedKMeans.pdf}
\caption{Distribution scheme of our cloud Batch K-Means.}
\end{figure}
\end{frame}

\begin{frame}
\begin{block}{Communication Modeling}
\begin{equation*}
T_{M}^{comm} = I\sqrt{M}\kappa dS(2T_{Blob}^{read} + T_{Blob}^{write}),
\end{equation*}
where \alert{$T_{Blob}^{read}$ (resp. $T_{Blob}^{write}$)} refers to the time needed by a given processing unit to \blue{download (resp. upload)} a blob from (resp. to) the storage per memory unit.
\end{block}

\vspace{0.6cm}

\begin{block}{Speed-up - Cloud architecture}
\begin{equation*}
SpeedUp(M,N) = \frac{3NT^{flop}}{\frac{3NT^{flop}}{M} + \sqrt{M}S(2T_{Blob}^{read} + T_{Blob}^{write})}.
\end{equation*}
\end{block}

\begin{block}{Optimal number of workers}
\begin{align*}
M^{*}(N)  =  \sqrt[2/3]{\frac{6NT^{flop}}{S(2T_{Blob}^{read} + T_{Blob}^{write})}}.
\end{align*}
\end{block}

\end{frame}

\begin{frame}
\begin{figure}[!h]
\begin{center}
\includegraphics[scale = 0.4, clip=true, trim = 0mm 25mm 0mm 25mm]{../../Pictures/Cloud/BandwidthReduce.pdf}
\end{center}
\caption{ Time to execute the Reduce phase per unit of memory ($2T_{Blob}^{read} + T_{Blob}^{write}$) in $~10^{-7}$sec/Byte in function of the number of communicating units.}
\label{fig:BandwidthReduce}
\end{figure}
\end{frame}

\begin{frame}
\begin{figure}
    \centering
    \includegraphics[scale=0.50]{../../texfiles/Nips2010Poster/Speedup.pdf}
    \caption{Charts of speedup performance curves with different data set size.}
    \label{fig_KMeansCloudScaleup1}
\end{figure}
\end{frame}

\begin{frame}
\begin{table}[!h]
    \centering
    \begin{scriptsize}
        \begin{tabular}{|c|c|c|c|c|c|c|c|}
            \hline
             & N & $M^{*}_{eff}$ & $M^{*}$  & Wall Time & Sequential & Effective  & Theoretical \\
             &  &  &  &  & theoretic time & Speedup &  Speedup ($=\frac{M^{*}}{3}$)\\
            \hline
            Exp. 1 & 62500 & 27 & 28 & 264 & 2798 & 10.6 & 9.34\\
            \hline
            Exp. 2 & 125000 & 45 & 45 & 306 & 5597 & 18.29 & 14.84\\
            \hline
            Exp. 3 & 250000 & 78 & 71 & 384 & 11194 & 29.15 & 23.55\\
            \hline
            Exp. 4 & 500000 & 95 & 112 & 521 & 22388 & 43.0 & 37.40\\
            \hline
        \end{tabular}

    \end{scriptsize}
    \caption{Comparison between the effective optimal number of processing units $M^{*}_{eff}$ and the theoretical optimal number of processing units $M^{*}$ for different data set size.}
    \label{table:speedupAchieved1}
\end{table}
\end{frame}

\begin{frame}
\begin{figure}[!h]
    \centering
    \includegraphics[scale=0.45]{../../texfiles/Nips2010Poster/Speedup2.pdf}
    \caption{Charts of speedup performance curves with different number of processing units. For each value of $M$, the value of $N$ is set accordingly so that the processing units are heavy loaded with data and computations.}
    \label{fig_KMeansCloudScaleup2}
\end{figure}
\end{frame}

\begin{frame}
\begin{figure}
    \centering
    \includegraphics[scale=0.5]{../../Pictures/DistributionSingleWorker.png}
    \caption{Distribution of the processing time (in second) for multiple runs of the same computation task for a single Virtual Machine (VM).}
    \label{fig_StragglerSingleDistribution}
\end{figure}
\end{frame}

\begin{frame}
\begin{figure}
    \centering
    \includegraphics[scale=0.5]{../../Pictures/Straggler.png}
    \caption{Distribution of the processing time (in second) for multiple runs of the same computation task for multiple VM.}
    \label{fig_StragglerDistribution}
\end{figure}
\end{frame}

\section{Distributed Vector Quantization algorithms}

\begin{frame}
    \frametitle{Asynchronous clustering: motivation\\
\begin{small}Joint work with Benoit Patra\end{small}}

\begin{block}{Every actions should be accounted once}
\begin{itemize}
\item No calculation should be discarded.
\item No calculation should be used more than once.
\item All the writes should result into prototypes update everywhere.
\item All the reads should be used locally.
\end{itemize}
\end{block}

\begin{block}{\textit{On War} from Clausewitz}
\begin{itemize}
\item Saturate bandwidth, memory, CPU, etc.
\item $\Longrightarrow$ \blue{Asynchronism}
\item $\Longrightarrow$ \alert{Online} or at least mini-batch (no more batch)
\end{itemize}
\end{block}
\end{frame}

\begin{frame}
    \frametitle{Sequential VQ algorithm}
\begin{block}{}
\begin{itemize}
\item Consists in \blue{incremental updates} of the $\rdkap$-valued prototypes $\{w(t)\}_{t=0}^\infty$.
\item Initiated from a random initial $w(0) \in \left(\real^d\right)^\kappa$.
\item Given a series of positive \emph{steps} $(\varepsilon_t)_{t>0}$, it produces a series of $w(t)$ by \alert{updating} $w$ at each step with a ``descent term''.
\end{itemize}
\end{block}

\begin{block}{}
\begin{equation*}\label{def:defH}
H(\zbf,w) = \left( \left(w_\ell - \zbf\right) \mathds{1}_{\left\{l= \argmin_{i = 1, \ldots, \kappa} \left\|\zbf  - w_i\right\|^2\right\}}\right)_{1 \leq \ell \leq \kappa}.
\end{equation*}
\end{block}

\begin{block}{}
\begin{equation*}\label{eq:finiteSample}
w(t+1)= w(t) - \varepsilon_{t+1}H\left(\zbf_{\{t+1 \text{ mod } n\}},w(t) \right), \quad t \geq 0.
\end{equation*}
\end{block}
\end{frame}

\begin{frame}
\begin{algorithm}[H]
    \caption{Sequential VQ algorithm}
    \begin{algorithmic}
        \STATE Select $\kappa$ initial prototypes $(w_{k})_{k=1}^{\kappa}$
        \STATE Set t=0
        \REPEAT
            \FOR{$k=1$ to $\kappa$}
                \STATE compute $||\zbf_{\{t+1 \text{ mod } n\}}-w_{k}||_{2}^{2}$
            \ENDFOR
            \STATE Deduce $H(\zbf_{\{t+1 \text{ mod } n\}},w)$
            \STATE Set $w(t+1)= w(t) - \varepsilon_{t+1}H\left(\zbf_{\{t+1 \text{ mod } n\}},w(t) \right)$
            \STATE increment $t$
        \UNTIL {the stopping criterion is met}
    \end{algorithmic}
\end{algorithm}
\end{frame}

\begin{frame}

\begin{block}{Our context}
\begin{itemize}
\item We assume that a \blue{satisfactory} VQ implementation has been found but too slow.
\item We will not be concerned with optimization of the several parameters (initialization, sequence of steps etc.)
\item We have access to a \alert{finite} dataset: $\left\{\zbf_{t}^i\right\}_{t=0}^n$, $i \in \{1, \ldots,M\}$ distributed over $M$ processing units.
\item \alert{When does a distributed VQ implementation perform better than the corresponding sequential one ?}
\end{itemize}
\end{block}
\end{frame}

\begin{frame}
\begin{block}{Definition of Speed-up for VQ algorithms}
\begin{itemize}
\item A ``reference'' prototypes version is made available in the shared-memory (BlobStorage), referred to as \blue{the prototypes shared version}: $w^{srd}$.
\item Performance is measured with the corresponding \blue{empirical distortion}: for all $\purple w \in \rdkap$,
\[  L_{N}(w)= \frac{1}{nM} \sum_{i=1}^M \sum_{t=1}^n{\min_{\ell = 1, \ldots, \kappa} \left\|\zbf^i_t  - w_\ell\right\|^2}\]
\item After any $t$ wall time seconds, the empirical distortion of the prototypes shared version should be lower than for the prototypes version produced by sequential algorithm.
\end{itemize}
\end{block}
\end{frame}

\begin{frame}
\begin{block}{Previous work}
\begin{itemize}
\item VQ as \blue{stochastic gradient descent} method
\item \blue{Shared-Memory} : interlaying the prototypes version updates
\item No Shared-Memory but \blue{Loss Convexity} : averaging the prototypes versions
\end{itemize}
\end{block}
\vspace{0.3cm}
\begin{block}{In our case}
\begin{itemize}
\item \alert{No efficient shared-memory}
\item \alert{No convexity of the loss function}
\end{itemize}
\end{block}

\begin{block}{Organization of our work}
\begin{itemize}
\item Simulated distributed architecture on a single machine.
\item Then cloud implementation
\end{itemize}
\end{block}

\end{frame}

\begin{frame}
\frametitle{First distributed scheme}

All the versions are set equal at time $t=0$, $w^1(0) = \ldots =w^M(0)$. For all $i \in \{1, \ldots,M\}$ and all $t \geq0$, we have the following iterations:
\begin{equation*}\label{eq:averaging}
\begin{cases}
w^i_{temp} = w^i(t) - \varepsilon_{t+1} H\left(\zbf^i_{\{t+1\text{ mod }n\}},w^i(t)\right)& \\ %\label{eq:processing}\\
w^i(t+1) = w^i_{temp} & \text{ if }  t \text{ mod } \tau  \neq 0 \text{ or } t=0,\\
    \begin{cases}
        w^{srd} =  \frac{1}{M}\sum_{j=1}^M{w^j_{temp}} & \\
        w^i(t+1) =  w^{srd}&\\
    \end{cases}& \text{ if } t \text{ mod } \tau =0 \text{ and } t \geq \tau.
\end{cases}
\end{equation*}

\end{frame}

\begin{frame}{A first basic parallelization scheme}
\begin{figure}
\begin{center}
\includegraphics[scale=0.18]{../../Pictures/parallelDALVQ/parallel.pdf}
\end{center}
\caption{A simple (and synchronous) scheme: whenever $\tau$ points are processed an averaging phase occurs.}
\end{figure}
\end{frame}

\begin{frame}{A first basic parallelization scheme}
\begin{figure}
\begin{center}
\includegraphics[scale=0.45]{../../Pictures/parallelDALVQ/pa10.pdf}
\end{center}
\caption{Charts of performance with different number of computing entities: $M=1,2,10$ and $\tau=10$.}
\end{figure}
\end{frame}

\begin{frame}{}
\begin{block}{A comparison between the previous parallel scheme and the sequential VQ}
For $t \text{ mod } \tau =0$ and $t > 0$. Then, for all $i\in\{1, \ldots,M\}$,
$\purple{w^i(t+1)} = \purple{w^i(t - \tau + 1)}- \sum_{t'= t- \tau +1}^{t}\varepsilon_{t'+1}\blue{\left(\frac{1}{M}\sum_{j=1}^M H\left(\zbf^j_{t'+1},w^j(t')\right)\right)} \quad \mbox{\alert{(parallel)}}$
\vspace{0.5cm}

$\purple{w(t+1)} = \purple{w(t - \tau + 1)}- \sum_{t'=t-\tau+1}^{t} \varepsilon_{t'+1}\blue{H\left(\zbf_{\{t'+1\text{ mod }n \}},w(t')\right)} \quad \mbox{\alert{(sequential)}}$

\end{block}
\vspace{0.5cm}
\begin{block}{}
Terms in \blue{blue} are estimators of the \alert{gradient}.
\end{block}
\end{frame}

\begin{frame}
\begin{block}{}
\begin{itemize}
\item Two SGD algorithms with the same \blue{sequence of steps} then, they have similar convergence speed.
\item Sequence of steps $\rightarrow$ \alert{learning rate} $\rightarrow$ trade-off exploration/convergence.
\end{itemize}
\end{block}
\begin{block}{Introducing displacement/descent terms}
For all $j \in \{1, \ldots, M\}$ and $t_2 \geq t_1 \geq 0$ set

\[
\Delta^j_{t_1\rightarrow t_2} = \sum_{t'=t_1 + 1}^{t_2} \varepsilon_{t'+1}H\left(\zbf^j_{\{t'+1\text{ mod }n \}},w^j(t')\right).
\]
corresponds to the \blue{displacement} of the prototypes computed by $j$ during $(t_1, t_2)$,
\end{block}
\end{frame}

\begin{frame}
\frametitle{Second distributed scheme}
\begin{equation*}\label{eq:correctAveraging}
\begin{cases}
w^i_{temp} = w^i(t) - \varepsilon_{t+1} H\left(\zbf^i_{\{t+1\text{ mod }n\}},w^i(t)\right)& \\ %\label{eq:processing}\\
w^i(t+1) = w^i_{temp} & \text{ if }  t \text{ mod } \tau  \neq 0 \text{ or } t=0,\\
    \begin{cases}
        w^{srd} = w^{srd} - \sum_{j=1}^M{\Delta^j_{t- \tau \rightarrow t} } & \\
        w^i(t+1) =  w^{srd} &\\
    \end{cases}& \text{ if } t \text{ mod } \tau =0 \text{ and } t \geq \tau.
\end{cases}
\end{equation*}
\end{frame}

\begin{frame}
\begin{figure}
\begin{center}
\includegraphics[scale = 0.18]{../../Pictures/ParallelDALVQ/gradientParallel.pdf}
\end{center}
\caption{Illustration of the parallelization scheme of VQ procedures described by equations \eqref{eq:correctAveraging}.}
\label{fig:illustrationGradient}
\end{figure}
\end{frame}

\begin{frame}
\begin{figure}
\begin{center}
\includegraphics[scale = 0.45]{../../Pictures/ParallelDALVQ/correctedpa10.pdf} \\
\end{center}
\caption{Charts of performance curves for a reviewed scheme $M=1,2,10$ and $\tau=10$.}
\label{fig:correctedParallel}
\end{figure}
\end{frame}

\begin{frame}
\frametitle{Delayed distributed scheme}
\begin{equation*}\label{eq:delayedAveraging}
\begin{cases}
w^i_{temp} = w^i(t) - \varepsilon_{t+1} H\left(\zbf^i_{\{t+1\text{ mod }n\}},w^i(t)\right)& \\ %\label{eq:processing}\\
w^i(t+1) = w^i_{temp} \;\;\;\;\;\;\;\;\;\;\;\;\;\;\;\;\;\;\;\;\;\;\;\;\;\;  \text{ if }  t \text{ mod } \tau  \neq 0 \text{ or } t=0,& \\
    \begin{cases}
        w^{srd} =   w^{srd} - \sum_{j=1}^M{\Delta^j_{t-2\tau \rightarrow t -\tau} } & \text{ if } t \text{ mod } \tau =0 \text{ and } t \geq 2\tau,\\
        w^i(t+1) =  w^{srd} - \Delta^i_{t -\tau \rightarrow t}&  \text{ if } t \text{ mod } \tau =0 \text{ and } t \geq \tau.\\
    \end{cases}
\end{cases}
\end{equation*}
\end{frame}

\begin{frame}
\begin{figure}
\begin{center}
\includegraphics[scale = 0.18]{../../Pictures/ParallelDALVQ/delayedGradientParallel.pdf}
\end{center}
\caption{Illustration of the parallelization scheme described by equations \eqref{eq:delayedAveraging}. The reducing phase is only drawn for processor $1$ where $t=2\tau$ and processor $4$ where $t=4\tau$.}
\label{fig:delayedAveraging}
\end{figure}
\end{frame}

\begin{frame}
\begin{figure}
\begin{center}
\includegraphics[scale = 0.45]{../../Pictures/ParallelDALVQ/delayedGradient10.pdf}
\end{center}
\caption{Charts of performance curves for iterations \eqref{eq:delayedAveraging} with different numbers of computing entities, $M=1,2,10$ and $\tau=10$.}
\label{fig:delayedParallel}
\end{figure}

\end{frame}

\begin{frame}{Simulated parallelization schemes first conclusions}
\begin{block}{}
\blue{Motto:} summing displacement term rather than averaging versions.
\end{block}
\vspace{0.5cm}
\begin{block}{Experimental results}
\begin{itemize}
\item Satisfactory speed-ups are \alert{recovered} for the later simulated parallel schemes.
\item \blue{Delays (determinist + random)} are also studied: reasonable [random] delays do not have sever impact on the convergence.
\item \alert{Good perspectives} for a true implementation on a could computing platform.
\end{itemize}
\end{block}
\end{frame}

\begin{frame}
\frametitle{The \texttt{CloudDALVQ} project}
\begin{itemize}
\item Scientific project for testing new large scale \purple{clustering/quantization} algorithms distributed on a Cloud Platform (\blue{MS Azure}).
\vspace{0.2cm}
\item \blue{Open source} written in C\#.NET released under new BSD Licence.\\
\begin{small}\blue{\texttt{http://code.google.com/p/clouddalvq/}}\end{small}
\end{itemize}
\end{frame}

\begin{frame}
\begin{figure}
\begin{center}
\includegraphics[scale = 0.40]{../../Pictures/CloudDALVQ/AsyncVQ.pdf}
\end{center}
\caption{Distribution scheme of our cloud VQ implementation.}
\label{fig:delayedParallel}
\end{figure}
\end{frame}

\begin{frame}
\begin{figure}
\begin{center}
\includegraphics[scale = 0.30]{../../Pictures/CloudDALVQ/ProcessService.pdf}
\end{center}
\label{fig:delayedParallel}
\end{figure}
\end{frame}

\begin{frame}
\begin{figure}[!h]
\begin{center}
\includegraphics[scale = 0.4]{../../Pictures/CloudDALVQ/scalabilityOneLayer.pdf}
\end{center}
\caption{Normalized quantization curves with $M=1,2,4,8,16$. Troubles appear with $M=16$ because the ReduceService is overloaded.}
\label{fig:onelayer}
\end{figure}
\end{frame}

\begin{frame}
\begin{figure}[!h]
\begin{center}
\includegraphics[scale = 0.4]{../../Pictures/CloudDALVQ/scalabilityTwoLayer.pdf}
\end{center}
\caption{Normalized quantization curves with $M=8,16,32,64$ with an extra layer for the so called ``reducing task''.}
\label{fig:twolayer}
\end{figure}
\end{frame}

\begin{frame}
\begin{figure}[!h]
\begin{center}
\includegraphics[scale = 0.4]{../../Pictures/CloudDALVQ/fightbatchM16.pdf}
\end{center}
\caption{This chart reports on the competition between our cloud DAVQ algorithm and the cloud Batch K-Means. The graph shows the empirical distortion of the algorithms over the time.}
\label{fig:twolayer}
\end{figure}
\end{frame}

\end{document}
