\documentclass{beamer}

\usepackage{algorithm}
\usepackage{algorithmic}
\usepackage{beamerthemesplit}
\usepackage{graphicx}
\usepackage[T1]{fontenc}
\usepackage[latin1]{inputenc}

\title{Algorithmes de classification répartis sur le cloud}
\author{Matthieu Durut}
\date{\sc Soutenance de thèse, 28 Septembre 2012}

\newcommand{\bZ}{\mathbf{Z}}
\newcommand{\bz}{\mathbf{z}}
\newcommand{\bX}{\mathbf{X}}
\newcommand{\bx}{\mathbf{x}}
\newcommand{\by}{\mathbf{y}}
\newcommand{\ind}{\mathbf{1}}
\newcommand{\card}[1]{|{#1}|}
\renewcommand{\L}{\mathcal L}
\newcommand{\R}{\mathbb R}
\newcommand{\de}{\mathrm d}
\newcommand{\jumpOnePage}{\newpage \thispagestyle{empty} \null \newpage}



\AtBeginSection[]
{
   \begin{frame}
       \frametitle{Outline}
       \tableofcontents[currentsection]
   \end{frame}
}

\AtBeginSubsection[]
{
   \begin{frame}
       \frametitle{Outline}
       \tableofcontents[currentsection,currentsubsection]
   \end{frame}
}

\begin{document}

\begin{frame}
\titlepage
\end{frame}

\section{Introduction.}

\begin{frame}
\frametitle{Lokad}
\begin{itemize}
\item Business Intelligence (BI) pour la grande distribution, les e-commerces, etc.
\item Moteur de prévisions pour séries temporelles
\end{itemize}
\end{frame}

\begin{frame}
\frametitle{Travail de thèse}
\begin{itemize}
\item Travail
\item 
\end{itemize}
\end{frame}


\begin{frame}
\frametitle{Quid de l'écriture et de la lecture d'un objet?}
\begin{itemize}
\item Lecture
\item Update
\end{itemize}
\end{frame}

\begin{frame}
\frametitle{Description du problème}
Étant donné un jeu de données de $N$ éléments $\{\zbf_{t}\}_{t=1}^N$ dans un espace de dimension $d$, nous voulons construire $K$ points $\{w_{k}\}_{k=1}^{K}$, désigné par la suite par prototypes ou centroïdes, de telle sorte que ces points minimisent la fonction de perte suivante, également appelée distortion :

\begin{equation*}
\label{eq:EmpiricalDistortion}
C_{N}(w) = \sum_{t=1}^N{\min_{\ell = 1, \ldots, K} \left\|\zbf_t  - w_\ell\right\|^2}, \quad \mbox{$w \in \left(\real^d\right)^K$}.
\end{equation*}

\end{frame}

\begin{frame}
\frametitle{Clustering aim}
\begin{itemize}

\item Let $x = (x_{i})_{i=1..n}$ be n points of $\mathbb{R}^{d} $ (data points)

\item Let $c= (c_{k})_{k=1..K}$ be k points of $\mathbb{R}^{d} $ (centroids)

\item We define the empirical loss by :
\begin{equation}
\Phi(x,c) = \sum_{i=1}^{n} \displaystyle \min_{k=1..K} (||x_{i}-c_{k}||_{2}^{2})
\end{equation}

\item and the optimal centroids by :

\begin{equation}
(c_{k})_{k=1..K}^{*} = \displaystyle{Argmin_{c\in \mathbb{R}^{d*K}}  \Phi(x,c) }
\end{equation}
\end{itemize}

\end{frame}

\begin{frame}
\frametitle{Some approximating algorithms}
\begin{itemize}
\item<1-> Empirical minimizer too long to compute.
\item<1-> Algorithms for approximating best clustering :\\
\indent \item<2-> K-Means
\indent \item<2-> Self-Organising Map
\indent \item<2-> Hierarchical Clustering...
\end{itemize}
\end{frame}

\begin{frame}
\frametitle{Batch K-Means}
\begin{itemize}
\item Batch K-Means steps :
    \begin{enumerate}[i]
    \item Initialisation of centroids
    \item Distance Calculation\\
            \indent for each $x_{i}$, get the distance $||x_{i}-c_{k}||_{2}$ and find the nearest centroid
    \item Centroid Recalculation\\
            \indent for each cluster, recompute centroid as the average of points assigned to this cluster
    \item Repeat steps ii and iii till convergence\\
    \end{enumerate}
\item Immediate evidence of the convergence of the algorithm
\end{itemize}
\end{frame}

\begin{frame}
\frametitle{Online K-Means}
\begin{itemize}
\item Online K-Means steps :
    \begin{enumerate}[i]
    \item Initialisation of centroids
    \item Get a dataset point. Select the nearest centroid. Update this centroid.
    \item Repeat steps ii till convergence\\
    \end{enumerate}
\item Probabilist result of convergence of the Online K-Means
\end{itemize}
\end{frame}

\begin{frame}
    \begin{algorithm}[H]
    \caption{Sequential Batch K-Means }
    \begin{algorithmic}
    \STATE Select K initial centroids $(c_{k})_{j=1..K}$
    \REPEAT
    \FOR{$i = 1$ to $n$}
    \FOR{$k=1$ to $K$}
    \STATE Compute $||x_{i}-c_{k}||_{2}^{2}$
    \ENDFOR
    \STATE Find the closest centroid $c_{k^{*}(i)}$ to $x_{i}$;
    \ENDFOR
    \FOR{$k=1$ to $K$}
    \STATE $c_{k} = \frac{1}{\# \{ i, k^{*}(i)=k \}} \sum_{\{ i, k^{*}(i)=k \}} x_{i}$
    \ENDFOR
    \UNTIL {no $c_{k}$ has changed since last iteration or empirical loss stabilizes}
    \end{algorithmic}
    \end{algorithm}
\end{frame}

\begin{frame}
\frametitle{K-Means Sequential cost}
The cost of a sequential Batch K-Means algorithm has been studied by Dhillon.
More precisely :
\begin{align*}
KMeans ~Sequential ~Cost \ & = I(n+K)d + IKd ~readings \\
\ & + InKd ~soustractions \\
\ & + InKd ~square ~operations\\
\ & + InK(d-1) + I(n-K)d  ~additions\\
\ & + IKd ~divisions\\
\ & + 2In + I*Kd ~writings\\
\ & + IKd ~double ~comparisons\\
\ & + I ~counts ~of ~K ~\displaystyle sets_{k=1..K} ~of ~size  ~n(k)\\
\ & ~where ~\displaystyle \sum_{k=1}^{K}n(k) = n
\end{align*}
\end{frame}

\begin{frame}
\frametitle{K-Means Sequential cost (2)}
\begin{align*}
KMeans ~Sequential ~Time \ & = (3Knd+Kn+Kd+nd)*I*T^{flop} \\
                         \ & \simeq 3Knd*I*T^{flop}
\end{align*}
\end{frame}

\begin{frame}
\frametitle{Distributing K-Means}
\begin{enumerate}
\item Different ways to split computation load
\item Splitting load without affinity (worker/cluster) : each worker responsible of n/P points
\item Splitting load with affinity : each worker responsible of K/P clusters
\end{enumerate}
\begin{itemize}
\item clustering without affinity seems more adequate.
\end{itemize}
\end{frame}

\begin{frame}
\begin{algorithm}[H]
\caption{Synchronous Distributed Batch K-Means without affinity}
\begin{footnotesize}
\begin{algorithmic}
\STATE $p$ = GetThisNodeId() (from 0 to P-1)
\STATE Get same initial centroids $(c_{k})_{k=1..K}$ in every node
\STATE Load into local memory $S_{p} = \{ x_{i}, i = p*(n/P) .. (p + 1)*(n/P) \}$
\REPEAT
\FOR{$x_{i} \in S_{p}$}
\FOR{$k=1$ to $K$}
\STATE Compute $||x_{i}-c_{k}||_{2}^{2}$
\ENDFOR
\STATE Find the closest centroid $c_{k^{*}(i)}$ to $x_{i}$
\ENDFOR
\FOR{$k=1$ to $K$}
    \STATE $c_{k,p} = \frac{1}{\# \{ i, ~x_i \in S_{p} ~\& ~k^{*}(i)=k \}} \sum_{\{ i, ~x_i \in S_{p} ~\& ~k^{*}(i)=k \}} {x_{i}}$
\ENDFOR
\STATE Wait for other processors to finish the for loops.
\FOR{$k=1$ to $K$}
\STATE Reduce through MPI the $(c_{k,p})_{p=0..P-1}$ with the corresponding weight : $\# \{ i, ~x_i \in S_{p} ~\& ~k^{*}(i)=k \}$
\STATE Register the value in $c_{k}$
\ENDFOR
\UNTIL {no $c_{k}$ has changed since last iteration or empirical loss stabilizes}
\end{algorithmic}
\end{footnotesize}
\end{algorithm}
\end{frame}

\begin{frame}
\frametitle{SMP Distributed K-Means costs}
\begin{itemize}
\item Distributed K-Means cost is dependant of hardware and how well workers can communicate.

\item SMP : Symmetric MultiProcessor  (shared memory)
    \begin{align*}
    \ & KMeans ~SMP ~Distributed ~Cost \\
    \ & = T_{P}^{comp}\\
    \ & = \frac{(3Knd+Kn+Kd+nd)*I*T^{flop}}{P}\\
    \ & \simeq \frac{3Knd*I*T^{flop}}{P}
    \end{align*}
\end{itemize}
\end{frame}

\begin{frame}
\frametitle{DMM Distributed K-Means costs}
\begin{itemize}
\item KMeans ~DMM ~Distributed  ~Cost
\begin{align*}
\ & = T_{P}^{comp} + T_{P}^{comm}\\
\ & = \frac{(3Knd+Kn+Kd+nd)*I*T^{flop}}{P} + T_{P}^{comm} \\
\ & \simeq \frac{3Knd*I*T^{flop}}{P} + O(log(P))
\end{align*}

\item $T_{P}^{comm} = O(log(P))$ comes from MPI according to Dhillon.
\item Issue : the constant is far greater than log(P) for reasonable P.
\end{itemize}
\end{frame}

\begin{frame}
\frametitle{Case Study : EDF load curves.}
\begin{itemize}
\item n = 20 000 000 series
\item d = 87600 (10 years of hourly series)
\item K = $\sqrt{n} = 4472$ clusters
\item P = 10000 processors
\item I = 100 iterations
\item $T^{flop} = \frac{1}{1 000 000 000} $ seconds
\end{itemize}
\end{frame}

\begin{frame}
\frametitle{Case study on SMP}
On SMP architecture (RAM limitations are not respected), we would get :\\
\begin{equation*}
T_{P, SMP}^{comp} = 235066 seconds
\end{equation*}
\\
\begin{equation*}
T_{P, SMP}^{comm} \simeq 0 seconds
\end{equation*}
\end{frame}

\begin{frame}
\frametitle{Case study on DMM using MPI}
On DMM architecture, we get :
\begin{equation*}
T_{P, DMM}^{comp} = 235066 seconds
\end{equation*}

For communication between 2 nodes, we can suppose : \\

\begin{align*}
\ & Centroids ~broadcast ~between ~2 ~processors ~time \\
\ & = \frac{I*Kd*size of 1 value}{bandwith} \\
\ & = I * \frac{5977 Mbytes}{20Mbytes/second} = 29800 seconds
\end{align*}

\begin{align*}
\ & Centroids ~merging ~time\\
\ & = I*kd*T^{flop} * 5 operations : (2 multiplications, 2 additions, 1 division)\\
\ & = 195.87 seconds
\end{align*}

\end{frame}

\begin{frame}
\frametitle{Communicating through Binary Tree}
\begin{center}
\includegraphics{BinaryWorkersTreeMPI.png}
\end{center}
\end{frame}

\begin{frame}
\frametitle{Estimation of  $T_{P,DMM}^{comm}$}
if MPI binary tree topology, $T_{P,DMM}^{comm}$ becomes :

\begin{align*}
\ & T_{P,DMM}^{comm} \\
\ & = (Centroids ~broadcast + Centroids ~merging ~time) * \lceil log_{2}(P) \rceil \\
\ & = (\frac{I*Kd*size of 1 value}{bandwith} + 5*I*Kd*T^{flop}) * \lceil log_{2}(P) \rceil \\
\ & \simeq 420000 seconds
\end{align*}

\end{frame}

\begin{frame}
\frametitle{Estimating when communication is a bottleneck}
\begin{equation*}
T_{P}^{comm} <= T_{P}^{comp}
\label{DMM1 GeneralCase}
\end{equation*}

\begin{small}
\begin{equation*}
(\frac{I*Kd*size of 1 value}{bandwith} + 5*I*Kd*T^{flop}) * \lceil log_{2}(P) \rceil <= \frac{(3nKd)*I*T^{flop}}{P}
\end{equation*}
\end{small}

\begin{equation*}
\frac{n}{P\lceil log_{2}(P) \rceil} >= \frac{ \frac{size of 1 value}{bandwith} + 5*T^{flop}}{3T^{flop}}
\end{equation*}

\begin{equation*}
\frac{n}{P\lceil log_{2}(P) \rceil} >= 255
\end{equation*}

\end{frame}

\begin{frame}
\frametitle{Empirical speed-up already observed}
\begin{itemize}
\item (Kantabutra, Couch) 2000, clustering with affinity : P=4 (workstations with ethernet) , D=2, K =4, N=900000, best speed-up of 2.1, concludes they have a O(K/2) speed-up.
\item (Kraj, Sharma, Garge, ...) 2008, (1 master, 7 nodes dualcore 3Ghz), D=200, K = 20, N=10000 genes, best speed-up 3
\item (Chu, Kim, Lin, Yu,...) 2006, (1 sun workstation, 16 nodes), N=from 30000 to 2500000, speed-up from 8 to 12.
\item (Dhillon, Modha) 1998, (1 IBM PowerParallel SP2 16 nodes (160Mhz)), D=8, K = 16, N= 2000000 then speed-up of 15.62 on 16 nodes, N = 2000, speed-up of 6 on 16 nodes
\end{itemize}
\end{frame}

\begin{frame}
\frametitle{Cloud Computing}
\begin{itemize}
\item Hardware resources on-demand for storage and computation
\end{itemize}
\end{frame}

\begin{frame}
\frametitle{Clustering on the cloud }
\begin{enumerate}
\item All data must transit through storage
\item Storage bandwith is limited
\item Bandwith, CPU power, latency are guaranteed on average only
\item Workers are likely to fail
\end{enumerate}
\begin{itemize}
\item Workers shouldn't wait for each other
\end{itemize}
\end{frame}

\begin{frame}
\begin{algorithm}[H]
\caption{Asynchronous Distributed K-Means without affinity}
\begin{footnotesize}
\begin{algorithmic}
\STATE $p$ = GetThisNodeId() (from 0 to P-1)
\STATE Get same initial centroids $(c_{k})_{k=1..K}$ in every node. Persist them on the Storage
\STATE Load into local memory $S_{p} = \{ x_{i}, i = p*(n/P) .. (p + 1)*(n/P) \}$
\REPEAT
\FOR{$x_{i} \in S_{p}$}
\FOR{$k=1$ to $K$}
\STATE Compute $||x_{i}-c_{k}||_{2}^{2}$
\ENDFOR
\STATE Find the closest centroid $c_{k^{*}(i)}$ to $x_{i}$
\ENDFOR
\FOR{$k=1$ to $K$}
    \STATE $c_{k,p} = \frac{1}{\# \{ i, ~x_i \in S_{p} ~\& ~k^{*}(i)=k \}} \sum_{\{ i, ~x_i \in S_{p} ~\& ~k^{*}(i)=k \}} {x_{i}}$
\ENDFOR
\STATE Don't wait for other processors to finish the for loops.
\STATE Retrieve centroids $(c_{k})_{k=1..K}$ from the storage
\FOR{$k=1$ to $K$}
\STATE Update $c_{k}$ using $c_{k,p}$
\ENDFOR
\STATE Update storage version of the centroids.
\UNTIL {empirical loss stabilizes}
\end{algorithmic}
\end{footnotesize}
\end{algorithm}
\end{frame}

\begin{frame}
\frametitle{Current work}
\begin{enumerate}
\item Synchronous K-Means
\item Asynchronous K-Means
\item Getting a speed-up (hopefully)
\end{enumerate}
\end{frame}

\begin{frame}
\frametitle{Present technical difficulties of coding on the cloud }
\begin{itemize}
\item Code Abstractions : Inversion of Control, SOA, Storage Garbage Collection, ...
\item Debugging the cloud : Mock Providers, Reporting System, ...
\item Profiling the cloud : no release date\\
\item Monitoring the cloud : Counting workers, Measuring utilization levels, ...
\end{itemize}
\end{frame}

\end{document}
