\documentclass{beamer}

\usepackage{algorithm}
\usepackage{algorithmic}
\usepackage{beamerthemesplit}
\usepackage{graphicx}
\usepackage[T1]{fontenc}
\usepackage[latin1]{inputenc}

\title{Algorithmes de classification répartis sur le cloud}
\author{Matthieu Durut}
\date{\sc Soutenance de thèse, 28 Septembre 2012}

\newcommand{\bZ}{\mathbf{Z}}
\newcommand{\bz}{\mathbf{z}}
\newcommand{\bX}{\mathbf{X}}
\newcommand{\bx}{\mathbf{x}}
\newcommand{\by}{\mathbf{y}}
\newcommand{\ind}{\mathbf{1}}
\newcommand{\card}[1]{|{#1}|}
\renewcommand{\L}{\mathcal L}
\newcommand{\R}{\mathbb R}
\newcommand{\de}{\mathrm d}
\newcommand{\jumpOnePage}{\newpage \thispagestyle{empty} \null \newpage}

%
%
%\AtBeginSection[]
%{
%   \begin{frame}
%       \frametitle{Outline}
%       \tableofcontents[currentsection]
%   \end{frame}
%}
%
%\AtBeginSubsection[]
%{
%   \begin{frame}
%       \frametitle{Outline}
%       \tableofcontents[currentsection,currentsubsection]
%   \end{frame}
%}

\begin{document}

\begin{frame}
\titlepage
\end{frame}

\section{Introduction.}

%\begin{frame}
%\frametitle{Lokad}
%\begin{itemize}
%\item Business Intelligence (BI) pour la grande distribution, les e-commerces, etc.
%\item Moteur de prévisions pour séries temporelles
%\end{itemize}
%\end{frame}
%
%\begin{frame}
%\frametitle{Travail de thèse}
%\begin{itemize}
%\item Travail
%\item 
%\end{itemize}
%\end{frame}
%
%
%\begin{frame}
%\frametitle{Quid de l'écriture et de la lecture d'un objet?}
%\begin{itemize}
%\item Lecture
%\item Update
%\end{itemize}
%\end{frame}
%
%\begin{frame}
%\frametitle{Description du problème}
%Étant donné un jeu de données de $N$ éléments $\{\zbf_{t}\}_{t=1}^N$ dans un espace de dimension $d$, nous voulons construire $K$ points $\{w_{k}\}_{k=1}^{K}$, désigné par la suite par prototypes ou centroïdes, de telle sorte que ces points minimisent la fonction de perte suivante, également appelée distortion :
%
%\begin{equation*}
%\label{eq:EmpiricalDistortion}
%C_{N}(w) = \sum_{t=1}^N{\min_{\ell = 1, \ldots, K} \left\|\zbf_t  - w_\ell\right\|^2}, \quad \mbox{$w \in \left(\real^d\right)^K$}.
%\end{equation*}
%
%\end{frame}
%
%\begin{frame}
%\frametitle{Clustering aim}
%\begin{itemize}
%
%\item Let $x = (x_{i})_{i=1..n}$ be n points of $\mathbb{R}^{d} $ (data points)
%
%\item Let $c= (c_{k})_{k=1..K}$ be k points of $\mathbb{R}^{d} $ (centroids)
%
%\item We define the empirical loss by :
%\begin{equation}
%\Phi(x,c) = \sum_{i=1}^{n} \displaystyle \min_{k=1..K} (||x_{i}-c_{k}||_{2}^{2})
%\end{equation}
%
%\item and the optimal centroids by :
%
%\begin{equation}
%(c_{k})_{k=1..K}^{*} = \displaystyle{Argmin_{c\in \mathbb{R}^{d*K}}  \Phi(x,c) }
%\end{equation}
%\end{itemize}
%
%\end{frame}
%
%\begin{frame}
%\frametitle{Some approximating algorithms}
%\begin{itemize}
%\item<1-> Empirical minimizer too long to compute.
%\item<1-> Algorithms for approximating best clustering :\\
%\indent \item<2-> K-Means
%\indent \item<2-> Self-Organising Map
%\indent \item<2-> Hierarchical Clustering...
%\end{itemize}
%\end{frame}
%
%\begin{frame}
%\frametitle{Batch K-Means}
%\begin{itemize}
%\item Batch K-Means steps :
%    \begin{enumerate}[i]
%    \item Initialisation of centroids
%    \item Distance Calculation\\
%            \indent for each $x_{i}$, get the distance $||x_{i}-c_{k}||_{2}$ and find the nearest centroid
%    \item Centroid Recalculation\\
%            \indent for each cluster, recompute centroid as the average of points assigned to this cluster
%    \item Repeat steps ii and iii till convergence\\
%    \end{enumerate}
%\item Immediate evidence of the convergence of the algorithm
%\end{itemize}
%\end{frame}
%
%\begin{frame}
%\frametitle{Online K-Means}
%\begin{itemize}
%\item Online K-Means steps :
%    \begin{enumerate}[i]
%    \item Initialisation of centroids
%    \item Get a dataset point. Select the nearest centroid. Update this centroid.
%    \item Repeat steps ii till convergence\\
%    \end{enumerate}
%\item Probabilist result of convergence of the Online K-Means
%\end{itemize}
%\end{frame}
%
%\begin{frame}
%    \begin{algorithm}[H]
%    \caption{Sequential Batch K-Means }
%    \begin{algorithmic}
%    \STATE Select K initial centroids $(c_{k})_{j=1..K}$
%    \REPEAT
%    \FOR{$i = 1$ to $n$}
%    \FOR{$k=1$ to $K$}
%    \STATE Compute $||x_{i}-c_{k}||_{2}^{2}$
%    \ENDFOR
%    \STATE Find the closest centroid $c_{k^{*}(i)}$ to $x_{i}$;
%    \ENDFOR
%    \FOR{$k=1$ to $K$}
%    \STATE $c_{k} = \frac{1}{\# \{ i, k^{*}(i)=k \}} \sum_{\{ i, k^{*}(i)=k \}} x_{i}$
%    \ENDFOR
%    \UNTIL {no $c_{k}$ has changed since last iteration or empirical loss stabilizes}
%    \end{algorithmic}
%    \end{algorithm}
%\end{frame}
%
%\begin{frame}
%\frametitle{K-Means Sequential cost}
%The cost of a sequential Batch K-Means algorithm has been studied by Dhillon.
%More precisely :
%\begin{align*}
%KMeans ~Sequential ~Cost \ & = I(n+K)d + IKd ~readings \\
%\ & + InKd ~soustractions \\
%\ & + InKd ~square ~operations\\
%\ & + InK(d-1) + I(n-K)d  ~additions\\
%\ & + IKd ~divisions\\
%\ & + 2In + I*Kd ~writings\\
%\ & + IKd ~double ~comparisons\\
%\ & + I ~counts ~of ~K ~\displaystyle sets_{k=1..K} ~of ~size  ~n(k)\\
%\ & ~where ~\displaystyle \sum_{k=1}^{K}n(k) = n
%\end{align*}
%\end{frame}
%
%\begin{frame}
%\frametitle{K-Means Sequential cost (2)}
%\begin{align*}
%KMeans ~Sequential ~Time \ & = (3Knd+Kn+Kd+nd)*I*T^{flop} \\
%                         \ & \simeq 3Knd*I*T^{flop}
%\end{align*}
%\end{frame}
%
%\begin{frame}
%\frametitle{Distributing K-Means}
%\begin{enumerate}
%\item Different ways to split computation load
%\item Splitting load without affinity (worker/cluster) : each worker responsible of n/P points
%\item Splitting load with affinity : each worker responsible of K/P clusters
%\end{enumerate}
%\begin{itemize}
%\item clustering without affinity seems more adequate.
%\end{itemize}
%\end{frame}
%
%\begin{frame}
%\begin{algorithm}[H]
%\caption{Synchronous Distributed Batch K-Means without affinity}
%\begin{footnotesize}
%\begin{algorithmic}
%\STATE $p$ = GetThisNodeId() (from 0 to P-1)
%\STATE Get same initial centroids $(c_{k})_{k=1..K}$ in every node
%\STATE Load into local memory $S_{p} = \{ x_{i}, i = p*(n/P) .. (p + 1)*(n/P) \}$
%\REPEAT
%\FOR{$x_{i} \in S_{p}$}
%\FOR{$k=1$ to $K$}
%\STATE Compute $||x_{i}-c_{k}||_{2}^{2}$
%\ENDFOR
%\STATE Find the closest centroid $c_{k^{*}(i)}$ to $x_{i}$
%\ENDFOR
%\FOR{$k=1$ to $K$}
%    \STATE $c_{k,p} = \frac{1}{\# \{ i, ~x_i \in S_{p} ~\& ~k^{*}(i)=k \}} \sum_{\{ i, ~x_i \in S_{p} ~\& ~k^{*}(i)=k \}} {x_{i}}$
%\ENDFOR
%\STATE Wait for other processors to finish the for loops.
%\FOR{$k=1$ to $K$}
%\STATE Reduce through MPI the $(c_{k,p})_{p=0..P-1}$ with the corresponding weight : $\# \{ i, ~x_i \in S_{p} ~\& ~k^{*}(i)=k \}$
%\STATE Register the value in $c_{k}$
%\ENDFOR
%\UNTIL {no $c_{k}$ has changed since last iteration or empirical loss stabilizes}
%\end{algorithmic}
%\end{footnotesize}
%\end{algorithm}
%\end{frame}
%
%\begin{frame}
%\frametitle{SMP Distributed K-Means costs}
%\begin{itemize}
%\item Distributed K-Means cost is dependant of hardware and how well workers can communicate.
%
%\item SMP : Symmetric MultiProcessor  (shared memory)
%    \begin{align*}
%    \ & KMeans ~SMP ~Distributed ~Cost \\
%    \ & = T_{P}^{comp}\\
%    \ & = \frac{(3Knd+Kn+Kd+nd)*I*T^{flop}}{P}\\
%    \ & \simeq \frac{3Knd*I*T^{flop}}{P}
%    \end{align*}
%\end{itemize}
%\end{frame}
%
%\begin{frame}
%\frametitle{DMM Distributed K-Means costs}
%\begin{itemize}
%\item KMeans ~DMM ~Distributed  ~Cost
%\begin{align*}
%\ & = T_{P}^{comp} + T_{P}^{comm}\\
%\ & = \frac{(3Knd+Kn+Kd+nd)*I*T^{flop}}{P} + T_{P}^{comm} \\
%\ & \simeq \frac{3Knd*I*T^{flop}}{P} + O(log(P))
%\end{align*}
%
%\item $T_{P}^{comm} = O(log(P))$ comes from MPI according to Dhillon.
%\item Issue : the constant is far greater than log(P) for reasonable P.
%\end{itemize}
%\end{frame}
%
%\begin{frame}
%\frametitle{Case Study : EDF load curves.}
%\begin{itemize}
%\item n = 20 000 000 series
%\item d = 87600 (10 years of hourly series)
%\item K = $\sqrt{n} = 4472$ clusters
%\item P = 10000 processors
%\item I = 100 iterations
%\item $T^{flop} = \frac{1}{1 000 000 000} $ seconds
%\end{itemize}
%\end{frame}
%
%\begin{frame}
%\frametitle{Case study on SMP}
%On SMP architecture (RAM limitations are not respected), we would get :\\
%\begin{equation*}
%T_{P, SMP}^{comp} = 235066 seconds
%\end{equation*}
%\\
%\begin{equation*}
%T_{P, SMP}^{comm} \simeq 0 seconds
%\end{equation*}
%\end{frame}
%
%\begin{frame}
%\frametitle{Case study on DMM using MPI}
%On DMM architecture, we get :
%\begin{equation*}
%T_{P, DMM}^{comp} = 235066 seconds
%\end{equation*}
%
%For communication between 2 nodes, we can suppose : \\
%
%\begin{align*}
%\ & Centroids ~broadcast ~between ~2 ~processors ~time \\
%\ & = \frac{I*Kd*size of 1 value}{bandwith} \\
%\ & = I * \frac{5977 Mbytes}{20Mbytes/second} = 29800 seconds
%\end{align*}
%
%\begin{align*}
%\ & Centroids ~merging ~time\\
%\ & = I*kd*T^{flop} * 5 operations : (2 multiplications, 2 additions, 1 division)\\
%\ & = 195.87 seconds
%\end{align*}
%
%\end{frame}
%
%\begin{frame}
%\frametitle{Communicating through Binary Tree}
%\begin{center}
%\includegraphics{BinaryWorkersTreeMPI.png}
%\end{center}
%\end{frame}
%
%\begin{frame}
%\frametitle{Estimation of  $T_{P,DMM}^{comm}$}
%if MPI binary tree topology, $T_{P,DMM}^{comm}$ becomes :
%
%\begin{align*}
%\ & T_{P,DMM}^{comm} \\
%\ & = (Centroids ~broadcast + Centroids ~merging ~time) * \lceil log_{2}(P) \rceil \\
%\ & = (\frac{I*Kd*size of 1 value}{bandwith} + 5*I*Kd*T^{flop}) * \lceil log_{2}(P) \rceil \\
%\ & \simeq 420000 seconds
%\end{align*}
%
%\end{frame}
%
%\begin{frame}
%\frametitle{Estimating when communication is a bottleneck}
%\begin{equation*}
%T_{P}^{comm} <= T_{P}^{comp}
%\label{DMM1 GeneralCase}
%\end{equation*}
%
%\begin{small}
%\begin{equation*}
%(\frac{I*Kd*size of 1 value}{bandwith} + 5*I*Kd*T^{flop}) * \lceil log_{2}(P) \rceil <= \frac{(3nKd)*I*T^{flop}}{P}
%\end{equation*}
%\end{small}
%
%\begin{equation*}
%\frac{n}{P\lceil log_{2}(P) \rceil} >= \frac{ \frac{size of 1 value}{bandwith} + 5*T^{flop}}{3T^{flop}}
%\end{equation*}
%
%\begin{equation*}
%\frac{n}{P\lceil log_{2}(P) \rceil} >= 255
%\end{equation*}
%
%\end{frame}
%
%\begin{frame}
%\frametitle{Empirical speed-up already observed}
%\begin{itemize}
%\item (Kantabutra, Couch) 2000, clustering with affinity : P=4 (workstations with ethernet) , D=2, K =4, N=900000, best speed-up of 2.1, concludes they have a O(K/2) speed-up.
%\item (Kraj, Sharma, Garge, ...) 2008, (1 master, 7 nodes dualcore 3Ghz), D=200, K = 20, N=10000 genes, best speed-up 3
%\item (Chu, Kim, Lin, Yu,...) 2006, (1 sun workstation, 16 nodes), N=from 30000 to 2500000, speed-up from 8 to 12.
%\item (Dhillon, Modha) 1998, (1 IBM PowerParallel SP2 16 nodes (160Mhz)), D=8, K = 16, N= 2000000 then speed-up of 15.62 on 16 nodes, N = 2000, speed-up of 6 on 16 nodes
%\end{itemize}
%\end{frame}
%
%\begin{frame}
%\frametitle{Cloud Computing}
%\begin{itemize}
%\item Hardware resources on-demand for storage and computation
%\end{itemize}
%\end{frame}
%
%\begin{frame}
%\frametitle{Clustering on the cloud }
%\begin{enumerate}
%\item All data must transit through storage
%\item Storage bandwith is limited
%\item Bandwith, CPU power, latency are guaranteed on average only
%\item Workers are likely to fail
%\end{enumerate}
%\begin{itemize}
%\item Workers shouldn't wait for each other
%\end{itemize}
%\end{frame}
%
%\begin{frame}
%\begin{algorithm}[H]
%\caption{Asynchronous Distributed K-Means without affinity}
%\begin{footnotesize}
%\begin{algorithmic}
%\STATE $p$ = GetThisNodeId() (from 0 to P-1)
%\STATE Get same initial centroids $(c_{k})_{k=1..K}$ in every node. Persist them on the Storage
%\STATE Load into local memory $S_{p} = \{ x_{i}, i = p*(n/P) .. (p + 1)*(n/P) \}$
%\REPEAT
%\FOR{$x_{i} \in S_{p}$}
%\FOR{$k=1$ to $K$}
%\STATE Compute $||x_{i}-c_{k}||_{2}^{2}$
%\ENDFOR
%\STATE Find the closest centroid $c_{k^{*}(i)}$ to $x_{i}$
%\ENDFOR
%\FOR{$k=1$ to $K$}
%    \STATE $c_{k,p} = \frac{1}{\# \{ i, ~x_i \in S_{p} ~\& ~k^{*}(i)=k \}} \sum_{\{ i, ~x_i \in S_{p} ~\& ~k^{*}(i)=k \}} {x_{i}}$
%\ENDFOR
%\STATE Don't wait for other processors to finish the for loops.
%\STATE Retrieve centroids $(c_{k})_{k=1..K}$ from the storage
%\FOR{$k=1$ to $K$}
%\STATE Update $c_{k}$ using $c_{k,p}$
%\ENDFOR
%\STATE Update storage version of the centroids.
%\UNTIL {empirical loss stabilizes}
%\end{algorithmic}
%\end{footnotesize}
%\end{algorithm}
%\end{frame}
%
%\begin{frame}
%\frametitle{Current work}
%\begin{enumerate}
%\item Synchronous K-Means
%\item Asynchronous K-Means
%\item Getting a speed-up (hopefully)
%\end{enumerate}
%\end{frame}
%
%\begin{frame}
%\frametitle{Present technical difficulties of coding on the cloud }
%\begin{itemize}
%\item Code Abstractions : Inversion of Control, SOA, Storage Garbage Collection, ...
%\item Debugging the cloud : Mock Providers, Reporting System, ...
%\item Profiling the cloud : no release date\\
%\item Monitoring the cloud : Counting workers, Measuring utilization levels, ...
%\end{itemize}
%\end{frame}

\end{document}
