\documentclass[11pt,amsfonts,amsmath]{article}
\usepackage{times}
\usepackage[latin1]{inputenc}
\usepackage{amsfonts,amsmath,amsthm,amssymb,stmaryrd}
\usepackage{algorithmic}
\usepackage{algorithm}
\usepackage{graphicx}
\usepackage{url}


\title{Designing Distributed K-Means for the cloud}
\author{}
\date{\today}

\begin{document}

\maketitle

\begin{abstract}
To cluster increasing amount of data, several implementations of distributed K-Means algorithm have been proposed. In this paper, we propose an implementation of K-Means on the cloud and describes how the cloud hardware affects the design of the algorithm. To build an efficient algorithm, we focus on communication bottleneck between workers. We refine speed-up modelisation initiated by Dhillon in ~\cite{Dhillon}, and show how communication affects theoretically the algorithm once developped on the cloud. We implemented our algorithm on the new cloud computing service of Microsoft, Azure, using a maximum of 64 Nodes.\\
\end{abstract}

\textbf{Keywords} : clustering, K-means, cloud computing, distributed algorithms

\section{Introduction}

Iterative clustering algorithms have been used in a wide variety of areas including visualization, pattern recognition, learning theory, computer graphics, computational geometry, DNA mining, text mining for Google News or Amazon recommendations, data compression,...\\

In this paper, we are considering a clustering problem where we want to find K subsets of a dataset, using the euclidian metric. Given a dataset of n points in $\Re^{d}$, we want to construct K points $(c_{k})_{k=1..K}$ also in $\Re^{d}$ as the minimum of the following program :
\begin{equation}
(c_{k}^{*})_{k=1..K} = \displaystyle{ argmin_{(c_{k})_{k=1..K}}} \sum_{i=1}^{n} \displaystyle \min_{k=1..K} (||x_{i}-c_{k}||_{2}^{2})
\end{equation}
The optimal existence of a solution is always defined because of we can reduce the problem to a compact issue. But finding the best solution is a NP-Hard problem, even if d=2 (~\cite{KMeansIsNPhard}). Several algorithms have been proposed to approximate the best solution within a reasonable amount of time. One of the most used is the Batch-K-Means, which we describe in section \ref{Sequential K-Means Algorithm Section}. \\

K-Means is supposed to be an embarrassingly distributed algorithm. Most of the computation time is spent in evaluating distance between points and centroids. These evaluations can be easily distributed on several workers. Distributed K-Means can then be viewed as an iterated Map-Reduce algorithm : first, we separate the n points into P subsets of equal length (where P is the number of processors), each processor (mapper) being responsible of computing distance between its points and centroids. At the end of the step, all processors computes a local version of the centroids, then broadcast this version to a unique processor responsible of gathering all local versions, computing the global centroids (reduce step) and sending it back to all units. Broadcasting the local centroids version can be done in several ways, depending on the hardware/software framework : broadcasting can be done using MPI (Message Passing Interface) implementation or web services for example.
Some example of parallel implementations with Message Passing Interface (MPI) with can be found here : ~\cite{Dhillon}, ~\cite{KMeansJoshi}, ~\cite{KMeansParallelInC}.
Implementation on a multi-core unique machine has been treated here : ~\cite{MapReduceForMachineLearning}.
Implementation through web services can be found here : ~\cite{ParaKMeans}\\

We now briefly outline the paper. In section \ref{Sequential K-Means Algorithm Section}, we present the sequential K-Means algorithm and its computational cost. In section \ref{Distributed K-Means Algorithm Section}, we describe how the Batch K-Means is often distributed on several processors and examine the new corresponding computational cost. In section \ref{Bandwidth_concerns_on_DMM_architectures}, we build some modeling of the real cost of a distributed K-Means on Distributed Memory Multiprocessors and develop a bandwidth condition inequality. Section \ref{Distributed K-Means Case Study} presents some case study to test the condition we presented in section \ref{Bandwidth_concerns_on_DMM_architectures}. Section \ref{Azure} is devoted to present some specificities of the cloud that prevent previous developments of Distributed Memory Multiprocessors K-Means cost to be applied on the cloud. Section \ref{Cloud Experiments} presents experimental results of a distributed K-Means on the cloud.

\clearpage
\section{Notations}

\begin{itemize}
\item K : number of clusters
\item n : number of points to cluster
\item d : dimension of the points
\item I : number of iterations
\item P : number of workers
\item $T^{flop}$ : time to run one operation on a float (addition, multiplication, ...)
\item $c_{i}$ : centroid number i
\item $x_{i}$ : points number i
\item $T_{P}^{comp}$ : ideal whole time spend on running computation on P workers
\item $T_{P}^{DMM, comm}$ : ideal whole time spend on communication on P workers
\item $T_{P}^{Cloud, comm}$ : ideal whole time spend on communication on P workers
\item $T_{P}^{reduce}$ : time required to perform a reduce operation on P nodes
\item SoD : size of a double in memory
\item $B_{P}^{DMM, MPI}$ : bandwidth between 2 workers to communicate through MPI while P/2 workers are communicating at the same time with P/2 other workers.
\item $B^{Cloud, read}$ : bandwidth from which a worker can access the storage
\item $B_{P}^{Cloud, read}$ : bandwidth from which a worker can access the storage, while P-1 other workers are accessing other blobs at the same time. We do not modelize whether other blobs are in the same container or not.
\item $B^{Cloud, write}$ : bandwidth from which a worker can write to the storage
\item $B_{P}^{Cloud, write}$ : bandwidth from which a worker can write to the storage, while P-1 other workers are writing on other blobs at the same time. We do not modelize whether other blobs are in the same container or not.
\end{itemize}

\clearpage
\section{Sequential K-Means : Algorithm and complexity cost}
\label{Sequential K-Means Algorithm Section}

Suppose we are given a dataset of n points in $\Re^{d}$, $(x_{i})_{i=1..n}$ . We want to construct K points $(c_{k})_{k=1..K}$ also in $\Re^{d}$ to minimize the objective function :
\begin{equation}
\sum_{i=1}^{n} \displaystyle \min_{k=1..K} (||x_{i}-c_{k}||_{2}^{2})
\end{equation}

Batch K-Means is an algorithm to find a local minimum of the objective function. The algorithm is composed of 2 phases we iterate : given a set of centroids, we find for each point in the data-set the nearest centroid. Second phase is run after every point has been assigned to the nearest centroid. We recompute each centroid as the average of all points in the data-set for which the nearest centroid is the one we want to recompute. After phase 2, we start again phase 1 and phase 2, until the centroids do not move anymore.\\

\begin{algorithm}
    \caption{Sequential Batch K-Means }
    \begin{algorithmic}
        \STATE Select K initial centroids $(c_{k})_{j=1..K}$
        \REPEAT
        \FOR{$i = 1$ to $n$}
        \FOR{$k=1$ to $K$}
        \STATE compute $||x_{i}-c_{k}||_{2}^{2}$
        \ENDFOR
        \STATE find the closest centroid $c_{k^{*}(i)}$ to $x_{i}$;
        \ENDFOR
        \FOR{$k=1$ to $K$}
        \STATE $c_{k} = \frac{1}{\# \{ i, k^{*}(i)=k \}} \sum_{\{ i, k^{*}(i)=k \}} x_{i}$
        \ENDFOR
        \UNTIL {no $c_{k}$ has changed since last iteration }
    \end{algorithmic}
\end{algorithm}

The cost of a sequential Batch K-Means algorithm has been studied by Dhillon ~\cite{Dhillon}. We describe below the total amount of operations we must achieve for a K-Means where we stop iterations if centroids update has not changed any centroids in last iteration. This cost is :\\

\begin{tabular}{ r c l }
\(KMeans ~Sequential ~Cost\) & \(=\) & \(I(K+n)d + IKd ~readings\) \\
   & \(+\) & \(IKnd ~substractions \) \\
   & \(+\) & \(IKnd ~square ~operations \) \\
   & \(+\) & \(IKn(d-1) + I(n-K)d  ~additions\) \\
   & \(+\) & \(IKd ~divisions\) \\
   & \(+\) & \(2In + IKd ~writings\) \\
   & \(+\) & \(IKd ~double ~comparisons\) \\
   & \(+\) & \(I ~counts ~of K ~\displaystyle {(sets)_{k=1..K}} ~of ~size  ~n(k)  ~where ~\displaystyle \sum_{k=1}^{K}n(k) = n\) \\
\end{tabular}

where I denotes the number of iterations.\\

One can simplify the previous equation by neglecting reading/writing time and double comparisons (we will come back to this approximation later, because it is not reasonable on the cloud hardware) and assuming that a multiplication cost is like an addition cost or a square operation cost. Finally, we can sum up the K-Means cost by a simplifying equation :\\

\begin{equation}
KMeans ~Sequential ~Time = (3Knd+Kn+Kd+nd)*I*T^{flop}
\label{sequential KMeans cost}
\end{equation}

where $T^{flop}$ denotes the time for a floating point operation and I the number of iterations.

\clearpage
\section{Distributed K-Means Algorithm on SMP and DMM architectures}
\label{Distributed K-Means Algorithm Section}

K-Means algorithm can be distributed on several processors. We describe here a version of synchronous distributed K-Means. Results of the K-Means stay unchanged compared with the sequential Batch K-Means.

\begin{algorithm}
    \caption{Distributed Batch K-Means}
    \begin{algorithmic}
    \STATE $p$ = GetThisNodeId() (from 0 to P-1)
    \STATE Get same initial centroids $(c_{k})_{k=1..K}$ in every node
    \STATE Load into local memory $S_{p} = \{ x_{i}, i = p*(n/P) .. (p + 1)*(n/P) \}$
    \REPEAT
    \FOR{$x_{i} \in S_{p}$}
    \FOR{$k=1$ to $K$}
        \STATE compute $||x_{i}-c_{k}||_{2}^{2}$
    \ENDFOR
    \STATE find the closest centroid $c_{k^{*}(i)}$ to $x_{i}$
    \ENDFOR
    \FOR{$k=1$ to $K$}
        \STATE $c_{k,p} = \frac{1}{\# \{ i, ~x_i \in S_{p} ~\& ~k^{*}(i)=k \}} \sum_{\{ i, ~x_i \in S_{p} ~\& ~k^{*}(i)=k \}} {x_{i}}$
    \ENDFOR
    \STATE Wait for other processors to finish the for loops.
    \FOR{$k=1$ to $K$}
    \STATE Reduce through MPI the $(c_{k,p})_{p=0..P-1}$ with the corresponding weight : $\# \{ i, ~x_i \in S_{p} ~\& ~k^{*}(i)=k \}$
    \STATE Register the value in $c_{k}$
    \ENDFOR
    \UNTIL {no $c_{k}$ has changed since last iteration}
    \end{algorithmic}
\end{algorithm}

How much can this distributed algorithm improve the whole time computation ? Now the algorithm is distributed, we have to take into account communication through the different processors.\\

\begin{equation}
Distributed ~KMeans ~Cost = T_{P}^{comp} + T_{P}^{comm}
\label{Synchronous Distributed K-Means Cost}
\end{equation}
where $T_{P}^{comm}$ is spent in broadcasting local centroids results.\\

On a SMP architecture (Symmetric Multi-Processors), we can neglect communication time ($T_{P}^{comm}$ = 0) and simplify the previous equation to only keep the predominant terms (additions, subtractions and square operations). We can then sum up the K-Means cost by a simplifying equation :

\begin{equation}
KMeans ~SMP ~Cost = T_{P}^{comp} = \displaystyle \frac{(3Knd+Kn+Kd+nd)*I*T^{flop}}{P}
\label{SmpCost}
\end{equation}

We then get a perfect theoretical speedUp on SMP architecture of a factor P.\\

The cost of the synchronous distributed Batch K-Means algorithm on DMM (Distributed Memory Multi-processors) has also been studied by Dhillon ~\cite{Dhillon} then by Joshi ~\cite{KMeansJoshi}.
This cost is :\\

\begin{tabular}{ r c l }
\(KMeans ~DMM ~Cost\) & \(=\) & \(T_{P}^{comp} + T_{P}^{comm}\) \\
   & \(=\) & \(\frac{(3Knd+Kn+Kd+nd)IT^{flop}}{P} + O(dKIT_{P}^{reduce}) \) \\
\end{tabular}

where  $T_{P}^{reduce}$ denotes the time required to perform a reduce operation on P nodes. \\

In most articles, the reduce step between processors in Distributed Memory Multi-processors is performed through an implementation of MPI (Message Passing Interface). MPI performs communication and broadcasts data between nodes using a tree-like topology, obtaining $T_{P}^{reduce}$ supposed to be performed in $O(log P)$.\\

The constant in front of $log P$ represents the cost of communication between processors and is supposed to be acceptable on most local networks, according to Dhillon ~\cite{Dhillon}. Yet, we think this constant should not be neglected, since communication latency (in bad case 100ms) and bandwidth between processors on a network can represent a lot of time comparing to high flops performance. Moreover, this constant is supposed to be much higher than $log(P)$ for reasonable amount of processors. Next section evaluates $T_{P}^{reduce}$ and computes some bandwidth condition to avoid communication to be a bottleneck.

\clearpage

\section{Bandwidth concerns on DMM architectures}
\label{Bandwidth_concerns_on_DMM_architectures}
\subsection{Modelisation of communication real costs}

Let's examine $T_{P}^{comm}$ in the case of DMM using MPI. Let's start by the cost of communicating local versions of centroids from one processor to another one. The $T_{P}^{comm}$ cost divides into 2 things : bandwidth and latency. Because bandwidth issue will be much stronger on large clusterings, we'll neglect latency. Size of 1 double in memory is supposed to be 8 bytes in the next sections.

\begin{align*}
2 ~workers ~communication ~time & = \frac{IKd*SoD}{B_{2}^{DMM, MPI}}\\
\label{DMM3 StudyCase1}
\end{align*}

%Besides, if the size of the local version of centroids does not fit into the RAM of the receiving processor, the time of $T_{P}^{comp}$ might be even greater.\\

Merging centroids time between 2 processors is a weighted sum of centroids (weights are the number of points in each processors (approximatively n/p)) :

\begin{align}
Centroids ~merging ~time & = IkdT^{flop} * 5 float ~operations\\
\label{DMM4 StudyCase1}
\end{align}

If we suppose the MPI implementation uses a binary tree topology, then total time to broadcast and reduce all local centroids version could be modelized  as :\\
\begin{align*}
T_{P}^{comm} & = (2*Centroids ~broadcast ~between ~2 ~processors ~time + Centroids merging ~time) * \lceil log_{2}(P) \rceil\\
& = \displaystyle \sum_{p=1}^{\lceil log_{2}(P) \rceil} (2\frac{IKd*SoD}{B_{\frac{P}{2^{p}}}^{DMM, MPI}} + 5IKd*T^{flop})\\
\end{align*}

where $\lceil log_{2}(P) \rceil$ denotes the integer part rounded up of $log_{2}(P)$.\\

Since bandwidth is rather difficult to observe when running an intensive execution, we will modelize time spent in communication in a simpler but more robust way :
\begin{align*}
T_{P}^{comm} & = (2*Centroids ~broadcast ~between ~2 ~processors ~time + Centroids merging ~time) * \lceil log_{2}(P) \rceil\\
& =  (2\frac{IKd*SoD}{{B_{P,avg}}^{DMM, MPI}} + 5IKd*T^{flop}) * \lceil log_{2}(P)\rceil\\
\end{align*}

\begin{figure}
    \centering
    \includegraphics[scale=0.6]{../../Pictures/HardwareSchemes/Merginglogic.png}
    \caption{Merging centroïds logic}
\end{figure}

We can then deduce an estimation of the speed-up rate on DMM architectures, like Dhillon did, but where $T_{P}^{comm}$ has been precised :

\begin{align}
\label{SpeedupFormula}
Speedup & = \frac{KMeans ~Sequential ~Cost}{KMeans DMM Distributed Cost}\\
& = \frac{KMeans Sequential Cost}{T_{P}^{comp} +T_{P}^{comm}}\\
& = \frac{3nKdIT^{flop}}{\frac{3nKdIT^{flop}}{P} +  \left(2 \frac{IKd*SoD}{B_{P}^{DMM, MPI}} + 5IKdT^{flop}\right) \lceil log_{2}(P) \rceil}\\
& = \frac{3nT^{flop}}{\frac{3nT^{flop}}{P} +  \left(2 \frac{SoD}{B_{P}^{DMM, MPI}} + 5T^{flop}\right) \lceil log_{2}(P) \rceil}\\
\end{align}

\noindent\textbf{NOTE 1: } The speed up might even be bigger, since we do not have take into account that the RAM is bounded. Using multiple processors might help everything fit into the RAM, whereas it would have been impossible on a single worker machine.\\
\textbf{NOTE 2: } We observe in this modelisation that speedup does not depend on the number of iterations, on the dimension of objects or on the number of cluster.\\
\textbf{NOTE 3: } Speedup is increasing with the number of points we want to cluster.
\textbf{NOTE 4: } No matter the bandwidth and CPU cadency of the hardware, we get
\begin{align}
\lim_{n\rightarrow +\infty} SpeedUp = P \\
\end{align}

\subsection{Bandwidth Condition}

We are now looking for a condition to ensure bandwidth will not be a bottleneck, that is to say we will get a good speedup. We will arbitrary say that bandwidth becomes a bottleneck if time spent in reducing operations exceed time spent in distance computation.\\
We will then say bandwidth is not a bottleneck if :\\

\begin{equation}
T_{P}^{comm} <= T_{P}^{comp}
\label{DMM1 GeneralCase}
\end{equation}

This condition guarantees we are going to get a speedup factor greater than half of optimal speedup factor.

\begin{equation*}
\left( \frac{IKd* SoD}{B_{P}^{DMM, MPI}} + 5IKdT^{flop}\right) * \lceil log_{2}(P) \rceil <= \frac{(3nKd+nK+dK+nd)*I*T^{flop}}{P}
\label{DMM2 GeneralCase}
\end{equation*}

We can get a sufficient condition to ensure bandwidth will not be a bottleneck by keeping only on the right side the largest term. We then get the sufficient condition :\\

\begin{equation*}
\left(\frac{IKd*SoD}{B_{P}^{DMM, MPI}} + 5*IKdT^{flop}\right) * \lceil log_{2}(P) \rceil <= \frac{3nKd*I*T^{flop}}{P}
\label{DMM3 GeneralCase}
\end{equation*}

Simplifying the inequations, one can get :\\

\begin{align}
\frac{n}{P\lceil log_{2}(P) \rceil} >= \frac{ \frac{SoD}{B_{P}^{DMM, MPI}} + 5*T^{flop}}{3T^{flop}}
\label{GeneralBandwidthCondition}
\end{align}

As an example, with $T^{flop} = 1/1000 000 000$ and $B_{P}^{DMM, MPI} = 100 Mbytes/second$, we get :

\begin{equation*}
\frac{n}{P\lceil log_{2}(P) \rceil} >= 215
\end{equation*}

So for P=50 for example, if each worker handles more than 1290 points, then there will be theoretically more time spent computing datapoints closest centroids than communicating local centroids results.

\clearpage

\section{Some Case study of K-Means on DMM architectures}
\label{Distributed K-Means Case Study}

Equation (\ref{GeneralBandwidthCondition}) sets general condition for avoiding bandwidth to be a bottleneck. When dealing with a local cluster, the design of the cluster can be chosen so the condition holds. In this section, we examine some case study and check if this condition holds or not.

\subsection{First case study}
First case study, we want to cluster through a K-Means all the electricity French load curves. The time series are observed every hour for 10 years. We want the number of clusters to increase with the number of series to cluster, but not as fast as linearly, so we will say we want as many clusters as the square root of the number of time series. We suppose for the modelisation that the clustering processus will stop after 100 iterations. The computation are run on a network of monocore processors of 1GFlops, which makes 1 000 000 000 operations per second.
Numbers in this experience are :

\begin{enumerate}
\item n = 20 000 000 time series
\item d = 24 hours * 365 days * 10 years = 87600 values per series
\item K = $\sqrt{n}$ = 4472 clusters
\end{enumerate}

Note : volume of timeseries is 12,3 TeraBytes.

Note : setting K as $\sqrt{n}$ is an heuristic to ensure K growths with n, but not so fast than n.\\

We will estimate cost of clustering theses data on 3 different hardware cases :

case 1 : a monocore 2GFlops processor (we do not take into account this amount of data cannot fit into memory). Using modelisation \ref{sequential KMeans cost} this would take 37 years to process.

case 2 : The hardware used at Yahoo \cite{TeraSortWinnerYahoo} to win the TeraSort contest \cite{SortBenchmark}.\\
This hardware is mainly caracterised by :
\begin{itemize}
\item 910 nodes
\item 4 dual core Xeons @ 2.0ghz per a node
\item 1 gigabit ethernet on each node
\item 40 nodes per a rack
\item 8 gigabit ethernet uplinks from each rack to the core
\end{itemize}

We would get :\\

\begin{equation}
DMM ~StudyCase1 ~Cost = T_{P}^{comp} + T_{P}^{comm}
\label{DMM1 StudyCase1}
\end{equation}

where $T_{P}^{comp}$ is :

\begin{align*}
T_{P}^{comp} & = \displaystyle \frac{(3nKd+nK+dK+nd)*I*T^{flop}}{P}\\
& = 322893 seconds\\
\label{DMM2 StudyCase1}
\end{align*}

and $T_{P}^{comm}$ became :

\begin{align*}
T_{P}^{comm} & = (2*~Broadcast ~time ~between ~2 ~processors + Merging ~time) * \lceil log_{2}(P) \rceil\\
& = (2\frac{IKd*SoD}{B_{P,avg}^{DMM, MPI}} + 5IKd*T^{flop}) * \lceil log_{2}(P) \rceil\\
& = 225866 seconds\\
\end{align*}

Therefore, it would take about 6 days to finish the computation and computation would take more time than communication .\\

case 3 : this time we suppose we can run on the same hardware, but the switch on top of all the racks has a limited aggregated bandwidth of 100Gbits/secs. Aggregated bandwidth is now limiting and  we can modelize  $B_{P,avg}^{DMM, MPI} = 109Mbits$.

\begin{align*}
T_{P}^{comm} & = (2*~Broadcast ~time ~between ~2 ~processors + Merging ~time) * \lceil log_{2}(P) \rceil\\
& = (2\frac{IKd*SoD}{B_{P,avg}^{DMM, MPI}} + 5IKd*T^{flop}) * \lceil log_{2}(P) \rceil\\
& = 434155 seconds\\
\end{align*}

ADD EVALUATION OF COST OF THE CLUSTER.\\
ADD EVALUATION OF COST ON THE CLOUD IF FOLLOWING ALGORITHM DEVELOPED IN SECTION 7\\

\subsection{Dhillon's experiments}

Dhillon's experiments have been run on a IBM SP2 with a maximum of 16 nodes. Each node is a IBM POWER2 processor running at 160MHz with 256 MBytes of main memory. Processors communicate through the High-Performance Switch (HPS) with HPS-2 adapters. Performance of the HPS has been discussed in those papers \cite{HPS_2}, \cite{HPS_1}. In \cite{HPS_1} for example, point-to-point bandwidth in a SP2 with HPS and HPS-2 adapters using MPI is reported to be about 35 MBytes/sec.\\
It is difficult to estimate bandwidth actually used during Dhillon's experiments, as it is difficult to estimate effective Flops. Dhillon has made no comment about bandwidth, yet he reported very fluctuent Flops : 1.2 GigaFlops in some experiment, 1.8 GFlops in another (maximum would be 16 * 160MFlops = 2,5GFlops). Yet, on smaller datasets Flops might be very under this level. Because we cannot guess the actual bandwidth and Flops during his experiments, we will use theoretical bandwidth and Flops, getting very approximative results.\\

Dhillon reports that he has obtained a speed-up factor of 6.22 on 16 processors when using $2^{11}$ points.
Using condition \eqref{SpeedupFormula}, we can estimate speed up when using $2^{11}$ points :

\begin{align*}
Estimated Speedup & = \frac{3nT^{flop}}{\frac{3nT^{flop}}{P} +  \left(2 \frac{SoD}{B_{P}^{DMM, MPI}} + 5T^{flop}\right) \lceil log_{2}(P) \rceil}\\
& \simeq 11
\end{align*}

We see the estimated speedup is not accurate, but we are in the range where communication is important enough to prevent a perfect speedup, but yet small enough so a significant speedup is observed.\\

When using $2^{21}$ points, we get :

\begin{align*}
Estimated Speedup & = \frac{3nT^{flop}}{\frac{3nT^{flop}}{P} +  \left(2 \frac{SoD}{B_{P}^{DMM, MPI}} + 5T^{flop}\right) \lceil log_{2}(P) \rceil}\\
& \simeq 15.996
\end{align*}

Dhillon reports that for $2^{21}$ points, he gets a 15.62 speed-up. Again, anticipated speed-up indicates there will be little issue to parallelize, and observed speed-up is indeed excellent.

Before we examine a bandwidth condition for cloud hardware, we give a short presentation of cloud.

\newpage

\section{"Azure OS" presentation}
\label{Azure}

We have been working on Microsoft's cloud : Azure. In this section we introduce cloud computing, Azure services and storage, give some key ideas of cloud apps design. Some specificities of the cloud computing hardware impose new constraints on our algorithm. We describe the hardware performances and how it impacts our algorithm, before running some cost modelisation and running experiments.

\subsection{Cloud/Azure Introduction}

Cloud computing is a set of innovative technologies (Virtualisation, distributed storage, MORE ON THIS, see \cite{lenk2009witcaamotcl} ...) merged together into a coherent framework to commoditize IT (information technology). As electricity became in the 1930's available to consumers when national electricity grid were developed, cloud computing is a crucial step to provide computation and storage supplies as a service to any customer. Many papers have been published recently about cloud HPC performances and how it can impact scientific research : \cite{BridgingtheGap}, \cite{BenchmarkingAmazonEC2}, \cite{HighPerformanceComputingWithClouds}, \cite{TOP500}, \cite{BenchmarkingCloudServices},... We recommend this very nice introduction to cloud computing \cite{lenk2009witcaamotcl}.
As reported in \cite{BenchmarkingCloudServices}, each cloud system has developed different strategies and API to address different needs. Thus, it is difficult to compare the different cloud offers. Yet, we can isolate different cloud levels :

\begin{itemize}
\item \textbf{Software as a Service} (SaaS) : applications are exposed as a service running on a cloud infrastructure. Little freedom is left to users except using the application build on top of the cloud. Examples include GMail, Salesforce.com, Zoho etc...
\item \textbf{Platform as a Service} (PaaS) : users are given abstractions and/or framework to develop applications. The framework provides them "easy" environment to run their code. Storage and/or workers are abstracted. Main examples are Amazon Simple Storage Service (S3), Microsoft Azure, BungeeLabs...
\item \textbf{Infrastructure as a Service} (IaaS) : computing resources (compute, storage, and network) are exposed as a capability. The users are relieved of the burden caused by owning, managing or controlling the underlying infrastructure. Main example is Amazon Elastic Cloud Compute (EC2).
\end{itemize}

Since SaaS offers little except applications already build, we are interesting in the choice between Paas and IaaS.
These cloud technologies show different choices on some tradeoffs :
\begin{itemize}
\item \textbf{abstraction/control tradeoff} : The higher the control users gets, the lower they can have on-the-shelves abstraction. PaaS will provide higher abstraction to develop applications than IaaS, but the application will have to conform and fit to the framework provided and some things couldn't be achieved in PaaS (GIVE AN EXAMPLE).
\item \textbf{scalability/cost to develop tradeoff} : Running your application on a PaaS grants you great scalability confort. Yet, this scalability only comes as the result of a strong redesign of your application so it fits the framework provided (Azure framework is explained below) : scalability comes as a side effect of the redesign of the application. On the contrary, running your application on an IaaS offer lets you choose : you can run your application as it is, and scalability is bounded by the present design of your application, or you can redesign your application and scalability is improved.
\end{itemize}

MAKE A PARAGRAPH HERE TO EXPLAIN WHY RUNNING AZURE AND INTRODUCE LOKAD.CLOUD.\\

Presently, Azure is designed to provide PaaS. As explained in Azure FAQs, Microsoft is planning to add Virtual Machines (VM) functionality to enrich the different uses of Azure. Therefore, Azure might also become Infrastructure as a Service (IaaS). Yet, since no communication or services have been released yet, we will assume Azure is only PaaS oriented.\\

\subsection{Description of some Azure abstractions}

To get a deeper introduction to some Azure abstractions, you can download documents on the Azure Storage official blog ~\cite{Azure_Storage_Resources}.

\begin{itemize}
\item \textbf{Azure Queues} provide a reliable message delivery mechanism through distributed queues, which can be used to connect different components of a cloud application. A queue of type T can be accessed by its name, and mainly supports 2 methods : void Queue(T instance) and T DeQueue(void). It has been designed to be scalable in the number of messages (presently it is not the best scalable part of Azure, see section \ref{Abstractions performance}), but not in the size of messages stored (Messages stored in queues must not excess 8KBytes \cite{AzureStorageResources} (6144 bytes more precisely, see \cite{AzureQueuesCannotBeLarger})). It is not supposed to respect FIFO logic (First In First Out) as standard queue. On small scale, the Azure queue will behave like a FIFO queue, but if it is more sought, it will adopt a different logic so it can better scale. Therefore, the FIFO assumption cannot be assumed. Azure queues are designed to provide asynchronous distributed communication between the different Azure services. Using queues to communicate helps building loosely coupled components. They also mitigate the impact of individual component failure. Our usage of the queues will be to store the different "Map instructions" or "Reduce instructions" we want the workers to run. Message stored in a queue are guaranteed to returned at least once, but there is no guarantee for messages to be returned only once (when a worker is failing while processing a message) : this possible behaviour will cause our jobs to be idempotent (more on this below). When a message is picked in a queue by a worker, message is not deleted but becomes invisible for other workers. If a worker is failing to process a message entirely (because it throws some exception or because the worker dies), the message becomes available after a certain period of time. Thus, we can make sure no job is lost because of a dying worker.
\item \textbf{Azure Blob Storage} enables applications to store large objects, up to 50GB each in the cloud.  It supports a massively scalable blob system, where hot blobs will be served from many servers to scale out and meet the traffic needs of our application. The data is replicated at least 3 times for durability \cite{AzureStorageResources}. In addition, strong consistency is provided to ensure that the object is immediately accessible once it is added or updated; a subsequent read will immediately see the changes made from a previously committed write. Blobs are stored using a key/value pair system.
\item \textbf{Azure Table Storage} provides scalable structured storage. Structured storage is provided in the form of Tables, which contain a set of Entities, which contains a set of named Properties. No schema is stored by Windows Azure Table, so all of the properties are stored as $<name, typed ~value>$ pairs. This means that two entities in the same table can have very different properties. Combined size of all data in an entity cannot exceed 1MB. Entities are stored inside the table using a two part key composed of a PartitionKey (primary key) and a RawKey (secondary key). An application can manipulate the entities and query over any of the properties stored in a Table.
\item \textbf{Azure Cloud Services} are classes containing the code to execute. Each service will be implementing a "Start method", taking a strongly-typed message of type T, and will execute instructions stored in the start method.
\end{itemize}

\subsection{ACID/BASE constraints on cloud/ on Azure}

Recently, the relational database model features, known as ACID, have been challenged. Brewer's CAP theorem (Consistency, Availability and Partition tolerance) \cite{CAP_Theorem} states that distributed storage won't be able to propose ACID properties in an acceptable way. Abadi recently explained some defaults of CAP theorem \cite{CLAP_Presentation}. During this time, web services delivered by very large companies like Google, Amazon, Yahoo, Microsoft... have been partially abandoning relational database and ACID properties, starting the NoSQL movement (NoSQL standing for Not Only SQL). Even some SQL-related people recently expressed their interest for NoSQL database, like Ayende (NHibernate contributor) did  \cite{Slaying_Relational_Dragons}. Most NoSQL data storages are censored here \cite{NoSQL}. The properties of these new data storage systems have been described as BASE (Basically available, Soft State, Eventually consistant) \cite{BASE}.\\

One of the key aspect of distributed data storage is data replication (Azure holds for each data 6 replicas : 3 replicas in a first datacenter and 3 others in a backup datacenter REFERENCE NEEDED). Since data servers are expected to fail from time to time, data are duplicated several times, so a data server can die without any data loss. This replication process (even more if replication is not stored on the same physical support) has a deep impact on consistency and performance as reported in \cite{CLAP_Presentation}. Some storage like Cassandra have been abandoning consistency guarantees for eventually consistency \cite{Eventually_Consistent}.\\

Azure does not provide (at least yet) consistency or availability guarantees. Since the Azure code is not disclosed, we cannot certify any of these properties. We have been noticing that system un-availability is not only theoretical. When accessing to the storage, we can get notified an Azure internal Exception. Therefore, we have to implement retry policies so we can try again to run a read or write operation. Since storage is designed to be accessed by internal workers but also by outside (we can think for example of a picture or a video stored and devoted to be downloaded a lot of time), it makes no sense to guarantee 100\% availability since internet connection does not give such guarantee.\\

To our knowledge, Azure has not been guaranteeing consistency too. Yet, we have not been able to experience any consistency issue. Three points here :
\begin{itemize}
\item \textbf{Block commits} : Data bigger than 4MBytes must be split into blocks before being uploaded. Once every block has been uploaded into Azure, the "transaction" is commited, and data is finally wrote on Azure side. This 2-phase upload (partial uploads, then integral commit) is TO BE FINISHED
\item \textbf{Etag} : As reported in the BlobStorage and TableStorage technical papers \cite{Azure_Storage_Resources}, Azure provides a kind of timestamp, called etag. When updating some data, we first download the data, update it, and upload it with the etag provided. When replacing the data, Azure checks that etag of data being replaced matches the etag provided by the upload. If it does not, the overwrite is rejected, an exception is thrown, so the user can retry. This mechanism helps to deal with multiple workers trying to modify a data at the same time, but is very probably also used inside Azure to ensure consistency between the different replica of a same data.
\item \textbf{Very few overwrites} Most applications do not need to overwrite data. Some typical usage could be for example a big video upload once, and hundreds or thousands of read request on this data (Facebook usecase). Clusterings we run on Azure required very few overwrite (mostly for counters), as it will be shown.
\end{itemize}

\subsection{other tradeoffs in Azure}

\textbf{Read/Write tradeoff} : As reported in \cite{BenchmarkingCloudServices}, distributed storage must choose a tradeoff between read throughput and write throughput. Taking the example of a blob in BlobStorage, a blob cannot be at the same time highly available in read operations and write operations (having a highly available blob for read operations implies the blob to be duplicated, so write operations must change several replicas, and write operations take more time). Azure let the traffic arbitrate this tradeoff : the more request to a blob per time unit, the more the blob is duplicated, the more it becomes available, and the more time it takes to run write operations.\\

\textbf{Responsivity/Efficiency} \cite{BenchmarkingCloudServices} is also reporting some tradeoff between services responsivity and efficiency. Efficiency, as defined as the ratio of work completed per hour per worker is a decreasing function of workers and an increasing function of total workload. Therefore, to achieve the best possible efficiency, we should make sure every worker is overworked. Yet, the more overworked workers are, the least responsive they can be on an additional task. Since the first goal of our algorithm is responsivity (we want to minimize amount of time spent processing the algorithm), we will make sure to provide a pool of workers slightly greater than the number of jobs we can parallelize, so we make sure all jobs that can be run parallelingly, are (having P jobs run by P-1 workers would lead approximately to twice the time P jobs running by P or P+1 workers). \\

\textbf{Synchronous Replication/Replication Latency tradeoff} : TO BE DIGGED. RELATED TO CONSISTENCY

\subsection{BlobStorage or TableStorage ?}

BlobStorage and TableStorage are both designed to store data. When should we use one or the other ? Here are some part of the answer :
\begin{itemize}
\item One part of the answer is the data locality management and atomicity. Every data object may they be blobs or table entities are stored in terms of their partition key. This partition key determines how Azure must load balance and partition the objects across Azure servers. Partition Key for TableStorage is the tableName + partitionkey (primary key), whereas PartitionKey for blobs are determined by containerName + blobName. Since the partition key is down to the blob name for blobStorage, access to different blobs can be load balanced  across as many servers as needed to scale. But no atomic transactions is authorised for multiple blobs. As explained below in design subsection \ref{Redesign of the algorithms}, jobs will need to be idempotent : no atomic transaction for blobStorage is another difficulty to guarantee the idempotency. Since partition key is up to the primary key in TableStorage, we can run atomic transactions up to 100 entities who share the same table + primarykey (Azure guarantees that all the entities will be changed the specified way, or no one will).
\item When dealing with really large objects, BlobStorage is easier to manipulate, since each blob can be arbitrarily large (up to 50 GB per blob). We do not need to break an object into a lot of entities, making sure each time each "part entity" respect the 1MB constraint. BUT WE HAVE TO DEAL WITH BLOCKS : SAY LOKAD.CLOUD HANDLES IT
\item When manipulating objects than can be stored both in TableStorage and BlobStorage, BlobStorage is reported by \cite{Early_Observations_On_Azure} to be more effective in insert operations on blobs larger than 4KBytes, which is below the average size of blobs we are going to use.
\item Price is another factor that might affect our choice. Using Azure storage, three things are charged : size of our storage, I/O communications, requests to the storage. A single query on TableStorage can load up to 100 elements at a time and costs as much as a single query on blobStorage, see \cite{Azure_Pricing}. If we have to store a lot of small objects on different places, using TableStorage will thus allow to run request on objects by packet of 100, so it would divide up to a factor 100 the bill of storage requests in comparison of running requests one by one on the BlobStorage. If we do not need to store the objects on different places, we could store them in one bigger blob, bounding the request to one. Yet, the more times we have to modify a small part of the storage, the less storing all the data in a single blob is the right thing, since we are going to download and upload the big blob each time a small part of the data need to be updated, making I/O costs predominant comparing to storage requests. So, using TableStorage or BlobStorage on the price criterion will depend on the size of your data, your application, the way the application is designed, your need to update small part of data or entire data each time, ...
\end{itemize}

To run our experiments, we chose to use the BlobStorage. Atomic updates on multiple object at once will not be a key point in our code, except for counters for which concurrency will be handled by etag mechanism. Price will be estimated but is not a criterion here. Besides, BlobStorage is easier to manipulate and due to its complexity, TableStorage suffered little bugs that affect performances when we ran our experiments (in June 2010).\\

\subsection{Abstractions performance}
\label{Abstractions performance}
The Azurescope website \cite{AzureScope} is reporting performance benchmark run from February/March 2010 to presently (June 2010) on Azure hardware.
Some paper have also been released very recently on same measures \cite{Early_Observations_On_Azure}.
Numbers reported in \cite{Early_Observations_On_Azure} seem to be more accurate than \cite{AzureScope}, and the methodology is better explained, so we will use these numbers as reference.
We will use the numbers observed to understand if bandwidth is an issue on Azure.

\begin{itemize}
\item BlobStorage Read Bandwidth : 13 MBytes/sec for 1 worker, BlobStorage aggregated read bandwidth bounded to 393 MBytes/sec when using more workers (each worker is using several threads to load data from storage). This quantity describes the bandwidth for several workers to download the same blob from the storage.
\item BlobStorage Write Bandwidth : 5 MBytes/sec for 1 worker, BlobStorage aggregated write bandwidth bounded to 124 MBytes/sec when using more workers (each worker is using several threads to upload data). This quantity describes the bandwidth for one or several workers to load data from different blobs but within the same container.
\item Communication throughput between 2 workers : 55 MBytes/sec on large messages on average (which is a very impressive bandwidth).
\item CPU power depends on type of VM allocated : from 1 core (on small VM) to 8 cores on extra-large VM
\item Queue throughput is presently 500 messages per second, but might be very increased soon.
\end{itemize}

Note : Since concurrent write on a blob is very costly, we will not write twice at a same place, unless when using counters.\\

We will therefore modelize and simplify download and upload Bandwidth when and without saturation as :

\begin{align*}
B^{read} & = 13 MBytes/sec\\
B_{P}^{read} & = \min(B^{read}, 393/P)\\
B^{write} & = 5MBytes/sec\\
B_{P}^{write} & = \min(B^{write}, 124/P)
\end{align*}

\subsection{Redesign of the algorithms}
\label{Redesign of the algorithms}

Before trying to estimate possible speed-up, we have to redesign the algorithm so it fits the hardware constraints. In this subsection, we will describe some key points in the design of cloud applications in general, and K-Means in particular :\\

\begin{itemize}
\item \textbf{Services Oriented Architecture} Cloud applications are inspired by SOA (Services Oriented Architecture) and will be divided into cloud services. Each cloud service will be associated to a specific queue and will be responsible for one logical part of the application. Services will communicate using queues and storage : when a service A is done and need to notify another service B it can start, service A will be pushing computation result in the storage into one or several blobs and will push a new message in queue associated to service B, the message containing the key of blobs pushed so service B can load back data from storage. This design helps the different services of the application to be loosely coupled and scalable. The storage on Azure has been designed not only to store data and results, but also to endorse the shared memory role in SMP architectures (with some limitations, though).
\item \textbf{No Master} DEVELOP THIS
\item \textbf{No worker is more efficient than another one for a specific task }. Work can be divided into tasks. No matter which worker is assigned on which task, because each worker can be as efficient as the others on each task. A very naive load balancer will dispatch available workers on the different queues to process the messages.
\item \textbf{No worker/data affinity is provided but it is necessary} Azure is not providing any affinity between data and workers. Data are abstracted into BlobStorage and TableStorage and it is impossible by design to try to run jobs on CPUS that are physically near the place data are stored. CITE GOOGLE MAP REDUCE. DEVELOP HOW WE WILL MANAGE THIS WHILE AVOIDING ITERATION THROUGH MESSAGES. LOOK FOR XMPP PROTOCOL.
\item \textbf{Scaling up is a developer initiative} Through the management API, administrator can change the number of workers available. Then, we are no more in a fixed hardware framework as K-Means run in other papers experiments. We can modulate the hardware so it better fits our need. Therefore we can ask a new question : how many workers do we need to minimize the whole time of the algorithm ?
\item \textbf{There are no strict guarantees on the number of workers actually running }.  Due to the huge amount of time necessary to set up workers \cite{Early_Observations_On_Azure} (which is going to be improved), workers will be instanciated before the experiments are run and their number is supposed to be constant over time. Yet, since a worker can be shutdown, a VM deplaced or paused arbitrarily by Azure without notification, the number of workers is not guaranteed to be exactly the number requested at each instant.
\item \textbf{Pinging queues as a tradeoff between cost and simplicity}. Workers will be pinging queues to detect if there are some messages to process. Since even QueueStorage operations are charged, we can wonder whether pinging queues can be avoided, and whether it is affordable. Indeed, we could open a TCP connection on each worker, set-up a listener so each worker is listening for notification. Each time a notification is sent, the listener tells the worker to ping the queue and get the message, or the notification is the message by itself and worker is directly told to process the message. In the first case, we do not ping queues unless a message needs to be processed. In the second case, we do not ping queues at all. We have been choosing to not implement listeners patterns and TCP connections, for simplicity reasons. Pinging a queue every 100 ms for a worker means a 315 millions request per worker per year, charged as 3000 dollars (see \cite{Azure_Pricing}). We have been adopting a back-off strategy : each time we ping an empty queue, we double time before pinging again the queue CHECK THIS IN LOKAD.CLOUD. IN THE EXPERIMENTATION PART, GIVE HOW MANY TIMES THE QUEUE IS PINGED
\item \textbf{Monitoring how many messages are delivered} It is not possible to get an exact count of how many messages are stored in some queue, we only can get an approximative count of queued messages when asking to the queue its metadata. When a service needs to process P messages, then need to push a message in another queue to launch a new service, we need to get an exact count of remaining messages to process. To achieve this, we are using blobcounters, blobs storing an integer and decremented by each worker using etag properties when the worker is done with a message. Since the worker can fail at any moment, and a failure means the message is requeued after some delay and is going to be processed again, we decrement the counter in the end, when the job is finished and the message is going to be deleted. Thus, the only way a single message can lead to decrement twice the counter is to make the worker fail exactly when it is noticing the queue to delete the message. Theoretically, the counter could therefore be decremented twice (with a very low probability). One way to deal with this issue would be to use an idempotent counter :
    \begin{enumerate}
    \item we push P messages in the queue. Each message has a unique Id between 1 and P
    \item we push a binary counter of P bits set to 1111..1111 in the blobStorage
    \item each time the job stored in message i is completed, we update using etag the counter and apply to the counter a \& operation 111110111111 where 0 is at index i. This way we made a counter where decrements will be idempotent.
    \end{enumerate}
    Even if getting number of messages stored in a queue is approximate, Azure is providing a transactional count of how many times a message is dequeued. FINISH HERE
\item \textbf{Jobs must be idempotent} Since messages can be consummed more than once, and some worker can stop in the middle of a message processing before another worker restart the message, we must design our jobs so jobs stored in messages are idempotent, that is to say running twice a job won't affect final result.
\item \textbf{Workers Communication}  On DMM architecture, we saw K-Means was distributed by using MPI to broadcast data across the different CPU. Communication between workers was the key point of the algorithm : the ratio bandwidth/CPU frequency was driving speed-up efficiency. When designing the algorithm for the cloud, we must keep in mind that we will run K-Means on a hardware where bandwidth will not be as good as in Dhillon's experiments, and CPU frequency higher. Therefore, on a same number of points to cluster, the speed-up factor will probably get worse.
    Here, we have 2 different designs available :
        \begin{itemize}
        \item Workers are communicating directly. Whenever a loop is completed, local centroids are communicated to another worker directly through TCP/IP. Workers are behaving like in a peer-to-peer environment.
        \item Local results are stored in the Azure storage, and workers are communicating through queues and storage to inform data have been processed and where we can find back the results.
        \end{itemize}

    This is a key point in the design on the algorithm. First version is the "direct" translation of DMM K-Means architecture. Yet, direct communication will come with a lot of code overhead :
        \begin{itemize}
        \item we have to make sure each worker has a list of other workers and their "address".
        \item we have to deal with failures. Especially, each worker must keep a track of available other workers, we must add code to handle failures, ...
        \end{itemize}

    Performance of "direct inter-worker communication KMeans" is expected to be better, because we do not have to use the storage as a middleman between each couple of communicating workers, and interworker communication bandwidth is reported to be better than storage communication bandwidth. Yet, code overhead to get a robust system using direct communication in a context of industrial constraints is so important that we are going to implement both design, and compare the two designs in term of performance.\\


\end{itemize}

\newpage

\subsection{Proposed Algorithms}

The first algorithm we propose is the most adapted to cloud services philosophy of Azure : we are running an Iterative MapReduce, messages queued indicating whether it is a map or reduce operation, and where the data can be retrieved.

\begin{algorithm}
    \caption{Synchronous Distributed Cloud Batch K-Means version 1 }
    \begin{algorithmic}
    \STATE Generate initial centroids $(c_{k})_{k=1..K}$, set them as current centroids and store them in a blob.
    \STATE List blobNames of blobs containing data points and split blobNames into P sets of blobNames $(S_{p})_{p=1..P}$.
    \STATE Create a job counter and set it to P. Put the counter into the blobStorage.
    \STATE Put in queues the P "MAP" messages containing : one set $S_{p}$, blobName of initial centroids version, blobName of the counter.
    \STATE Each worker will run the following actions :
    \REPEAT
        \STATE Ping the queue dedicated to clustering on intervals discussed previously. When the worker got a message :
        \IF {message p is "MAP"}
            \STATE Load data $(x_{i}^{p})_{i,x_{i}^{p} \in S_{p} }$ using blobNames of message p.
            \STATE Load centroids $(c_{k})_{k=1..K}$ from previous iteration (or initial version).
            \FOR{$x_{i}^{p} \in S_{p}$}
                \FOR{$k=1$ to $K$}
                    \STATE compute $||x_{i}^{p}-c_{k}||_{2}^{2}$
                \ENDFOR
                \STATE find the closest centroid $c_{k^{*}(i,p)}$ to $x_{i}^{p}$
                \STATE Update on line the sum of points belonging to cluster $k^{*}(i,p)$ with $x_{i}$
                \STATE Increment element counter of cluster $k^{*}(i,p)$
            \ENDFOR
            \FOR{$k=1$ to $K$}
                \STATE Compute each centroïd by dividing the sum by the element counter
            \ENDFOR
            \STATE Write into the storage local computed centroids
            \STATE Decrement job counter stored in the blob using etag properties.
            \IF {decremented job counter is 0}
                \STATE Push a "REDUCE" message in queue
            \ENDIF
        \ELSE
            \STATE Message is "REDUCE"
            \STATE List all local computed centroids stored in the storage using the adequate prefix
            \STATE merge all sets of centroids into a merged set of centroids.
            \STATE update current centroids into the storage
            \STATE Reset counter to P
            \STATE Push P "MAP" messages into the queue
        \ENDIF

        \STATE One worker load from the storage all the local computed centroids, merge them with the corresponding weight : $\# \{ i, ~x_i \in S_{p} ~\& ~k^{*}(i)=k \}$, overwrite blob with current centroids with the new computed centroids, and throws P messages in queues
    \UNTIL {no $c_{k}$ has changed since last iteration}
    \end{algorithmic}
\end{algorithm}

Description of Synchronous Distributed Cloud Batch K-Means version 1 :
\begin{itemize}
\item This version is the most adapted to services design in Azure : jobs are launched using queues, each worker can process any of the messages.
\item Updating local centroïds version on each point and not in the end of the iteration makes the algorithm to be locally on-line, so even if data processed in each worker does not fit into memory it's ok (NOTE : if data do not fit into memory, the algorithm is going to take YEARS).
\item Pushing the Reduce message in the queue instead of processing it directly in the last worker makes the processus more resilient to failures. QUESTIONABLE BUT TRUE. DEVELOP
\end{itemize}

Yet, in this version each worker has to load again at each iteration a part of the entire dataset, since the worker is not supposed to process the same part of the dataset at each iteration. Due to time spent in loading data, we can avoid a lot of time wasting by switching algorithm version 1 into algorithm version 2 :

\begin{algorithm}
    \caption{Synchronous Distributed Cloud Batch K-Means version 2 }
    \begin{algorithmic}
    \STATE Generate initial centroids $(c_{k})_{k=1..K}$, set them as current centroids and store them in the "current centroids version blob".
    \STATE List blobNames of blobs containing data points and split blobNames into P sets of blobNames $(S_{p})_{p=1..P}$.
    \STATE Set an integer as the number of workers running to P, and save it into the blobStorage.
    \STATE Put in queues the P sets in messages containing : one set $S_{p}$ and the "current centroids version blobName", the counter blobName.
    \FOR{each worker}
        \STATE Dequeue a message.
        \STATE Load into local memory data $(x_{i}^{p})_{(i,p),x_{i}^{p} \in S_{p} }$ using blobNames stored in message p once and for all.
        \REPEAT
            \STATE load the current centroids $(c_{k})_{k=1..K}$ from "current centroids version blobName".
            \FOR{$x_{i}^{p} \in S_{p}$}
                \FOR{$k=1$ to $K$}
                    \STATE compute $||x_{i}^{p}-c_{k}||_{2}^{2}$
                \ENDFOR
                \STATE find the closest centroid $c_{k^{*}(i,p)}$ to $x_{i}^{p}$
            \ENDFOR
            \FOR{$k=1$ to $K$}
                \STATE $c_{k,p} = \frac{1}{\# \{ i, ~x_i \in S_{p} ~\& ~k^{*}(i)=k \}} \sum_{\{ i, ~x_i \in S_{p} ~\& ~k^{*}(i)=k \}} {x_{i}}$
            \ENDFOR
            \STATE Write into the storage local computed centroids.
            \STATE Acquire a lock on the counter, decrement it
            \IF {counter is equal to 0}
                \STATE List all local computed centroids blobs, load them, merge them using corresponding weights, and overwrite "current centroids version blob" with the result
            \ENDIF
        \UNTIL {no $c_{k}$ has changed since last iteration}
    \ENDFOR
    \end{algorithmic}
\end{algorithm}

\begin{figure}
    \centering
    \includegraphics[scale=0.3]{../../Pictures/HardwareSchemes/centroidComputation1.png}
    \caption{centroidComputation1}
\end{figure}

\begin{figure}
    \centering
    \includegraphics[scale=0.3]{../../Pictures/HardwareSchemes/centroidComputation2.png}
    \caption{centroidComputation2}
\end{figure}

\begin{figure}
    \centering
    \includegraphics[scale=0.3]{../../Pictures/HardwareSchemes/centroidComputation3.png}
    \caption{centroidComputation3}
\end{figure}

\subsection{SpeedUp theoretical estimation}

We will try to give an estimation of the speed-up we can get using last algorithm version.
Yet, the very nature of the cloud will make this estimation rather difficult to make :
\begin{itemize}
\item Bandwidth reading and writing on the storage introduce a lot of uncertainty in the estimation. Because it varies over time. But also because before loading up data or after downloading data we must serialize/deserialize them, compress/uncompress them, ... Therefore, bandwidth capacity is not exactly what expected.
\item To download and upload data, using multi-threads improves a lot total throughput, as observed here \ref{AzureScope}. Throwing multiple threads (up to 30 as advised) comes with some little overhead.
\item Workers are pinging queues to detect if there are messages to process. To limit the number of network pings, we schedule the pinging process only once per several seconds, introducing again uncertainty and latency.
\item Virtual Machines upon workers might be shutdown or die, but work will still be executed once the VM is moved or reboot or another worker takes the job. Therefore, it is impossible to note that a VM has been unavailable for a part of the time, yet it will affect the result.
\item Storage is sometimes unavailable for short periods of time. Retry policies are implemented on different layers of the abstractions so we do not have to  be careful. Yet, time to upload and download will be affected and we won't be able to notice, unless we build our algorithms on lower level of abstraction, increasing a lot the difficulty of the design.
\item When running this algorithm, workers wait for the others to complete before running the next iteration. On the cloud, stragglers issue are stronger than on a DMM machine. Therefore, synchronisation barrier will affect a lot the process, and time consumption will be affected by every bottleneck on every worker.
\end{itemize}

We will keep the same value for $T_{P}^{comp}$ :

\begin{align*}
T_{P}^{comp} & = \frac{(3nKd+nK+dK+nd)*I*T^{flop}}{P}\\
\end{align*}

Workers on the cloud will need to load data from the storage, a cost that has not been taken into account in last modelisations.
If a worker is loading all the data from the storage, we can modelize time to load the data (if they fit into memory) :

\begin{align}
T^{Load} & = \frac{nd*SoD}{P * Bandwidth^{read}}\\
\end{align}

and if we divide the amount of data to load on P workers, loading time becomes :

\begin{align}
T_{P}^{Load} & = \frac{nd*SoD}{P * \max(Bandwidth^{read}, \frac{MaxReadThroughput}{P})}\\
\end{align}

Then, we can neglect time spent to load data in speed up modelisation if :

\begin{align*}
T_{P}^{Load} << T_{P}^{Comp}
\end{align*}

which means :

\begin{align*}
\frac{SoD}{3IK*T^{flop} * \max(Bandwidth^{read}, \frac{MaxReadThroughput}{P})} << 1
\end{align*}

As we will verify in numerical applications, this is always the case.

We will modelize time spent by workers to communicate by :

\begin{align*}
T_{P}^{comm, per iteration} & = ~Time ~to ~read ~1 ~blob ~"simultaneously" ~by ~P ~workers\\
& + ~Time ~to ~write ~P ~blobs ~"simultaneously"\\
& + ~Time ~to ~read ~P ~blobs ~by ~1 ~worker\\
& + ~Time ~to ~merge ~the ~different ~centroids ~version\\
& + ~Time ~to ~write ~1 ~blob ~per ~1 ~worker\\
\end{align*}

\begin{align*}
T_{P}^{comm, per iteration} & = \frac{Kd *SoD}{Min(Bandwidth^{read}, \frac{BlobReadThroughput}{P} )} \\
& + \frac{Kd * SoD}{Min(Bandwidth^{write}, \frac{MaxWriteThroughput}{P})}\\
& + \frac{PKd* SoD}{Bandwidth^{read}} \\
& + (P-1)Kd * 5floatOperations * T^{flop}\\
& + \frac{Kd * SoD}{Bandwidth^{write}}\\
\end{align*}

\begin{align*}
T_{P}^{comm, per iteration} & = Kd*SoD * ( \frac{1}{Min(Bandwidth^{read}, \frac{BlobReadThroughput}{P} )} \\
& + \frac{1}{Min(Bandwidth^{write}, \frac{MaxWriteThroughput}{P})}\\
& + \frac{P}{Bandwidth^{read}} \\
& + \frac{(P-1) * 5floatOperations * T^{flop}}{SoD}\\
& + \frac{1}{Bandwidth^{write}})\\
\end{align*}

Keeping the most significant term, we get :

\begin{align*}
T_{P}^{comm} & \simeq \frac{IPKd*SoD}{Bandwidth^{read}} \\
\end{align*}

Then we can deduce an "idea" of the speed-up factor :

\begin{align*}
SpeedUp & = \frac{T_{1}^{comp}}{T_{P}^{Comp} + T_{P}{comm}}\\
& \simeq \frac{3IKdnT^{flop}}{\frac{3IKdnT^{flop}}{P} +\frac{IPKd*SoD}{Bandwidth^{read}} } \\
& \simeq \frac{3nT^{flop}}{\frac{3nT^{flop}}{P} + \frac{P*SoD}{Bandwidth^{read}}}
\end{align*}

Using this modelisation, which is very naive, we can see there is an optimal number of workers to use.
This number is obtained by having time spent by communication equals time spent by computation.

\begin{align*}
P^{*}  = \sqrt{\frac{3nT^{flop}Bandwidth^{read}}{SoD}}
\end{align*}

Remarks : P must grow proportionately with $\sqrt{Bandwidth^{read}}$, $\sqrt{n}$ and $\frac{1}{\sqrt{CPU cadency}}$

Numbers in the next experimentation :

\begin{itemize}
\item n = 432 000
\item K = 657
\item d = 168
\item I = 100
\item PMax = 64
\item $Bandwidth^{read} = 20 Mbytes/sec  (resp. 5Mbytes/sec )$
\end{itemize}

Then we get those results :

\begin{align*}
Sequential ~KMeans ~time = 14300 seconds
\end{align*}

\begin{align*}
Optimal ~number ~of ~workers = 57 (resp 28)
\end{align*}

\begin{align*}
SpeedUp = 28,5 (resp 14)
\end{align*}

\newpage

\section{Experiments}
\label{Cloud Experiments}

\newpage

\cite{*}

\bibliographystyle{plain}
\bibliography{DistributedKMeans}

\end{document} 