\documentclass{beamer}

\usepackage{algorithm}
\usepackage{algorithmic}
\usepackage{beamerthemesplit}
\usepackage{graphicx}

\title{Designing K-Means for the cloud}
\author{Matthieu Durut, Fabrice Rossi}
\date{\today}

\begin{document}

\begin{frame}
\titlepage
\end{frame}

\begin{frame}
\frametitle{Few reminders about distributed K-Means}
\begin{align*}
KMeans ~Sequential ~Time \ & = (3Knd+Kn+Kd+nd)*I*T^{flop} \\
                         \ & \simeq 3Knd*I*T^{flop}
\end{align*}
\end{frame}

\begin{frame}
\frametitle{SMP Cost}
\begin{align*}
KMeans ~SMP ~Distributed ~Cost & = T_{P}^{comp}\\
    \ & = \frac{(3Knd+Kn+Kd+nd)*I*T^{flop}}{P}\\
    \ & \simeq \frac{3Knd*I*T^{flop}}{P}
\end{align*}
\end{frame}

\begin{frame}
\frametitle{DMM Cost}
\begin{align*}
& KMeans ~DMM ~Cost\\
& = T_{P}^{comp} + T_{P}^{comm}\\
& = \frac{(3Knd+Kn+Kd+nd)*I*T^{flop}}{P} + T_{P}^{comm} \\
& \simeq \frac{3Knd*I*T^{flop}}{P} + O(log(P))
\end{align*}
\end{frame}

\begin{frame}
\begin{figure}
    \centering
    \includegraphics[scale=0.6]{../../Pictures/HardwareSchemes/Merginglogic.pdf}
    \caption{Merging centroïds logic}
\end{figure}
\end{frame}

\begin{frame}
\frametitle{classical issue : pipes}
The size of the biggest pipe (on top of the tree) and the aggregated bandwidth.
\end{frame}

\begin{frame}
\frametitle{modelisation of $T_{P}^{comm}$ in DMM}
\begin{align*}
T_{P}^{comm} = \displaystyle \sum_{p=1}^{\lceil log_{2}(P) \rceil} (2\frac{IKd*SoD}{B_{\frac{P}{2^{p}}}^{DMM, MPI}} + 5IKd*T^{flop})\\
\end{align*}
\end{frame}

\begin{frame}
\frametitle{modelisation of $T_{P}^{comm}$ in DMM (2)}
\begin{align*}
T_{P}^{comm} = (2\frac{IKd*SoD}{{B_{P,avg}}^{DMM, MPI}} + 5IKd*T^{flop}) * \lceil log_{2}(P)\rceil\\
\end{align*}
\end{frame}

\begin{frame}
\frametitle{SpeedUp}
\begin{align*}
Speedup & = \frac{KMeans ~Sequential ~Cost}{KMeans DMM Distributed Cost}\\
& = \frac{KMeans Sequential Cost}{T_{P}^{comp} +T_{P}^{comm}}\\
& = \frac{3nKdIT^{flop}}{\frac{3nKdIT^{flop}}{P} +  \left(2 \frac{IKd*SoD}{B_{P}^{DMM, MPI}} + 5IKdT^{flop}\right) \lceil log_{2}(P) \rceil}\\
& = \frac{3nT^{flop}}{\frac{3nT^{flop}}{P} +  \left(2 \frac{SoD}{B_{P}^{DMM, MPI}} + 5T^{flop}\right) \lceil log_{2}(P) \rceil}\\
\end{align*}
\end{frame}

\begin{frame}
\frametitle{Some comments}
\begin{itemize}
\item RAM is not modelize $\Rightarrow$ speed up might be greater
\item speedup independant of K, I and d
\item winner paper model !
\begin{align}
\lim_{n\rightarrow +\infty} SpeedUp = P \\
\end{align}
\end{itemize}
\end{frame}

\begin{frame}
\frametitle{First case study : EDF}
\begin{itemize}
\item n = 20 000 000 time series
\item d = 24 hours * 365 days * 10 years = 87600 values per series
\item K = $\sqrt{n}$ = 4472 clusters
\item Note : volume of timeseries is 12.3 TeraBytes.
\end{itemize}
\end{frame}

\begin{frame}
\frametitle{1 monocore}
37 years
\end{frame}

\begin{frame}
\frametitle{Yahoo terasort winner}
\begin{itemize}
\item 910 nodes
\item 4 dual core Xeons @ 2.0ghz per a node
\item 1 gigabit ethernet on each node
\item 40 nodes per a rack
\item 8 gigabit ethernet uplinks from each rack to the core
\end{itemize}
\end{frame}

\begin{frame}
\frametitle{Yahoo terasort winner(2)}
\begin{align*}
T_{P}^{comp} & = \displaystyle \frac{(3nKd+nK+dK+nd)*I*T^{flop}}{P}\\
& = 322893 seconds\\
\label{DMM2 StudyCase1}
\end{align*}

\begin{align*}
T_{P}^{comm} & = (2*~Broadcast ~time ~between ~2 ~processors + Merging ~time) * \lceil log_{2}(P) \rceil\\
& = (2\frac{IKd*SoD}{B_{P,avg}^{DMM, MPI}} + 5IKd*T^{flop}) * \lceil log_{2}(P) \rceil\\
& = 225866 seconds\\
\end{align*}
\end{frame}

\begin{frame}
\frametitle{Yahoo terasort winner(2) bounded by aggregated bandwidth}
$B_{P,avg}^{DMM, MPI} = 109Mbits$
\begin{align*}
T_{P}^{comm} & = (2*~Broadcast ~time ~between ~2 ~processors + Merging ~time) * \lceil log_{2}(P) \rceil\\
& = (2\frac{IKd*SoD}{B_{P,avg}^{DMM, MPI}} + 5IKd*T^{flop}) * \lceil log_{2}(P) \rceil\\
& = 434155 seconds\\
\end{align*}
\end{frame}

\begin{frame}
\frametitle{Pricing on Azure}
\begin{itemize}
\item Storage price : 1845 dollars/Month (0,15 dollars /GB/Month)
\item No charge for internal transfers ?
\item update : 10 GBytes/Month  : 3 dollars (0.3dollars /GB)
\item cost of running the KMeans : (very rough) 0.12dollars/hour * 324120 = 40 000dollars
\item storage requests : ? from 0.01 dollars to 11 millions dollars
\end{itemize}
\end{frame}

\begin{frame}
\begin{figure}
    \centering
    \includegraphics[scale=0.6]{../../Pictures/HardwareSchemes/PaaS.png}
    \caption{Paas, Iaas, SaaS}
\end{figure}
\end{frame}

\begin{frame}
\frametitle{Paas and Iaas : some tradeoffs}
\begin{itemize}
\item abstraction/control tradeoff
\item scalability/cost to develop tradeoff
\end{itemize}
\end{frame}

\begin{frame}
\frametitle{Introducing Lokad.Cloud}
\begin{itemize}
\item higher level abstractions
\end{itemize}
\end{frame}

\begin{frame}
\frametitle{Azure Abstractions}
\begin{itemize}
\item Azure Queues
\item Azure BlobStorage
\item Azure TableStorage
\item Azure SQLStorage
\end{itemize}
\end{frame}

\begin{frame}
\frametitle{Azure Queues}
\begin{itemize}
\item loosely coupled asynchronous communication mechanism
\item not scalable yet (in size and number of messages)
\item FIFO when not sought
\item messages can be processed several times if worker fails (see idempotency)
\end{itemize}
\end{frame}

\begin{frame}
\frametitle{Azure BlobStorage}
\begin{itemize}
\item objects up to 50 GB
\item consistency ?
\item Key/value system
\end{itemize}
\end{frame}

\begin{frame}
\frametitle{Azure TableStorage}
\begin{itemize}
\item Tables/Entities/Properties
\item key/value pair also
\end{itemize}
\end{frame}

\begin{frame}
\frametitle{Azure QueueServices}
\begin{itemize}
\item SOA
\item Queue consumers
\end{itemize}
\end{frame}

\begin{frame}
\frametitle{ACID/BASE constraints on Cloud}
\begin{itemize}
\item ACID
\item BASE
\item CAP Theorem
\item CLAP system
\end{itemize}
\end{frame}

\begin{frame}
\frametitle{ACID/BASE constraints on Azure}
\begin{itemize}
\item Availability : No
\item Internet and Electricity neither
\item Consistency : ?
\end{itemize}
\end{frame}

\begin{frame}
\frametitle{Consistency on Azure : replicas and consistency}
\begin{itemize}
\item Block commits
\item Etag
\item very few overwrites
\end{itemize}
\end{frame}

\begin{frame}
\frametitle{Consistency on Azure : transactionality}
\begin{itemize}
\item The SQL trick
\end{itemize}
\end{frame}

\begin{frame}
\frametitle{Azure tradeoffs choices}
\begin{itemize}
\item read/write tradeoff
\item Responsivity/efficiency
\item Synchronous replication/Asynchronous replication ?
\end{itemize}
\end{frame}

\begin{frame}
\frametitle{BlobStorage or TableStorage ?}
\begin{itemize}
\item Partition Key and locality
\item size of objects
\item efficiency of Blobstorage for objects larger than 4Kbytes
\item Price
\end{itemize}
\end{frame}

\begin{frame}
\frametitle{Abstractions performances}
\begin{itemize}
\item BlobStorage Read : 13MBytes/sec/worker. Aggregated bounded to 393 MBytes/sec
\item BlobStorage Write : 5MBytes/sec/worker. Aggregated bounded to 124 MBytes/sec
\item Direct worker communication : 55MBytes/sec on very large messages
\item VM : from 1 to 8 cores
\item Queues throughput : 500 messages/sec
\end{itemize}
\end{frame}


\begin{frame}
\frametitle{Redesign of the algorithm}
\begin{itemize}
\item SOA
\item No Master
\item All workers can be switched
\item No worker/data per affinity but ...
\item Scaling up as an initiative in PaaS
\item No exact workers count
\item Pinging queues as a tradeoff between cost and simplicity
\item Queue Messages count
\item idempotent jobs
\end{itemize}
\end{frame}

\begin{frame}
\frametitle{Redesign of the algorithm (2)}
\begin{itemize}
\item Idempotent Counters
\item Workers communication ?
\end{itemize}
\end{frame}

\begin{frame}
\frametitle{Proposed Algorithm}
\begin{itemize}
\item Scheme
\end{itemize}
\end{frame}

\begin{frame}
\frametitle{Difficulty to anticipate speedUp}
\begin{itemize}
\item Serialization/Deserialization. DataContract ? BinaryFormat ? GoogleProtoBuf ?
\item variability of the bandwidth
\item latency due to pinging queues
\item a worker can fail, a VM can be killed
\item Storage unavailability
\item Stragglers issues ?
\end{itemize}
\end{frame}

\begin{frame}
\frametitle{Is time to load data important ?}
\begin{align*}
T_{P}^{Load} & = \frac{nd*SoD}{P * \max(Bandwidth^{read}, \frac{MaxReadThroughput}{P})}\\
\end{align*}
\begin{align*}
T_{P}^{Load} << T_{P}^{Comp}
\end{align*}
Implicit supposition : loading is run once.
\begin{align*}
\frac{SoD}{3IK*T^{flop} * \max(Bandwidth^{read}, \frac{MaxReadThroughput}{P})} << 1
\end{align*}

\end{frame}


\begin{frame}
\frametitle{Communication modelisation}
\begin{align*}
T_{P}^{comm, per iteration} & = ~Time ~to ~read ~1 ~blob ~"simultaneously" ~by ~P ~workers\\
& + ~Time ~to ~write ~P ~blobs ~"simultaneously"\\
& + ~Time ~to ~read ~P ~blobs ~by ~1 ~worker\\
& + ~Time ~to ~merge ~the ~different ~centroids ~version\\
& + ~Time ~to ~write ~1 ~blob ~per ~1 ~worker\\
\end{align*}
\end{frame}

\begin{frame}
\frametitle{Communication modelisation(2)}
\begin{align*}
T_{P}^{comm, per iteration} & = \frac{Kd *SoD}{Min(Bandwidth^{read}, \frac{BlobReadThroughput}{P} )} \\
& + \frac{Kd * SoD}{Min(Bandwidth^{write}, \frac{MaxWriteThroughput}{P})}\\
& + \frac{PKd* SoD}{Bandwidth^{read}} \\
& + (P-1)Kd * 5floatOperations * T^{flop}\\
& + \frac{Kd * SoD}{Bandwidth^{write}}\\
\end{align*}
\end{frame}

\begin{frame}
\frametitle{Communication modelisation(3)}
\begin{align*}
T_{P}^{comm} & \simeq \frac{IPKd*SoD}{Bandwidth^{read}} \\
\end{align*}
\end{frame}

\begin{frame}
\frametitle{SpeedUp}
\begin{align*}
SpeedUp & = \frac{T_{1}^{comp}}{T_{P}^{Comp} + T_{P}{comm}}\\
& \simeq \frac{3IKdnT^{flop}}{\frac{3IKdnT^{flop}}{P} +\frac{IPKd*SoD}{Bandwidth^{read}} } \\
& \simeq \frac{3nT^{flop}}{\frac{3nT^{flop}}{P} + \frac{P*SoD}{Bandwidth^{read}}}
\end{align*}
\end{frame}

\begin{frame}
\frametitle{Optimal Number of workers}
\begin{align*}
P^{*}  = \sqrt{\frac{3nT^{flop}Bandwidth^{read}}{SoD}}
\end{align*}
\end{frame}

\begin{frame}
\frametitle{Numerical Example}
\begin{itemize}
\item n = 432 000
\item K = 657
\item d = 168
\item I = 100
\item PMax = 64
\item $Bandwidth^{read} = 20 Mbytes/sec  (resp. 5Mbytes/sec )$
\end{itemize}
\end{frame}

\begin{frame}
\frametitle{Results}
\begin{align*}
Sequential ~KMeans ~time = 14300 seconds
\end{align*}
\begin{align*}
Optimal ~number ~of ~workers = 57 (resp 28)
\end{align*}
\begin{align*}
SpeedUp = 28,5 (resp 14)
\end{align*}
\end{frame}

\begin{frame}
\frametitle{Improvements : more reducers ?}
scheme
\end{frame}

\begin{frame}
\frametitle{Direct worker communication ?}
\end{frame}

\end{document} 