\documentclass{esannV2}
%\usepackage[dvips]{graphicx}
\usepackage{graphicx} %WARNING: We used graphicx instead of [dvips]{graphix}
\usepackage[latin1]{inputenc}
\usepackage{amssymb,amsmath,array}
\usepackage{float}
\usepackage{amsfonts}
\usepackage{theorem}
\usepackage{amscd}
\usepackage{latexsym}
\usepackage{dsfont}
\usepackage[english]{babel}

\voffset 0 cm \hoffset 0 cm \addtolength{\textwidth}{0cm}
\addtolength{\textheight}{0cm}\addtolength{\leftmargin}{0cm}

%TODO: check that the final size of the top and
%bottom margins is 5.2 cm and of the left and right margins is 4.4 cm.

\DeclareMathOperator*{\argmin}{argmin}
\DeclareMathOperator*{\argminloc}{argminloc}
\DeclareMathOperator*{\diam}{diam}
\DeclareMathOperator*{\dist}{dist}
\DeclareMathOperator*{\supp}{supp}
\newcommand{\Ind}{\mathds{1}}
\newcommand{\Hrond}{\mathcal{H}}
\newcommand{\FrondT}{\mathcal{F}_T}
\newcommand{\zbf}{{\bf z}}
\newcommand{\Hrondi}{{\Hrond_i}}
\newcommand {\real}{\mathbb{R}}
\newcommand{\pinteger}{\mathbb{N}}
\newcommand{\esp}{\mathbb{E}}
\newcommand{\prob}{\mathbb{P}}
\newcommand{\var}{\mathbb{V}}
\newcommand{\vertbar}{\left\vert\vphantom{\frac{1}{1}}\right.}
\newcommand{\proofend}{\begin{flushright} $\square$ \end{flushright}}
\newcommand{\wch}{\check{w}}
\newcommand{\Gcal}{\mathcal{G}}
\newcommand{\Dcal}{\mathcal{D}}
\newcommand{\rdkap}{(\real^d)^\kappa}

\begin{document}
\title{A Discussion on Parallelization Schemes for Vector Quantization Algorithms}

\author{Matthieu Durut$^{1,3}$, Benoit Patra$^{2,3}$, Fabrice Rossi$^{4}$
%
%
\vspace{.3cm}\\
%
%
1- Telecom ParisTech - INFRES, 46 rue Barrault - Paris - France
%
%
\vspace{.1cm}\\
2- Université Pierre et Marie Curie - LSTA, 4 place Jussieu - Paris - France
%
%
\vspace{.1cm}\\
3- Lokad , 10 rue Philippe de Champaigne - Paris - France
%
%
\vspace{.1cm}\\
4- Université Paris I Panthéon-Sorbonne - SAMM, 90 rue de Tolbiac - Paris - France
}
%
%

\maketitle

\begin{abstract}
This paper studies parallelization schemes of the Vector Quantization algorithm in order to obtain time speed-ups using distributed resources. We show that the most intuitive parallelization scheme does not lead to better performance than the sequential algorithm. Another distributed scheme is therefore introduced which recovers the expected speed-ups. Then, it is improved to fit implementation on distributed architectures where communications are slow and inter-machines synchronization too costly. The schemes are tested with simulated distributed architectures and, for the last one, with Microsoft Windows Azure platform obtaining speed-ups up to $32$ VMs.
\end{abstract}

\section{Introduction}
Motivated by the problem of executing clustering algorithms on very large datasets, we investigate parallelization schemes of the Vector Quantization (VQ) method (also called online $k$-means). This procedure is known for its good statistical properties but, as any on-line procedure, it is inherently slow. In this paper, we assume to have access to a satisfactory sequential implementation of the VQ algorithm. Then, our goal is to distribute efficiently this algorithm in order to bring speed-ups, i.e., gain of time execution brought by more computing resources compared to a sequential execution. Such theoretical parallel algorithms are developed in \cite{PAT1}. However, they may cover many different practical implementations. Practical considerations lead us to the present analysis.\\

The VQ technique computes a summary of a dataset $\{\zbf_{t}\}_{t=1}^n$ of $d$ dimensional samples using the following iterations on a vector $w=(w_1, \ldots,w_\kappa) \in \rdkap$, whose components are called the prototypes,
\begin{equation}\label{eq:finiteSample}
w(t+1)= w(t) - \varepsilon_{t+1}H\left(\zbf_{\{t+1 \text{ mod } n\}},w(t) \right), \quad t \geq 0.
\end{equation}
In the equation above, $w(0) \in \left(\real^d\right)^\kappa$, the $\varepsilon_t $ are positive reals called \emph{steps} and $H(\zbf,w)$ is defined by
\begin{equation}\label{def:defH}
H(\zbf,w) = \left( \left(w_\ell - \zbf\right) \mathds{1}_{\left\{l= \argmin_{i = 1, \ldots, \kappa} \left\|\zbf  - w_i\right\|^2\right\}}\right)_{1 \leq \ell \leq \kappa}, \quad \mbox{$\zbf \in \real^d$ and $w \in \rdkap$.}
\end{equation}
The mod operator stands for the remainder of an integer division operation. For a discussion on ties that may appear in equation (\ref{def:defH}) and for a satisfactory almost sure convergence theorem of the VQ procedure, the reader is referred to \cite{PAG1}. Remark that the term $H(\zbf,w)$ is accessible with a simple distance computation and can also be thought as a sample of the gradient of the clustering criterion, namely the distortion (see for instance \cite{PAG1}). Therefore, the VQ technique also belongs to the class of stochastic gradient descent algorithms.\\

This paper follows the VQ ideas presented in \cite{PAT1}. We assume having access to $M$ computing entities, each of them executing concurrent VQ procedures. These executions are performed on a dataset, split among the local memory of the computing instances, and represented by the sequences $\{\zbf_t^i\}_{t=1}^n$, $i\in\{1,\ldots,M\}$. The prototype iterations computed by the multiple VQ techniques are denoted by $\{w^{i}(t)\}_{t=0}^\infty$ and called \emph{versions}. We use the following normalized criterion to measure the speed-up ability of our investigated schemes.
\begin{equation}\label{eq:empiricalDisto}
C_{n,M}(w) = \frac{1}{nM} \sum_{i=1}^M \sum_{t=1}^n{\min_{\ell = 1, \ldots, \kappa} \left\|\zbf^i_t  - w_\ell\right\|^2}, \quad \mbox{$w \in \rdkap$}.
\end{equation}

Algorithms are tested using simulated distributed architecture and synthetic vectorial data\footnote{Source code is available at the address \texttt{http://code.google.com/p/clouddalvq/}} in this paper. In addition, it is organized as follows. First, Section \ref{sec:firstScheme} provides empirical evidences that the most simple scheme cannot bring speed-ups. Then, some insights to explain the previous non satisfactory situation are provided in Section \ref{sec:secondScheme}. Consequently, we design a new scheme and prove by practice its ability to bring speed-ups. Finally, in Section \ref{sec:delaysScheme}, we present an asynchronous adaptation of this latter scheme which fits better slow communication architectures such as Cloud Computing.

\section{A first distributed scheme}\label{sec:firstScheme}
Our investigation starts with the most intuitive parallelization scheme given by the system of equations (\ref{eq:averaging}). All versions are set equal at time $t=0$,\\ $w^1(0) = \ldots =w^M(0)$. For $i \in \{1, \ldots,M\}$ and $t \geq0$, we have the following iterations:
\begin{equation}\label{eq:averaging}
\begin{cases}
w^i_{temp} = w^i(t) - \varepsilon_{t+1} H\left(\zbf^i_{\{t+1\text{ mod }n\}},w^i(t)\right)& \\
w^i(t+1) = w^i_{temp} & \text{ if }  t \text{ mod } \tau  \neq 0 \text{ or } t=0,\\
    \begin{cases}
        w^{srd} =  \frac{1}{M}\sum_{j=1}^M{w^j_{temp}} & \\
        w^i(t+1) =  w^{srd}&\\
    \end{cases}& \text{ if } t \text{ mod } \tau =0 \text{ and } t \geq \tau.
\end{cases}
\end{equation}

The averaging phase described by the braced inner equations is executed only whenever $\tau$ points have been processed by every concurrent processors. It produces a shared version of the prototypes, namely $w^{srd}$, which is broadcasted to each processing unit. The Figure \ref{fig:parallel10} shows that multiple resources do not bring speed-ups for convergence. Even if more data are processed, no gain in term of wall clock time is provided using this distributed scheme.

\begin{figure}[h]
\begin{center}
\includegraphics[scale = 0.35]{pa10NoTitle}
\end{center}
\caption{Charts of performance curves for iterations (\ref{eq:averaging}) with $\tau=10$ and different number
of computing entities: M = 1, 2, 10.}
\label{fig:parallel10}
\end{figure}


\section{Towards a better scheme}\label{sec:secondScheme}

The investigation of the previous non-satisfactory result starts by rewriting both the sequential and the distributed scheme. On the one hand, the sequential VQ iterations (\ref{eq:finiteSample}) can be rewritten:
\begin{equation}\label{eq:iteration}
w(t+1) = w(t - \tau + 1)- \sum_{t'=t-\tau+1}^{t} \varepsilon_{t'+1}H\left(\zbf_{\{t'+1\text{ mod }n \}},w(t')\right),\quad \mbox{$t \geq \tau$}.
\end{equation}

On the other hand, if $t \text{ mod } \tau =0$ and $t > 0$ then, for all $i\in\{1, \ldots,M\}$, iterations (\ref{eq:averaging}) write
\begin{equation}\label{eq:iteration2}
w^i(t+1) = w^i(t - \tau + 1) - \sum_{t'= t- \tau +1}^{t}\varepsilon_{t'+1}\left(\frac{1}{M}\sum_{j=1}^M H\left(\zbf^j_{\{t'+1\text{ mod }n\}},w^j(t')\right)\right)
\end{equation}
Assuming that $w^j(t')\thickapprox w^i(t')$, for all $(i,j) \in \{1,\ldots,M\}^2$ and $t' \geq 0$, the mean in parenthesis is an estimator of the gradient of the distortion at $w^i(t')$. Consequently, the two algorithms can be thought as stochastic gradient descent procedures with different estimators but driven by the same learning rate which is given by the sequence $\{\varepsilon_{t}\}_{t=1}^{\infty}$.\\

The convergence speed of a non-fixed step gradient descent procedure is essentially driven by the decreasing speed of the sequence of steps (see for instance \cite{BEN1}). The choice of this sequence is subject to an exploration/convergence trade-off. Since the two procedures above share the same learning rate with respect to the iterations $t \geq 0$, they share the same convergence speed with respect to the wall clock time (time measured by an exterior observer). Yet, the distributed scheme of Section \ref{sec:firstScheme} has a much lower learning rate with respect to the number of samples processed, favoring exploration to the detriment of the convergence. The multiple resources therefore lead to better exploration but to similar convergence speed with respect to wall clock time.\\

Remind that we suppose to have a satisfactory VQ implementation. Consequently we seek for a distributed scheme that have the same learning rate evolution in term of processed samples and which convergence speed with respect to iterations is accelerated.\\
Set,
\begin{equation}\label{eq:displacement}
\Delta^j_{t_1\rightarrow t_2} = \sum_{t'=t_1 + 1}^{t_2} \varepsilon_{t'+1}H\left(\zbf^j_{\{t'+1\text{ mod }n \}},w^j(t')\right), \;\; \mbox{ $j \in \{1, \ldots, M\}$ and $t_2 > t_1 \geq 0$}.
\end{equation}

At time $t=0$, $w^1(0) = \ldots =w^M(0)=w^{srd}$. For all $i \in \{1, \ldots,M\}$ and all $t \geq0$, consider the distributed scheme given by
\begin{equation}\label{eq:correctAveraging}
\begin{cases}
w^i_{temp} = w^i(t) - \varepsilon_{t+1} H\left(\zbf^i_{\{t+1\text{ mod }n\}},w^i(t)\right)& \\  %\label{eq:processing}\\
w^i(t+1) = w^i_{temp} & \text{ if }  t \text{ mod } \tau  \neq 0 \text{ or } t=0,\\
    \begin{cases}
        w^{srd} = w^{srd} - \sum_{j=1}^M{\Delta^j_{t- \tau \rightarrow t} } & \\
        w^i(t+1) =  w^{srd} &\\
    \end{cases}& \text{ if } t \text{ mod } \tau =0 \text{ and } t \geq \tau.
\end{cases}
\end{equation}

The results of the simulations are displayed in the charts of Figure \ref{fig:correctedParallel}. The charts show that substantial speed-ups are obtained with distributed resources. The acceleration is greater when the reducing phase (described by the braced inner equations) is frequent. Indeed, if $\tau$ is large then more autonomy has been granted to the concurrent executions, they could be attracted to different regions that would slow down the consensus and the convergence.

\begin{figure}[h]
\begin{center}
\includegraphics[scale = 0.35]{correctedPa10NoTitle}
\end{center}
\caption{Charts of performance curves for iterations (\ref{eq:correctAveraging}) with $\tau=10$ and different number
of computing entities: M = 1, 2, 10.}
\label{fig:correctedParallel}
\end{figure}

\section{A model with stochastic delays}
\label{sec:delaysScheme}

The previous parallelization schemes do not deal with communication costs introduced by update exchanges between machines. In the context of cloud computing, no efficient shared memory is available and these costs introduce delays. The effect of delays for parallel stochastic gradient descent has already been studied (see for instance \cite{LAN1}). However in the previous paper the computing architecture is endowed with an efficient shared memory. Moreover, the unreliability of the cloud computing hardware introduces strong straggler issues and makes the synchronization process inappropriate. In this subsection, we improve the model of iterations (\ref{eq:correctAveraging}) with random communication costs that follows a geometric distribution and we remove the synchronization process of reducing phase, resulting in the more realistic iterations (\ref{eq:delayedAveraging}) below. For each time $t \geq0$, let $\tau^{i}(t)$ be the latest time before t when the unit $i$ finished to send its updates and received the shared version. At time $t=0$ we have $w^1(0) = \ldots =w^M(0)=w^{srd}$, and for all $i \in \{1, \ldots,M\}$ and all $t \geq0$,

\begin{equation}\label{eq:delayedAveraging}
\begin{cases}
w^i_{temp} = w^i(t) - \varepsilon_{t+1} H\left(\zbf^i_{\{t+1\text{ mod }n\}},w^i(t)\right)& \\ %\label{eq:processing}\\
w^i(t+1) = w^{i}_{temp} & \text{ if }  t \neq \tau^{i}(t) \\
w^i(t+1) = w^{srd}(\tau^{i}(t-1)) - \Delta^i_{\tau^{i}(t-1) \rightarrow t}& \text{ if }  t = \tau^{i}(t) \\
w^{srd}(t+1) = w^{srd}(t) - \displaystyle\sum_{j, t = \tau^{j}(t)}{\Delta^j_{\tau^{j}(\tau^{j}(t-1) -1) \rightarrow \tau^{j}(t-1)}}& \\
\end{cases}
\end{equation}

\begin{figure}[h]
\begin{center}
\includegraphics[scale = 0.35]{pa10RandomdelayedNoTitle}
\end{center}
\caption{Charts of performance curves for iterations (\ref{eq:delayedAveraging}) with $\tau=10$ and different number
of computing entities: M = 1, 2, 10.}
\label{fig:delayedParallel}
\end{figure}

There are no more synchronization between processing units: each machine uploads its updates and downloads the shared version as soon as its previous uploads and downloads are completed. A dedicated unit permanently modifies the shared version with the latest updates received from the other machines without any synchronization barrier. The figure \ref{fig:delayedParallel} shows that the introduction of small delays and asynchronism only slightly impacts performances, compared to the scheme given by equations (\ref{eq:correctAveraging}). The figure \ref{fig:cloudParallel} shows the results obtained by our cloud implementation of the iterations (\ref{eq:delayedAveraging}) using 32 real processing units. A future paper will describe more precisely this cloud implementation.

\begin{figure}[h]
\begin{center}
\includegraphics[scale = 0.35]{scalabilityNoTitle}
\end{center}
\caption{Charts of performance curves for iterations (\ref{eq:delayedAveraging}) on our cloud implementation and different number
of computing entities.}
\label{fig:cloudParallel}
\end{figure}

\section{Conclusion}

In this paper we show that the naive parallelization scheme proposed in section \ref{sec:firstScheme} does not provide better performance than the sequential scheme. This surprising result derives from the fact that our first parallel scheme leads to a decrease of the learning rate per data points processed. We therefore propose a new parallelization scheme relying on asynchronous updates of a common "shared version". This latter algorithm is very well suited for parallel computation on slow communication networks such as cloud computing platforms. Our implementation on Azure show significant scale-up, up to 32 machines.

\begin{footnotesize}

\bibliographystyle{unsrt}
\bibliography{dalvq_esann_2012}

\end{footnotesize}

% ****************************************************************************
% END OF BIBLIOGRAPHY AREA
% ****************************************************************************

\end{document} 