\begin{thebibliography}{100} \bibitem{AmazonBusiness} Amazon cloud to break the 1 billion dollars barrier? \newblock \url{http://www.crn.com/news/cloud/231002515/amazon-cloud-to-break-the-1-billion-barrier.htm} read the 29/03/2012. \bibitem{AppDomainTrick} Appdomain trick. \newblock \url{http://code.google.com/p/lokad-cloud/wiki/ExceptionHandling} read the 26/04/2012. \bibitem{Azure_Pricing} Azure pricing. \newblock \url{http://www.microsoft.com/windowsazure/pricing/}. \bibitem{AzurePricingCalculator} Azure pricing calculator. \newblock \url{http://www.windowsazure.com/en-us/pricing/calculator/advanced/} read the 25/04/2012. \bibitem{AzurePricingDetails} Azure pricing details. \newblock \url{http://www.windowsazure.com/en-us/pricing/details/} read the 25/04/2012. \bibitem{AzureScope} {Azure Scope}. \newblock \url{http://azurescope.cloudapp.net/}. \bibitem{AzureStorageResources} Azure storage resources. \newblock \url{http://blogs.msdn.com/b/windowsazurestorage/archive/2010/03/28/windows-azure-storage-resources.aspx}. \bibitem{CloudSurvey} Cloud survey. \newblock \url{http://assets1.csc.com/newsroom/downloads/CSC_Cloud_Usage_Index_Report.pdf} read the 28/03/2012. \bibitem{NoSQL} http://nosql-database.org/. \newblock \url{http://nosql-database.org/ censored NoSQL databases}. \bibitem{Spolsky_LeakyAbstractions} Leaky abstractions. \newblock \url{http://www.joelonsoftware.com/articles/LeakyAbstractions.html}. \bibitem{LokadCloud} Lokad-cloud. \newblock \url{http://code.google.com/p/lokad-cloud/}. \bibitem{LokadCQRS} Lokad-cqrs. \newblock \url{http://lokad.github.com/lokad-cqrs/}. \bibitem{CloudInvestment} Microsoft cloud investment. \newblock \url{http://www.bloomberg.com/news/2011-04-06/microsoft-s-courtois-says-to-spend-90-of-r-d-on-cloud-strategy.html} read the 28/03/2012. \bibitem{MpiEc2} {MPI} cluster on {EC2}. \newblock \url{http://datawrangling.s3.amazonaws.com/elasticwulf_pycon_talk.pdf} read the 03/05/2012. \bibitem{NoSQLmeansNoAcid} Problems with {ACID} and how to fix them. \newblock \url{http://dbmsmusings.blogspot.com/2010/08/problems-with-acid-and-how-to-fix-them.html} read the 28/03/2012. \bibitem{AsyncAPI} Should {I} expose synchronous wrappers for asynchronous methods? \newblock \url{http://blogs.msdn.com/b/pfxteam/archive/2012/04/13/10293638.aspx} read the 26/04/2012. \bibitem{WebSiteMonitoring} Websitemonitoring. \newblock \url{http://www.website-monitoring.com/}. \bibitem{DatacentersDesign} {\em VL2: A Scalable and Flexible Data Center Network}, 2011. \bibitem{CLAP} D.~Abadi. \newblock Problems with {CAP}, and {Yahoo's} little known {NoSQL} system. \newblock \url{http://dbmsmusings.blogspot.com/2010/04/problems-with-cap-and-yahoos-little.html} read the 28/03/2012. \bibitem{HPS_1} G.~A Abandah and E.~S. Davidson. \newblock Modeling the communication and computation performance of the {IBM SP2}. \newblock {\em Proceedings of the 10th International Parallel Processing Symposium}, (April), 1996. \bibitem{ABR} C.~Abraham, P.~A. Cornillon, E.~Matzner-Løber, and N.~Molinari. \newblock Unsupervised curve clustering using {B-Splines}. \newblock {\em Scandinavian Journal of Statistics}, 30:581--595, 2003. \bibitem{ParallelMiningAssociationRules} R.~Agrawal and J.~C. Shafer. \newblock Parallel mining of association rules: Design, implementation, and experience. \newblock {\em IEEE Trans. Knowledge and Data Eng., 8(6):962-969}, 1996. \bibitem{LooseSynchronization} J.~Albrecht, C.~Tuttle, A.~C. Snoeren, and A.~Vahdat. \newblock Loose synchronization for large-scale networked systems. \newblock In {\em Proceedings of the annual conference on USENIX '06 Annual Technical Conference}, ATEC '06, pages 28--28, Berkeley, CA, USA, 2006. USENIX Association. \bibitem{BOINC} D.~P. Anderson. \newblock Boinc: A system for public-resource computing and storage. \newblock In {\em 5th IEEE/ACM International Workshop on Grid Computing}, pages 4--10, 2004. \bibitem{AboveTheCloud} M.~Armbrust, A.~Fox, R.~Griffith, A.~D. Joseph, R.~Katz, A.~Konwinski, G.~Lee, D.~Patterson, A.~Rabkin, I.~Stoica, and M.~Zaharia. \newblock Above the clouds: A berkeley view of cloud computing. \newblock Technical Report UCB/EECS-2009-28, EECS Department, University of California, Berkeley, Feb 2009. \bibitem{KMeansSlow} D.~Arthur and S.~Vassilvitskii. \newblock How slow is the k-means method? \newblock {\em Construction}, 2006. \bibitem{AsynchronousHeterogeneousClusters} J.~Bahi, S.~Contassot-Vivier, and R.~Couturier. \newblock Evaluation of the asynchronous iterative algorithms in the context of distant heterogeneous clusters. \newblock {\em Parallel Computing}, 31(5):439--461, 2005. \bibitem{BBL} R.~Bekkerman, M.~Bilenko, and J.~Langford. \newblock {\em Scaling up Machine Learning}. \newblock Cambridge University Press, 2012. \bibitem{BEN1} A.~Benveniste, M.~M\'{e}tivier, and P.~Priouret. \newblock {\em Adaptive algorithms and stochastic approximations}. \newblock Springer-Verlag, 1990. \bibitem{BER3} S.~Bermejo and J.~Cabestany. \newblock The effect of finite sample size on on-line $k$-means. \newblock {\em Neurocomputing}, 48:511--539, 2002. \bibitem{BiauKMeans} G{\'e}rard Biau, Luc Devroye, and G{\'a}bor Lugosi. \newblock On the performance of clustering in hilbert spaces. \newblock {\em IEEE Transactions on Information Theory}, 54(2):781--790, 2008. \bibitem{BottouBengioKMeans} L.~Bottou and Y.~Bengio. \newblock Convergence properties of the k-means algorithms. \newblock In {\em Advances in Neural Information Processing Systems 7}, pages 585--592. MIT Press, 1995. \bibitem{BOT5} L.~Bottou and Y.~LeCun. \newblock Large scale online learning. \newblock In {\em Advances in Neural Information Processing Systems 16}. MIT Press, 2004. \bibitem{BOT6} L.~Bottou and Y.~LeCun. \newblock On-line learning for very large datasets. \newblock {\em Applied Stochastic Models in Business and Industry}, 21:137--151, 2005. \bibitem{BRA1} P.~S. Bradley and U.~M. Fayyad. \newblock {\em Refining Initial Points for K-Means Clustering}, volume 727, pages 91--99. \newblock Morgan Kaufmann, San Francisco, CA, 1998. \bibitem{AzureStorage} B.~Calder, J.~Wang, A.~Ogus, N.~Nilakantan, A.~Skjolsvold, S.~McKelvie, Y.~Xu, S.~Srivastav, J.~Wu, H.~Simitci, J.~Haridas, C.~Uddaraju, H.~Khatri, A.~Edwards, V.~Bedekar, S.~Mainali, R.~Abbasi, A.~Agarwal, M.~F. Haq, M.~I. Haq, D.~Bhardwaj, S.~Dayanand, A.~Adusumilli, M.~McNett, S.~Sankaran, K.~Manivannan, and L.~Rigas. \newblock {Windows Azure Storage}: a highly available cloud storage service with strong consistency. \newblock In {\em Proceedings of the Twenty-Third ACM Symposium on Operating Systems Principles}, SOSP '11, pages 143--157, New York, NY, USA, 2011. ACM. \bibitem{grid5000} F.~Cappello, E.~Caron, M.~Dayde, F.~Desprez, Y.~Jegou, P.~Primet, E.~Jeannot, S.~Lanteri, J.~Leduc, N.~Melab, G.~Mornet, R.~Namyst, B.~Quetier, and O.~Richard. \newblock Grid'5000: A large scale and highly reconfigurable grid experimental testbed. \newblock In {\em Proceedings of the 6th IEEE/ACM International Workshop on Grid Computing}, GRID '05, pages 99--106, Washington, DC, USA, 2005. IEEE Computer Society. \bibitem{SCOPE} R.~Chaiken, B.~Jenkins, P.~Larson, B.~Ramsey, D.~Shakib, S.~Weaver, and J.~Zhou. \newblock Scope: easy and efficient parallel processing of massive data sets. \newblock {\em Proc. VLDB Endow.}, 1(2):1265--1276, August 2008. \bibitem{bigtable} F.~Chang, J.~Dean, S.~Ghemawat, W.~C. Hsieh, D.~A. Wallach, M.~Burrows, T.~Chandra, A.~Fikes, and R.~E. Gruber. \newblock Bigtable: A distributed storage system for structured data. \newblock In {\em in proceedings of the 7th conference on usenix symposium on operating systems design and implementation - volume 7}, pages 205--218, 2006. \bibitem{MapReduceForMachineLearning} C.~T. Chu, S.~K. Kim, Y.~A. Lin, Y.~Yu, G.~R. Bradski, A.~Y. Ng, and K.~Olukotun. \newblock {Map-Reduce for Machine Learning on Multicore}. \newblock In Bernhard Sch\"{o}lkopf, John~C. Platt, and Thomas Hoffman, editors, {\em NIPS}, pages 281--288. MIT Press, 2006. \bibitem{freenet} I.~Clarke, O.~Sandberg, B.~Wiley, and T.~W. Hong. \newblock Freenet: A distributed anonymous information storage and retrieval system. \newblock In {\em International workshop on designing privacy enhancing technologies: design issues in anonymity and unobservability}, pages 46--66. Springer-Verlag New York, Inc., 2001. \bibitem{AsynchronismGPU} S.~Contassot-Vivier, T.~Jost, and S.~Vialle. \newblock Impact of asynchronism on {GPU} accelerated parallel iterative computations. \newblock In {\em PARA 2010: State of the Art in Scientific and Parallel Computing}, LNCS. Springer, Heidelberg, 2011. \bibitem{BenchmarkingCloudServices} B.~F. Cooper, A.~Silberstein, E.~Tam, R.~Ramakrishnan, and R.~Sears. \newblock Benchmarking cloud serving systems with {YCSB}. \newblock In {\em Proceedings of the 1st ACM symposium on Cloud computing}, SoCC '10, pages 143--154, New York, NY, USA, 2010. ACM. \bibitem{CloudDP} J.~Dai and B.~Huang. \newblock {\em Design Patterns for Cloud Services}, volume~74, pages 31--56. \newblock Springer Berlin Heidelberg, 2011. \bibitem{DEB} C.~de~Boor. \newblock On calculating with b-splines. \newblock {\em Journal of Approximation Theory}, 6:50--62, 1972. \bibitem{DEB1} C.~de~Boor. \newblock {\em A practical guide to splines}. \newblock Springer-Verlag, 1978. \bibitem{MapReduce2004} J.~Dean and S.~Ghemawat. \newblock Mapreduce: simplified data processing on large clusters. \newblock In {\em OSDI'04: Proceedings of the 6th conference on Symposium on Opearting Systems Design \& Implementation}, pages 10--10, Berkeley, CA, USA, 2004. USENIX Association. \bibitem{MapReduce2008} J.~Dean and S.~Ghemawat. \newblock Mapreduce: simplified data processing on large clusters. \newblock In {\em Communications of the ACM, vol. 51, no. 1}, pages 107--113, 2008. \bibitem{DekelShamir} O.~Dekel, R.~Gilad-Bachrach, O~Shamir, and L.~Xiao. \newblock Optimal distributed online prediction using mini-batches. \newblock {\em Journal of Machine Learning Research}, 13:165--202, March 2012. \bibitem{Delalleau_Bengio_2007} O.~Delalleau and Y.~Bengio. \newblock Parallel stochastic gradient descent. \newblock {\em Proceedings of SPIE}, 6711:67110F--67110F--14, 2007. \bibitem{DhillonModha2000} I.~S. Dhillon and D.~S. Modha. \newblock A data-clustering algorithm on distributed memory multiprocessors. \newblock In {\em Revised Papers from Large-Scale Parallel Data Mining, Workshop on Large-Scale Parallel KDD Systems, SIGKDD}, pages 245--260, London, UK, 2000. Springer-Verlag. \bibitem{Drepper} U.~Drepper. \newblock What every programmer should know about memory, 2007. \bibitem{BatchVsOnline2} J.C. Fort, M.~Cottrell, and P.~Letremy. \newblock Stochastic on-line algorithm versus batch algorithm for quantization and self organizing maps. \newblock {\em Neural Networks for Signal Processing XI Proceedings of the 2001 IEEE Signal Processing Society Workshop IEEE Piscataway NJ USA}, 00(C):43--52, 2001. \bibitem{FRE1} A.~Freeman. \newblock {\em Pro .NET 4 Parallel Programming in C\#}. \newblock Apress, 2010. \bibitem{GER1} A.~Gersho and R.~M. Gray. \newblock {\em Vector quantization and signal compression}. \newblock Kluwer, 1992. \bibitem{GoogleFileSystem} S.~Ghemawat, H.~Gobioff, and S.~Leung. \newblock The google file system. \newblock {\em SIGOPS Oper. Syst. Rev.}, 37(5):29--43, October 2003. \bibitem{CAP_Theorem} S.~Gilbert and N.~Lynch. \newblock Brewer's conjecture and the feasibility of consistent, available, partition-tolerant web services. \newblock {\em SIGACT News}, 33(2):51--59, 2002. \bibitem{Goodman89efficientsynchronization} James~R. Goodman, Mary~K. Vernon, and Philip~J. Woest. \newblock Efficient synchronization primitives for large-scale cache-coherent multiprocessors. \newblock {\em SIGARCH Comput. Archit. News}, 17(2):64--75, April 1989. \bibitem{CloudCost} A.~Greenberg, J.~Hamilton, D.~A. Maltz, and P.~Patel. \newblock The cost of a cloud: research problems in data center networks. \newblock {\em SIGCOMM Comput. Commun. Rev.}, 39(1):68--73, December 2008. \bibitem{ShardingCounters} J.~Gregorio. \newblock Sharding counters. \newblock \url{https://developers.google.com/appengine/articles/sharding_counters} read the 17/05/2012. \bibitem{GRE} W.~H. Greub. \newblock {\em Linear algebra}. \newblock Springer-Verlag, 4th edition, 1975. \bibitem{MPI_performance_measurements} W.~Gropp and E.~Lusk. \newblock Reproducible measurements of {MPI} performance characteristics. \newblock In {\em Proceedings of the 6th European PVM/MPI Users' Group Meeting on Recent Advances in Parallel Virtual Machine and Message Passing Interface}, pages 11--18, London, UK, UK, 1999. Springer-Verlag. \bibitem{DataEffectiveness} A.~Halevy, P.~Norvig, and F.~Pereira. \newblock The unreasonable effectiveness of data. \newblock {\em IEEE Intelligent Systems}, 24:8--12, 2009. \bibitem{Helland} P.~Helland. \newblock Life beyond distributed transactions: an apostate's opinion. \newblock In {\em {CIDR} 2007, Third Biennial Conference on Innovative Data Systems Research, Asilomar, {CA, USA}, January 7-10, 2007, Online Proceedings}, pages 132--141. \bibitem{Hennig99} C.~Hennig. \newblock Models and methods for clusterwise linear regression. \newblock In {\em Proceedings in Computational Statistics}, pages 3--0. Springer, 1999. \bibitem{Herlihy95scalableconcurrent} M.~Herlihy, B.~H. Lim, and N.~Shavit. \newblock Scalable concurrent counting. \newblock {\em ACM Transactions on Computer Systems}, 13:343--364, 1995. \bibitem{AMP} M.~Herlihy and N.~Shavit. \newblock {\em The Art of Multiprocessor Programming}. \newblock Morgan Kaufmann Publishers Inc., San Francisco, CA, USA, 2008. \bibitem{FourthParadigm} T.~Hey, S.~Tansley, and K.~Tolle. \newblock {\em The Fourth Paradigm: Data-Intensive Scientific Discovery}. \newblock Microsoft Research, Redmond, Washington, 2009. \bibitem{Early_Observations_On_Azure} Z.~Hill, J.~Li, M.~Mao, A.~Ruiz-Alvarez, and M.~Humphrey. \newblock Early observations on the performance of {Windows Azure}. \newblock {\em Sci. Program.}, 19(2-3):121--132, April 2011. \bibitem{MPI_performance} T.~Hoefler, W.~Gropp, R.~Thakur, and J.~L. Träff. \newblock Toward performance models of {MPI} implementations for understanding application scaling issues. \newblock In {\em Proceedings of the 17th European {MPI} users' group meeting conference on Recent advances in the message passing interface}, EuroMPI'10, pages 21--30, Berlin, Heidelberg, 2010. Springer-Verlag. \bibitem{AsynchronousMultiThreads} B.~Hong and Z.~He. \newblock An asynchronous multithreaded algorithm for the maximum network flow problem with nonblocking global relabeling heuristic. \newblock {\em IEEE Transactions on Parallel and Distributed Systems}, 22:1025--1033, 2011. \bibitem{CloudAutomation} M.~Isard. \newblock Autopilot: automatic data center management. \newblock {\em SIGOPS Oper. Syst. Rev.}, 41(2):60--67, April 2007. \bibitem{Dryad2007} M.~Isard, M.~Budiu, Y.~Yu, A.~Birrell, and D.~Fetterly. \newblock Dryad: distributed data-parallel programs from sequential building blocks. \newblock In {\em EuroSys '07: Proceedings of the 2nd ACM SIGOPS/EuroSys European Conference on Computer Systems 2007}, pages 59--72, New York, NY, USA, 2007. ACM. \bibitem{DataClusteringReview} A.~K. Jain, M.~N. Murty, and P.~J. Flynn. \newblock Data clustering: A review. \newblock In {\em ACM computing surveys, Vol.31, no.3, September}, 1999. \bibitem{KMeansJoshi} M.~N. Joshi. \newblock Parallel {K-means} algorithm on {Distributed Memory Multiprocessors}. \newblock Technical report, Computer Science Department University of {Minnesota}, Twin Cities, 2003. \bibitem{KOH1} T.~Kohonen. \newblock Analysis of a simple self-organizing process. \newblock {\em Biological Cybernetics}, 44:135--140, 1982. \bibitem{KUS1} H.~J. Kushner and D.~S. Clark. \newblock {\em Stochastic approximation for constrained and unconstrained systems}. \newblock Springer-Verlag, 1978. \bibitem{folding@home} S.~M. Larson, C.~D. Snow, M.~Shirts, and V.~S. Pande. \newblock Folding@home and genome@home: Using distributed computing to tackle previously intractable problems in computational biology. \newblock {\em Security}, 2009. \bibitem{lenk2009witcaamotcl} A.~Lenk, M.~Klems, J.~Nimis, S.~Tai, and T.~Sandholm. \newblock What's inside the cloud? an architectural map of the cloud landscape. \newblock In {\em ICSE Workshop on Software Engineering Challenges of Cloud Computing, 2009. CLOUD 09.} IEEE Press, Mai 2009. \bibitem{StragglerIssuesInMapReduce} J.~Lin. \newblock {The Curse of Zipf and Limits to Parallelization: A Look at the Stragglers Problem in MapReduce}. \newblock In {\em LSDS-IR workshop}, 2009. \bibitem{MapReduceForTextMining} J.~Lin and C.~Dyer. \newblock {\em Data-intensive text processing with MapReduce}. \newblock Morgan \& Claypool Publishers, 2010. \bibitem{CloudMapReduce2010} H.~Liu and D.~Orban. \newblock Cloud mapreduce: a mapreduce implementation on top of a cloud operating system. \newblock Technical report, Accenture Technology Labs, 2009. \newblock \url{http://code.google.com/p/cloudmapreduce/}. \bibitem{Lloyd} S.~Lloyd. \newblock Least squares quantization in {PCM}. \newblock {\em IEEE Transactions on Information Theory}, 28:129--137, 1982. \bibitem{LOU1} G.~Louppe and P.~Geurts. \newblock A zealous parallel gradient descent algorithm. \newblock In {\em NIPS 2010 Workshop on Learning on Cores, Clusters and Clouds}, 2010. \bibitem{Graphlab2010} Y.~Low, J.~Gonzalez, A.~Kyrola, D.~Bickson, C.~Guestrin, and J.~M. Hellerstein. \newblock Graphlab: A new parallel framework for machine learning. \newblock In {\em Conference on Uncertainty in Artificial Intelligence (UAI)}, Catalina Island, California, July 2010. \bibitem{MacQueen} J.~B. MacQueen. \newblock Some methods of classification and analysis of multivariate observations. \newblock In {\em Proceedings of the Fifth Berkeley Symposium on Mathematical Statistics and Probability}, 1967. \bibitem{KMeansIsNPhard} M.~Mahajan, P.~Nimbhorkar, and K.~Varadarajan. \newblock The planar k-means problem is np-hard. \newblock In {\em Proceedings of the 3rd International Workshop on Algorithms and Computation}, WALCOM '09, pages 274--285, Berlin, Heidelberg, 2009. Springer-Verlag. \bibitem{CloudMarinescu} D.~C. Marinescu. \newblock {\em Cloud Computing: Theory and Practice}. \newblock 2012. \bibitem{MIL1} G.~W. Milligan and P.~D. Isaac. \newblock The validation of four ultrametric clustering algorithms. \newblock {\em Pattern Recognition}, 12:41--50, 1980. \bibitem{MIR1} B.~Mirkin. \newblock {\em Clustering for data mining: a data recovery approach}. \newblock Chapman \& Hall/CRC, 2005. \bibitem{Nesterov} Y.~Nesterov. \newblock {\em Introductory lectures on convex optimization: A basic course}, volume~87. \newblock Kluwer Academic Publishers, 2004. \bibitem{TeraSortWinnerYahoo} O.~O~Malley. \newblock Terabyte sort on apache hadoop. \newblock \url{http://www.hpl.hp.com/hosted/sortbenchmark/YahooHadoop.pdf}. \bibitem{piglatin} C.~Olston, B.~Reed, U.~Srivastava, R.~Kumar, and A.~Tomkins. \newblock Pig latin: a not-so-foreign language for data processing. \newblock In {\em Proceedings of the 2008 ACM SIGMOD international conference on Management of data}, SIGMOD '08, pages 1099--1110, New York, NY, USA, 2008. ACM. \bibitem{PAG1} G.~Pag\`es. \newblock A space vector quantization for numerical integration. \newblock {\em Journal of Applied and Computational Mathematics}, 89:1--38, 1997. \bibitem{PatraDALVQ} B.~{Patra}. \newblock Convergence of distributed asynchronous learning vector quantization algorithms. \newblock {\em Journal of Machine Learning Research}, 12:3431--3466, 2010. \bibitem{PET1} A.~D. Peterson, A.~P. Ghosh, and R.~Maitra. \newblock A systematic evaluation of different methods for initializing the $k$-means clustering algorithm. \newblock Technical report, 2010. \bibitem{sawzall} R.~Pike, S.~Dorward, R.~Griesemer, and S.~Quinlan. \newblock Interpreting the data: Parallel analysis with sawzall. \newblock {\em Sci. Program.}, 13(4):277--298, October 2005. \bibitem{Pollard} D.~Pollard. \newblock Strong consistency of k-means clustering. \newblock {\em The Annals of Statistics}, January 1981. \bibitem{BASE} D.~Pritchett. \newblock Base: An acid alternative. \newblock {\em Queue}, 6(3):48--55, May 2008. \bibitem{TwoPhaseCommitProtocol} Y.~Raz. \newblock The dynamic two phase commitment (d2pc) protocol. \newblock In {\em Database Theory - ICDT '95, Lecture Notes in Computer Science, Volume 893 Springer, ISBN 978-3-540-58907-5}, pages 162--176, 1995. \bibitem{ROS1} F.~Rossi, B.~Conan-Guez, and A.~El~Golli. \newblock Clustering functional data with the {SOM} algorithm. \newblock In {\em Proceedings of ESANN 2004}, pages 305--312, Bruges, Belgium, April 2004. \newblock \url{http://apiacoa.org/publications/2004/som-esann04.pdf}. \bibitem{ScalableParallelClassifier} J.~C. Shafer, R.~Agrawal, and M.~Mehta. \newblock A scalable parallel classifier for data mining. \newblock {\em Proc. 22nd International Conference on VLDB, Mumbai, India}, 1996. \bibitem{MPI1996} M.~Snir, S.~Otto, S.~Huss-Lederman, W.~David, and J.~Dongarra. \newblock {\em {MPI}: The Complete Reference}. \newblock MIT Press, Boston, 1996. \bibitem{PVM} V.~S. Sunderam. \newblock {PVM}: A framework for parallel distributed computing. \newblock {\em Concurrency: Practice and Experience}, 2:315--339, 1990. \bibitem{Condor} D.~Thain, T.~Tannenbaum, and M.~Livny. \newblock Distributed computing in practice: The condor experience. \newblock {\em Concurrency and Computation: Practice and Experience}, 17:2--4, 2005. \bibitem{TSI1} J.~Tsitsiklis, D.~Bertsekas, and M.~Athans. \newblock Distributed asynchronous deterministic and stochastic gradient optimization algorithms. \newblock {\em IEEE Transactions on Automatic Control}, 31:803--812, 1986. \bibitem{BridgingModel} L.~G. Valiant. \newblock A bridging model for parallel computation. \newblock {\em Communications of the ACM}, 33(8):103--111, 1990. \bibitem{Eventually_Consistent} W.~Vogels. \newblock Eventually consistent. \newblock {\em Queue}, 6(6):14--19, October 2008. \bibitem{CloudConsistency} H.~Wada, A.~Fekete, L.~Zhao, K.~Lee, and A.~Liu. \newblock Data consistency properties and the trade-offs in commercial cloud storages : the consumers'perspective. \newblock {\em Reading}, pages 134--143, 2011. \bibitem{BenchmarkingAmazonEC2} E.~Walker. \newblock {Benchmarking Amazon EC2 for high-performance scientific computing}. \newblock {\em LOGIN}, 33(5):18--23, October 2008. \bibitem{The_Impact_Of_Virtualization} Guohui Wang and T.~S.~Eugene Ng. \newblock The impact of virtualization on network performance of amazon ec2 data center. \newblock In {\em Proceedings of the 29th conference on Information communications}, INFOCOM'10, pages 1163--1171, Piscataway, NJ, USA, 2010. IEEE Press. \bibitem{Hadoop} Tom White. \newblock {\em Hadoop: The Definitive Guide}. \newblock O'Reilly Media, Inc., 1st edition, 2009. \bibitem{MathematicsNature} E.~Wigner. \newblock The unreasonable effectiveness of mathematics in the natural sciences. \newblock In {\em Communications in Pure and Applied Mathematics vol. 13, No. I February}, 1960. \bibitem{BatchVsOnline} D.Randall Wilson and Tony~R. Martinez. \newblock The general inefficiency of batch training for gradient descent learning. \newblock {\em Neural Networks}, 16(10):1429 -- 1451, 2003. \bibitem{HotSpot} P.C. Yew, N.F. Tzeng, and D.H. Lawrie. \newblock Distributing hot-spot addressing in large-scale multiprocessors. \newblock {\em IEEE Transactions on Computers}, pages 388--395, April 1987. \bibitem{DryadLINQ} Y.~Yu, M.~Isard, D.~Fetterly, M.~Budiu, U.~Erlingsson, P.~Kumar, and G.~J. Currey. \newblock Dryadlinq: a system for general-purpose distributed data-parallel computing using a high-level language. \newblock In {\em Proceedings of the 8th USENIX conference on Operating systems design and implementation}, OSDI'08, pages 1--14, Berkeley, CA, USA, 2008. USENIX Association. \bibitem{HPS_2} X.~Zhiwei and K.~Hwang. \newblock Modeling communication overhead: {MPI} and {MPL} performance on the {IBM SP2}. \newblock {\em Parallel Distributed Technology Systems Applications IEEE}, 4(1):9--23, 1996. \bibitem{SlowLearners} M.~Zinkevich, A.~Smola, and J.~Langford. \newblock Slow learners are fast. \newblock In {\em Advances in Neural Information Processing Systems 22}, pages 2331--2339, 2009. \bibitem{LAN2} M.~Zinkevich, M.~Weimer, A.~Smola, and L.~Li. \newblock Parallelized stochastic gradient descent. \newblock In {\em Advances in Neural Information Processing Systems 23}, 2010. \end{thebibliography}