Merge pull request #45 from PyDataBlog/experimental

Patch release `0.1.1` offering interface support for MLJModels.

Author: PyDataBlog

Project.toml

@@ -1,19 +1,22 @@
 name = "ParallelKMeans"
 uuid = "42b8e9d4-006b-409a-8472-7f34b3fb58af"
 authors = ["Bernard Brenyah", "Andrey Oskin"]
-version = "0.1.0"
+version = "0.1.1"
 
 [deps]
+Distances = "b4f34e82-e78d-54a5-968a-f98e89d6e8f7"
+MLJModelInterface = "e80e1ace-859a-464e-9ed9-23947d8ae3ea"
 StatsBase = "2913bbd2-ae8a-5f71-8c99-4fb6c76f3a91"
 
 [compat]
 StatsBase = "0.32, 0.33"
-julia = "1.3"
+julia = "1.3, 1.4"
 
 [extras]
+MLJBase = "a7f614a8-145f-11e9-1d2a-a57a1082229d"
 Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c"
-Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40"
 Suppressor = "fd094767-a336-5f1f-9728-57cf17d0bbfb"
+Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40"
 
 [targets]
-test = ["Test", "Random", "Suppressor"]
+test = ["Test", "Random", "Suppressor", "MLJBase"]

README.md

@@ -10,6 +10,7 @@ ________________________________________________________________________________
 _________________________________________________________________________________________________________
 
 ## Table Of Content
+
 1. [Documentation](#Documentation)
 2. [Installation](#Installation)
 3. [Features](#Features)
@@ -18,13 +19,15 @@ ________________________________________________________________________________
 _________________________________________________________________________________________________________
 
 ### Documentation
+
 - Stable Documentation: [![Stable](https://img.shields.io/badge/docs-stable-blue.svg)](https://PyDataBlog.github.io/ParallelKMeans.jl/stable)
 
 - Experimental Documentation: [![Dev](https://img.shields.io/badge/docs-dev-blue.svg)](https://PyDataBlog.github.io/ParallelKMeans.jl/dev)
 
 _________________________________________________________________________________________________________
 
 ### Installation
+
 You can grab the latest stable version of this package by simply running in Julia.
 Don't forget to Julia's package manager with `]`
 
@@ -39,9 +42,11 @@ pkg> dev [email protected]:PyDataBlog/ParallelKMeans.jl.git
 ```
 
 Don't forget to checkout the experimental branch and you are good to go with bleeding edge features and breaks!
+
 ```bash
 git checkout experimental
 ```
+
 _________________________________________________________________________________________________________
 
 ### Features

docs/src/index.md

@@ -5,26 +5,29 @@ Depth = 4
 ```
 
 ## Motivation
+
 It's actually a funny story led to the development of this package.
-What started off as a personal toy project trying to re-construct the K-Means algorithm in native Julia blew up after a heated discussion on the Julia Discourse forum when I asked for Julia optimizaition tips. Long story short, Julia community is an amazing one! Andrey offered his help and together, we decided to push the speed limits of Julia with a parallel implementation of the most famous clustering algorithm. The initial results were mind blowing so we have decided to tidy up the implementation and share with the world as a maintained Julia pacakge. 
+What started off as a personal toy project trying to re-construct the K-Means algorithm in native Julia blew up after a heated discussion on the Julia Discourse forum when I asked for Julia optimizaition tips. Long story short, Julia community is an amazing one! Andrey offered his help and together, we decided to push the speed limits of Julia with a parallel implementation of the most famous clustering algorithm. The initial results were mind blowing so we have decided to tidy up the implementation and share with the world as a maintained Julia pacakge.
 
 Say hello to `ParallelKMeans`!
 
 This package aims to utilize the speed of Julia and parallelization (both CPU & GPU) to offer an extremely fast implementation of the K-Means clustering algorithm and its variations via a friendly interface for practioners.
 
-In short, we hope this package will eventually mature as the "one stop" shop for everything KMeans on both CPUs and GPUs.
+In short, we hope this package will eventually mature as the "one stop" shop for everything K-Means on both CPUs and GPUs.
 
 ## K-Means Algorithm Implementation Notes
+
 Since Julia is a column major language, the input (design matrix) expected by the package in the following format;
 
 - Design matrix X of size n×m, the i-th column of X `(X[:, i])` is a single data point in n-dimensional space.
 - Thus, the rows of the design design matrix represents the feature space with the columns representing all the training examples in this feature space.
 
-One of the pitfalls of K-Means algorithm is that it can fall into a local minima. 
+One of the pitfalls of K-Means algorithm is that it can fall into a local minima.
 This implementation inherits this problem like every implementation does.
 As a result, it is useful in practice to restart it several times to get the correct results.
 
 ## Installation
+
 You can grab the latest stable version of this package from Julia registries by simply running;
 
 *NB:* Don't forget to Julia's package manager with `]`
@@ -40,32 +43,35 @@ dev [email protected]:PyDataBlog/ParallelKMeans.jl.git
 ```
 
 Don't forget to checkout the experimental branch and you are good to go with bleeding edge features and breaks!
+
 ```bash
 git checkout experimental
 ```
 
 ## Features
+
 - Lightening fast implementation of Kmeans clustering algorithm even on a single thread in native Julia.
-- Support for multi-theading implementation of Kmeans clustering algorithm.
+- Support for multi-theading implementation of K-Means clustering algorithm.
 - 'Kmeans++' initialization for faster and better convergence.
-- Modified version of Elkan's Triangle inequality to speed up K-Means algorithm.
-
+- Implementation of available classic and contemporary variants of the K-Means algorithm.
 
 ## Pending Features
-- [X] Implementation of [Hamerly implementation](https://www.researchgate.net/publication/220906984_Making_k-means_Even_Faster). 
+
+- [X] Implementation of [Hamerly implementation](https://www.researchgate.net/publication/220906984_Making_k-means_Even_Faster).
+- [X] Interface for inclusion in Alan Turing Institute's [MLJModels](https://github.com/alan-turing-institute/MLJModels.jl#who-is-this-repo-for).
 - [ ] Full Implementation of Triangle inequality based on [Elkan - 2003 Using the Triangle Inequality to Accelerate K-Means"](https://www.aaai.org/Papers/ICML/2003/ICML03-022.pdf).
 - [ ] Implementation of [Geometric methods to accelerate k-means algorithm](http://cs.baylor.edu/~hamerly/papers/sdm2016_rysavy_hamerly.pdf).
-- [ ] Support for DataFrame inputs.
+- [ ] Native support for tabular data inputs outside of MLJModels' interface.
 - [ ] Refactoring and finalizaiton of API desgin.
 - [ ] GPU support.
-- [ ] Even faster Kmeans implementation based on current literature.
+- [ ] Even faster Kmeans implementation based on recent literature.
 - [ ] Optimization of code base.
 - [ ] Improved Documentation
 - [ ] More benchmark tests
 
-
 ## How To Use
-Taking advantage of Julia's brilliant multiple dispatch system, the package exposes users to a very easy to use API. 
+
+Taking advantage of Julia's brilliant multiple dispatch system, the package exposes users to a very easy to use API.
 
 ```julia
 using ParallelKMeans
@@ -83,7 +89,7 @@ The main design goal is to offer all available variations of the KMeans algorith
 some_results = kmeans([algo], input_matrix, k; kwargs)
 
 # example
-r = kmeans(Lloyd(), X, 3)  # same result as the default 
+r = kmeans(Lloyd(), X, 3)  # same result as the default
 ```
 
 ```julia
@@ -95,30 +101,31 @@ r.iterations            # number of elapsed iterations
 r.converged             # whether the procedure converged
 ```
 
-### Supported KMeans algorithm variations.
-- [Lloyd()](https://cs.nyu.edu/~roweis/csc2515-2006/readings/lloyd57.pdf) 
-- [Hamerly()](https://www.researchgate.net/publication/220906984_Making_k-means_Even_Faster) 
+### Supported KMeans algorithm variations
+
+- [Lloyd()](https://cs.nyu.edu/~roweis/csc2515-2006/readings/lloyd57.pdf)
+- [Hamerly()](https://www.researchgate.net/publication/220906984_Making_k-means_Even_Faster)
 - [Geometric()](http://cs.baylor.edu/~hamerly/papers/sdm2016_rysavy_hamerly.pdf) - (Coming soon)
-- [Elkan()](https://www.aaai.org/Papers/ICML/2003/ICML03-022.pdf) - (Coming soon) 
+- [Elkan()](https://www.aaai.org/Papers/ICML/2003/ICML03-022.pdf) - (Coming soon)
 - [MiniBatch()](https://www.eecs.tufts.edu/~dsculley/papers/fastkmeans.pdf) - (Coming soon)
 
-
 ### Practical Usage Examples
+
 Some of the common usage examples of this package are as follows:
 
 #### Clustering With A Desired Number Of Groups
 
-```julia 
+```julia
 using ParallelKMeans, RDatasets, Plots
 
 # load the data
-iris = dataset("datasets", "iris"); 
+iris = dataset("datasets", "iris");
 
 # features to use for clustering
-features = collect(Matrix(iris[:, 1:4])'); 
+features = collect(Matrix(iris[:, 1:4])');
 
 # various artificats can be accessed from the result ie assigned labels, cost value etc
-result = kmeans(features, 3); 
+result = kmeans(features, 3);
 
 # plot with the point color mapped to the assigned cluster index
 scatter(iris.PetalLength, iris.PetalWidth, marker_z=result.assignments,
@@ -129,6 +136,7 @@ scatter(iris.PetalLength, iris.PetalWidth, marker_z=result.assignments,
 ![Image description](iris_example.jpg)
 
 #### Elbow Method For The Selection Of optimal number of clusters
+
 ```julia
 using ParallelKMeans
 
@@ -140,21 +148,18 @@ c = [ParallelKMeans.kmeans(X, i; tol=1e-6, max_iters=300, verbose=false).totalco
 
 ```
 
-
 ## Benchmarks
-Currently, this package is benchmarked against similar implementation in both Python and Julia. All reproducible benchmarks can be found in [ParallelKMeans/extras](https://github.com/PyDataBlog/ParallelKMeans.jl/tree/master/extras) directory. More tests in various languages are planned beyond the initial release version (`0.1.0`).
-
-*Note*: All benchmark tests are made on the same computer to help eliminate any bias. 
 
+Currently, this package is benchmarked against similar implementation in both Python and Julia. All reproducible benchmarks can be found in [ParallelKMeans/extras](https://github.com/PyDataBlog/ParallelKMeans.jl/tree/master/extras) directory. More tests in various languages are planned beyond the initial release version (`0.1.0`).
 
-Currently, the benchmark speed tests are based on the search for optimal number of clusters using the [Elbow Method](https://en.wikipedia.org/wiki/Elbow_method_(clustering)) since this is a practical use case for most practioners employing the K-Means algorithm. 
+*Note*: All benchmark tests are made on the same computer to help eliminate any bias.
 
+Currently, the benchmark speed tests are based on the search for optimal number of clusters using the [Elbow Method](https://en.wikipedia.org/wiki/Elbow_method_(clustering)) since this is a practical use case for most practioners employing the K-Means algorithm.
 
 ### Benchmark Results
 
 ![benchmark_image.png](benchmark_image.png)
 
-
 _________________________________________________________________________________________________________
 
 | 1 million (ms) | 100k (ms) | 10k (ms) | 1k (ms) | package                 | language |
@@ -168,17 +173,18 @@ ________________________________________________________________________________
 
 _________________________________________________________________________________________________________
 
+## Release History
 
-## Release History 
 - 0.1.0 Initial release
-
+- 0.1.1 Added interface for MLJ
 
 ## Contributing
+
 Ultimately, we see this package as potentially the one stop shop for everything related to KMeans algorithm and its speed up variants. We are open to new implementations and ideas from anyone interested in this project.
 
 Detailed contribution guidelines will be added in upcoming releases.
 
-<!--- Insert Contribution Guidelines Below --->
+<!--- TODO: Contribution Guidelines --->
 
 ```@index
 ```

src/ParallelKMeans.jl

@@ -1,13 +1,16 @@
 module ParallelKMeans
 
 using StatsBase
+using MLJModelInterface
 import Base.Threads: @spawn
+import Distances
 
 include("seeding.jl")
 include("kmeans.jl")
 include("lloyd.jl")
 include("light_elkan.jl")
 include("hamerly.jl")
+include("mlj_interface.jl")
 
 export kmeans
 export Lloyd, LightElkan, Hamerly

src/hamerly.jl

@@ -41,12 +41,13 @@ function kmeans!(alg::Hamerly, containers, design_matrix, k;
     @parallelize n_threads ncol chunk_initialize!(alg, containers, centroids, design_matrix)
 
     converged = false
-    niters = 1
+    niters = 0
     J_previous = 0.0
     p = containers.p
 
     # Update centroids & labels with closest members until convergence
-    while niters <= max_iters
+    while niters < max_iters
+        niters += 1
         update_containers!(containers, alg, centroids, n_threads)
         @parallelize n_threads ncol chunk_update_centroids!(centroids, containers, alg, design_matrix)
         collect_containers(alg, containers, n_threads)
@@ -58,7 +59,7 @@ function kmeans!(alg::Hamerly, containers, design_matrix, k;
         @parallelize n_threads ncol chunk_update_bounds!(containers, r1, r2, pr1, pr2)
 
         if verbose
-            # Show progress and terminate if J stopped decreasing.
+            # Show progress and terminate if J stops decreasing as specified by the tolerance level.
             println("Iteration $niters: Jclust = $J")
         end
 
@@ -69,7 +70,7 @@ function kmeans!(alg::Hamerly, containers, design_matrix, k;
         end
 
         J_previous = J
-        niters += 1
+
     end
 
     @parallelize n_threads ncol sum_of_squares(containers, design_matrix, containers.labels, centroids)

src/kmeans.jl

@@ -173,7 +173,7 @@ end
 
 
 """
-    Kmeans!(alg::AbstractKMeansAlg, containers, design_matrix, k; n_threads = nthreads(), k_init="k-means++", max_iters=300, tol=1e-6, verbose=true)
+    Kmeans!(alg::AbstractKMeansAlg, containers, design_matrix, k; n_threads = nthreads(), k_init="k-means++", max_iters=300, tol=1e-6, verbose=false)
 
 Mutable version of `kmeans` function. Definition of arguments and results can be
 found in `kmeans`.
@@ -189,12 +189,14 @@ function kmeans!(alg, containers, design_matrix, k;
     centroids = init == nothing ? smart_init(design_matrix, k, n_threads, init=k_init).centroids : deepcopy(init)
 
     converged = false
-    niters = 1
+    niters = 0
     J_previous = 0.0
 
     # Update centroids & labels with closest members until convergence
 
-    while niters <= max_iters
+    while niters < max_iters
+        niters += 1
+
         update_containers!(containers, alg, centroids, n_threads)
         J = update_centroids!(centroids, containers, alg, design_matrix, n_threads)
 
@@ -210,7 +212,7 @@ function kmeans!(alg, containers, design_matrix, k;
         end
 
         J_previous = J
-        niters += 1
+
     end
 
     totalcost = sum_of_squares(design_matrix, containers.labels, centroids)