Merge pull request #185 from LCSB-BioCore/develop

Develop

Author: laurentheirendt

.github/workflows/docker.yml

@@ -1,7 +1,9 @@
 name: Publish Docker image
+
 on:
   release:
-    types: [published]
+    types: [published, created]
+
 jobs:
   push_to_registry:
     name: Push Docker image to GitHub Packages
@@ -14,9 +16,12 @@ jobs:
         with:
           registry: docker.pkg.github.com
           username: cylon-x
-          password: ${{ secrets.DOCKER_TOKEN }}
-      - uses: docker/build-push-action@v2
+          password: ${{ secrets.docker_token }}
+      - name: Push to GitHub Registry
+        uses: mr-smithers-excellent/docker-build-push@v5
         with:
-         push: true
-         tags: lcsb-biocore/gigasom:latest
-
+          image: gigasom.jl
+          tags: latest
+          registry: ghcr.io
+          username: cylon-x
+          password: ${{ secrets.docker_token }}

.github/workflows/docs.yml

@@ -6,7 +6,6 @@ on:
     branches:
       - develop
     tags: '*'
-  pull_request:
   release:
     types: [published, created]
 

Project.toml

@@ -1,6 +1,6 @@
 name = "GigaSOM"
 uuid = "a03a9c34-069e-5582-a11c-5c984cab887c"
-version = "0.6.4"
+version = "0.6.5"
 
 [deps]
 CSV = "336ed68f-0bac-5ca0-87d4-7b16caf5d00b"
@@ -23,10 +23,10 @@ XLSX = "fdbf4ff8-1666-58a4-91e7-1b58723a45e0"
 
 [compat]
 CSV = "^0.5.12, 0.6, 0.7, 0.8"
-DataFrames = "0.20, 0.21, 0.22"
+DataFrames = "0.20, 0.21, 0.22, 1.0"
 Distances = "^0.8.2, 0.9, 0.10"
 DistributedArrays = "^0.6.4"
-Distributions = "^0.21.1, 0.22, 0.23, 0.24"
+Distributions = "^0.21.1, 0.22, 0.23, 0.24, 0.25"
 FCSFiles = "^0.1.1"
 FileIO = "^1.0.7"
 JSON = "0.21"

src/analysis/core.jl

@@ -82,13 +82,19 @@ end
 
 
 """
-    trainGigaSOM(som::Som, dInfo::LoadedDataInfo;
-                 kernelFun::Function = gaussianKernel,
-                 metric = Euclidean(),
-                 somDistFun = distMatrix(Chebyshev()),
-                 knnTreeFun = BruteTree,
-                 rStart = 0.0, rFinal=0.1, radiusFun=expRadius(-5.0),
-                 epochs = 20)
+    trainGigaSOM(
+        som::Som,
+        dInfo::LoadedDataInfo;
+        kernelFun::Function = gaussianKernel,
+        metric = Euclidean(),
+        somDistFun = distMatrix(Chebyshev()),
+        knnTreeFun = BruteTree,
+        rStart = 0.0,
+        rFinal = 0.1,
+        radiusFun = expRadius(-5.0),
+        epochs = 20,
+        eachEpoch = (e, r, som) -> nothing,
+    )
 
 # Arguments:
 - `som`: object of type Som with an initialised som
@@ -102,6 +108,9 @@ end
 - `rFinal`: target radius at the last epoch, defaults to 0.1
 - `radiusFun`: Function that generates radius decay, e.g. `linearRadius` or `expRadius(10.0)`
 - `epochs`: number of SOM training iterations (default 10)
+- `eachEpoch`: a function to call back after each epoch, accepting arguments
+  `(epochNumber, radius, som)`. For simplicity, this gets additionally called
+  once before the first epoch, with `epochNumber` set to zero.
 """
 function trainGigaSOM(
     som::Som,
@@ -114,6 +123,7 @@ function trainGigaSOM(
     rFinal = 0.1,
     radiusFun = expRadius(-5.0),
     epochs = 20,
+    eachEpoch = (e, r, som) -> nothing,
 )
 
     # set the default radius
@@ -125,75 +135,51 @@ function trainGigaSOM(
     # get the SOM neighborhood distances
     dm = somDistFun(som.grid)
 
-    codes = som.codes
+    result_som = copy(som)
+    result_som.codes = copy(som.codes) # prevent rewriting by reference
+
+    eachEpoch(0, rStart, result_som)
 
-    for j = 1:epochs
-        @debug "Epoch $j..."
+    for epoch = 1:epochs
+        @debug "Epoch $epoch..."
 
         numerator, denominator = distributedEpoch(
             dInfo,
-            codes,
-            knnTreeFun(Array{Float64,2}(transpose(codes)), metric),
+            result_som.codes,
+            knnTreeFun(Array{Float64,2}(transpose(result_som.codes)), metric),
         )
 
-        r = radiusFun(rStart, rFinal, j, epochs)
+        r = radiusFun(rStart, rFinal, epoch, epochs)
         @debug "radius: $r"
         if r <= 0
             @error "Sanity check failed: radius must be positive"
             error("Radius check")
         end
 
         wEpoch = kernelFun(dm, r)
-        codes = (wEpoch * numerator) ./ (wEpoch * denominator)
-    end
+        result_som.codes = (wEpoch * numerator) ./ (wEpoch * denominator)
 
-    som.codes = copy(codes)
+        eachEpoch(epoch, r, result_som)
+    end
 
-    return som
+    return result_som
 end
 
 """
     trainGigaSOM(som::Som, train;
-                 kernelFun::Function = gaussianKernel,
-                 metric = Euclidean(),
-                 somDistFun = distMatrix(Chebyshev()),
-                 knnTreeFun = BruteTree,
-                 rStart = 0.0, rFinal=0.1, radiusFun=expRadius(-5.0),
-                 epochs = 20)
+                 kwargs...)
 
 Overload of `trainGigaSOM` for simple DataFrames and matrices. This slices the
-data using `DistributedArrays`, sends them the workers, and runs normal
-`trainGigaSOM`. Data is `undistribute`d after the computation.
+data, distributes them to the workers, and runs normal `trainGigaSOM`. Data is
+`undistribute`d after the computation.
 """
-function trainGigaSOM(
-    som::Som,
-    train;
-    kernelFun::Function = gaussianKernel,
-    metric = Euclidean(),
-    somDistFun = distMatrix(Chebyshev()),
-    knnTreeFun = BruteTree,
-    rStart = 0.0,
-    rFinal = 0.1,
-    radiusFun = expRadius(-5.0),
-    epochs = 20,
-)
+function trainGigaSOM(som::Som, train; kwargs...)
 
     train = Matrix{Float64}(train)
 
     #this slices the data into parts and and sends them to workers
     dInfo = distribute_array(:GigaSOMtrainDataVar, train, workers())
-    som_res = trainGigaSOM(
-        som,
-        dInfo,
-        kernelFun = kernelFun,
-        metric = metric,
-        somDistFun = somDistFun,
-        knnTreeFun = knnTreeFun,
-        rStart = rStart,
-        rFinal = rFinal,
-        radiusFun = radiusFun,
-        epochs = epochs,
-    )
+    som_res = trainGigaSOM(som, dInfo; kwargs...)
     undistribute(dInfo)
     return som_res
 end

src/base/structs.jl

@@ -28,6 +28,14 @@ mutable struct Som
     ) = new(codes, xdim, ydim, numCodes, grid)
 end
 
+Base.copy(som::Som) = Som(
+    codes = som.codes,
+    xdim = som.xdim,
+    ydim = som.ydim,
+    numCodes = som.numCodes,
+    grid = som.grid,
+)
+
 """
     LoadedDataInfo