add docs

Author: t-rutten

lib/cifar10.ex

@@ -14,6 +14,35 @@ defmodule AxonDatasets.CIFAR10 do
     end
   end
 
+  @doc """
+  Downloads the CIFAR10 dataset or fetches it locally.
+  ## Options
+  * `datapath` - path where the dataset .gz should be stored locally
+  * `transform_images/1` - accepts accept a tuple like
+      `{binary_data, tensor_type, data_shape}` which can be used for
+      converting the `binary_data` to a tensor with a function like
+          fn {labels_binary, type, _shape} ->
+            labels_binary
+            |> Nx.from_binary(type)
+            |> Nx.new_axis(-1)
+            |> Nx.equal(Nx.tensor(Enum.to_list(0..9)))
+            |> Nx.to_batched_list(32)
+          end
+  * `transform_labels/1` - similar to `transform_images/1` but applied to
+      dataset labels
+
+  Examples:
+    iex> AxonDatasets.CIFAR10.download()
+    Fetching cifar-10-binary.tar.gz from https://www.cs.toronto.edu/~kriz/
+
+    {{<<59, 43, 50, 68, 98, 119, 139, 145, 149, 149, 131, 125, 142, 144, 137, 129,
+    137, 134, 124, 139, 139, 133, 136, 139, 152, 163, 168, 159, 158, 158, 152,
+    148, 16, 0, 18, 51, 88, 120, 128, 127, 126, 116, 106, 101, 105, 113, 109,
+    112, ...>>, {:u, 8}, {50000, 3, 32, 32}},
+    {<<6, 9, 9, 4, 1, 1, 2, 7, 8, 3, 4, 7, 7, 2, 9, 9, 9, 3, 2, 6, 4, 3, 6, 6, 2,
+        6, 3, 5, 4, 0, 0, 9, 1, 3, 4, 0, 3, 7, 3, 3, 5, 2, 2, 7, 1, 1, 1, ...>>,
+      {:u, 8}, {50000}}}
+  """
   def download(opts \\ []) do
     data_path = opts[:data_path] || @default_data_path
     transform_images = opts[:transform_images] || fn out -> out end

lib/fashionmnist.ex

@@ -18,14 +18,71 @@ defmodule AxonDatasets.FashionMNIST do
 
   defp download_labels(opts) do
     data_path = opts[:data_path] || @default_data_path
-    transform = opts[:transform_images] || fn out -> out end
+    transform = opts[:transform_labels] || fn out -> out end
 
     <<_::32, n_labels::32, labels::binary>> =
       Utils.unzip_cache_or_download(@base_url, @label_file, data_path)
 
     transform.({labels, {:u, 8}, {n_labels}})
   end
 
+  @doc """
+  Downloads the FashionMNIST dataset or fetches it locally.
+  ## Options
+  * `datapath` - path where the dataset .gz should be stored locally
+  * `transform_images/1` - accepts accept a tuple like
+      `{binary_data, tensor_type, data_shape}` which can be used for
+      converting the `binary_data` to a tensor with a function like
+          fn {labels_binary, type, _shape} ->
+            labels_binary
+            |> Nx.from_binary(type)
+            |> Nx.new_axis(-1)
+            |> Nx.equal(Nx.tensor(Enum.to_list(0..9)))
+            |> Nx.to_batched_list(32)
+          end
+  * `transform_labels/1` - similar to `transform_images/1` but applied to
+      dataset labels
+
+  Examples:
+    iex> AxonDatasets.FashionMNIST.download()
+    Fetching train-images-idx3-ubyte.gz from http://fashion-mnist.s3-website.eu-central-1.amazonaws.com/
+
+    Fetching train-labels-idx1-ubyte.gz from http://fashion-mnist.s3-website.eu-central-1.amazonaws.com/
+
+    {{<<0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+        0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, ...>>,
+      {:u, 8}, {60000, 28, 28}},
+    {<<9, 0, 0, 3, 0, 2, 7, 2, 5, 5, 0, 9, 5, 5, 7, 9, 1, 0, 6, 4, 3, 1, 4, 8, 4,
+        3, 0, 2, 4, 4, 5, 3, 6, 6, 0, 8, 5, 2, 1, 6, 6, 7, 9, 5, 9, 2, 7, ...>>,
+      {:u, 8}, {60000}}}
+
+    iex> transform_labels = fn {labels_binary, type, _shape} ->
+    iex>             labels_binary
+    iex>             |> Nx.from_binary(type)
+    iex>             |> Nx.new_axis(-1)
+    iex>             |> Nx.equal(Nx.tensor(Enum.to_list(0..9)))
+    iex>             |> Nx.to_batched_list(32)
+    iex>         end
+    #Function<7.126501267/1 in :erl_eval.expr/5>
+    iex> AxonDatasets.FashionMNIST.download(transform_labels: transform_labels)
+    Using train-images-idx3-ubyte.gz from tmp/fashionmnist
+
+    Using train-labels-idx1-ubyte.gz from tmp/fashionmnist
+
+    {{<<0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+      0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, ...>>,
+    {:u, 8}, {60000, 28, 28}}, #Nx.Tensor<
+    u8[60000][10]
+    [
+      [0, 0, 0, 0, 0, 0, 0, 0, 0, 1],
+      [1, 0, 0, 0, 0, 0, 0, 0, 0, 0],
+      [1, 0, 0, 0, 0, 0, 0, 0, 0, 0],
+      [0, 0, 0, 1, 0, 0, 0, 0, 0, 0],
+      [1, 0, 0, 0, 0, 0, 0, 0, ...],
+      ...
+    ]
+    >}
+  """
   def download(opts \\ []),
     do: {download_images(opts), download_labels(opts)}
 end

lib/mnist.ex

@@ -26,6 +26,24 @@ defmodule AxonDatasets.MNIST do
     transform.({labels, {:u, 8}, {n_labels}})
   end
 
+  @doc """
+  Downloads the MNIST dataset or fetches it locally.
+
+  ## Options
+  * `datapath` - path where the dataset .gz should be stored locally
+  * `transform_images/1` - accepts accept a tuple like
+      `{binary_data, tensor_type, data_shape}` which can be used for
+      converting the `binary_data` to a tensor with a function like
+          fn {labels_binary, type, _shape} ->
+            labels_binary
+            |> Nx.from_binary(type)
+            |> Nx.new_axis(-1)
+            |> Nx.equal(Nx.tensor(Enum.to_list(0..9)))
+            |> Nx.to_batched_list(32)
+          end
+  * `transform_labels/1` - similar to `transform_images/1` but applied to
+      dataset labels
+  """
   def download(opts \\ []),
     do: {download_images(opts), download_labels(opts)}
 end