Skip to content

Commit b02f2b8

Browse files
authored
Merge pull request #37 from joshmoore/jzarr
jzarr read/write (close #36)
2 parents 349019d + 0fd59c7 commit b02f2b8

File tree

7 files changed

+266
-16
lines changed

7 files changed

+266
-16
lines changed

.github/workflows/build.yml

+8
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,14 @@ jobs:
2828
env:
2929
ACTIONS_ALLOW_UNSECURE_COMMANDS: true
3030

31+
- name: Cache local Maven repository
32+
uses: actions/cache@v2
33+
with:
34+
path: ~/.m2/repository
35+
key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}
36+
restore-keys: |
37+
${{ runner.os }}-maven-
38+
3139
- name: Run tests
3240
shell: bash -l {0}
3341
run: make

Makefile

+5-1
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,10 @@ test: data
44
data/reference_image.png:
55
python generate_reference_image.py
66

7+
.PHONY: jzarr
8+
jzarr: data/reference_image.png
9+
bash generate_data/jzarr/generate_data.sh
10+
711
.PHONY: n5java
812
n5java: data/reference_image.png
913
bash generate_data/n5-java/generate_data.sh
@@ -33,7 +37,7 @@ xtensor_zarr: data/reference_image.png
3337
bash generate_data/xtensor_zarr/generate_data.sh
3438

3539
.PHONY: data
36-
data: n5java pyn5 z5py zarr js xtensor_zarr zarrita
40+
data: jzarr n5java pyn5 z5py zarr js xtensor_zarr zarrita
3741

3842
.PHONY: test
3943

generate_data/jzarr/generate_data.sh

+19
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,19 @@
1+
# cd to this directory
2+
# https://stackoverflow.com/a/6393573/2700168
3+
cd "${0%/*}"
4+
5+
set -e
6+
set -u
7+
set -x
8+
9+
MVN_FLAGS=${MVN_FLAGS:-"--no-transfer-progress"}
10+
mvn "${MVN_FLAGS}" clean package
11+
12+
java -cp target/jzarr-1.0.0.jar zarr_implementations.jzarr.App "$@" && {
13+
# Workaround for: https://github.com/bcdev/jzarr/issues/25
14+
find ../../data/jzarr* -name .zarray -exec sed -ibak 's/>u1/|u1/' {} \;
15+
} || {
16+
echo jzarr failed
17+
exit 2
18+
}
19+

generate_data/jzarr/pom.xml

+71
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,71 @@
1+
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
2+
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
3+
<modelVersion>4.0.0</modelVersion>
4+
<groupId>zarr_implementations</groupId>
5+
<artifactId>jzarr</artifactId>
6+
<packaging>jar</packaging>
7+
<version>1.0.0</version>
8+
<name>jzarr</name>
9+
<url>http://maven.apache.org</url>
10+
<dependencies>
11+
<dependency>
12+
<groupId>junit</groupId>
13+
<artifactId>junit</artifactId>
14+
<version>4.13.1</version>
15+
<scope>test</scope>
16+
</dependency>
17+
<dependency>
18+
<groupId>com.bc.zarr</groupId>
19+
<artifactId>jzarr</artifactId>
20+
<version>0.3.3</version>
21+
</dependency>
22+
</dependencies>
23+
<repositories>
24+
<repository>
25+
<id>bc-nexus-repo</id>
26+
<name>Brockmann-Consult Public Maven Repository</name>
27+
<url>https://nexus.senbox.net/nexus/content/groups/public/</url>
28+
</repository>
29+
</repositories>
30+
<build>
31+
<plugins>
32+
<plugin>
33+
<artifactId>maven-enforcer-plugin</artifactId>
34+
<executions>
35+
<execution>
36+
<id>enforce-rules</id>
37+
<phase>none</phase>
38+
</execution>
39+
</executions>
40+
</plugin>
41+
<plugin>
42+
<groupId>org.apache.maven.plugins</groupId>
43+
<artifactId>maven-compiler-plugin</artifactId>
44+
<version>3.3</version>
45+
<configuration>
46+
<source>1.8</source>
47+
<target>1.8</target>
48+
</configuration>
49+
</plugin>
50+
<plugin>
51+
<groupId>org.apache.maven.plugins</groupId>
52+
<artifactId>maven-shade-plugin</artifactId>
53+
<version>2.4.3</version>
54+
<executions>
55+
<execution>
56+
<phase>package</phase>
57+
<goals>
58+
<goal>shade</goal>
59+
</goals>
60+
<configuration>
61+
<createDependencyReducedPom>true</createDependencyReducedPom>
62+
<dependencyReducedPomLocation>
63+
${java.io.tmpdir}/dependency-reduced-pom.xml
64+
</dependencyReducedPomLocation>
65+
</configuration>
66+
</execution>
67+
</executions>
68+
</plugin>
69+
</plugins>
70+
</build>
71+
</project>
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,129 @@
1+
package zarr_implementations.jzarr;
2+
3+
import com.bc.zarr.ArrayParams;
4+
import com.bc.zarr.CompressorFactory;
5+
import com.bc.zarr.DataType;
6+
import com.bc.zarr.ZarrArray;
7+
import com.bc.zarr.ZarrGroup;
8+
9+
import javax.imageio.ImageIO;
10+
import java.awt.*;
11+
import java.awt.image.BufferedImage;
12+
import java.awt.image.DataBuffer;
13+
import java.awt.image.DataBufferByte;
14+
import java.awt.image.DataBufferInt;
15+
import java.awt.image.WritableRaster;
16+
import java.io.ByteArrayOutputStream;
17+
import java.io.File;
18+
import java.io.IOException;
19+
import java.nio.ByteBuffer;
20+
import java.nio.file.Path;
21+
import java.nio.file.Paths;
22+
import java.util.Arrays;
23+
import java.util.stream.IntStream;
24+
25+
26+
public class App {
27+
28+
enum Compression {
29+
raw("null"),
30+
zlib("zlib"),
31+
blosc("blosc");
32+
33+
private final String value;
34+
35+
private Compression(final String value) {
36+
this.value = value;
37+
}
38+
39+
@Override
40+
public String toString() {
41+
return value;
42+
}
43+
}
44+
45+
// NOTE for now we use 100, 100, 1 as block-size in all examples
46+
// maybe it's a better idea to make this more irregular though
47+
private static final int WIDTH = 512;
48+
private static final int HEIGHT = 512;
49+
private static final int CHANNELS = 3;
50+
private static final int[] CHUNKS = new int[]{100, 100, 1};
51+
private static final int[] SHAPE = new int[] {WIDTH, HEIGHT, CHANNELS};
52+
private static final Path IN_PATH = Paths.get("..", "..", "data", "reference_image.png");
53+
private static final Path OUT_PATH = Paths.get("..", "..", "data", "jzarr_flat.zr");
54+
55+
private static int[] getTestData() throws IOException {
56+
final BufferedImage image = ImageIO.read(new File(IN_PATH.toString()));
57+
int[] result = new int[WIDTH * HEIGHT * CHANNELS];
58+
for (int i = 0; i < WIDTH; i++) {
59+
for (int j = 0; j < HEIGHT; j++) {
60+
Color color = new Color(image.getRGB(i, j));
61+
int index = (WIDTH*3*j) + (3*i);
62+
result[index + 0] = color.getRed();
63+
result[index + 1] = color.getGreen();
64+
result[index + 2] = color.getBlue();
65+
}
66+
}
67+
return result;
68+
}
69+
70+
71+
private static int[] getArrayData(ZarrArray zarr) throws Exception {
72+
int[] data = new int[WIDTH * HEIGHT * CHANNELS];
73+
zarr.read(data, SHAPE, new int[]{0, 0, 0});
74+
int[] unsigned = new int[data.length];
75+
for (int i = 0; i < data.length; i++) {
76+
unsigned[i] = data[i] & 0xff;
77+
}
78+
return unsigned;
79+
}
80+
81+
public static void main(String args[]) throws Exception {
82+
83+
if (args.length != 0 && args.length != 3) {
84+
System.out.println("usage: App");
85+
System.out.println("usage: App -verify fpath dsname");
86+
System.exit(2); // EARLY EXIT
87+
} else if (args.length == 3) {
88+
String fpath = args[1];
89+
String dsname = args[2];
90+
ZarrArray verification = ZarrGroup.open(fpath).openArray(dsname);
91+
int[] shape = verification.getShape();
92+
if (!Arrays.equals(SHAPE, shape)) {
93+
throw new RuntimeException(String.format(
94+
"shape-mismatch expected:%s found:%s",
95+
Arrays.toString(SHAPE), Arrays.toString(shape)
96+
));
97+
}
98+
99+
int[] test = getTestData();
100+
int[] verify = getArrayData(verification);
101+
if (!Arrays.equals(test, verify)) {
102+
throw new RuntimeException(String.format(
103+
"values don't match"));
104+
}
105+
return; // EARLY EXIT
106+
}
107+
108+
int[] data = getTestData();
109+
110+
final ZarrGroup container = ZarrGroup.create(OUT_PATH);
111+
for (final Compression compressionType : Compression.values()) {
112+
ArrayParams arrayParams = new ArrayParams()
113+
.shape(SHAPE)
114+
.chunks(CHUNKS)
115+
.dataType(DataType.u1)
116+
// .nested(nested) FIXME: requires a different branch
117+
.compressor(CompressorFactory.create(compressionType.toString())); // jzarr name, "null"
118+
119+
String dsname = compressionType.name(); // zarr_implementation name, "raw"
120+
if ("blosc".equals(dsname)) {
121+
dsname = "blosc/lz4"; // FIXME: better workaround?
122+
}
123+
Path subdir = OUT_PATH.resolve(dsname);
124+
ZarrArray zArray = ZarrArray.create(subdir, arrayParams);
125+
// final ZarrArray zarr = ZarrArray.open(getRootPath().resolve(pathName));
126+
zArray.write(data, SHAPE, new int[]{0, 0, 0});
127+
}
128+
}
129+
}

generate_data/n5-java/generate_data.sh

+3-1
Original file line numberDiff line numberDiff line change
@@ -2,5 +2,7 @@
22
# https://stackoverflow.com/a/6393573/2700168
33
cd "${0%/*}"
44

5-
mvn clean package
5+
MVN_FLAGS=${MVN_FLAGS:-"--no-transfer-progress"}
6+
mvn "${MVN_FLAGS}" clean package
7+
68
java -cp target/n5_java-1.0.0.jar zarr_implementations.n5_java.App

test/test_read_all.py

+31-14
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,7 @@
77
* READABLE_CODECS.{library_name}.{format_name}[{codec1}, {codec2}, ...]
88
* Write a function which takes a container path and dataset name,
99
and returns a numpy-esque array
10-
* Add it to READ_FNS under the {library_name} key
10+
* Add it to _get_read_fn under the {library_name} key
1111
1212
The matrix of tests is automatically generated,
1313
and individual tests correctly fail on unavailable imports.
@@ -43,6 +43,11 @@
4343

4444

4545
READABLE_CODECS: Dict[str, Dict[str, List[str]]] = {
46+
"jzarr": {
47+
"zarr": ["blosc", "raw", "zlib"],
48+
"zarr-v3": [],
49+
"N5": [],
50+
},
4651
"z5py": {
4752
"zarr": ["blosc", "gzip", "raw", "zlib"],
4853
"zarr-v3": [],
@@ -71,6 +76,20 @@
7176
}
7277

7378

79+
def read_with_jzarr(fpath, ds_name, nested=None):
80+
if ds_name == "blosc":
81+
ds_name = "blosc/lz4"
82+
83+
cmd = (
84+
f"generate_data/jzarr/generate_data.sh "
85+
f"-verify {str(fpath)} {ds_name}"
86+
)
87+
88+
# will raise subprocess.CalledProcessError if return code is not 0
89+
subprocess.check_output(cmd, shell=True)
90+
return None
91+
92+
7493
def read_with_zarr(fpath, ds_name, nested):
7594
import zarr
7695
if ds_name == "blosc":
@@ -122,15 +141,6 @@ def read_with_xtensor_zarr(fpath, ds_name, nested):
122141
return np.load(fname)["a"]
123142

124143

125-
READ_FNS = {
126-
"zarr": read_with_zarr,
127-
"zarrita": read_with_zarrita,
128-
"pyn5": read_with_pyn5,
129-
"z5py": read_with_z5py,
130-
"xtensor_zarr": read_with_xtensor_zarr,
131-
}
132-
133-
134144
EXTENSIONS = {"zarr": ".zr", "N5": ".n5", "zarr-v3": ".zr3"}
135145
HERE = Path(__file__).resolve().parent
136146
DATA_DIR = HERE.parent / "data"
@@ -221,6 +231,7 @@ def create_params():
221231

222232
def _get_read_fn(reading_library):
223233
read_fn = {
234+
"jzarr": read_with_jzarr,
224235
"zarr": read_with_zarr,
225236
"pyn5": read_with_pyn5,
226237
"z5py": read_with_z5py,
@@ -244,8 +255,10 @@ def test_correct_read(fmt, writing_library, reading_library, codec, nested,
244255
"using 'make data'"
245256
)
246257
test = read_fn(fpath, codec, nested)
247-
assert test.shape == reference.shape
248-
assert np.allclose(test, reference)
258+
# Assume if None is returned, the read function has verified.
259+
if test is not None:
260+
assert test.shape == reference.shape
261+
assert np.allclose(test, reference)
249262

250263

251264
def tabulate_test_results(params, per_codec_tables=False):
@@ -262,8 +275,12 @@ def tabulate_test_results(params, per_codec_tables=False):
262275
fail_type = f"{type(e).__name__}: {e}"
263276

264277
if fail_type is None:
265-
result = test.shape == reference.shape
266-
result = result and np.allclose(test, reference)
278+
if test is None:
279+
# Assume implementation handled the verification
280+
result = True
281+
else:
282+
result = test.shape == reference.shape
283+
result = result and np.allclose(test, reference)
267284
else:
268285
result = fail_type
269286

0 commit comments

Comments
 (0)