Skip to content

Commit 58e65ad

Browse files
huizhiluhuizhilu
authored
Cherry-pick and adapt ZOOKEEPER-2260 for getChildren pagination (#5)
Apply the patch of ZOOKEEPER-2260: paginated getChildren call (apache#50) And adjust code style to pass the check style verification Credit goest to the original author of ZOOKEEPER-2260
1 parent 5fbe83a commit 58e65ad

File tree

11 files changed

+978
-0
lines changed

11 files changed

+978
-0
lines changed

‎zookeeper-jute/src/main/resources/zookeeper.jute

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -51,6 +51,10 @@ module org.apache.zookeeper.data {
5151
long ephemeralOwner; // owner id if ephemeral, 0 otw
5252
long pzxid; // last modified children
5353
}
54+
class PathWithStat {
55+
ustring path;
56+
org.apache.zookeeper.data.Stat stat;
57+
}
5458
}
5559

5660
module org.apache.zookeeper.proto {
@@ -157,6 +161,13 @@ module org.apache.zookeeper.proto {
157161
ustring path;
158162
boolean watch;
159163
}
164+
class GetChildrenPaginatedRequest {
165+
ustring path;
166+
int maxReturned;
167+
long minCzxId;
168+
long czxIdOffset;
169+
boolean watch;
170+
}
160171
class CheckVersionRequest {
161172
ustring path;
162173
int version;
@@ -228,6 +239,10 @@ module org.apache.zookeeper.proto {
228239
vector<ustring> children;
229240
org.apache.zookeeper.data.Stat stat;
230241
}
242+
class GetChildrenPaginatedResponse {
243+
vector<org.apache.zookeeper.data.PathWithStat> children;
244+
org.apache.zookeeper.data.Stat stat;
245+
}
231246
class GetACLResponse {
232247
vector<org.apache.zookeeper.data.ACL> acl;
233248
org.apache.zookeeper.data.Stat stat;

‎zookeeper-server/src/main/java/org/apache/zookeeper/ChildrenBatchIterator.java

Lines changed: 101 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,101 @@
1+
/**
2+
* Licensed to the Apache Software Foundation (ASF) under one
3+
* or more contributor license agreements. See the NOTICE file
4+
* distributed with this work for additional information
5+
* regarding copyright ownership. The ASF licenses this file
6+
* to you under the Apache License, Version 2.0 (the
7+
* "License"); you may not use this file except in compliance
8+
* with the License. You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing, software
13+
* distributed under the License is distributed on an "AS IS" BASIS,
14+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15+
* See the License for the specific language governing permissions and
16+
* limitations under the License.
17+
*/
18+
19+
package org.apache.zookeeper;
20+
21+
import java.util.LinkedList;
22+
import java.util.List;
23+
import java.util.NoSuchElementException;
24+
import org.apache.zookeeper.data.PathWithStat;
25+
26+
/**
27+
* Iterator over children nodes of a given path.
28+
*/
29+
class ChildrenBatchIterator implements RemoteIterator<PathWithStat> {
30+
31+
private final ZooKeeper zooKeeper;
32+
private final String path;
33+
private final Watcher watcher;
34+
private final int batchSize;
35+
private final LinkedList<PathWithStat> childrenQueue;
36+
private long nextBatchMinZxid;
37+
private int nextBatchZxidOffset;
38+
39+
40+
ChildrenBatchIterator(ZooKeeper zooKeeper, String path, Watcher watcher, int batchSize, long minZxid)
41+
throws KeeperException, InterruptedException {
42+
this.zooKeeper = zooKeeper;
43+
this.path = path;
44+
this.watcher = watcher;
45+
this.batchSize = batchSize;
46+
this.nextBatchZxidOffset = 0;
47+
this.nextBatchMinZxid = minZxid;
48+
49+
this.childrenQueue = new LinkedList<>();
50+
51+
List<PathWithStat> firstChildrenBatch = zooKeeper.getChildren(path, watcher, batchSize, nextBatchMinZxid, nextBatchZxidOffset);
52+
childrenQueue.addAll(firstChildrenBatch);
53+
54+
updateOffsetsForNextBatch(firstChildrenBatch);
55+
}
56+
57+
@Override
58+
public boolean hasNext() {
59+
60+
// next() never lets childrenQueue empty unless we iterated over all children
61+
return !childrenQueue.isEmpty();
62+
}
63+
64+
@Override
65+
public PathWithStat next() throws KeeperException, InterruptedException, NoSuchElementException {
66+
67+
if (!hasNext()) {
68+
throw new NoSuchElementException("No more children");
69+
}
70+
71+
// If we're down to the last element, backfill before returning it
72+
if (childrenQueue.size() == 1) {
73+
74+
List<PathWithStat> childrenBatch = zooKeeper.getChildren(path, watcher, batchSize, nextBatchMinZxid, nextBatchZxidOffset);
75+
childrenQueue.addAll(childrenBatch);
76+
77+
updateOffsetsForNextBatch(childrenBatch);
78+
}
79+
80+
PathWithStat returnChildren = childrenQueue.pop();
81+
82+
return returnChildren;
83+
}
84+
85+
/**
86+
* Prepare minZxid and zkidOffset for the next batch request based on the children returned in the current
87+
*/
88+
private void updateOffsetsForNextBatch(List<PathWithStat> children) {
89+
90+
for (PathWithStat child : children) {
91+
long childZxid = child.getStat().getCzxid();
92+
93+
if (nextBatchMinZxid == childZxid) {
94+
++nextBatchZxidOffset;
95+
} else {
96+
nextBatchZxidOffset = 1;
97+
nextBatchMinZxid = childZxid;
98+
}
99+
}
100+
}
101+
}

‎zookeeper-server/src/main/java/org/apache/zookeeper/RemoteIterator.java

Lines changed: 42 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,42 @@
1+
/**
2+
* Licensed to the Apache Software Foundation (ASF) under one
3+
* or more contributor license agreements. See the NOTICE file
4+
* distributed with this work for additional information
5+
* regarding copyright ownership. The ASF licenses this file
6+
* to you under the Apache License, Version 2.0 (the
7+
* "License"); you may not use this file except in compliance
8+
* with the License. You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing, software
13+
* distributed under the License is distributed on an "AS IS" BASIS,
14+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15+
* See the License for the specific language governing permissions and
16+
* limitations under the License.
17+
*/
18+
19+
package org.apache.zookeeper;
20+
21+
import java.util.NoSuchElementException;
22+
23+
/**
24+
* An iterator over a collection whose elements need to be fetched remotely.
25+
*/
26+
public interface RemoteIterator<E> {
27+
28+
/**
29+
* Returns true if the iterator has more elements.
30+
* @return true if the iterator has more elements, false otherwise.
31+
*/
32+
boolean hasNext();
33+
34+
/**
35+
* Returns the next element in the iteration.
36+
* @return the next element in the iteration.
37+
* @throws InterruptedException if the thread is interrupted
38+
* @throws KeeperException if an error is encountered server-side
39+
* @throws NoSuchElementException if the iteration has no more elements
40+
*/
41+
E next() throws InterruptedException, KeeperException, NoSuchElementException;
42+
}

‎zookeeper-server/src/main/java/org/apache/zookeeper/ZooDefs.java

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -79,6 +79,8 @@ public interface OpCode {
7979

8080
int multiRead = 22;
8181

82+
int getChildrenPaginated = 71;
83+
8284
int auth = 100;
8385

8486
int setWatches = 101;

‎zookeeper-server/src/main/java/org/apache/zookeeper/ZooKeeper.java

Lines changed: 110 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -53,6 +53,7 @@
5353
import org.apache.zookeeper.client.ZooKeeperSaslClient;
5454
import org.apache.zookeeper.common.PathUtils;
5555
import org.apache.zookeeper.data.ACL;
56+
import org.apache.zookeeper.data.PathWithStat;
5657
import org.apache.zookeeper.data.Stat;
5758
import org.apache.zookeeper.proto.AddWatchRequest;
5859
import org.apache.zookeeper.proto.CheckWatchesRequest;
@@ -69,6 +70,8 @@
6970
import org.apache.zookeeper.proto.GetAllChildrenNumberResponse;
7071
import org.apache.zookeeper.proto.GetChildren2Request;
7172
import org.apache.zookeeper.proto.GetChildren2Response;
73+
import org.apache.zookeeper.proto.GetChildrenPaginatedRequest;
74+
import org.apache.zookeeper.proto.GetChildrenPaginatedResponse;
7275
import org.apache.zookeeper.proto.GetChildrenRequest;
7376
import org.apache.zookeeper.proto.GetChildrenResponse;
7477
import org.apache.zookeeper.proto.GetDataRequest;
@@ -2748,6 +2751,113 @@ public List<String> getChildren(final String path, Watcher watcher) throws Keepe
27482751
return response.getChildren();
27492752
}
27502753

2754+
/**
2755+
* Returns the a subset (a "page") of the children node of the given path.
2756+
*
2757+
* <p>
2758+
* The returned list contains up to {@code maxReturned} ordered by czxid.
2759+
* </p>
2760+
*
2761+
* <p>
2762+
* If {@code watcher} is non-null, a watch on children creation/deletion is set when reaching the end of pagination
2763+
* </p>
2764+
*
2765+
* @param path
2766+
* - the path of the node
2767+
* @param watcher
2768+
* - a concrete watcher or null
2769+
* @param maxReturned
2770+
* - the maximum number of children returned
2771+
* @param minCzxId
2772+
* - only return children whose creation zkid is equal or greater than {@code minCzxId}
2773+
* @param czxIdOffset
2774+
* - how many children with zkid == minCzxId to skip server-side, as they were returned in previous pages
2775+
* @return
2776+
* an ordered list of children nodes, up to {@code maxReturned}, ordered by czxid
2777+
* @throws KeeperException
2778+
* if the server signals an error with a non-zero error code.
2779+
* @throws IllegalArgumentException
2780+
* if any of the following is true:
2781+
* <ul>
2782+
* <li> {@code path} is invalid
2783+
* <li> {@code maxReturned} is less than 1
2784+
* </ul>
2785+
*
2786+
* @throws InterruptedException
2787+
* if the server transaction is interrupted.
2788+
*
2789+
* @since 3.6.2
2790+
*/
2791+
public List<PathWithStat> getChildren(final String path, Watcher watcher, final int maxReturned, final long minCzxId, final int czxIdOffset)
2792+
throws KeeperException, InterruptedException {
2793+
final String clientPath = path;
2794+
PathUtils.validatePath(clientPath);
2795+
2796+
if (maxReturned <= 0) {
2797+
throw new IllegalArgumentException("Cannot return less than 1 children");
2798+
}
2799+
2800+
// the watch contains the un-chroot path
2801+
WatchRegistration wcb = null;
2802+
if (watcher != null) {
2803+
wcb = new ChildWatchRegistration(watcher, clientPath);
2804+
}
2805+
2806+
final String serverPath = prependChroot(clientPath);
2807+
2808+
RequestHeader h = new RequestHeader();
2809+
h.setType(ZooDefs.OpCode.getChildrenPaginated);
2810+
GetChildrenPaginatedRequest request = new GetChildrenPaginatedRequest();
2811+
request.setPath(serverPath);
2812+
request.setWatch(watcher != null);
2813+
request.setMaxReturned(maxReturned);
2814+
request.setMinCzxId(minCzxId);
2815+
request.setCzxIdOffset(czxIdOffset);
2816+
GetChildrenPaginatedResponse response = new GetChildrenPaginatedResponse();
2817+
ReplyHeader r = cnxn.submitRequest(h, request, response, wcb);
2818+
if (r.getErr() != 0) {
2819+
throw KeeperException.create(KeeperException.Code.get(r.getErr()),
2820+
clientPath);
2821+
}
2822+
return response.getChildren();
2823+
}
2824+
2825+
/**
2826+
* Returns the a RemoteIterator over the children nodes of the given path.
2827+
*
2828+
* <p>
2829+
* If {@code watcher} is non-null, a watch on children creation/deletion is set when reaching the end the iterator
2830+
* </p>
2831+
*
2832+
* @param path
2833+
* - the path of the node
2834+
* @param watcher
2835+
* - a concrete watcher or null
2836+
* @param batchSize
2837+
* - the maximum number of children returned by each background call to the server
2838+
* @param minCzxId
2839+
* - only return children whose creation zkid is equal or greater than {@code minCzxId}
2840+
*
2841+
* @return
2842+
* an iterator on children node, ordered by czxid
2843+
* @throws KeeperException
2844+
* if the server signals an error with a non-zero error code.
2845+
* @throws IllegalArgumentException
2846+
* if any of the following is true:
2847+
* <ul>
2848+
* <li> {@code path} is invalid
2849+
* <li> {@code maxReturned} is less than 1
2850+
* </ul>
2851+
* @throws InterruptedException
2852+
* if the server transaction is interrupted.
2853+
*
2854+
* @since 3.6.2
2855+
*/
2856+
public RemoteIterator<PathWithStat> getChildrenIterator(String path, Watcher watcher, int batchSize, long minCzxId)
2857+
throws KeeperException, InterruptedException {
2858+
return new ChildrenBatchIterator(this, path, watcher, batchSize, minCzxId);
2859+
}
2860+
27512861
/**
27522862
* Return the list of the children of the node of the given path.
27532863
* <p>

0 commit comments

Comments
 (0)