From 5c6a1c6e5232b90617e42f818a173762f0f3bf5c Mon Sep 17 00:00:00 2001 From: bfoster Date: Wed, 1 Aug 2012 10:22:48 -0400 Subject: [PATCH 1/2] Support limit/skip in MongoInputSplit --- .../main/java/com/mongodb/hadoop/input/MongoInputSplit.java | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/core/src/main/java/com/mongodb/hadoop/input/MongoInputSplit.java b/core/src/main/java/com/mongodb/hadoop/input/MongoInputSplit.java index 1acc1a5a..673b39cc 100644 --- a/core/src/main/java/com/mongodb/hadoop/input/MongoInputSplit.java +++ b/core/src/main/java/com/mongodb/hadoop/input/MongoInputSplit.java @@ -136,9 +136,8 @@ public void readFields( DataInput in ) throws IOException{ DBCursor getCursor(){ // Return the cursor with the split's query, etc. already slotted in for // them. - // todo - support limit/skip if ( _cursor == null ){ - _cursor = MongoConfigUtil.getCollection( _mongoURI ).find( _querySpec, _fieldSpec ).sort( _sortSpec ); + _cursor = MongoConfigUtil.getCollection( _mongoURI ).find( _querySpec, _fieldSpec ).sort( _sortSpec ).limit(_limit).skip(_skip); if (_notimeout) _cursor.setOptions( Bytes.QUERYOPTION_NOTIMEOUT ); _cursor.slaveOk(); } @@ -160,7 +159,7 @@ BSONDecoder getBSONDecoder(){ @Override public String toString(){ - return "MongoInputSplit{URI=" + _mongoURI + ", keyField=" + _keyField + ", query=" + _querySpec + ", sort=" + _sortSpec + ", fields=" + _fieldSpec + '}'; + return "MongoInputSplit{URI=" + _mongoURI + ", keyField=" + _keyField + ", query=" + _querySpec + ", sort=" + _sortSpec + ", fields=" + _fieldSpec + ", limit=" + _limit + ", skip=" + _skip +'}'; } public MongoInputSplit(){ } From 155c3db88a639ed17450245e5a57b28f2eea24cd Mon Sep 17 00:00:00 2001 From: bfoster Date: Wed, 1 Aug 2012 16:54:40 -0400 Subject: [PATCH 2/2] Don't skip() if _skip is 0. I don't know if this is necessary, but playing it safe --- .../main/java/com/mongodb/hadoop/input/MongoInputSplit.java | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/core/src/main/java/com/mongodb/hadoop/input/MongoInputSplit.java b/core/src/main/java/com/mongodb/hadoop/input/MongoInputSplit.java index 673b39cc..146bdc5d 100644 --- a/core/src/main/java/com/mongodb/hadoop/input/MongoInputSplit.java +++ b/core/src/main/java/com/mongodb/hadoop/input/MongoInputSplit.java @@ -137,7 +137,10 @@ DBCursor getCursor(){ // Return the cursor with the split's query, etc. already slotted in for // them. if ( _cursor == null ){ - _cursor = MongoConfigUtil.getCollection( _mongoURI ).find( _querySpec, _fieldSpec ).sort( _sortSpec ).limit(_limit).skip(_skip); + _cursor = MongoConfigUtil.getCollection( _mongoURI ).find( _querySpec, _fieldSpec ).sort( _sortSpec ).limit(_limit); + if (_skip > 0) { + _cursor.skip(_skip); + } if (_notimeout) _cursor.setOptions( Bytes.QUERYOPTION_NOTIMEOUT ); _cursor.slaveOk(); }