From 6e3a359783c62752506e593a20d8aefe8b4efbdf Mon Sep 17 00:00:00 2001 From: Yan Ma Date: Thu, 8 Aug 2019 16:19:10 -0700 Subject: [PATCH 1/4] sorting index creation --- .../data/redis/core/IndexWriter.java | 20 ++++- .../data/redis/core/RedisQueryEngine.java | 5 +- .../convert/IndexedDataFactoryProvider.java | 27 +++++- .../convert/SortingIndexedPropertyValue.java | 30 +++++++ .../CompositeSortingIndexDefinition.java | 69 +++++++++++++++ .../redis/core/index/IndexNameHandler.java | 7 ++ .../redis/core/index/IndexValueHandler.java | 7 ++ .../core/index/SortingIndexDefinition.java | 48 ++++++++++ .../SortingIndexedPropertyValueTest.java | 23 +++++ .../core/index/RedisRepositoryIndexTest.java | 88 +++++++++++++++++++ .../data/redis/core/index/TestAddress.java | 25 ++++++ .../data/redis/core/index/TestGender.java | 5 ++ .../core/index/TestIndexConfiguration.java | 32 +++++++ .../data/redis/core/index/TestPerson.java | 31 +++++++ .../core/index/TestPersonRepository.java | 13 +++ .../data/redis/core/index/TestState.java | 5 ++ 16 files changed, 431 insertions(+), 4 deletions(-) create mode 100644 src/main/java/org/springframework/data/redis/core/convert/SortingIndexedPropertyValue.java create mode 100644 src/main/java/org/springframework/data/redis/core/index/CompositeSortingIndexDefinition.java create mode 100644 src/main/java/org/springframework/data/redis/core/index/IndexNameHandler.java create mode 100644 src/main/java/org/springframework/data/redis/core/index/IndexValueHandler.java create mode 100644 src/main/java/org/springframework/data/redis/core/index/SortingIndexDefinition.java create mode 100644 src/test/java/org/springframework/data/redis/core/convert/SortingIndexedPropertyValueTest.java create mode 100644 src/test/java/org/springframework/data/redis/core/index/RedisRepositoryIndexTest.java create mode 100644 src/test/java/org/springframework/data/redis/core/index/TestAddress.java create mode 100644 src/test/java/org/springframework/data/redis/core/index/TestGender.java create mode 100644 src/test/java/org/springframework/data/redis/core/index/TestIndexConfiguration.java create mode 100644 src/test/java/org/springframework/data/redis/core/index/TestPerson.java create mode 100644 src/test/java/org/springframework/data/redis/core/index/TestPersonRepository.java create mode 100644 src/test/java/org/springframework/data/redis/core/index/TestState.java diff --git a/src/main/java/org/springframework/data/redis/core/IndexWriter.java b/src/main/java/org/springframework/data/redis/core/IndexWriter.java index e288f45bc5..83cb039dcf 100644 --- a/src/main/java/org/springframework/data/redis/core/IndexWriter.java +++ b/src/main/java/org/springframework/data/redis/core/IndexWriter.java @@ -25,6 +25,7 @@ import org.springframework.data.redis.core.convert.RedisConverter; import org.springframework.data.redis.core.convert.RemoveIndexedData; import org.springframework.data.redis.core.convert.SimpleIndexedPropertyValue; +import org.springframework.data.redis.core.convert.SortingIndexedPropertyValue; import org.springframework.data.redis.util.ByteUtils; import org.springframework.lang.Nullable; import org.springframework.util.Assert; @@ -39,6 +40,7 @@ * * @author Christoph Strobl * @author Rob Winch + * @author Yan Ma * @since 1.7 */ class IndexWriter { @@ -106,7 +108,7 @@ private void createOrUpdateIndexes(Object key, @Nullable Iterable i if (indexValues.iterator().hasNext()) { IndexedData data = indexValues.iterator().next(); - if (data != null) { + if (data != null && data.getKeyspace() != null) { removeKeyFromIndexes(data.getKeyspace(), binKey); } } @@ -179,6 +181,8 @@ protected void removeKeyFromExistingIndexes(byte[] key, IndexedData indexedData) if (indexedData instanceof GeoIndexedPropertyValue) { connection.geoRemove(existingKey, key); + } else if(indexedData instanceof SortingIndexedPropertyValue){ + connection.zRem(existingKey, key); } else { connection.sRem(existingKey, key); } @@ -222,7 +226,19 @@ protected void addKeyToIndex(byte[] key, IndexedData indexedData) { // keep track of indexes used for the object connection.sAdd(ByteUtils.concatAll(toBytes(indexedData.getKeyspace() + ":"), key, toBytes(":idx")), indexKey); - } else if (indexedData instanceof GeoIndexedPropertyValue) { + } else if(indexedData instanceof SortingIndexedPropertyValue ) { + SortingIndexedPropertyValue sortingIndexedData = (SortingIndexedPropertyValue) indexedData; + String indexName = sortingIndexedData.getIndexName(); + if(indexName == null) return; + Double score = sortingIndexedData.getScore(); + if(score == null) return; + + byte[] indexKey = toBytes(indexedData.getKeyspace() + ":" + indexName); + connection.zAdd(indexKey , score, key); + + // keep track of indexes used for the object + connection.sAdd(ByteUtils.concatAll(toBytes(indexedData.getKeyspace() + ":"), key, toBytes(":idx")), indexKey); + } else if (indexedData instanceof GeoIndexedPropertyValue) { GeoIndexedPropertyValue geoIndexedData = ((GeoIndexedPropertyValue) indexedData); diff --git a/src/main/java/org/springframework/data/redis/core/RedisQueryEngine.java b/src/main/java/org/springframework/data/redis/core/RedisQueryEngine.java index 74594c5a50..708816fcf1 100644 --- a/src/main/java/org/springframework/data/redis/core/RedisQueryEngine.java +++ b/src/main/java/org/springframework/data/redis/core/RedisQueryEngine.java @@ -45,6 +45,7 @@ * * @author Christoph Strobl * @author Mark Paluch + * @author Yan Ma * @since 1.7 */ class RedisQueryEngine extends QueryEngine> { @@ -78,7 +79,9 @@ public Collection execute(RedisOperationChain criteria, Comparator sor String keyspace, Class type) { if (criteria == null - || (CollectionUtils.isEmpty(criteria.getOrSismember()) && CollectionUtils.isEmpty(criteria.getSismember())) + || (CollectionUtils.isEmpty(criteria.getOrSismember()) + && CollectionUtils.isEmpty(criteria.getSismember())) +// && CollectionUtils.isEmpty(criteria.getRanges())) && criteria.getNear() == null) { return (Collection) getAdapter().getAllOf(keyspace, offset, rows); } diff --git a/src/main/java/org/springframework/data/redis/core/convert/IndexedDataFactoryProvider.java b/src/main/java/org/springframework/data/redis/core/convert/IndexedDataFactoryProvider.java index 837e7b8d0a..70f89be2de 100644 --- a/src/main/java/org/springframework/data/redis/core/convert/IndexedDataFactoryProvider.java +++ b/src/main/java/org/springframework/data/redis/core/convert/IndexedDataFactoryProvider.java @@ -18,9 +18,11 @@ import lombok.RequiredArgsConstructor; import org.springframework.data.geo.Point; +import org.springframework.data.redis.core.index.CompositeSortingIndexDefinition; import org.springframework.data.redis.core.index.GeoIndexDefinition; import org.springframework.data.redis.core.index.IndexDefinition; import org.springframework.data.redis.core.index.SimpleIndexDefinition; +import org.springframework.data.redis.core.index.SortingIndexDefinition; import org.springframework.lang.Nullable; /** @@ -40,14 +42,37 @@ IndexedDataFactory getIndexedDataFactory(IndexDefinition definition) { return new SimpleIndexedPropertyValueFactory((SimpleIndexDefinition) definition); } else if (definition instanceof GeoIndexDefinition) { return new GeoIndexedPropertyValueFactory(((GeoIndexDefinition) definition)); - } + } else if (definition instanceof SortingIndexDefinition){ + return new SortingIndexedPropertyValueFactory(((SortingIndexDefinition) definition)); + } return null; } static interface IndexedDataFactory { IndexedData createIndexedDataFor(Object value); } + + static class SortingIndexedPropertyValueFactory implements IndexedDataFactory { + final SortingIndexDefinition indexDefinition; + + public SortingIndexedPropertyValueFactory(SortingIndexDefinition indexDefinition) { + this.indexDefinition = indexDefinition; + } + @Override + public IndexedData createIndexedDataFor(Object value) { + if (indexDefinition instanceof CompositeSortingIndexDefinition) { + CompositeSortingIndexDefinition csid = (CompositeSortingIndexDefinition) indexDefinition; + return new SortingIndexedPropertyValue(indexDefinition.getKeyspace(), csid.getIndexName(value), + csid.getIndexValue(value)); + } else { + return new SortingIndexedPropertyValue(indexDefinition.getKeyspace(), indexDefinition.getIndexName(), + indexDefinition.valueTransformer().convert(value)); + } + } + + } + /** * @author Christoph Strobl * @since 1.8 diff --git a/src/main/java/org/springframework/data/redis/core/convert/SortingIndexedPropertyValue.java b/src/main/java/org/springframework/data/redis/core/convert/SortingIndexedPropertyValue.java new file mode 100644 index 0000000000..7a186f9009 --- /dev/null +++ b/src/main/java/org/springframework/data/redis/core/convert/SortingIndexedPropertyValue.java @@ -0,0 +1,30 @@ +package org.springframework.data.redis.core.convert; + +import org.springframework.data.redis.core.convert.IndexedData; + +public class SortingIndexedPropertyValue implements IndexedData { + + private final String keyspace; + private final String indexName; + private final double score; + + public SortingIndexedPropertyValue(String keyspace, String indexName, Object value) { + this.keyspace = keyspace; + this.indexName = indexName; + this.score = (Double) value; + } + + @Override + public String getIndexName() { + return indexName; + } + + @Override + public String getKeyspace() { + return keyspace; + } + + public Double getScore() { + return score; + } +} diff --git a/src/main/java/org/springframework/data/redis/core/index/CompositeSortingIndexDefinition.java b/src/main/java/org/springframework/data/redis/core/index/CompositeSortingIndexDefinition.java new file mode 100644 index 0000000000..c163f0925d --- /dev/null +++ b/src/main/java/org/springframework/data/redis/core/index/CompositeSortingIndexDefinition.java @@ -0,0 +1,69 @@ +package org.springframework.data.redis.core.index; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +/** + * @author Yan Ma + */ +public class CompositeSortingIndexDefinition extends SortingIndexDefinition { + private static final Logger LOG = LoggerFactory.getLogger(CompositeSortingIndexDefinition.class); + public IndexNameHandler indexNameHandler; + public IndexValueHandler indexValueHandler; + + public CompositeSortingIndexDefinition(String keyspace, String path, IndexNameHandler indexNameHandler, + IndexValueHandler indexValueHandler) { + + super(keyspace, path, path); + this.indexNameHandler = indexNameHandler; + this.indexValueHandler = indexValueHandler; + setValueTransformer(new CompositeSortingIndexValueTransformer(this.indexValueHandler)); + } + + public String getIndexName(T obj) { + + T typedObj = obj; + String indexName = null; + try { + indexName = indexNameHandler.getIndexName(typedObj); + } catch (Exception e) { + LOG.error("Thrown exception in getting index name: {}", e.getMessage()); + } + LOG.debug("Got the index name: {}", indexName); + return indexName; + } + + public Object getIndexValue(T value) { + + T typedValue = (T) value; + Double indexValue = null; + try { + indexValue = indexValueHandler.getValue(typedValue); + } catch (Exception e) { + LOG.error("Thrown exception in getting index value: {}", e.getMessage()); + } + LOG.debug("Got the index value: {}", indexValue); + return indexValue; + } + + class CompositeSortingIndexValueTransformer implements IndexValueTransformer { + IndexValueHandler indexValueHandler; + + public CompositeSortingIndexValueTransformer(IndexValueHandler indexValueHandler) { + this.indexValueHandler = indexValueHandler; + } + + @Override + public Object convert(Object source) { + + Double value = null; + try { + value = indexValueHandler.getValue((T) source); + } catch (Exception e) { + LOG.error("Thrown exception in transforming the value: {}", e.getMessage()); + } + LOG.debug("Got the transformed value: {}", value); + return value; + } + + } +} diff --git a/src/main/java/org/springframework/data/redis/core/index/IndexNameHandler.java b/src/main/java/org/springframework/data/redis/core/index/IndexNameHandler.java new file mode 100644 index 0000000000..6d86a29c36 --- /dev/null +++ b/src/main/java/org/springframework/data/redis/core/index/IndexNameHandler.java @@ -0,0 +1,7 @@ +package org.springframework.data.redis.core.index; +/** + * @author Yan Ma + */ +public interface IndexNameHandler { + String getIndexName(T t); +} diff --git a/src/main/java/org/springframework/data/redis/core/index/IndexValueHandler.java b/src/main/java/org/springframework/data/redis/core/index/IndexValueHandler.java new file mode 100644 index 0000000000..2acf693c09 --- /dev/null +++ b/src/main/java/org/springframework/data/redis/core/index/IndexValueHandler.java @@ -0,0 +1,7 @@ +package org.springframework.data.redis.core.index; +/** + * @author Yan Ma + */ +public interface IndexValueHandler { + public Double getValue(Input input); +} diff --git a/src/main/java/org/springframework/data/redis/core/index/SortingIndexDefinition.java b/src/main/java/org/springframework/data/redis/core/index/SortingIndexDefinition.java new file mode 100644 index 0000000000..241cc6cb45 --- /dev/null +++ b/src/main/java/org/springframework/data/redis/core/index/SortingIndexDefinition.java @@ -0,0 +1,48 @@ +package org.springframework.data.redis.core.index; + +import java.math.BigDecimal; +import java.util.Date; + +/** + * @author Yan Ma + */ +public class SortingIndexDefinition extends RedisIndexDefinition implements PathBasedRedisIndexDefinition { + public SortingIndexDefinition(String keyspace, String path) { + super(keyspace, path, path); + setValueTransformer(new DoubleValueTransformer()); + } + + public SortingIndexDefinition(String keyspace, String path, String indexName) { + super(keyspace, path, indexName); + setValueTransformer(new DoubleValueTransformer()); + } + + static class DoubleValueTransformer implements IndexValueTransformer { + + @Override + public Double convert(Object source) { + + if (source instanceof Date) { + Date date = (Date) source; + return (double) date.getTime(); + } else if (source instanceof Integer) { + Integer integer = (Integer) source; + return (double) integer.intValue(); + } else if (source instanceof Long) { + Long l = (Long) source; + return (double) l.doubleValue(); + } else if (source instanceof BigDecimal) { + BigDecimal bd = (BigDecimal) source; + return bd.toBigInteger().doubleValue(); + } else if (source instanceof Double) { + return (Double) source; + } else if (source instanceof Float) { + return (Double) source; + } else { + return null; + } + } + + } + +} diff --git a/src/test/java/org/springframework/data/redis/core/convert/SortingIndexedPropertyValueTest.java b/src/test/java/org/springframework/data/redis/core/convert/SortingIndexedPropertyValueTest.java new file mode 100644 index 0000000000..457aaafbe2 --- /dev/null +++ b/src/test/java/org/springframework/data/redis/core/convert/SortingIndexedPropertyValueTest.java @@ -0,0 +1,23 @@ +package org.springframework.data.redis.core.convert; + +import static org.junit.Assert.*; + +import org.junit.Test; +import org.springframework.data.redis.core.index.SortingIndexDefinition; + +/** + * DATAREDIS-814 + * + * @author Yan Ma + */ +public class SortingIndexedPropertyValueTest { + + @Test + public void test() { + + SortingIndexDefinition sid = new SortingIndexDefinition("AccountTransaction", "createdTimestamp"); + SortingIndexedPropertyValue sipv = new SortingIndexedPropertyValue("AccountTransaction", "createdTimestamp", + "createdTimestamp"); + } + +} diff --git a/src/test/java/org/springframework/data/redis/core/index/RedisRepositoryIndexTest.java b/src/test/java/org/springframework/data/redis/core/index/RedisRepositoryIndexTest.java new file mode 100644 index 0000000000..4f0875bbc3 --- /dev/null +++ b/src/test/java/org/springframework/data/redis/core/index/RedisRepositoryIndexTest.java @@ -0,0 +1,88 @@ +package org.springframework.data.redis.core.index; + +import static org.junit.Assert.*; + +import java.util.Date; +import java.util.List; +import java.util.Optional; +import java.util.UUID; + + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.data.redis.SettingsUtils; +import org.springframework.data.redis.connection.RedisConnectionFactory; +import org.springframework.data.redis.connection.jedis.JedisConnectionFactory; +import org.springframework.data.redis.core.RedisTemplate; +import org.springframework.data.redis.repository.configuration.EnableRedisRepositories; +import org.springframework.data.redis.test.util.RelaxedJUnit4ClassRunner; +import org.springframework.test.context.ContextConfiguration; + +import redis.clients.jedis.JedisPoolConfig; + +/** + * @author Yan Ma + */ +@RunWith(RelaxedJUnit4ClassRunner.class) +@ContextConfiguration +public class RedisRepositoryIndexTest { + + final Log logger = LogFactory.getLog(getClass()); + + @Configuration + @EnableRedisRepositories(indexConfiguration=TestIndexConfiguration.class) + public static class Config { + + @Bean + RedisConnectionFactory connectionFactory() { + + JedisConnectionFactory connectionFactory = new JedisConnectionFactory(); + connectionFactory.setHostName(SettingsUtils.getHost()); + connectionFactory.setPort(SettingsUtils.getPort()); + + JedisPoolConfig poolConfig = new JedisPoolConfig(); + poolConfig.setMaxWaitMillis(2000L); + connectionFactory.setPoolConfig(poolConfig); + connectionFactory.afterPropertiesSet(); + + return connectionFactory; + } + + @Bean + RedisTemplate redisTemplate(RedisConnectionFactory connectionFactory) { + RedisTemplate template = new RedisTemplate<>(); + template.setConnectionFactory(connectionFactory); + return template; + } + } + + @Autowired + TestPersonRepository repo; + + @Test + public void test_index_creation() { + TestPerson p = new TestPerson(); + p.id = UUID.randomUUID(); + p.firstName = "my.first,name_" + p.id; + String lastName = "my.last.name"; + p.lastName = lastName; + p.address = new TestAddress(); + p.address.city = "Pleasanton"; + p.address.state = TestState.CA; + p.address.streetNumber = 6220; + p.address.street = "stoneridge mall rd"; + p.gender = TestGender.FEMALE; + p.age = 25; + repo.save(p); + logger.info("after save: "+ p); + Optional findOne = repo.findById(p.id); + assertNotNull(findOne); + List findByLastName = repo.findByFirstName(lastName); + assertEquals(1, findByLastName.size()); + } +} diff --git a/src/test/java/org/springframework/data/redis/core/index/TestAddress.java b/src/test/java/org/springframework/data/redis/core/index/TestAddress.java new file mode 100644 index 0000000000..d10a828560 --- /dev/null +++ b/src/test/java/org/springframework/data/redis/core/index/TestAddress.java @@ -0,0 +1,25 @@ +package org.springframework.data.redis.core.index; + +import org.springframework.data.redis.core.index.Indexed; + +public class TestAddress { + + public String street; + @Indexed + public String city; + public TestState state; + public int streetNumber; + public static String getStateCategory(TestState state) { + String stateCategory; + switch(state){ + case CA: + case OR: + case WA: + stateCategory = "West"; + break; + default: + stateCategory = "others"; + } + return stateCategory; + } +} diff --git a/src/test/java/org/springframework/data/redis/core/index/TestGender.java b/src/test/java/org/springframework/data/redis/core/index/TestGender.java new file mode 100644 index 0000000000..e99f144012 --- /dev/null +++ b/src/test/java/org/springframework/data/redis/core/index/TestGender.java @@ -0,0 +1,5 @@ +package org.springframework.data.redis.core.index; + +public enum TestGender { + MALE, FEMALE +} diff --git a/src/test/java/org/springframework/data/redis/core/index/TestIndexConfiguration.java b/src/test/java/org/springframework/data/redis/core/index/TestIndexConfiguration.java new file mode 100644 index 0000000000..f024503ce6 --- /dev/null +++ b/src/test/java/org/springframework/data/redis/core/index/TestIndexConfiguration.java @@ -0,0 +1,32 @@ +package org.springframework.data.redis.core.index; + +import java.util.ArrayList; + +public class TestIndexConfiguration extends IndexConfiguration { + private static final String TESTPERSON_KEYSPACE = "TestPerson"; + + @Override + protected Iterable initialConfiguration() { + ArrayList indexes = new ArrayList(); + indexes.add(new SimpleIndexDefinition(TESTPERSON_KEYSPACE, "firstName")); + + //The sorted key on age + indexes.add(new SortingIndexDefinition(TESTPERSON_KEYSPACE, "age")); + //The sorted key on createdTimestamp + indexes.add(new SortingIndexDefinition(TESTPERSON_KEYSPACE, "createdTimestamp")); + + //The composite index + indexes.add(new CompositeSortingIndexDefinition(TESTPERSON_KEYSPACE, "", new IndexNameHandler(){ + @Override + public String getIndexName(TestPerson p) { + return TestAddress.getStateCategory(p.address.state); + } + }, new IndexValueHandler(){ + @Override + public Double getValue(TestPerson p) { + return (double) p.createdTimestamp.getTime(); + } + })); + return indexes; + } +} diff --git a/src/test/java/org/springframework/data/redis/core/index/TestPerson.java b/src/test/java/org/springframework/data/redis/core/index/TestPerson.java new file mode 100644 index 0000000000..a1b909e2dc --- /dev/null +++ b/src/test/java/org/springframework/data/redis/core/index/TestPerson.java @@ -0,0 +1,31 @@ +package org.springframework.data.redis.core.index; + +import java.util.Date; +import java.util.List; +import java.util.UUID; + +import org.springframework.data.annotation.Id; +import org.springframework.data.redis.core.RedisHash; +import org.springframework.data.redis.core.index.Indexed; + +@RedisHash("TestPerson") +public class TestPerson { + + @Id + public UUID id; + // Simple index defined in indexConfiguration + public String firstName; + @Indexed + public String lastName; + // Composite sorting index sample. + // The index name is derived from state name in the address + // while sorting by persont's created time. + public TestAddress address; + public TestGender gender; + // SortingIndex defined in indexConfigration + public int age; + public List children; + // Sorting index defined in indexConfiguration + public Date createdTimestamp = new Date(); + public Date updatedTimestamp = new Date(); +} diff --git a/src/test/java/org/springframework/data/redis/core/index/TestPersonRepository.java b/src/test/java/org/springframework/data/redis/core/index/TestPersonRepository.java new file mode 100644 index 0000000000..89fd4afc1c --- /dev/null +++ b/src/test/java/org/springframework/data/redis/core/index/TestPersonRepository.java @@ -0,0 +1,13 @@ +package org.springframework.data.redis.core.index; + +import java.util.List; +import java.util.UUID; + +import org.springframework.data.repository.CrudRepository; + +interface TestPersonRepository extends CrudRepository { + + List findByLastName(String lastname); + List findByAddress_City(String city); + List findByFirstName(String firstName); +} diff --git a/src/test/java/org/springframework/data/redis/core/index/TestState.java b/src/test/java/org/springframework/data/redis/core/index/TestState.java new file mode 100644 index 0000000000..f0084e4aa4 --- /dev/null +++ b/src/test/java/org/springframework/data/redis/core/index/TestState.java @@ -0,0 +1,5 @@ +package org.springframework.data.redis.core.index; + +public enum TestState { +AL, AK, AZ, AR, CA, CO, CT, DE, FL, GA, HI, ID, IL, IN, IA, KS, KY, LA, ME, MD, MA, MI, MN, MS, MO, MT, NE, NV, NH, MJ, NM, NY, NC, ND, OH, OK, OR, PA, RI, SC, SD, TN, TX, UT, VT, VA, WA, WV, WI, WY +} From 821159c9e4982ccba4cce2734d0d1f474218eff0 Mon Sep 17 00:00:00 2001 From: Yan Ma Date: Wed, 14 Aug 2019 16:36:31 -0700 Subject: [PATCH 2/4] sortingIndex creation and query --- .../data/redis/core/RedisQueryEngine.java | 172 ++++++++++++- .../repository/query/RedisOperationChain.java | 30 +++ .../repository/query/RedisQueryCreator.java | 43 +++- .../core/index/RedisRepositoryIndexTest.java | 239 ++++++++++++++---- .../core/index/TestPersonRepository.java | 13 +- 5 files changed, 434 insertions(+), 63 deletions(-) diff --git a/src/main/java/org/springframework/data/redis/core/RedisQueryEngine.java b/src/main/java/org/springframework/data/redis/core/RedisQueryEngine.java index 708816fcf1..42da20435f 100644 --- a/src/main/java/org/springframework/data/redis/core/RedisQueryEngine.java +++ b/src/main/java/org/springframework/data/redis/core/RedisQueryEngine.java @@ -15,13 +15,18 @@ */ package org.springframework.data.redis.core; +import java.nio.ByteBuffer; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collection; import java.util.Collections; import java.util.Comparator; +import java.util.HashSet; +import java.util.Iterator; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; +import java.util.Set; import org.springframework.data.geo.Circle; import org.springframework.data.geo.GeoResult; @@ -30,7 +35,9 @@ import org.springframework.data.keyvalue.core.QueryEngine; import org.springframework.data.keyvalue.core.SortAccessor; import org.springframework.data.keyvalue.core.query.KeyValueQuery; +import org.springframework.data.redis.connection.RedisConnection; import org.springframework.data.redis.connection.RedisGeoCommands.GeoLocation; +import org.springframework.data.redis.connection.RedisZSetCommands.Range; import org.springframework.data.redis.core.convert.GeoIndexedPropertyValue; import org.springframework.data.redis.core.convert.RedisData; import org.springframework.data.redis.repository.query.RedisOperationChain; @@ -81,7 +88,6 @@ public Collection execute(RedisOperationChain criteria, Comparator sor if (criteria == null || (CollectionUtils.isEmpty(criteria.getOrSismember()) && CollectionUtils.isEmpty(criteria.getSismember())) -// && CollectionUtils.isEmpty(criteria.getRanges())) && criteria.getNear() == null) { return (Collection) getAdapter().getAllOf(keyspace, offset, rows); } @@ -90,11 +96,11 @@ public Collection execute(RedisOperationChain criteria, Comparator sor List allKeys = new ArrayList<>(); if (!criteria.getSismember().isEmpty()) { - allKeys.addAll(connection.sInter(keys(keyspace + ":", criteria.getSismember()))); + allKeys.addAll(getKeysFromIsMembers(connection, keyspace + ":", criteria.getSismember())); } if (!criteria.getOrSismember().isEmpty()) { - allKeys.addAll(connection.sUnion(keys(keyspace + ":", criteria.getOrSismember()))); + allKeys.addAll(getKeysFromOrMembers(connection, keyspace + ":", criteria.getOrSismember())); } if (criteria.getNear() != null) { @@ -145,6 +151,166 @@ public Collection execute(RedisOperationChain criteria, Comparator sor return result; } + + private Collection getKeysFromIsMembers(RedisConnection connection, String prefix, + Set members) { + + // get simple types + Set simpleQueries = new HashSet(); + // get range types + Set rangeQueries = new HashSet(); + for (PathAndValue curr : members) { + if (curr.getFirstValue() instanceof Range) { + rangeQueries.add(curr); + } else { + simpleQueries.add(curr); + } + } + + Set sInter = new HashSet(); + // To limit the range query size should not exceeds 1. + if(rangeQueries.size() > 1) { + return sInter; + } + + if (!simpleQueries.isEmpty()) { + sInter = connection.sInter(keys(prefix, simpleQueries)); + } + + if (!simpleQueries.isEmpty() && sInter.isEmpty()) { + // we do have something in simple queries but nothing found in the intersections. + // no need of further checks. just return the empty set + return sInter; + } + + boolean rangeQueryOnly = simpleQueries.isEmpty(); + for (PathAndValue pathAndValue : rangeQueries) {// 0 or 1 loop only + byte[] keyInByte = getAdapter().getConverter().getConversionService().convert(prefix + pathAndValue.getPath(), + byte[].class); + Set zRangeByScore = connection.zRangeByScore(keyInByte, (Range) pathAndValue.getFirstValue()); + if(rangeQueryOnly){ + // no simple query but range query only. + sInter.addAll(zRangeByScore); + } else { + if (sInter.isEmpty()) { + // in case we support multiple range queries later this could be a quick return + // no more simple query overlapping with range query any more + // return the empty set + return sInter; + } else { + // remain intersections only + Iterator itSimpleQuery = sInter.iterator(); + while (itSimpleQuery.hasNext()) { + Iterator itTarget = zRangeByScore.iterator(); + byte[] source = itSimpleQuery.next(); + boolean isContained = false; + while (itTarget.hasNext()) { + byte[] target = itTarget.next(); + if (Arrays.equals(source, target)) { + isContained = true; + break; + } + } + if (!isContained) { + itSimpleQuery.remove(); + } + } + } + } + } + + return sInter; + } + + private Collection getKeysFromOrMembers(RedisConnection connection, String prefix, + Set members) { + + // get simple types + Set simpleQueries = new HashSet(); + // get range types + Set rangeQueries = new HashSet(); + + for (PathAndValue curr : members) { + if (curr.getFirstValue() instanceof Range) { + rangeQueries.add(curr); + } else { + simpleQueries.add(curr); + } + } + + Set sUnion = new HashSet(); + // To limit the range query size should not exceeds 1. + if(rangeQueries.size() > 1) return sUnion; + + Set simpleUnion = connection.sUnion(keys(prefix, simpleQueries)); + Set tmpSet = new HashSet(); + for (PathAndValue pathAndValue : rangeQueries) {// 0 or 1 loop only + byte[] keyInByte = getAdapter().getConverter().getConversionService() + .convert(prefix + pathAndValue.getPath(), byte[].class); + Set zRangeByScore = connection.zRangeByScore(keyInByte, (Range) pathAndValue.getFirstValue()); + // To merge range query with simple query union + // O(n) + O(m), n = the number fo range query results, m = the number simple query union + for(byte[] entry : zRangeByScore){ + tmpSet.add(ByteBuffer.wrap(entry)); + } + } + for(byte[] entry: simpleUnion){ + tmpSet.add(ByteBuffer.wrap(entry)); + } + for(ByteBuffer bb : tmpSet){ + sUnion.add(bb.array()); + } + // this method time complexity is O(n x m) +// Set extra = new HashSet(); +// Iterator itUnion = (Iterator) sUnion.iterator(); +// while (itUnion.hasNext()) { +// Iterator itRange = (Iterator) zRangeByScore.iterator(); +// byte[] source = itUnion.next(); +// boolean isContained = false; +// while (itRange.hasNext()) { +// byte[] target = itRange.next(); +// if (Arrays.equals(source, target)) { +// isContained = true; +// break; +// } +// } +// if (!isContained) { +// extra.add(source); +// } +// } + return sUnion; + } +// private Collection getKeysFromOrMembers(RedisConnection connection, String prefix, +// Set> members) { +// +// Set sUnion = new HashSet(); +// for (Set isMemberSet : members) { +// Collection keysFromIsMembers = getKeysFromIsMembers(connection, prefix, isMemberSet); +// if (sUnion.isEmpty()) { +// sUnion.addAll(keysFromIsMembers); +// } else { +// // merge +// Iterator it = (Iterator) keysFromIsMembers.iterator(); +// while (it.hasNext()) { +// Iterator itTarget = (Iterator) sUnion.iterator(); +// byte[] source = it.next(); +// boolean isContained = false; +// while (itTarget.hasNext()) { +// byte[] target = itTarget.next(); +// if (Arrays.equals(source, target)) { +// isContained = true; +// break; +// } +// } +// if (!isContained) { +// sUnion.add(source); +// } +// } +// } +// } +// return sUnion; +// } + /* * (non-Javadoc) * @see org.springframework.data.keyvalue.core.QueryEngine#execute(java.lang.Object, java.lang.Object, int, int, java.lang.String) diff --git a/src/main/java/org/springframework/data/redis/repository/query/RedisOperationChain.java b/src/main/java/org/springframework/data/redis/repository/query/RedisOperationChain.java index 14ed992db6..46d9ae2917 100644 --- a/src/main/java/org/springframework/data/redis/repository/query/RedisOperationChain.java +++ b/src/main/java/org/springframework/data/redis/repository/query/RedisOperationChain.java @@ -28,12 +28,14 @@ import org.springframework.data.geo.Point; import org.springframework.lang.Nullable; import org.springframework.util.Assert; +import org.springframework.util.ObjectUtils; /** * Simple set of operations required to run queries against Redis. * * @author Christoph Strobl * @author Mark Paluch + * @author Yan Ma * @since 1.7 */ public class RedisOperationChain { @@ -125,6 +127,34 @@ public Object getFirstValue() { public String toString() { return path + ":" + (isSingleValue() ? getFirstValue() : values); } + + @Override + public int hashCode() { + + int result = ObjectUtils.nullSafeHashCode(path); + result += ObjectUtils.nullSafeHashCode(values); + return result; + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + if (obj == null) { + return false; + } + if (!(obj instanceof PathAndValue)) { + return false; + } + PathAndValue that = (PathAndValue) obj; + if (!ObjectUtils.nullSafeEquals(this.path, that.path)) { + return false; + } + + return ObjectUtils.nullSafeEquals(this.values, that.values); + } + } /** diff --git a/src/main/java/org/springframework/data/redis/repository/query/RedisQueryCreator.java b/src/main/java/org/springframework/data/redis/repository/query/RedisQueryCreator.java index 1edcd7b4eb..594bcb9ce5 100644 --- a/src/main/java/org/springframework/data/redis/repository/query/RedisQueryCreator.java +++ b/src/main/java/org/springframework/data/redis/repository/query/RedisQueryCreator.java @@ -15,7 +15,10 @@ */ package org.springframework.data.redis.repository.query; +import java.math.BigDecimal; +import java.util.Date; import java.util.Iterator; +import java.util.LinkedHashSet; import org.springframework.dao.InvalidDataAccessApiUsageException; import org.springframework.data.domain.Sort; @@ -24,7 +27,9 @@ import org.springframework.data.geo.Metrics; import org.springframework.data.geo.Point; import org.springframework.data.keyvalue.core.query.KeyValueQuery; +import org.springframework.data.redis.connection.RedisZSetCommands.Range; import org.springframework.data.redis.repository.query.RedisOperationChain.NearPath; +import org.springframework.data.redis.repository.query.RedisOperationChain.PathAndValue; import org.springframework.data.repository.query.ParameterAccessor; import org.springframework.data.repository.query.parser.AbstractQueryCreator; import org.springframework.data.repository.query.parser.Part; @@ -36,6 +41,7 @@ * * @author Christoph Strobl * @author Mark Paluch + * @author Yan Ma * @since 1.7 */ public class RedisQueryCreator extends AbstractQueryCreator, RedisOperationChain> { @@ -69,6 +75,22 @@ private RedisOperationChain from(Part part, Iterator iterator, RedisOper case NEAR: sink.near(getNearPath(part, iterator)); break; + case GREATER_THAN: + sink.sismember(part.getProperty().toDotPath(), new Range().gt(getComparableValue(iterator.next()))); + break; + case GREATER_THAN_EQUAL: + sink.sismember(part.getProperty().toDotPath(), new Range().gte(getComparableValue(iterator.next()))); + break; + case LESS_THAN: + sink.sismember(part.getProperty().toDotPath(), new Range().lt(getComparableValue(iterator.next()))); + break; + case LESS_THAN_EQUAL: + sink.sismember(part.getProperty().toDotPath(), new Range().lte(getComparableValue(iterator.next()))); + break; + case BETWEEN: + sink.sismember(part.getProperty().toDotPath(), + new Range().gte(getComparableValue(iterator.next())).lte(getComparableValue(iterator.next()))); + break; default: throw new IllegalArgumentException(String.format("%s is not supported for Redis query derivation!", part.getType())); } @@ -76,7 +98,22 @@ private RedisOperationChain from(Part part, Iterator iterator, RedisOper return sink; } - /* + /** + * Derives a comparable value from the object + * @param obj + * @return + */ + private Object getComparableValue(Object obj) { + if(obj instanceof Date){ + return ((Date) obj).getTime(); + } +// if(obj instanceof Integer || obj instanceof Long || obj instanceof Double || obj instanceof Float || obj instanceof BigDecimal){ +// return obj; +// } + return obj; + } + + /* * (non-Javadoc) * @see org.springframework.data.repository.query.parser.AbstractQueryCreator#and(org.springframework.data.repository.query.parser.Part, java.lang.Object, java.util.Iterator) */ @@ -123,8 +160,8 @@ private NearPath getNearPath(Part part, Iterator iterator) { Object o = iterator.next(); - Point point; - Distance distance; + Point point = null; + Distance distance = null; if (o instanceof Circle) { diff --git a/src/test/java/org/springframework/data/redis/core/index/RedisRepositoryIndexTest.java b/src/test/java/org/springframework/data/redis/core/index/RedisRepositoryIndexTest.java index 4f0875bbc3..8b441fd173 100644 --- a/src/test/java/org/springframework/data/redis/core/index/RedisRepositoryIndexTest.java +++ b/src/test/java/org/springframework/data/redis/core/index/RedisRepositoryIndexTest.java @@ -7,9 +7,10 @@ import java.util.Optional; import java.util.UUID; - import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; +import org.junit.After; +import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; import org.springframework.beans.factory.annotation.Autowired; @@ -23,6 +24,7 @@ import org.springframework.data.redis.test.util.RelaxedJUnit4ClassRunner; import org.springframework.test.context.ContextConfiguration; +import kotlin.random.Random; import redis.clients.jedis.JedisPoolConfig; /** @@ -31,58 +33,185 @@ @RunWith(RelaxedJUnit4ClassRunner.class) @ContextConfiguration public class RedisRepositoryIndexTest { - - final Log logger = LogFactory.getLog(getClass()); - - @Configuration - @EnableRedisRepositories(indexConfiguration=TestIndexConfiguration.class) - public static class Config { - - @Bean - RedisConnectionFactory connectionFactory() { - - JedisConnectionFactory connectionFactory = new JedisConnectionFactory(); - connectionFactory.setHostName(SettingsUtils.getHost()); - connectionFactory.setPort(SettingsUtils.getPort()); - - JedisPoolConfig poolConfig = new JedisPoolConfig(); - poolConfig.setMaxWaitMillis(2000L); - connectionFactory.setPoolConfig(poolConfig); - connectionFactory.afterPropertiesSet(); - - return connectionFactory; - } - - @Bean - RedisTemplate redisTemplate(RedisConnectionFactory connectionFactory) { - RedisTemplate template = new RedisTemplate<>(); - template.setConnectionFactory(connectionFactory); - return template; - } - } - - @Autowired - TestPersonRepository repo; - - @Test - public void test_index_creation() { - TestPerson p = new TestPerson(); - p.id = UUID.randomUUID(); - p.firstName = "my.first,name_" + p.id; - String lastName = "my.last.name"; - p.lastName = lastName; - p.address = new TestAddress(); - p.address.city = "Pleasanton"; - p.address.state = TestState.CA; - p.address.streetNumber = 6220; - p.address.street = "stoneridge mall rd"; - p.gender = TestGender.FEMALE; - p.age = 25; - repo.save(p); - logger.info("after save: "+ p); - Optional findOne = repo.findById(p.id); - assertNotNull(findOne); - List findByLastName = repo.findByFirstName(lastName); - assertEquals(1, findByLastName.size()); - } + + final Log logger = LogFactory.getLog(getClass()); + + @Configuration + @EnableRedisRepositories(indexConfiguration = TestIndexConfiguration.class) + public static class Config { + + @Bean + RedisConnectionFactory connectionFactory() { + + JedisConnectionFactory connectionFactory = new JedisConnectionFactory(); + connectionFactory.setHostName(SettingsUtils.getHost()); + connectionFactory.setPort(SettingsUtils.getPort()); + + JedisPoolConfig poolConfig = new JedisPoolConfig(); + poolConfig.setMaxWaitMillis(2000L); + connectionFactory.setPoolConfig(poolConfig); + connectionFactory.afterPropertiesSet(); + + return connectionFactory; + } + + @Bean + RedisTemplate redisTemplate(RedisConnectionFactory connectionFactory) { + RedisTemplate template = new RedisTemplate<>(); + template.setConnectionFactory(connectionFactory); + return template; + } + } + + @Autowired TestPersonRepository repo; + + @Before + public void setup(){ + repo.deleteAll(); + } + + @After + public void cleanup(){ + repo.deleteAll(); + } + + @Test + public void test_index_creation_and_simple_queries() { + + int random = Random.Default.nextInt(10); + Date start = new Date(); + while (random == 0) + random = Random.Default.nextInt(10); + logger.info("random number = " + random); + String lastName = "my.last.name" + new Date(); + String city = "Pleasanton"; + for (int i = 0; i < random; i++) { + TestPerson p = new TestPerson(); + p.id = UUID.randomUUID(); + String firstName = "my.first,name_" + p.id; + p.firstName = firstName; + p.lastName = lastName; + p.address = new TestAddress(); + p.address.city = city; + p.address.state = TestState.CA; + p.address.streetNumber = 6220; + p.address.street = "stoneridge mall rd"; + p.gender = TestGender.FEMALE; + p.age = 25; + repo.save(p); + logger.info("after save: " + p); + // test single match query by primary key + Optional findOne = repo.findById(p.id); + assertNotNull(findOne); + // test single match query by specify key value + List findByFirstName = repo.findByFirstName(firstName); + assertEquals(1, findByFirstName.size()); + } + Date end = new Date(); + // test range query + List findByCreatedTimestampBetween = repo.findByCreatedTimestampBetween(start, end); + assertEquals(random, findByCreatedTimestampBetween.size()); + long oneMinuteAgo = start.getTime() - 60000; + long oneSecondAgo = start.getTime() - 1000; + List findByCreatedTimestampBetween2 = repo.findByCreatedTimestampBetween(new Date(oneMinuteAgo), new Date(oneSecondAgo)); + assertEquals(0, findByCreatedTimestampBetween2.size()); + long oneSecondAfter = end.getTime() + 1000; + long oneMinuteAfter = end.getTime() + 60000; + List findByCreatedTimestampBetween3 = repo.findByCreatedTimestampBetween(new Date(oneSecondAfter), new Date(oneMinuteAfter)); + assertEquals(0, findByCreatedTimestampBetween3.size()); + // test less than + List findByCreatedTimestampLessThan = repo.findByCreatedTimestampLessThan(end); + assertEquals(random, findByCreatedTimestampLessThan.size()); + List findByCreatedTimestampLessThan1 = repo.findByCreatedTimestampLessThan(start); + assertEquals(0, findByCreatedTimestampLessThan1.size()); + // test less than equal + List findByCreatedTimestampLessThanEqual = repo.findByCreatedTimestampLessThanEqual(end); + assertEquals(random, findByCreatedTimestampLessThanEqual.size()); + List findByCreatedTimestampLessThanEqual1 = repo.findByCreatedTimestampLessThanEqual(start); + assertEquals(0, findByCreatedTimestampLessThanEqual1.size()); + // test batch simple query on simple attributes + List findByLastName = repo.findByLastName(lastName); + assertEquals(random, findByLastName.size()); + List findByLastName2 = repo.findByLastName("new_last_name"+new Date()); + assertEquals(0, findByLastName2.size()); + // test batch simple query on nested attributes + List findByAddressCity = repo.findByAddressCity(city); + assertEquals(random, findByAddressCity.size()); + } + + @Test + public void test_complex_queries() throws InterruptedException { + Date start = new Date(); + TestPerson p1 = new TestPerson(); + p1.id = UUID.randomUUID(); + String firstName1 = "f1"; + p1.firstName = firstName1; + String lastName = "last"; + p1.lastName = lastName; + p1.address = new TestAddress(); + String cityName = "city"; + p1.address.city = cityName; + repo.save(p1); + TestPerson p2 = new TestPerson(); + p2.id = UUID.randomUUID(); + p2.firstName = "f2"; + p2.lastName = lastName; + p2.address = new TestAddress(); + p2.address.city = cityName; + repo.save(p2); + TestPerson p3 = new TestPerson(); + p3.id = UUID.randomUUID(); + p3.firstName = "f3"; + p3.lastName = lastName; + p3.address = new TestAddress(); + p3.address.city = cityName; + repo.save(p3); + + Date end = new Date(); + String SOME_POSTFIX = "something_else"; + // To test List findByFirstNameAndLastName(String firstName, String lastName); + List findByFirstNameAndLastName = repo.findByFirstNameAndLastName(firstName1, lastName); + assertEquals(1, findByFirstNameAndLastName.size()); + List findByFirstNameAndLastName1 = repo.findByFirstNameAndLastName(firstName1+SOME_POSTFIX, lastName); + assertEquals(0, findByFirstNameAndLastName1.size()); + List findByFirstNameAndLastName2 = repo.findByFirstNameAndLastName(firstName1, lastName+SOME_POSTFIX); + assertEquals(0, findByFirstNameAndLastName2.size()); + // To test List findByFirstNameOrLastName(String firstName, String lastName); + List findByFirstNameOrLastName = repo.findByFirstNameOrLastName(firstName1, lastName); + assertEquals(3, findByFirstNameOrLastName.size()); + List findByFirstNameOrLastName1 = repo.findByFirstNameOrLastName(firstName1+SOME_POSTFIX, lastName); + assertEquals(3, findByFirstNameOrLastName1.size()); + List findByFirstNameOrLastName2 = repo.findByFirstNameOrLastName(firstName1, lastName+SOME_POSTFIX); + assertEquals(1, findByFirstNameOrLastName2.size()); + // To test List findByAddressCityAndCreatedTimestampBetween(String city, Date start, Date end); + List findByAddressCityAndCreatedTimestampBetween = repo.findByAddressCityAndCreatedTimestampBetween(cityName, start, end); + assertEquals(3, findByAddressCityAndCreatedTimestampBetween.size()); + List findByAddressCityAndCreatedTimestampBetween1 = repo.findByAddressCityAndCreatedTimestampBetween(cityName+SOME_POSTFIX, start, end); + assertEquals(0, findByAddressCityAndCreatedTimestampBetween1.size()); + Date oneMinuteAgo = new Date(start.getTime() - 60000); + Date oneSecondAgo = new Date(start.getTime() - 1000); + List findByAddressCityAndCreatedTimestampBetween2 = repo.findByAddressCityAndCreatedTimestampBetween(cityName, oneMinuteAgo, oneSecondAgo); + assertEquals(0, findByAddressCityAndCreatedTimestampBetween2.size()); + Date oneSecondAfter = new Date(end.getTime() + 1000); + Date oneMinuteAfter = new Date(end.getTime() + 60000); + List findByAddressCityAndCreatedTimestampBetween3 = repo.findByAddressCityAndCreatedTimestampBetween(cityName, oneSecondAfter, oneMinuteAfter); + assertEquals(0, findByAddressCityAndCreatedTimestampBetween3.size()); + List findByAddressCityAndCreatedTimestampBetween4 = repo.findByAddressCityAndCreatedTimestampBetween(cityName, oneMinuteAgo, oneMinuteAfter); + assertEquals(3, findByAddressCityAndCreatedTimestampBetween4.size()); + // To test List findByAddressCityOrCreatedTimestampGreaterThan(String city, Date start); + List findByAddressCityOrCreatedTimestampGreaterThan = repo.findByAddressCityOrCreatedTimestampGreaterThan(cityName, start); + assertEquals(3, findByAddressCityOrCreatedTimestampGreaterThan.size()); + List findByAddressCityOrCreatedTimestampGreaterThan1 = repo.findByAddressCityOrCreatedTimestampGreaterThan(cityName+SOME_POSTFIX, start); + assertEquals(2, findByAddressCityOrCreatedTimestampGreaterThan1.size()); + List findByAddressCityOrCreatedTimestampGreaterThan11 = repo.findByAddressCityOrCreatedTimestampGreaterThanEqual(cityName+SOME_POSTFIX, start); + assertEquals(3, findByAddressCityOrCreatedTimestampGreaterThan11.size()); + List findByAddressCityOrCreatedTimestampGreaterThan2 = repo.findByAddressCityOrCreatedTimestampGreaterThan(cityName, oneSecondAgo); + assertEquals(3, findByAddressCityOrCreatedTimestampGreaterThan2.size()); + List findByAddressCityOrCreatedTimestampGreaterThan3 = repo.findByAddressCityOrCreatedTimestampGreaterThan(cityName, oneSecondAfter); + assertEquals(3, findByAddressCityOrCreatedTimestampGreaterThan3.size()); + List findByAddressCityOrCreatedTimestampGreaterThan4 = repo.findByAddressCityOrCreatedTimestampGreaterThan(cityName+SOME_POSTFIX, oneSecondAgo); + assertEquals(3, findByAddressCityOrCreatedTimestampGreaterThan4.size()); + List findByAddressCityOrCreatedTimestampGreaterThan5 = repo.findByAddressCityOrCreatedTimestampGreaterThan(cityName+SOME_POSTFIX, oneSecondAfter); + assertEquals(0, findByAddressCityOrCreatedTimestampGreaterThan5.size()); + } } diff --git a/src/test/java/org/springframework/data/redis/core/index/TestPersonRepository.java b/src/test/java/org/springframework/data/redis/core/index/TestPersonRepository.java index 89fd4afc1c..40887a5903 100644 --- a/src/test/java/org/springframework/data/redis/core/index/TestPersonRepository.java +++ b/src/test/java/org/springframework/data/redis/core/index/TestPersonRepository.java @@ -1,5 +1,6 @@ package org.springframework.data.redis.core.index; +import java.util.Date; import java.util.List; import java.util.UUID; @@ -7,7 +8,15 @@ interface TestPersonRepository extends CrudRepository { - List findByLastName(String lastname); - List findByAddress_City(String city); + List findByLastName(String lastName); + List findByAddressCity(String city); List findByFirstName(String firstName); + List findByCreatedTimestampBetween(Date start, Date end); + List findByCreatedTimestampLessThan(Date end); + List findByCreatedTimestampLessThanEqual(Date end); + List findByFirstNameAndLastName(String firstName, String lastName); + List findByFirstNameOrLastName(String firstName, String lastName); + List findByAddressCityAndCreatedTimestampBetween(String city, Date start, Date end); + List findByAddressCityOrCreatedTimestampGreaterThan(String city, Date start); + List findByAddressCityOrCreatedTimestampGreaterThanEqual(String city, Date start); } From c886d82d60863b1cc8d5412f0e0ed3b30c7e7077 Mon Sep 17 00:00:00 2001 From: Yan Ma Date: Thu, 15 Aug 2019 15:29:19 -0700 Subject: [PATCH 3/4] CompositeSortingIndex and unit test cases. --- .../redis/core/convert/PathIndexResolver.java | 32 ++++++++++++++++++- .../CompositeSortingIndexDefinition.java | 12 ++++--- .../core/index/RedisRepositoryIndexTest.java | 29 +++++++++++++++-- 3 files changed, 66 insertions(+), 7 deletions(-) diff --git a/src/main/java/org/springframework/data/redis/core/convert/PathIndexResolver.java b/src/main/java/org/springframework/data/redis/core/convert/PathIndexResolver.java index 7c62e602bf..957031af3a 100644 --- a/src/main/java/org/springframework/data/redis/core/convert/PathIndexResolver.java +++ b/src/main/java/org/springframework/data/redis/core/convert/PathIndexResolver.java @@ -16,6 +16,7 @@ package org.springframework.data.redis.core.convert; import java.util.Arrays; +import java.util.Collection; import java.util.HashSet; import java.util.LinkedHashSet; import java.util.Map; @@ -194,9 +195,38 @@ private TypeInformation updateTypeHintForActualValue(TypeInformation typeH } }); - + + // customized code for top level index + indexes.addAll(resolveCompositeIndexes(keyspace, path, typeInformation, value)); + return indexes; } + + private Collection resolveCompositeIndexes(String keyspace, String path, + TypeInformation typeInformation, Object value) { + + Set data = new LinkedHashSet(); + if (indexConfiguration.hasIndexFor(keyspace, path)) { + IndexingContext context = new IndexingContext(keyspace, path, typeInformation); + + for (IndexDefinition indexDefinition : indexConfiguration.getIndexDefinitionsFor(keyspace, path)) { + if (!verifyConditions(indexDefinition.getConditions(), value, context)) { + continue; + } + Object transformedValue = indexDefinition.valueTransformer().convert(value); + + IndexedData indexedData = null; + if (transformedValue == null) { + indexedData = new RemoveIndexedData(indexedData); + } else { + indexedData = indexedDataFactoryProvider.getIndexedDataFactory(indexDefinition).createIndexedDataFor(value); + } + data.add(indexedData); + } + } + + return data; + } protected Set resolveIndex(String keyspace, String propertyPath, @Nullable PersistentProperty property, @Nullable Object value) { diff --git a/src/main/java/org/springframework/data/redis/core/index/CompositeSortingIndexDefinition.java b/src/main/java/org/springframework/data/redis/core/index/CompositeSortingIndexDefinition.java index c163f0925d..872f153d57 100644 --- a/src/main/java/org/springframework/data/redis/core/index/CompositeSortingIndexDefinition.java +++ b/src/main/java/org/springframework/data/redis/core/index/CompositeSortingIndexDefinition.java @@ -2,7 +2,11 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; + /** + * CompositeSortingIndexDefinition serves the requirement that key name and value derive from different attributes of the + * same parent object. + * * @author Yan Ma */ public class CompositeSortingIndexDefinition extends SortingIndexDefinition { @@ -12,7 +16,7 @@ public class CompositeSortingIndexDefinition extends SortingIndexDefinition { public CompositeSortingIndexDefinition(String keyspace, String path, IndexNameHandler indexNameHandler, IndexValueHandler indexValueHandler) { - + super(keyspace, path, path); this.indexNameHandler = indexNameHandler; this.indexValueHandler = indexValueHandler; @@ -20,7 +24,7 @@ public CompositeSortingIndexDefinition(String keyspace, String path, IndexNameHa } public String getIndexName(T obj) { - + T typedObj = obj; String indexName = null; try { @@ -33,7 +37,7 @@ public String getIndexName(T obj) { } public Object getIndexValue(T value) { - + T typedValue = (T) value; Double indexValue = null; try { @@ -54,7 +58,7 @@ public CompositeSortingIndexValueTransformer(IndexValueHandler indexValueHand @Override public Object convert(Object source) { - + Double value = null; try { value = indexValueHandler.getValue((T) source); diff --git a/src/test/java/org/springframework/data/redis/core/index/RedisRepositoryIndexTest.java b/src/test/java/org/springframework/data/redis/core/index/RedisRepositoryIndexTest.java index 8b441fd173..420906bc74 100644 --- a/src/test/java/org/springframework/data/redis/core/index/RedisRepositoryIndexTest.java +++ b/src/test/java/org/springframework/data/redis/core/index/RedisRepositoryIndexTest.java @@ -5,6 +5,7 @@ import java.util.Date; import java.util.List; import java.util.Optional; +import java.util.Set; import java.util.UUID; import org.apache.commons.logging.Log; @@ -25,6 +26,7 @@ import org.springframework.test.context.ContextConfiguration; import kotlin.random.Random; +import redis.clients.jedis.Jedis; import redis.clients.jedis.JedisPoolConfig; /** @@ -64,7 +66,7 @@ RedisConnectionFactory connectionFactory() { } @Autowired TestPersonRepository repo; - + Jedis jedis = null; @Before public void setup(){ repo.deleteAll(); @@ -73,6 +75,9 @@ public void setup(){ @After public void cleanup(){ repo.deleteAll(); + if(jedis!=null){ + jedis.close(); + } } @Test @@ -85,6 +90,7 @@ public void test_index_creation_and_simple_queries() { logger.info("random number = " + random); String lastName = "my.last.name" + new Date(); String city = "Pleasanton"; + TestState testState = TestState.CA; for (int i = 0; i < random; i++) { TestPerson p = new TestPerson(); p.id = UUID.randomUUID(); @@ -93,7 +99,7 @@ public void test_index_creation_and_simple_queries() { p.lastName = lastName; p.address = new TestAddress(); p.address.city = city; - p.address.state = TestState.CA; + p.address.state = testState; p.address.streetNumber = 6220; p.address.street = "stoneridge mall rd"; p.gender = TestGender.FEMALE; @@ -137,6 +143,21 @@ public void test_index_creation_and_simple_queries() { // test batch simple query on nested attributes List findByAddressCity = repo.findByAddressCity(city); assertEquals(random, findByAddressCity.size()); + // To test the CompositeSortingIndex + // The key set name should be West + jedis = jedis==null? new Jedis() :jedis; + String zset = "TestPerson" + ":" + TestAddress.getStateCategory(testState); + assertEquals("zset", jedis.type(zset)); + // The total number of values should be the same as the random int + assertTrue(random == jedis.zcard(zset)); + // They should be sorted per createdTimestamp + Set zrange = jedis.zrange(zset, 0, -1); + double prev = 0; + for(String key : zrange){ + double curr = Double.valueOf(jedis.hget("TestPerson" + ":" + key, "createdTimestamp")); + assertTrue(prev < curr); + prev = curr; + } } @Test @@ -151,6 +172,7 @@ public void test_complex_queries() throws InterruptedException { p1.address = new TestAddress(); String cityName = "city"; p1.address.city = cityName; + p1.address.state = TestState.CA; repo.save(p1); TestPerson p2 = new TestPerson(); p2.id = UUID.randomUUID(); @@ -158,6 +180,7 @@ public void test_complex_queries() throws InterruptedException { p2.lastName = lastName; p2.address = new TestAddress(); p2.address.city = cityName; + p2.address.state = TestState.CA; repo.save(p2); TestPerson p3 = new TestPerson(); p3.id = UUID.randomUUID(); @@ -165,6 +188,7 @@ public void test_complex_queries() throws InterruptedException { p3.lastName = lastName; p3.address = new TestAddress(); p3.address.city = cityName; + p3.address.state = TestState.CA; repo.save(p3); Date end = new Date(); @@ -214,4 +238,5 @@ public void test_complex_queries() throws InterruptedException { List findByAddressCityOrCreatedTimestampGreaterThan5 = repo.findByAddressCityOrCreatedTimestampGreaterThan(cityName+SOME_POSTFIX, oneSecondAfter); assertEquals(0, findByAddressCityOrCreatedTimestampGreaterThan5.size()); } + } From 59f970a3a076e57567e32216fb27216a6119c819 Mon Sep 17 00:00:00 2001 From: Yan Ma Date: Thu, 15 Aug 2019 16:36:03 -0700 Subject: [PATCH 4/4] unit tests --- .../SortingIndexedPropertyValueTest.java | 23 ------------------- .../core/index/RedisRepositoryIndexTest.java | 2 +- 2 files changed, 1 insertion(+), 24 deletions(-) delete mode 100644 src/test/java/org/springframework/data/redis/core/convert/SortingIndexedPropertyValueTest.java diff --git a/src/test/java/org/springframework/data/redis/core/convert/SortingIndexedPropertyValueTest.java b/src/test/java/org/springframework/data/redis/core/convert/SortingIndexedPropertyValueTest.java deleted file mode 100644 index 457aaafbe2..0000000000 --- a/src/test/java/org/springframework/data/redis/core/convert/SortingIndexedPropertyValueTest.java +++ /dev/null @@ -1,23 +0,0 @@ -package org.springframework.data.redis.core.convert; - -import static org.junit.Assert.*; - -import org.junit.Test; -import org.springframework.data.redis.core.index.SortingIndexDefinition; - -/** - * DATAREDIS-814 - * - * @author Yan Ma - */ -public class SortingIndexedPropertyValueTest { - - @Test - public void test() { - - SortingIndexDefinition sid = new SortingIndexDefinition("AccountTransaction", "createdTimestamp"); - SortingIndexedPropertyValue sipv = new SortingIndexedPropertyValue("AccountTransaction", "createdTimestamp", - "createdTimestamp"); - } - -} diff --git a/src/test/java/org/springframework/data/redis/core/index/RedisRepositoryIndexTest.java b/src/test/java/org/springframework/data/redis/core/index/RedisRepositoryIndexTest.java index 420906bc74..fe1015bc18 100644 --- a/src/test/java/org/springframework/data/redis/core/index/RedisRepositoryIndexTest.java +++ b/src/test/java/org/springframework/data/redis/core/index/RedisRepositoryIndexTest.java @@ -134,7 +134,7 @@ public void test_index_creation_and_simple_queries() { List findByCreatedTimestampLessThanEqual = repo.findByCreatedTimestampLessThanEqual(end); assertEquals(random, findByCreatedTimestampLessThanEqual.size()); List findByCreatedTimestampLessThanEqual1 = repo.findByCreatedTimestampLessThanEqual(start); - assertEquals(0, findByCreatedTimestampLessThanEqual1.size()); + assertTrue(findByCreatedTimestampLessThanEqual1.size() <= 1); // test batch simple query on simple attributes List findByLastName = repo.findByLastName(lastName); assertEquals(random, findByLastName.size());